From ffa3434abc6ec16f6865ea31c79ac3b53af66ea6 Mon Sep 17 00:00:00 2001 From: Emanuele Bardelli Date: Fri, 7 Nov 2025 12:32:23 -0800 Subject: [PATCH 01/22] Fix type discovery when the first row has NULL values --- packages/datasources/duckdb/index.cjs | 12 ++- packages/datasources/duckdb/test/test.js | 112 +++++++++++++++++++++++ 2 files changed, 123 insertions(+), 1 deletion(-) diff --git a/packages/datasources/duckdb/index.cjs b/packages/datasources/duckdb/index.cjs index 0dae79c8e3..8e7e7cbf98 100644 --- a/packages/datasources/duckdb/index.cjs +++ b/packages/datasources/duckdb/index.cjs @@ -29,6 +29,11 @@ function standardizeRow(obj) { * @returns {EvidenceType | undefined} */ function nativeTypeToEvidenceType(data) { + // Handle null/undefined explicitly: let callers decide (they will mark as INFERRED) + if (data === null || data === undefined) return undefined; + // Some runtimes may yield bigint; normalize higher-level handling elsewhere but + // treat here as a number type. + if (typeof data === 'bigint') return EvidenceType.NUMBER; switch (typeof data) { case 'number': return EvidenceType.NUMBER; @@ -40,7 +45,9 @@ function nativeTypeToEvidenceType(data) { if (data instanceof Date) { return EvidenceType.DATE; } - throw new Error(`Unsupported object type: ${data}`); + // For other object types (arrays, plain objects, buffers, etc.) return + // undefined so the caller will mark the column as inferred (STRING). + return undefined; default: return EvidenceType.STRING; } @@ -51,6 +58,9 @@ function nativeTypeToEvidenceType(data) { * @returns {import('@evidence-dev/db-commons').ColumnDefinition[]} */ const mapResultsToEvidenceColumnTypes = function (rows) { + // If there are no rows, return an empty column list. Caller may provide DESCRIBE-based types. + if (!rows || rows.length === 0) return []; + return Object.entries(rows[0]).map(([name, value]) => { /** @type {TypeFidelity} */ let typeFidelity = TypeFidelity.PRECISE; diff --git a/packages/datasources/duckdb/test/test.js b/packages/datasources/duckdb/test/test.js index 3b6f043044..b788a77012 100644 --- a/packages/datasources/duckdb/test/test.js +++ b/packages/datasources/duckdb/test/test.js @@ -181,6 +181,118 @@ test('query runs', async () => { } }); +test('handles nulls in first row', async () => { + try { + const { rows: row_generator, columnTypes } = await runQuery( + `select + NULL as boolean_col, + CAST(NULL AS TINYINT) as tinyint_col, + CAST(NULL AS SMALLINT) as smallint_col, + CAST(NULL AS INTEGER) as int_col, + CAST(NULL AS BIGINT) as bigint_col, + CAST(NULL AS HUGEINT) as hugeint_col, + CAST(NULL AS UTINYINT) as utinyint_col, + CAST(NULL AS USMALLINT) as usmallint_col, + CAST(NULL AS UINTEGER) as uint_col, + CAST(NULL AS UBIGINT) as ubigint_col, + CAST(NULL AS DATE) as date_col, + CAST(NULL AS TIME) as time_col, + CAST(NULL AS TIMESTAMP) as timestamp_col, + CAST(NULL AS TIMESTAMP_S) as timestamp_s_col, + CAST(NULL AS TIMESTAMP_MS) as timestamp_ms_col, + CAST(NULL AS TIMESTAMP_NS) as timestamp_ns_col, + CAST(NULL AS TIME WITH TIME ZONE) as time_with_tz_col, + CAST(NULL AS TIMESTAMP WITH TIME ZONE) as timestamp_with_tz_col, + CAST(NULL AS FLOAT) as float_col, + CAST(NULL AS DOUBLE) as double_col, + CAST(NULL AS DECIMAL(4,1)) as decimal_4_1_col, + CAST(NULL AS DECIMAL(9,4)) as decimal_9_4_col, + CAST(NULL AS DECIMAL(18,6)) as decimal_18_6_col, + CAST(NULL AS DECIMAL(38,10)) as decimal_38_10_col, + CAST(NULL AS UUID) as uuid_col, + CAST(NULL AS INTERVAL) as interval_col, + CAST(NULL AS VARCHAR) as varchar_col, + CAST(NULL AS BLOB) as blob_col, + CAST(NULL AS BIT) as bit_col + UNION ALL + select + true as boolean_col, + CAST(127 AS TINYINT) as tinyint_col, + CAST(32767 AS SMALLINT) as smallint_col, + CAST(2147483647 AS INTEGER) as int_col, + CAST(9223372036854775807 AS BIGINT) as bigint_col, + CAST('9223372036854775807000000' AS HUGEINT) as hugeint_col, + CAST(255 AS UTINYINT) as utinyint_col, + CAST(65535 AS USMALLINT) as usmallint_col, + CAST(4294967295 AS UINTEGER) as uint_col, + CAST(18446744073709551615 AS UBIGINT) as ubigint_col, + CURRENT_DATE as date_col, + CURRENT_TIME as time_col, + CURRENT_TIMESTAMP as timestamp_col, + CAST('2023-12-13' AS TIMESTAMP_S) as timestamp_s_col, + CAST('2023-12-13 12:34:56.789' AS TIMESTAMP_MS) as timestamp_ms_col, + CAST('2023-12-13 12:34:56.789123456' AS TIMESTAMP_NS) as timestamp_ns_col, + TIMETZ '1992-09-20 11:30:00.123456-02:00' as time_with_tz_col, + TIMESTAMPTZ '1992-09-20 11:30:00.123456' as timestamp_with_tz_col, + CAST(3.14 AS FLOAT) as float_col, + CAST(3.14 AS DOUBLE) as double_col, + CAST(3.14 AS DECIMAL(4,1)) as decimal_4_1_col, + CAST(3.14 AS DECIMAL(9,4)) as decimal_9_4_col, + CAST(3.14 AS DECIMAL(18,6)) as decimal_18_6_col, + CAST(3.14 AS DECIMAL(38,10)) as decimal_38_10_col, + CAST('550e8400-e29b-41d4-a716-446655440000' AS UUID) as uuid_col, + CAST('1 day 2 hours 30 minutes' AS INTERVAL) as interval_col, + 'Evidence' as varchar_col, + CAST('SGVsbG8gd29ybGQ=' AS BLOB) as blob_col, + CAST(1 AS BIT) as bit_col + ` + ); + + const rows = await batchedAsyncGeneratorToArray(row_generator); + assert.instance(rows, Array); + assert.instance(columnTypes, Array); + assert.type(rows[0], 'object'); + assert.equal(columnTypes.length, 29); + + const actualColumnTypes = columnTypes.map((columnType) => columnType.evidenceType); + const actualColumnNames = columnTypes.map((columnType) => columnType.name); + const actualTypePrecisions = columnTypes.map((columnType) => columnType.typeFidelity); + + const expectedColumnTypes = [ + 'boolean', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', + 'date', 'string', 'date', 'date', 'date', 'date', 'string', 'date', 'number', 'number', + 'number', 'number', 'number', 'number', 'string', 'string', 'string', 'string', 'string' + ]; + + const expectedColumnNames = [ + 'boolean_col','tinyint_col','smallint_col','int_col','bigint_col','hugeint_col','utinyint_col','usmallint_col','uint_col','ubigint_col', + 'date_col','time_col','timestamp_col','timestamp_s_col','timestamp_ms_col','timestamp_ns_col','time_with_tz_col','timestamp_with_tz_col','float_col','double_col', + 'decimal_4_1_col','decimal_9_4_col','decimal_18_6_col','decimal_38_10_col','uuid_col','interval_col','varchar_col','blob_col','bit_col' + ]; + + const expectedTypePrecision = Array(29).fill(TypeFidelity.PRECISE); + + assert.equal( + true, + expectedColumnTypes.length === actualColumnTypes.length && + expectedColumnTypes.every((value, index) => value === actualColumnTypes[index]) + ); + assert.equal( + true, + expectedColumnNames.length === actualColumnNames.length && + expectedColumnNames.every((value, index) => value === actualColumnNames[index]) + ); + assert.equal( + true, + expectedTypePrecision.length === actualTypePrecisions.length && + expectedTypePrecision.every((value, index) => value === actualTypePrecisions[index]) + ); + } catch (e) { + console.error('Error in null-first-row test:', e.message); + throw e; + } +}); + test('query batches results properly', async () => { try { const { rows, expectedRowCount } = await runQuery( From 8cba8a6e2efb1246dfe06d8cb39efdbe913de1c4 Mon Sep 17 00:00:00 2001 From: Emanuele Bardelli Date: Fri, 7 Nov 2025 12:15:13 -0800 Subject: [PATCH 02/22] Write a unique tmp file at each run --- .../lib/universal-sql/src/build-parquet.js | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/packages/lib/universal-sql/src/build-parquet.js b/packages/lib/universal-sql/src/build-parquet.js index 42dd46709a..39c20b9977 100644 --- a/packages/lib/universal-sql/src/build-parquet.js +++ b/packages/lib/universal-sql/src/build-parquet.js @@ -95,6 +95,12 @@ export async function buildMultipartParquet( let tmpFilenames = []; let rowCount = 0; + // Unique id for this build run to avoid reusing the same temp filenames across + // consecutive builds in the same session. DuckDB (wasm) may keep handles or + // mappings to filenames; creating unique filenames per build avoids races or + // protocol errors when files are replaced. + const buildId = `${Date.now()}.${Math.random().toString(36).slice(2, 8)}`; + const flush = async (results) => { log.debug(`Flushing batch ${batchNum} with ${results.length} rows`); let { meta, done } = log.measure('flush'); @@ -124,11 +130,20 @@ export async function buildMultipartParquet( const parquetBuffer = writeParquet(Table.fromIPCStream(IPC), writerProperties); - const tempFilename = path.join(tmpDir, `${outputSubpath}.${batchNum}.parquet`); - await fs.mkdir(path.dirname(tempFilename), { recursive: true }); - await fs.writeFile(tempFilename, parquetBuffer); + const finalTempFilename = path.join(tmpDir, `${outputSubpath}.${buildId}.${batchNum}.parquet`); + // Write to a unique temporary file first, then atomically rename into place. + // This avoids a race where DuckDB may attempt to read a file while it's being written + // (e.g., during rapid dev refreshes). Using an atomic rename ensures readers only + // see the completed file. + const uniqueSuffix = `.${Date.now()}.${Math.random().toString(36).slice(2, 8)}.tmp`; + const writeTempFilename = finalTempFilename + uniqueSuffix; + + await fs.mkdir(path.dirname(finalTempFilename), { recursive: true }); + await fs.writeFile(writeTempFilename, parquetBuffer); + // Atomically move into final location + await fs.rename(writeTempFilename, finalTempFilename); - tmpFilenames.push(tempFilename); + tmpFilenames.push(finalTempFilename); rowCount += results.length; done(); From 53c08b8df9653ec58c5ce6cb7448a11a72d08d57 Mon Sep 17 00:00:00 2001 From: Emanuele Bardelli Date: Fri, 7 Nov 2025 12:59:23 -0800 Subject: [PATCH 03/22] Run formatter --- packages/datasources/duckdb/test/test.js | 64 +++++++++++++++++-- .../lib/universal-sql/src/build-parquet.js | 28 ++++---- packages/ui/core-components/CHANGELOG.md | 1 - 3 files changed, 72 insertions(+), 21 deletions(-) diff --git a/packages/datasources/duckdb/test/test.js b/packages/datasources/duckdb/test/test.js index b788a77012..6c89129974 100644 --- a/packages/datasources/duckdb/test/test.js +++ b/packages/datasources/duckdb/test/test.js @@ -259,15 +259,67 @@ test('handles nulls in first row', async () => { const actualTypePrecisions = columnTypes.map((columnType) => columnType.typeFidelity); const expectedColumnTypes = [ - 'boolean', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', 'number', - 'date', 'string', 'date', 'date', 'date', 'date', 'string', 'date', 'number', 'number', - 'number', 'number', 'number', 'number', 'string', 'string', 'string', 'string', 'string' + 'boolean', + 'number', + 'number', + 'number', + 'number', + 'number', + 'number', + 'number', + 'number', + 'number', + 'date', + 'string', + 'date', + 'date', + 'date', + 'date', + 'string', + 'date', + 'number', + 'number', + 'number', + 'number', + 'number', + 'number', + 'string', + 'string', + 'string', + 'string', + 'string' ]; const expectedColumnNames = [ - 'boolean_col','tinyint_col','smallint_col','int_col','bigint_col','hugeint_col','utinyint_col','usmallint_col','uint_col','ubigint_col', - 'date_col','time_col','timestamp_col','timestamp_s_col','timestamp_ms_col','timestamp_ns_col','time_with_tz_col','timestamp_with_tz_col','float_col','double_col', - 'decimal_4_1_col','decimal_9_4_col','decimal_18_6_col','decimal_38_10_col','uuid_col','interval_col','varchar_col','blob_col','bit_col' + 'boolean_col', + 'tinyint_col', + 'smallint_col', + 'int_col', + 'bigint_col', + 'hugeint_col', + 'utinyint_col', + 'usmallint_col', + 'uint_col', + 'ubigint_col', + 'date_col', + 'time_col', + 'timestamp_col', + 'timestamp_s_col', + 'timestamp_ms_col', + 'timestamp_ns_col', + 'time_with_tz_col', + 'timestamp_with_tz_col', + 'float_col', + 'double_col', + 'decimal_4_1_col', + 'decimal_9_4_col', + 'decimal_18_6_col', + 'decimal_38_10_col', + 'uuid_col', + 'interval_col', + 'varchar_col', + 'blob_col', + 'bit_col' ]; const expectedTypePrecision = Array(29).fill(TypeFidelity.PRECISE); diff --git a/packages/lib/universal-sql/src/build-parquet.js b/packages/lib/universal-sql/src/build-parquet.js index 39c20b9977..3577ffd0b1 100644 --- a/packages/lib/universal-sql/src/build-parquet.js +++ b/packages/lib/universal-sql/src/build-parquet.js @@ -130,20 +130,20 @@ export async function buildMultipartParquet( const parquetBuffer = writeParquet(Table.fromIPCStream(IPC), writerProperties); - const finalTempFilename = path.join(tmpDir, `${outputSubpath}.${buildId}.${batchNum}.parquet`); - // Write to a unique temporary file first, then atomically rename into place. - // This avoids a race where DuckDB may attempt to read a file while it's being written - // (e.g., during rapid dev refreshes). Using an atomic rename ensures readers only - // see the completed file. - const uniqueSuffix = `.${Date.now()}.${Math.random().toString(36).slice(2, 8)}.tmp`; - const writeTempFilename = finalTempFilename + uniqueSuffix; - - await fs.mkdir(path.dirname(finalTempFilename), { recursive: true }); - await fs.writeFile(writeTempFilename, parquetBuffer); - // Atomically move into final location - await fs.rename(writeTempFilename, finalTempFilename); - - tmpFilenames.push(finalTempFilename); + const finalTempFilename = path.join(tmpDir, `${outputSubpath}.${buildId}.${batchNum}.parquet`); + // Write to a unique temporary file first, then atomically rename into place. + // This avoids a race where DuckDB may attempt to read a file while it's being written + // (e.g., during rapid dev refreshes). Using an atomic rename ensures readers only + // see the completed file. + const uniqueSuffix = `.${Date.now()}.${Math.random().toString(36).slice(2, 8)}.tmp`; + const writeTempFilename = finalTempFilename + uniqueSuffix; + + await fs.mkdir(path.dirname(finalTempFilename), { recursive: true }); + await fs.writeFile(writeTempFilename, parquetBuffer); + // Atomically move into final location + await fs.rename(writeTempFilename, finalTempFilename); + + tmpFilenames.push(finalTempFilename); rowCount += results.length; done(); diff --git a/packages/ui/core-components/CHANGELOG.md b/packages/ui/core-components/CHANGELOG.md index 0718e0b6e2..4ad9a2d7b7 100644 --- a/packages/ui/core-components/CHANGELOG.md +++ b/packages/ui/core-components/CHANGELOG.md @@ -7,7 +7,6 @@ - 6d2782e64: Fixed base path not being applied to data file (parquet) URLs. This resolves an issue where applications deployed with a base path would fail to load data files, resulting in 404 errors. The fix restores the dependency injection pattern for the `addBasePath` function in `setParquetURLs`, ensuring that base paths are correctly applied to all data requests in both monorepo and published package environments. - - @evidence-dev/component-utilities@4.0.10 - @evidence-dev/tailwind@3.1.1 From f6b1fb7450c84b77c77b28942a129ae22dacca9a Mon Sep 17 00:00:00 2001 From: Emanuele Bardelli Date: Fri, 7 Nov 2025 13:02:24 -0800 Subject: [PATCH 04/22] Add changeset --- .changeset/warm-drinks-collect.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/warm-drinks-collect.md diff --git a/.changeset/warm-drinks-collect.md b/.changeset/warm-drinks-collect.md new file mode 100644 index 0000000000..0020f198f7 --- /dev/null +++ b/.changeset/warm-drinks-collect.md @@ -0,0 +1,7 @@ +--- +'@evidence-dev/duckdb': minor +'@evidence-dev/universal-sql': minor +'@evidence-dev/core-components': patch +--- + +Fix NULL in first row bug and temporary parquet files bugs From 54fa16f51ddff2115556cb77b0aaa0318bf672b1 Mon Sep 17 00:00:00 2001 From: Emanuele Bardelli Date: Fri, 7 Nov 2025 14:51:45 -0800 Subject: [PATCH 05/22] Fix error handling for mssql data source --- packages/datasources/mssql/index.cjs | 49 +++++++++++++++--- packages/datasources/mssql/test/test.js | 68 ++++++++++++++++++++++++- 2 files changed, 107 insertions(+), 10 deletions(-) diff --git a/packages/datasources/mssql/index.cjs b/packages/datasources/mssql/index.cjs index 7360078080..fd56064495 100755 --- a/packages/datasources/mssql/index.cjs +++ b/packages/datasources/mssql/index.cjs @@ -144,18 +144,51 @@ const runQuery = async (queryString, database = {}, batchSize = 100000) => { const request = new mssql.Request(); request.stream = true; + + // Promise that resolves when recordset is emitted or rejects on request error + const recordsetPromise = new Promise((resolve, reject) => { + request.once('recordset', resolve); + request.once('error', reject); + }); + + // Start the streaming query request.query(queryString); - const columns = await new Promise((res) => request.once('recordset', res)); + try { + const columns = await recordsetPromise; - const stream = request.toReadableStream(); - const results = await asyncIterableToBatchedAsyncGenerator(stream, batchSize, { - closeConnection: () => pool.close() - }); - results.columnTypes = mapResultsToEvidenceColumnTypes(columns); - results.expectedRowCount = expected_row_count; + const stream = request.toReadableStream(); + + // Ensure any stream errors are handled to avoid unhandled 'error' events + stream.on('error', async (streamErr) => { + try { + await pool.close(); + } catch (_) { + // ignore close errors + } + // Nothing else to do here; the asyncIterableToBatchedAsyncGenerator + // consumer will observe the error as a rejection when iterating. + }); + + const results = await asyncIterableToBatchedAsyncGenerator(stream, batchSize, { + closeConnection: () => pool.close() + }); + results.columnTypes = mapResultsToEvidenceColumnTypes(columns); + results.expectedRowCount = expected_row_count; + + return results; + } catch (err) { + // Close pool when we hit errors from the request/stream and normalize the error + try { + await pool.close(); + } catch (_) {} - return results; + if (err && err.message) { + throw err.message.replace(/\n|\r/g, ' '); + } else { + throw ('' + err).replace(/\n|\r/g, ' '); + } + } } catch (err) { if (err.message) { throw err.message.replace(/\n|\r/g, ' '); diff --git a/packages/datasources/mssql/test/test.js b/packages/datasources/mssql/test/test.js index a5be8b17f3..5952074f86 100644 --- a/packages/datasources/mssql/test/test.js +++ b/packages/datasources/mssql/test/test.js @@ -1,9 +1,10 @@ import { test } from 'uvu'; import * as assert from 'uvu/assert'; -import runQuery from '../index.cjs'; -import { TypeFidelity, batchedAsyncGeneratorToArray } from '@evidence-dev/db-commons'; +import { createRequire } from 'module'; import 'dotenv/config'; +const require = createRequire(import.meta.url); + test('query runs', async () => { if (process.env.MSSQL_DATABASE) { try { @@ -99,4 +100,67 @@ test('query batches results properly', async () => { } }); +test('runQuery returns normalized error on SQL syntax error', async () => { + // Create a fake mssql module and inject before loading index.cjs + const fakeMssql = { + TYPES: {}, + connect: async () => ({ + request: () => ({ + query: async (q) => { + // If it's the COUNT(*) wrapper, return 0 + if (typeof q === 'string' && q.trim().toUpperCase().startsWith('SELECT COUNT(*)')) { + return { recordset: [{ expected_row_count: 0 }] }; + } + // otherwise, return empty + return { recordset: [] }; + } + }), + close: async () => {} + }), + Request: function () { + const EventEmitter = require('events'); + const r = new EventEmitter(); + r.stream = false; + r.query = function (q) { + // simulate an async error emitted by the request (syntax error) + process.nextTick(() => { + const err = new Error("Incorrect syntax near the keyword 'select'."); + err.code = 'EREQUEST'; + r.emit('error', err); + }); + }; + r.toReadableStream = function () { + const { Readable } = require('stream'); + // A readable that immediately errors when read + const s = new Readable({ objectMode: true, read() {} }); + process.nextTick(() => s.emit('error', new Error('stream error'))); + return s; + }; + return r; + } + }; + + const path = require('path'); + const mssqlModulePath = path.join(process.cwd(), 'node_modules', 'mssql', 'index.js'); + require.cache[mssqlModulePath] = { + id: mssqlModulePath, + filename: mssqlModulePath, + loaded: true, + exports: fakeMssql + }; + + const runQuery = require('../index.cjs'); + + try { + await runQuery('select * from', {}, 10); + assert.unreachable('Expected runQuery to throw'); + } catch (e) { + // The function normalizes errors to strings + assert.ok(typeof e === 'string' || e instanceof String || e.message, 'error should be string or have message'); + const msg = typeof e === 'string' ? e : e.message || String(e); + // Message should be non-empty and mention syntax/select or be an error code + assert.ok(msg && msg.length > 0, 'error message should be non-empty'); + } +}); + test.run(); From 73b1441edeb6a525d327a4fbd98e2ce41ac8b7fd Mon Sep 17 00:00:00 2001 From: Emanuele Bardelli Date: Fri, 7 Nov 2025 14:53:58 -0800 Subject: [PATCH 06/22] Add changeset --- .../datasources/mssql/.changeset/purple-readers-live.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 packages/datasources/mssql/.changeset/purple-readers-live.md diff --git a/packages/datasources/mssql/.changeset/purple-readers-live.md b/packages/datasources/mssql/.changeset/purple-readers-live.md new file mode 100644 index 0000000000..6595225692 --- /dev/null +++ b/packages/datasources/mssql/.changeset/purple-readers-live.md @@ -0,0 +1,8 @@ +--- +'@evidence-dev/duckdb': minor +'@evidence-dev/mssql': minor +'@evidence-dev/universal-sql': minor +'@evidence-dev/core-components': minor +--- + +Fix various data connector bugs From bd1df3d9105057a07f1466fff780166c57417c76 Mon Sep 17 00:00:00 2001 From: Emanuele Bardelli Date: Fri, 7 Nov 2025 15:22:49 -0800 Subject: [PATCH 07/22] Refactor assertions to verify filenames in buildMultipartParquet tests use the buildId identifier --- .../universal-sql/src/build-parquet.spec.js | 34 ++++++++++--------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/packages/lib/universal-sql/src/build-parquet.spec.js b/packages/lib/universal-sql/src/build-parquet.spec.js index 620f987bc5..7c9ef885b9 100644 --- a/packages/lib/universal-sql/src/build-parquet.spec.js +++ b/packages/lib/universal-sql/src/build-parquet.spec.js @@ -73,10 +73,13 @@ describe('buildMultipartParquet', () => { // Make sure it contains data expect(stat.size).toBeGreaterThan(0); expect(fs.rm).toHaveBeenCalledOnce(); - expect(fs.rm).toHaveBeenCalledWith( - adaptFilePath('.evidence/template/.evidence-queries/intermediate-parquet/out.0.parquet'), - { force: true } + // verify the rm call targeted the intermediate directory and a tempfile + const calledPath = fs.rm.mock.calls[0][0]; + expect(path.dirname(calledPath)).toBe( + adaptFilePath('.evidence/template/.evidence-queries/intermediate-parquet') ); + // basename should look like out...0.parquet + expect(path.basename(calledPath)).toMatch(/^out\.\d+\.[0-9a-z]{6}\.0\.parquet$/); }); it('should write two temporary files when given enough rows for two batches', async () => { @@ -105,16 +108,16 @@ describe('buildMultipartParquet', () => { // Make sure it contains data expect(stat.size).toBeGreaterThan(0); expect(fs.rm).toHaveBeenCalledTimes(2); - expect(fs.rm).toHaveBeenNthCalledWith( - 1, - adaptFilePath('.evidence/template/.evidence-queries/intermediate-parquet/out.0.parquet'), - { force: true } + const calledPath1 = fs.rm.mock.calls[0][0]; + const calledPath2 = fs.rm.mock.calls[1][0]; + expect(path.dirname(calledPath1)).toBe( + adaptFilePath('.evidence/template/.evidence-queries/intermediate-parquet') ); - expect(fs.rm).toHaveBeenNthCalledWith( - 2, - adaptFilePath('.evidence/template/.evidence-queries/intermediate-parquet/out.1.parquet'), - { force: true } + expect(path.dirname(calledPath2)).toBe( + adaptFilePath('.evidence/template/.evidence-queries/intermediate-parquet') ); + expect(path.basename(calledPath1)).toMatch(/^out\.\d+\.[0-9a-z]{6}\.0\.parquet$/); + expect(path.basename(calledPath2)).toMatch(/^out\.\d+\.[0-9a-z]{6}\.1\.parquet$/); }); it('should accept an array as the data argument', async () => { @@ -133,12 +136,11 @@ describe('buildMultipartParquet', () => { // Make sure it contains data expect(stat.size).toBeGreaterThan(0); expect(fs.rm).toHaveBeenCalledOnce(); - expect(fs.rm).toHaveBeenCalledWith( - adaptFilePath('.evidence/template/.evidence-queries/intermediate-parquet/out.0.parquet'), - { - force: true - } + const calledPathA = fs.rm.mock.calls[0][0]; + expect(path.dirname(calledPathA)).toBe( + adaptFilePath('.evidence/template/.evidence-queries/intermediate-parquet') ); + expect(path.basename(calledPathA)).toMatch(/^out\.\d+\.[0-9a-z]{6}\.0\.parquet$/); }); it('should handle a very large number of batches', async () => { From 68cf209714f87516ad36d092ad8b8e7d1cdf98d7 Mon Sep 17 00:00:00 2001 From: Emanuele Bardelli Date: Tue, 11 Nov 2025 18:51:32 -0800 Subject: [PATCH 08/22] Update duckdb packages to latest version --- packages/datasources/duckdb/index.cjs | 2 +- packages/datasources/duckdb/package.json | 2 +- packages/lib/universal-sql/package.json | 2 +- pnpm-lock.yaml | 290 +++++++++-------------- 4 files changed, 115 insertions(+), 181 deletions(-) diff --git a/packages/datasources/duckdb/index.cjs b/packages/datasources/duckdb/index.cjs index 0dae79c8e3..3a2fd2d085 100644 --- a/packages/datasources/duckdb/index.cjs +++ b/packages/datasources/duckdb/index.cjs @@ -5,7 +5,7 @@ const { cleanQuery, exhaustStream } = require('@evidence-dev/db-commons'); -const { Database } = require('duckdb-async'); +const { Database } = require('@duckdb/node-api'); const path = require('path'); const fs = require('fs/promises'); diff --git a/packages/datasources/duckdb/package.json b/packages/datasources/duckdb/package.json index 41f37839f3..dbeea12d0d 100644 --- a/packages/datasources/duckdb/package.json +++ b/packages/datasources/duckdb/package.json @@ -10,7 +10,7 @@ }, "dependencies": { "@evidence-dev/db-commons": "workspace:*", - "duckdb-async": "1.1.3" + "@duckdb/node-api": "^1.4.1-r.5" }, "devDependencies": { "dotenv": "^16.0.1" diff --git a/packages/lib/universal-sql/package.json b/packages/lib/universal-sql/package.json index a31635a994..8e2d9393ab 100644 --- a/packages/lib/universal-sql/package.json +++ b/packages/lib/universal-sql/package.json @@ -14,7 +14,7 @@ }, "type": "module", "dependencies": { - "@duckdb/duckdb-wasm": "1.29.0", + "@duckdb/duckdb-wasm": "^1.31.1-dev1.0", "apache-arrow": "17.0.0", "chalk": "^5.2.0", "cli-progress": "^3.12.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 18ac486acc..6fbaae1c2d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -639,12 +639,12 @@ importers: packages/datasources/duckdb: dependencies: + '@duckdb/node-api': + specifier: ^1.4.1-r.5 + version: 1.4.1-r.5 '@evidence-dev/db-commons': specifier: workspace:* version: link:../../lib/db-commons - duckdb-async: - specifier: 1.1.3 - version: 1.1.3 devDependencies: dotenv: specifier: ^16.0.1 @@ -1270,8 +1270,8 @@ importers: packages/lib/universal-sql: dependencies: '@duckdb/duckdb-wasm': - specifier: 1.29.0 - version: 1.29.0 + specifier: ^1.31.1-dev1.0 + version: 1.31.1-dev1.0 apache-arrow: specifier: 17.0.0 version: 17.0.0 @@ -3838,6 +3838,69 @@ packages: resolution: {integrity: sha512-8Zq7vafQuIz9gklC/9375KE38UlkaS2n8+yvG+/JK7irm3DjwYNJHL4xfplIj0bSHFIg6we5XhWYFqtE/vO3+Q==} dependencies: apache-arrow: 17.0.0 + dev: true + + /@duckdb/duckdb-wasm@1.31.1-dev1.0: + resolution: {integrity: sha512-q2QiJ7I+D9reSSmHeI+J5nMldeH4xUtYMuFJfzEhyiddsAsH2omF7oqWjC6JyVTodsaCfFx9rtfEvBHxa5IXcg==} + dependencies: + apache-arrow: 17.0.0 + dev: false + + /@duckdb/node-api@1.4.1-r.5: + resolution: {integrity: sha512-RBDVqlMp3/GYKF/VhHcUrIYGJoOOMmVN2Ha7FPpm0Qpbqzh5RAY+QSfq30PTkRvIav4IwhM/FOu8gJNUqYCZGw==} + dependencies: + '@duckdb/node-bindings': 1.4.1-r.5 + dev: false + + /@duckdb/node-bindings-darwin-arm64@1.4.1-r.5: + resolution: {integrity: sha512-7LhYHy98ZqkGsK5XwDT/p7tCnGBTmZuxiEeNGJ/sdqvLJeDfbJuFnkXwnKMAue8ilgpI1pdET1QmqfdiKgkJyw==} + cpu: [arm64] + os: [darwin] + requiresBuild: true + dev: false + optional: true + + /@duckdb/node-bindings-darwin-x64@1.4.1-r.5: + resolution: {integrity: sha512-LIHHNPecG5Yk5JS+CnUMvq0dUaJ9glyRyV7w9GedyjxRYqT9d4Sbxl5UCj3a6Pu6Sf8IZrWtWXQB0C/4e6ITKw==} + cpu: [x64] + os: [darwin] + requiresBuild: true + dev: false + optional: true + + /@duckdb/node-bindings-linux-arm64@1.4.1-r.5: + resolution: {integrity: sha512-G1dp7PY7gQR0svheXVAQdrsOKv61jQxaM6jhp3FhhSNt2cHlsHld8kjnjXVCDr3OVlTP0vez/1mcbWpLZ7eB+w==} + cpu: [arm64] + os: [linux] + requiresBuild: true + dev: false + optional: true + + /@duckdb/node-bindings-linux-x64@1.4.1-r.5: + resolution: {integrity: sha512-BVkWzA7Lrl5VQcPNh3k8mb/83kyH+csZTdHM2sePjxCts0zvji9hnGnBIIn33xarHSd0+3igBDijeqxxiFiPYg==} + cpu: [x64] + os: [linux] + requiresBuild: true + dev: false + optional: true + + /@duckdb/node-bindings-win32-x64@1.4.1-r.5: + resolution: {integrity: sha512-Y/+vsLVGXIx9UMMyYDpYCSY3m0hXC1RjSaYfb7jkim8gcWQg0eRlbjTlDnmhth/8Z+eQ2sXXy8UVBd5IZ/bqYw==} + cpu: [x64] + os: [win32] + requiresBuild: true + dev: false + optional: true + + /@duckdb/node-bindings@1.4.1-r.5: + resolution: {integrity: sha512-HUncdmpfjT4HxbH5NO6myazzudhddPc9lly7AgsYBdQSfTr1LCaQGb45cGiS5jEafpod9VOlk0bwJVbAIaBDWw==} + optionalDependencies: + '@duckdb/node-bindings-darwin-arm64': 1.4.1-r.5 + '@duckdb/node-bindings-darwin-x64': 1.4.1-r.5 + '@duckdb/node-bindings-linux-arm64': 1.4.1-r.5 + '@duckdb/node-bindings-linux-x64': 1.4.1-r.5 + '@duckdb/node-bindings-win32-x64': 1.4.1-r.5 + dev: false /@esbuild/aix-ppc64@0.25.12: resolution: {integrity: sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==} @@ -4336,7 +4399,9 @@ packages: /@gar/promisify@1.1.3: resolution: {integrity: sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw==} + requiresBuild: true dev: false + optional: true /@google-cloud/bigquery@6.2.0: resolution: {integrity: sha512-jkyu8n5WR5HoF4saVc/ukfAE8uEjqkWRu3Bn3+hhJ7RTNR8unloS9J6JKzsGtLlA8gS3gHSceRg/CpCl5WFg+g==} @@ -5300,14 +5365,6 @@ packages: dev: false optional: true - /@npmcli/fs@2.1.2: - resolution: {integrity: sha512-yOJKRvohFOaLqipNtwYB9WugyZKhC/DZC4VYPmpaCzDBrA8YpK3qHZ8/HGscMnE4GqbkLNuVcCnxkeQEdGt6LQ==} - engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} - dependencies: - '@gar/promisify': 1.1.3 - semver: 7.7.3 - dev: false - /@npmcli/move-file@1.1.2: resolution: {integrity: sha512-1SUf/Cg2GzGDyaf15aR9St9TWlb+XvbZXWpDx8YKs7MLzMH/BCeopv+y9vzrzgkfykCGuWOlSu3mZhj2+FQcrg==} engines: {node: '>=10'} @@ -5319,15 +5376,6 @@ packages: dev: false optional: true - /@npmcli/move-file@2.0.1: - resolution: {integrity: sha512-mJd2Z5TjYWq/ttPLLGqArdtnC74J6bOzg4rMDnN+p1xTacZ2yPRCk2y0oSWQtygLR9YVQXgOcONrwtnk3JupxQ==} - engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} - deprecated: This functionality has been moved to @npmcli/fs - dependencies: - mkdirp: 1.0.4 - rimraf: 3.0.2 - dev: false - /@parcel/bundler-default@2.16.0(@parcel/core@2.16.0): resolution: {integrity: sha512-8kY+TUhir7qm+TgSMeMd8CP2PVoZjXamiZ8+mbXws4jKw6IrIVDQf8TkBZKGk7ncKJEteiX4ybbmiPjho8cHuA==} engines: {node: '>= 16.0.0', parcel: ^2.16.0} @@ -9336,6 +9384,7 @@ packages: /abbrev@1.1.1: resolution: {integrity: sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==} + requiresBuild: true dev: false /abort-controller@3.0.0: @@ -9387,17 +9436,21 @@ packages: /agentkeepalive@4.6.0: resolution: {integrity: sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==} engines: {node: '>= 8.0.0'} + requiresBuild: true dependencies: humanize-ms: 1.2.1 dev: false + optional: true /aggregate-error@3.1.0: resolution: {integrity: sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==} engines: {node: '>=8'} + requiresBuild: true dependencies: clean-stack: 2.2.0 indent-string: 4.0.0 dev: false + optional: true /ajv-formats@2.1.1(ajv@8.17.1): resolution: {integrity: sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==} @@ -9557,10 +9610,12 @@ packages: resolution: {integrity: sha512-QZW4EDmGwlYur0Yyf/b2uGucHQMa8aFUP7eu9ddR73vvhFyt4V0Vl3QHPcTNJ8l6qYOBdxgXdnBXQrHilfRQBg==} engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} deprecated: This package is no longer supported. + requiresBuild: true dependencies: delegates: 1.0.0 readable-stream: 3.6.2 dev: false + optional: true /arg@5.0.2: resolution: {integrity: sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==} @@ -10141,32 +10196,6 @@ packages: dev: false optional: true - /cacache@16.1.3: - resolution: {integrity: sha512-/+Emcj9DAXxX4cwlLmRI9c166RuL3w30zp4R7Joiv2cQTtTtA+jeuCAjH3ZlGnYS3tKENSrKhAzVVP9GVyzeYQ==} - engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} - dependencies: - '@npmcli/fs': 2.1.2 - '@npmcli/move-file': 2.0.1 - chownr: 2.0.0 - fs-minipass: 2.1.0 - glob: 8.1.0 - infer-owner: 1.0.4 - lru-cache: 7.18.3 - minipass: 3.3.6 - minipass-collect: 1.0.2 - minipass-flush: 1.0.5 - minipass-pipeline: 1.2.4 - mkdirp: 1.0.4 - p-map: 4.0.0 - promise-inflight: 1.0.1 - rimraf: 3.0.2 - ssri: 9.0.1 - tar: 6.2.1 - unique-filename: 2.0.1 - transitivePeerDependencies: - - bluebird - dev: false - /call-bind-apply-helpers@1.0.2: resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==} engines: {node: '>= 0.4'} @@ -10418,6 +10447,7 @@ packages: engines: {node: '>=6'} requiresBuild: true dev: false + optional: true /cli-cursor@4.0.0: resolution: {integrity: sha512-VGtlMu3x/4DOtIUwEkRezxUZ2lBacNJCHash0N0WeZDBS+7Ux1dm3XWAgWYxLJFMMdOeXMHXorshEFhbMSGelg==} @@ -11295,29 +11325,6 @@ packages: engines: {node: '>=4'} dev: false - /duckdb-async@1.1.3: - resolution: {integrity: sha512-8Q8BYTjfzjPFJ3J8bafLGUcdxI2ZXPxpw3u8x+SFr7s3T7JM0Nn26LKis4rA0i4TCBSZ9ilhB8NOJ/9HrPGSlw==} - dependencies: - duckdb: 1.1.3 - transitivePeerDependencies: - - bluebird - - encoding - - supports-color - dev: false - - /duckdb@1.1.3: - resolution: {integrity: sha512-tIpZr2NsSkYmfGC1ETl75RuVsaDyjvR3yAOrECcIyw7bdluzcyzEXOXoiuT+4t54hT+CppZv43gk/HiZdKW9Vw==} - requiresBuild: true - dependencies: - '@mapbox/node-pre-gyp': 1.0.11 - node-addon-api: 7.1.1 - node-gyp: 9.4.1 - transitivePeerDependencies: - - bluebird - - encoding - - supports-color - dev: false - /dunder-proto@1.0.1: resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==} engines: {node: '>= 0.4'} @@ -11471,7 +11478,9 @@ packages: /env-paths@2.2.1: resolution: {integrity: sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==} engines: {node: '>=6'} + requiresBuild: true dev: false + optional: true /envinfo@7.20.0: resolution: {integrity: sha512-+zUomDcLXsVkQ37vUqWBvQwLaLlj8eZPSi61llaEFAVBY5mhcXdaSw1pSJVl4yTYD5g/gEfpNl28YYk4IPvrrg==} @@ -11483,6 +11492,7 @@ packages: resolution: {integrity: sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA==} requiresBuild: true dev: false + optional: true /error-ex@1.3.4: resolution: {integrity: sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ==} @@ -12128,10 +12138,6 @@ packages: jest-util: 29.7.0 dev: true - /exponential-backoff@3.1.3: - resolution: {integrity: sha512-ZgEeZXj30q+I0EN+CbSSpIyPaJ5HVQD18Z1m+u1FXbAeT94mr1zw50q4q6jiiC447Nl/YTcIYSAftiGqetwXCA==} - dev: false - /export-to-csv@0.2.1: resolution: {integrity: sha512-KTbrd3CAZ0cFceJEZr1e5uiMasabeCpXq1/5uvVxDl53o4jXJHnltasQoj2NkzrxD8hU9kdwjnMhoir/7nNx/A==} @@ -12530,6 +12536,7 @@ packages: resolution: {integrity: sha512-f9m+BEN5jkg6a0fZjleidjN51VE1X+mPFQ2DJ0uv1V39oCLCbsGe6yjbBnp7eK7z/+GAon99a3nHuqbuuthyPg==} engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} deprecated: This package is no longer supported. + requiresBuild: true dependencies: aproba: 2.1.0 color-support: 1.1.3 @@ -12540,6 +12547,7 @@ packages: strip-ansi: 6.0.1 wide-align: 1.1.5 dev: false + optional: true /gaxios@5.1.3: resolution: {integrity: sha512-95hVgBRgEIRQQQHIbnxBXeHbW4TqFk4ZDJW7wmVtvYar72FdhRIo1UGOLS2eRAKCPEdPBWu+M7+A33D9CdX9rA==} @@ -12783,6 +12791,7 @@ packages: inherits: 2.0.4 minimatch: 5.1.6 once: 1.4.0 + dev: true /globals@13.24.0: resolution: {integrity: sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ==} @@ -13068,7 +13077,9 @@ packages: /http-cache-semantics@4.2.0: resolution: {integrity: sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ==} + requiresBuild: true dev: false + optional: true /http-errors@2.0.0: resolution: {integrity: sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==} @@ -13172,6 +13183,7 @@ packages: dependencies: ms: 2.1.3 dev: false + optional: true /hyperdyperid@1.2.0: resolution: {integrity: sha512-Y93lCzHYgGWdrJ66yIktxiaGULYc6oGiABxhcO5AufBeOyoIdZF7bIfLaOrbM0iGIOXQQgxxRrFEnb+Y6w1n4A==} @@ -13244,7 +13256,9 @@ packages: /infer-owner@1.0.4: resolution: {integrity: sha512-IClj+Xz94+d7irH5qRyfJonOdfTzuDaifE6ZPWfx0N0+/ATZCbuTPq2prFl526urkQd90WyUKIh1DfBQ2hMz9A==} + requiresBuild: true dev: false + optional: true /inflight@1.0.6: resolution: {integrity: sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==} @@ -13457,7 +13471,9 @@ packages: /is-lambda@1.0.1: resolution: {integrity: sha512-z7CMFGNrENq5iFB9Bqo64Xk6Y9sg+epq1myIcdHaGnbMTYOxvzsEtdYqQUylB7LxfkvgrrjP32T6Ywciio9UIQ==} + requiresBuild: true dev: false + optional: true /is-map@2.0.3: resolution: {integrity: sha512-1Qed0/Hr2m+YqxnM09CjA2d/i6YZNfF6R2oRAOj36eUdS6qIV/huPJNSEpKbupewFs+ZsJlxsjjPbc0/afW6Lw==} @@ -14890,31 +14906,6 @@ packages: dependencies: semver: 7.7.3 - /make-fetch-happen@10.2.1: - resolution: {integrity: sha512-NgOPbRiaQM10DYXvN3/hhGVI2M5MtITFryzBGxHM5p4wnFxsVCbxkrBrDsk+EZ5OB4jEOT7AjDxtdF+KVEFT7w==} - engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} - dependencies: - agentkeepalive: 4.6.0 - cacache: 16.1.3 - http-cache-semantics: 4.2.0 - http-proxy-agent: 5.0.0 - https-proxy-agent: 5.0.1 - is-lambda: 1.0.1 - lru-cache: 7.18.3 - minipass: 3.3.6 - minipass-collect: 1.0.2 - minipass-fetch: 2.1.2 - minipass-flush: 1.0.5 - minipass-pipeline: 1.2.4 - negotiator: 0.6.4 - promise-retry: 2.0.1 - socks-proxy-agent: 7.0.0 - ssri: 9.0.1 - transitivePeerDependencies: - - bluebird - - supports-color - dev: false - /make-fetch-happen@9.1.0: resolution: {integrity: sha512-+zopwDy7DNknmwPQplem5lAZX/eCOzSvSNNcSKm5eVwTkOBzoktEfXsa9L23J/GIRhxRsaxzkPEhrJEpE2F4Gg==} engines: {node: '>= 10'} @@ -15140,6 +15131,7 @@ packages: engines: {node: '>=10'} dependencies: brace-expansion: 2.0.2 + dev: true /minimatch@9.0.3: resolution: {integrity: sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg==} @@ -15169,9 +15161,11 @@ packages: /minipass-collect@1.0.2: resolution: {integrity: sha512-6T6lH0H8OG9kITm/Jm6tdooIbogG9e0tLgpY6mphXSm/A9u8Nq1ryBG+Qspiub9LjWlBPsPS3tWQ/Botq4FdxA==} engines: {node: '>= 8'} + requiresBuild: true dependencies: minipass: 3.3.6 dev: false + optional: true /minipass-fetch@1.4.1: resolution: {integrity: sha512-CGH1eblLq26Y15+Azk7ey4xh0J/XfJfrCox5LDJiKqI2Q2iwOLOKrlmIaODiSQS8d18jalF6y2K2ePUm0CmShw==} @@ -15186,37 +15180,32 @@ packages: dev: false optional: true - /minipass-fetch@2.1.2: - resolution: {integrity: sha512-LT49Zi2/WMROHYoqGgdlQIZh8mLPZmOrN2NdJjMXxYe4nkN6FUyuPuOAOedNJDrx0IRGg9+4guZewtp8hE6TxA==} - engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} - dependencies: - minipass: 3.3.6 - minipass-sized: 1.0.3 - minizlib: 2.1.2 - optionalDependencies: - encoding: 0.1.13 - dev: false - /minipass-flush@1.0.5: resolution: {integrity: sha512-JmQSYYpPUqX5Jyn1mXaRwOda1uQ8HP5KAT/oDSLCzt1BYRhQU0/hDtsB1ufZfEEzMZ9aAVmsBw8+FWsIXlClWw==} engines: {node: '>= 8'} + requiresBuild: true dependencies: minipass: 3.3.6 dev: false + optional: true /minipass-pipeline@1.2.4: resolution: {integrity: sha512-xuIq7cIOt09RPRJ19gdi4b+RiNvDFYe5JH+ggNvBqGqpQXcru3PcRmOZuHBKWK1Txf9+cQ+HMVN4d6z46LZP7A==} engines: {node: '>=8'} + requiresBuild: true dependencies: minipass: 3.3.6 dev: false + optional: true /minipass-sized@1.0.3: resolution: {integrity: sha512-MbkQQ2CTiBMlA2Dm/5cY+9SWFEN8pzzOXi6rlM5Xxq0Yqbda5ZQy9sU75a673FE9ZK0Zsbr6Y5iP6u9nktfg2g==} engines: {node: '>=8'} + requiresBuild: true dependencies: minipass: 3.3.6 dev: false + optional: true /minipass@3.3.6: resolution: {integrity: sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==} @@ -15445,7 +15434,9 @@ packages: /negotiator@0.6.4: resolution: {integrity: sha512-myRT3DiWPHqho5PrJaIRyaMv2kgYf0mUVgBNOYMuCH5Ki1yEiQaf/ZJuQ62nvpc44wL5WDbTX7yGJi1Neevw8w==} engines: {node: '>= 0.6'} + requiresBuild: true dev: false + optional: true /neo-async@2.6.2: resolution: {integrity: sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==} @@ -15485,6 +15476,7 @@ packages: /node-addon-api@7.1.1: resolution: {integrity: sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==} + dev: true /node-domexception@1.0.0: resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} @@ -15554,27 +15546,6 @@ packages: dev: false optional: true - /node-gyp@9.4.1: - resolution: {integrity: sha512-OQkWKbjQKbGkMf/xqI1jjy3oCTgMKJac58G2+bjZb3fza6gW2YrCSdMQYaoTb70crvE//Gngr4f0AgVHmqHvBQ==} - engines: {node: ^12.13 || ^14.13 || >=16} - hasBin: true - dependencies: - env-paths: 2.2.1 - exponential-backoff: 3.1.3 - glob: 7.2.3 - graceful-fs: 4.2.11 - make-fetch-happen: 10.2.1 - nopt: 6.0.0 - npmlog: 6.0.2 - rimraf: 3.0.2 - semver: 7.7.3 - tar: 6.2.1 - which: 2.0.2 - transitivePeerDependencies: - - bluebird - - supports-color - dev: false - /node-int64@0.4.0: resolution: {integrity: sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==} @@ -15589,14 +15560,6 @@ packages: abbrev: 1.1.1 dev: false - /nopt@6.0.0: - resolution: {integrity: sha512-ZwLpbTgdhuZUnZzjd7nb1ZV+4DoiC6/sfiVKok72ym/4Tlf+DFdlHYmT2JPmcNNWV6Pi3SDf1kT+A4r9RTuT9g==} - engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} - hasBin: true - dependencies: - abbrev: 1.1.1 - dev: false - /normalize-package-data@2.5.0: resolution: {integrity: sha512-/5CMN3T0R4XTj4DcGaexo+roZSdSFW/0AOOTROrjxzCG1wrWXEsGbRKevjlIL+ZDE4sZlJr5ED4YW0yqmkK+eA==} dependencies: @@ -15683,12 +15646,14 @@ packages: resolution: {integrity: sha512-/vBvz5Jfr9dT/aFWd0FIRf+T/Q2WBsLENygUaFUqstqsycmZAP/t5BvFJTK0viFmSUxiUKTUplWy5vt+rvKIxg==} engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} deprecated: This package is no longer supported. + requiresBuild: true dependencies: are-we-there-yet: 3.0.1 console-control-strings: 1.1.0 gauge: 4.0.4 set-blocking: 2.0.0 dev: false + optional: true /nth-check@2.1.1: resolution: {integrity: sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==} @@ -15952,9 +15917,11 @@ packages: /p-map@4.0.0: resolution: {integrity: sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==} engines: {node: '>=10'} + requiresBuild: true dependencies: aggregate-error: 3.1.0 dev: false + optional: true /p-timeout@3.2.0: resolution: {integrity: sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==} @@ -16016,9 +15983,6 @@ packages: resolution: {integrity: sha512-4sgnoYixTR6Qq6265tjmufXQj7wxvZo4VJHrYfbnfWQWfW5CgF80IiM+dy050pYgtBAMvh+8vJDDYiSto1YPUA==} engines: {node: '>= 16.0.0'} hasBin: true - peerDependenciesMeta: - '@parcel/core': - optional: true dependencies: '@parcel/config-default': 2.16.0(@parcel/core@2.16.0) '@parcel/core': 2.16.0 @@ -16589,20 +16553,24 @@ packages: /promise-inflight@1.0.1: resolution: {integrity: sha512-6zWPyEOFaQBJYcGMHBKTKJ3u6TBsnMFOIZSa6ce1e/ZrrsOlnHRHbabMjLiBYKp+n44X9eUI6VUPaukCXHuG4g==} + requiresBuild: true peerDependencies: bluebird: '*' peerDependenciesMeta: bluebird: optional: true dev: false + optional: true /promise-retry@2.0.1: resolution: {integrity: sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==} engines: {node: '>=10'} + requiresBuild: true dependencies: err-code: 2.0.3 retry: 0.12.0 dev: false + optional: true /prompts@2.4.2: resolution: {integrity: sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==} @@ -17060,6 +17028,7 @@ packages: engines: {node: '>= 4'} requiresBuild: true dev: false + optional: true /retry@0.13.1: resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==} @@ -17546,17 +17515,6 @@ packages: dev: false optional: true - /socks-proxy-agent@7.0.0: - resolution: {integrity: sha512-Fgl0YPZ902wEsAyiQ+idGd1A7rSFx/ayC1CQVMw5P+EQx2V0SgpGtf6OKFhVjPflPUl9YMmEOnmfjCdMUsygww==} - engines: {node: '>= 10'} - dependencies: - agent-base: 6.0.2 - debug: 4.4.3 - socks: 2.8.7 - transitivePeerDependencies: - - supports-color - dev: false - /socks-proxy-agent@8.0.5: resolution: {integrity: sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==} engines: {node: '>= 14'} @@ -17672,9 +17630,6 @@ packages: /sqlite3@5.1.6: resolution: {integrity: sha512-olYkWoKFVNSSSQNvxVUfjiVbz3YtBwTJj+mfV5zpHmqW3sELx2Cf4QCdirMelhM5Zh+KDVaKgQHqCxrqiWHybw==} requiresBuild: true - peerDependenciesMeta: - node-gyp: - optional: true dependencies: '@mapbox/node-pre-gyp': 1.0.11 node-addon-api: 4.3.0 @@ -17711,13 +17666,6 @@ packages: dev: false optional: true - /ssri@9.0.1: - resolution: {integrity: sha512-o57Wcn66jMQvfHG1FlYbWeZWW/dHZhJXjpIcTfXldXEk5nz5lStPo3mK0OJQfGR3RbZUlbISexbljkJzuEj/8Q==} - engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} - dependencies: - minipass: 3.3.6 - dev: false - /stack-trace@0.0.10: resolution: {integrity: sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==} dev: false @@ -19000,13 +18948,6 @@ packages: dev: false optional: true - /unique-filename@2.0.1: - resolution: {integrity: sha512-ODWHtkkdx3IAR+veKxFV+VBkUMcN+FaqzUUd7IZzt+0zhDZFPFxhlqwPF3YQvMHx1TD0tdgYl+kuPnJ8E6ql7A==} - engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} - dependencies: - unique-slug: 3.0.0 - dev: false - /unique-slug@2.0.2: resolution: {integrity: sha512-zoWr9ObaxALD3DOPfjPSqxt4fnZiWblxHIgeWqW8x7UqDzEtHEQLzji2cuJYQFCU6KmoJikOYAZlrTHHebjx2w==} requiresBuild: true @@ -19015,13 +18956,6 @@ packages: dev: false optional: true - /unique-slug@3.0.0: - resolution: {integrity: sha512-8EyMynh679x/0gqE9fT9oilG+qEt+ibFyqjuVTsZn1+CMxH+XLlpvr2UZx4nVcCwTpx81nICr2JQFkM+HPLq4w==} - engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} - dependencies: - imurmurhash: 0.1.4 - dev: false - /unist-util-is@4.1.0: resolution: {integrity: sha512-ZOQSsnce92GrxSqlnEEseX0gi7GH9zTJZ0p9dtu87WRb/37mMPO2Ilx1s/t9vBHrFhbgweUwb+t7cIn5dxPhZg==} From 266bde9baf28f413b9e2f7abc133b9bb6610bdff Mon Sep 17 00:00:00 2001 From: Emanuele Bardelli Date: Tue, 11 Nov 2025 19:24:29 -0800 Subject: [PATCH 09/22] Update to new duckdb/node-api --- packages/datasources/duckdb/index.cjs | 80 +++++++++++++++++++++------ 1 file changed, 64 insertions(+), 16 deletions(-) diff --git a/packages/datasources/duckdb/index.cjs b/packages/datasources/duckdb/index.cjs index 3a2fd2d085..873d2b00be 100644 --- a/packages/datasources/duckdb/index.cjs +++ b/packages/datasources/duckdb/index.cjs @@ -5,7 +5,7 @@ const { cleanQuery, exhaustStream } = require('@evidence-dev/db-commons'); -const { Database } = require('@duckdb/node-api'); +const { DuckDBInstance } = require('@duckdb/node-api'); const path = require('path'); const fs = require('fs/promises'); @@ -129,7 +129,8 @@ const runQuery = async (queryString, database, batchSize = 100000) => { } const mode = filename !== ':memory:' ? 'READ_ONLY' : 'READ_WRITE'; - const db = await Database.create(filename, { + // Create a DuckDB instance and connection using the new node-api + const db = await DuckDBInstance.create(filename, { access_mode: mode, custom_user_agent: 'evidence-dev' }); @@ -141,27 +142,74 @@ const runQuery = async (queryString, database, batchSize = 100000) => { const initScript = await fs.readFile(path.resolve(database.directory, 'initialize.sql'), { encoding: 'utf-8' }); - await conn.exec(initScript); + // run the initialization script + await conn.run(initScript).catch(() => null); } } - const stream = conn.stream(queryString); - + // Prepare a reader that can stream chunks const count_query = `WITH root as (${cleanQuery(queryString)}) SELECT COUNT(*) FROM root`; - const expected_count = await db.all(count_query).catch(() => null); - const expected_row_count = expected_count?.[0]['count_star()']; + let expected_row_count = null; + try { + const countReader = await conn.runAndReadAll(count_query); + await countReader.readAll(); + const countRow = countReader.getRowObjectsJS()?.[0]; + if (countRow) { + // take the first value from the row (column name may vary) + expected_row_count = Object.values(countRow)[0]; + } + } catch (e) { + expected_row_count = null; + } const column_query = `DESCRIBE ${cleanQuery(queryString)}`; - const column_types = await db - .all(column_query) - .then(duckdbDescribeToEvidenceType) - .catch(() => null); - - const results = await asyncIterableToBatchedAsyncGenerator(stream, batchSize, { - mapResultsToEvidenceColumnTypes: - column_types == null ? mapResultsToEvidenceColumnTypes : undefined, + let column_types = null; + try { + const colReader = await conn.runAndReadAll(column_query); + await colReader.readAll(); + const describeRows = colReader.getRowObjectsJS(); + column_types = duckdbDescribeToEvidenceType(describeRows); + } catch (e) { + column_types = null; + } + + // Create an async generator that yields batches using the DuckDBResultReader + const reader = await conn.streamAndRead(queryString); + + const rowsAsyncIterable = (async function* () { + try { + let prev = 0; + // keep reading until done + while (!reader.done) { + const target = prev + batchSize; + await reader.readUntil(target); + const allRows = reader.getRowObjectsJS(); + const newRows = allRows.slice(prev).map(standardizeRow); + if (newRows.length > 0) yield newRows; + prev = reader.currentRowCount; + } + } finally { + // disconnect the connection and close the instance synchronously + try { + conn.disconnectSync(); + } catch (err) {} + try { + db.closeSync(); + } catch (err) {} + } + })(); + + const results = await asyncIterableToBatchedAsyncGenerator(rowsAsyncIterable, batchSize, { + mapResultsToEvidenceColumnTypes: column_types == null ? mapResultsToEvidenceColumnTypes : undefined, standardizeRow, - closeConnection: () => db.close() + closeConnection: () => { + try { + conn.disconnectSync(); + } catch (err) {} + try { + db.closeSync(); + } catch (err) {} + } }); if (column_types != null) { results.columnTypes = column_types; From 26c3d7a88d3eec27e38b495ce2b845aec1874602 Mon Sep 17 00:00:00 2001 From: Emanuele Bardelli Date: Tue, 11 Nov 2025 20:35:04 -0800 Subject: [PATCH 10/22] Yield rows individually --- packages/datasources/duckdb/index.cjs | 8 +++++++- packages/datasources/duckdb/test/test.js | 17 +++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/packages/datasources/duckdb/index.cjs b/packages/datasources/duckdb/index.cjs index 873d2b00be..0687eefb00 100644 --- a/packages/datasources/duckdb/index.cjs +++ b/packages/datasources/duckdb/index.cjs @@ -125,6 +125,9 @@ const runQuery = async (queryString, database, batchSize = 100000) => { } else if (database.directory) { // Local database stored in source directory filename = path.join(database.directory, database.filename); + } else { + // filename provided without a directory; use as given (may be absolute or relative) + filename = database.filename; } } @@ -185,7 +188,10 @@ const runQuery = async (queryString, database, batchSize = 100000) => { await reader.readUntil(target); const allRows = reader.getRowObjectsJS(); const newRows = allRows.slice(prev).map(standardizeRow); - if (newRows.length > 0) yield newRows; + // yield rows one-by-one (asyncIterableToBatchedAsyncGenerator expects single-row yields) + for (const r of newRows) { + yield r; + } prev = reader.currentRowCount; } } finally { diff --git a/packages/datasources/duckdb/test/test.js b/packages/datasources/duckdb/test/test.js index 3b6f043044..9c393a5880 100644 --- a/packages/datasources/duckdb/test/test.js +++ b/packages/datasources/duckdb/test/test.js @@ -1,9 +1,26 @@ import { test } from 'uvu'; import * as assert from 'uvu/assert'; +import path from 'path'; +import { fileURLToPath } from 'url'; import runQuery from '../index.cjs'; import { batchedAsyncGeneratorToArray, TypeFidelity } from '@evidence-dev/db-commons'; import 'dotenv/config'; +test('basic select from needful_things.duckdb', async () => { + // Resolve the database file path relative to the repository root (from this test file) + const __filename = fileURLToPath(import.meta.url); + const __dirname = path.dirname(__filename); + const dbPath = path.join(__dirname, '..', '..', '..', '..', 'needful_things.duckdb'); + // Select the rows from sqlite_master; this test database contains 7 entries + const query = 'SELECT name FROM sqlite_master'; + const { rows: rowGen, expectedRowCount } = await runQuery(query, { filename: dbPath }); + const rows = await batchedAsyncGeneratorToArray(rowGen); + assert.instance(rows, Array); + assert.type(rows[0], 'object'); + // Expect exactly 7 rows in this test database + assert.equal(rows.length, 7); +}); + // Types to test // BOOLEAN // TINYINT From e23de237a7efcee4d35f0ba811c7966c2d8da0e7 Mon Sep 17 00:00:00 2001 From: Emanuele Bardelli Date: Wed, 12 Nov 2025 12:13:25 -0800 Subject: [PATCH 11/22] Add splitSQLStatements to db-commons --- packages/datasources/duckdb/index.cjs | 81 +++++++++---- packages/datasources/duckdb/test/test.js | 142 +++++++++++++++++++++++ packages/lib/db-commons/index.cjs | 83 +++++++++++++ 3 files changed, 281 insertions(+), 25 deletions(-) diff --git a/packages/datasources/duckdb/index.cjs b/packages/datasources/duckdb/index.cjs index 0687eefb00..046d7266a9 100644 --- a/packages/datasources/duckdb/index.cjs +++ b/packages/datasources/duckdb/index.cjs @@ -3,12 +3,14 @@ const { TypeFidelity, asyncIterableToBatchedAsyncGenerator, cleanQuery, - exhaustStream + exhaustStream, + splitSQLStatements } = require('@evidence-dev/db-commons'); const { DuckDBInstance } = require('@duckdb/node-api'); const path = require('path'); const fs = require('fs/promises'); + /** * Converts BigInt values to Numbers in an object. * @param {Record} obj - The input object with potential BigInt values. @@ -150,49 +152,78 @@ const runQuery = async (queryString, database, batchSize = 100000) => { } } - // Prepare a reader that can stream chunks - const count_query = `WITH root as (${cleanQuery(queryString)}) SELECT COUNT(*) FROM root`; + // Split the incoming SQL into statements and treat all but the last as prefix + // statements which should be executed before metadata queries or the main + // streaming query. This allows SET/USE/CREATE statements to affect the + // session. + const statements = splitSQLStatements(queryString); + const prefixStatements = statements.slice(0, -1); + const mainStatement = statements.length > 0 ? statements[statements.length - 1] : ''; + + const count_query = mainStatement + ? `WITH root as (${cleanQuery(mainStatement)}) SELECT COUNT(*) FROM root` + : null; let expected_row_count = null; try { - const countReader = await conn.runAndReadAll(count_query); - await countReader.readAll(); - const countRow = countReader.getRowObjectsJS()?.[0]; - if (countRow) { - // take the first value from the row (column name may vary) - expected_row_count = Object.values(countRow)[0]; + // Execute prefix statements first so they apply to the session + for (const ps of prefixStatements) { + try { + await conn.run(ps).catch(() => null); + } catch (err) { + // ignore errors from prefix statements to mimic existing behavior + } + } + + if (count_query) { + const countReader = await conn.runAndReadAll(count_query); + await countReader.readAll(); + const countRow = countReader.getRowObjectsJS()?.[0]; + if (countRow) { + // take the first value from the row (column name may vary) + expected_row_count = Object.values(countRow)[0]; + } } } catch (e) { expected_row_count = null; } - const column_query = `DESCRIBE ${cleanQuery(queryString)}`; + const column_query = mainStatement ? `DESCRIBE ${cleanQuery(mainStatement)}` : null; let column_types = null; try { - const colReader = await conn.runAndReadAll(column_query); - await colReader.readAll(); - const describeRows = colReader.getRowObjectsJS(); - column_types = duckdbDescribeToEvidenceType(describeRows); + if (column_query) { + const colReader = await conn.runAndReadAll(column_query); + await colReader.readAll(); + const describeRows = colReader.getRowObjectsJS(); + column_types = duckdbDescribeToEvidenceType(describeRows); + } } catch (e) { column_types = null; } // Create an async generator that yields batches using the DuckDBResultReader - const reader = await conn.streamAndRead(queryString); + // Stream only the main statement (last statement). If there is no main + // statement, create an empty generator. + let reader = null; + if (mainStatement) { + reader = await conn.streamAndRead(mainStatement); + } const rowsAsyncIterable = (async function* () { try { let prev = 0; - // keep reading until done - while (!reader.done) { - const target = prev + batchSize; - await reader.readUntil(target); - const allRows = reader.getRowObjectsJS(); - const newRows = allRows.slice(prev).map(standardizeRow); - // yield rows one-by-one (asyncIterableToBatchedAsyncGenerator expects single-row yields) - for (const r of newRows) { - yield r; + if (reader) { + // keep reading until done + while (!reader.done) { + const target = prev + batchSize; + await reader.readUntil(target); + const allRows = reader.getRowObjectsJS(); + const newRows = allRows.slice(prev).map(standardizeRow); + // yield rows one-by-one (asyncIterableToBatchedAsyncGenerator expects single-row yields) + for (const r of newRows) { + yield r; + } + prev = reader.currentRowCount; } - prev = reader.currentRowCount; } } finally { // disconnect the connection and close the instance synchronously diff --git a/packages/datasources/duckdb/test/test.js b/packages/datasources/duckdb/test/test.js index 9c393a5880..3008e506dc 100644 --- a/packages/datasources/duckdb/test/test.js +++ b/packages/datasources/duckdb/test/test.js @@ -4,6 +4,7 @@ import path from 'path'; import { fileURLToPath } from 'url'; import runQuery from '../index.cjs'; import { batchedAsyncGeneratorToArray, TypeFidelity } from '@evidence-dev/db-commons'; +import fs from 'fs/promises'; import 'dotenv/config'; test('basic select from needful_things.duckdb', async () => { @@ -198,6 +199,20 @@ test('query runs', async () => { } }); +test('expectedRowCount present for UNION with NULL-first-row', async () => { + // Simple reproduction: first row has NULL, second row has a value + const query = `select NULL as a UNION ALL select 1 as a`; + const { rows, expectedRowCount } = await runQuery(query, undefined, 2); + // consume rows to ensure the stream runs + const arr = []; + for await (const batch of rows()) { + arr.push(...batch); + } + assert.equal(arr.length, 2); + assert.type(expectedRowCount, 'number'); + assert.equal(expectedRowCount, 2); +}); + test('query batches results properly', async () => { try { const { rows, expectedRowCount } = await runQuery( @@ -220,4 +235,131 @@ test('query batches results properly', async () => { } }); +test('handles leading SET statements before main query', async () => { + try { + const query = ` + SET VARIABLE VAR1 = DATE '2023-10-04'; + SET VARIABLE VAR2 = '23'; + select 1 union all select 2 union all select 3; + `; + const { rows, expectedRowCount } = await runQuery(query, undefined, 2); + const arr = []; + for await (const batch of rows()) { + arr.push(batch); + } + // should batch into [2,1] + assert.equal(arr[0].length, 2); + assert.equal(arr[1].length, 1); + assert.equal(expectedRowCount, 3); + } catch (e) { + throw Error(e); + } +}); + +test('semicolon inside single-quoted string should not split statements', async () => { + try { + const query = "SET V='this;is;a;string'; select 1 union all select 2;"; + const { rows, expectedRowCount } = await runQuery(query, undefined, 2); + const arr = []; + for await (const batch of rows()) { + arr.push(batch); + } + assert.equal(expectedRowCount, 2); + assert.equal(arr[0].length, 2); + } catch (e) { + throw Error(e); + } +}); + +test('semicolon inside block comment should not split statements', async () => { + try { + const query = "/* comment; still comment; */ select 1 union all select 2;"; + const { rows, expectedRowCount } = await runQuery(query, undefined, 2); + const arr = []; + for await (const batch of rows()) { + arr.push(batch); + } + assert.equal(expectedRowCount, 2); + assert.equal(arr[0].length, 2); + } catch (e) { + throw Error(e); + } +}); + +test('handles leading SET statements before main query', async () => { + try { + const query = ` + SET VARIABLE VAR1 = DATE '2023-10-04'; + SET VARIABLE VAR2 = '23'; + select 1 union all select 2 union all select 3; + `; + const { rows, expectedRowCount } = await runQuery(query, undefined, 2); + const arr = []; + for await (const batch of rows()) { + arr.push(batch); + } + // should batch into [2,1] + assert.equal(arr[0].length, 2); + assert.equal(arr[1].length, 1); + assert.equal(expectedRowCount, 3); + } catch (e) { + throw Error(e); + } +}); + +test('semicolon inside single-quoted string should not split statements', async () => { + try { + const query = "SET V='this;is;a;string'; select 1 union all select 2;"; + const { rows, expectedRowCount } = await runQuery(query, undefined, 2); + const arr = []; + for await (const batch of rows()) { + arr.push(batch); + } + assert.equal(expectedRowCount, 2); + assert.equal(arr[0].length, 2); + } catch (e) { + throw Error(e); + } +}); + +test('semicolon inside block comment should not split statements', async () => { + try { + const query = '/* comment; still comment; */ select 1 union all select 2;'; + const { rows, expectedRowCount } = await runQuery(query, undefined, 2); + const arr = []; + for await (const batch of rows()) { + arr.push(batch); + } + assert.equal(expectedRowCount, 2); + assert.equal(arr[0].length, 2); + } catch (e) { + throw Error(e); + } +}); + +test('USE statement before query should apply to metadata queries', async () => { + try { + // Create a temporary on-disk DuckDB file to test that USE affects + // subsequent queries in the same session. We create a schema and table + // in prefix statements, then USE the schema and select from the table. + // Use an in-memory database so we can CREATE schema/table in the test + const dbFile = ':memory:'; + const query = ` + CREATE SCHEMA s; + CREATE TABLE s.t AS SELECT 1 AS x UNION ALL SELECT 2 AS x; + USE s; + SELECT x FROM t; + `; + const { rows, expectedRowCount } = await runQuery(query, { filename: dbFile }, 2); + const arr = []; + for await (const batch of rows()) { + arr.push(batch); + } + assert.equal(expectedRowCount, 2); + assert.equal(arr[0].length, 2); + } catch (e) { + throw Error(e); + } +}); + test.run(); diff --git a/packages/lib/db-commons/index.cjs b/packages/lib/db-commons/index.cjs index 88464614a9..93745dcb4b 100644 --- a/packages/lib/db-commons/index.cjs +++ b/packages/lib/db-commons/index.cjs @@ -197,6 +197,88 @@ const cleanQuery = (query) => { return cleanedString + '\n'; }; +/** + * Split a SQL script into individual statements, respecting single/double/backtick + * quotes and block/line comments so semicolons inside those constructs don't split. + * Returns an array of statements (strings) without trailing semicolons and trimmed. + * @param {string} sql + * @returns {string[]} + */ +const splitSQLStatements = function (sql) { + const statements = []; + let cur = ''; + let inSingle = false; + let inDouble = false; + let inBacktick = false; + let inLineComment = false; + let inBlockComment = false; + for (let i = 0; i < sql.length; i++) { + const ch = sql[i]; + const next = sql[i + 1]; + + if (inLineComment) { + cur += ch; + if (ch === '\n') { + inLineComment = false; + } + continue; + } + if (inBlockComment) { + cur += ch; + if (ch === '*' && next === '/') { + cur += next; + i++; + inBlockComment = false; + } + continue; + } + + // start of line comment + if (!inSingle && !inDouble && !inBacktick && ch === '-' && next === '-') { + cur += ch; + inLineComment = true; + continue; + } + // start of block comment + if (!inSingle && !inDouble && !inBacktick && ch === '/' && next === '*') { + cur += ch; + inBlockComment = true; + continue; + } + + // quotes + if (!inDouble && !inBacktick && ch === "'") { + inSingle = !inSingle; + cur += ch; + continue; + } + if (!inSingle && !inBacktick && ch === '"') { + inDouble = !inDouble; + cur += ch; + continue; + } + if (!inSingle && !inDouble && ch === '`') { + inBacktick = !inBacktick; + cur += ch; + continue; + } + + // semicolon splits statements only when not inside quotes or comments + if (ch === ';' && !inSingle && !inDouble && !inBacktick && !inLineComment && !inBlockComment) { + const s = cur.trim(); + if (s.length > 0) statements.push(s); + cur = ''; + continue; + } + + cur += ch; + } + + const last = cur.trim(); + if (last.length > 0) statements.push(last); + return statements; +}; + /** * @param {QueryResult} stream * @returns {Promise} @@ -217,5 +299,6 @@ exports.asyncIterableToBatchedAsyncGenerator = asyncIterableToBatchedAsyncGenera exports.batchedAsyncGeneratorToArray = batchedAsyncGeneratorToArray; exports.cleanQuery = cleanQuery; exports.exhaustStream = exhaustStream; +exports.splitSQLStatements = splitSQLStatements; exports.getEnv = require('./src/getEnv.cjs').getEnv; From 9785bb0eccddad8b6e1012a897ff613cc5e66243 Mon Sep 17 00:00:00 2001 From: Emanuele Bardelli Date: Wed, 12 Nov 2025 15:03:18 -0800 Subject: [PATCH 12/22] Change `splitSQLStatements` to singular --- packages/datasources/duckdb/index.cjs | 4 ++-- packages/lib/db-commons/index.cjs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/datasources/duckdb/index.cjs b/packages/datasources/duckdb/index.cjs index 046d7266a9..aa374bed3b 100644 --- a/packages/datasources/duckdb/index.cjs +++ b/packages/datasources/duckdb/index.cjs @@ -4,7 +4,7 @@ const { asyncIterableToBatchedAsyncGenerator, cleanQuery, exhaustStream, - splitSQLStatements + splitSQLStatement } = require('@evidence-dev/db-commons'); const { DuckDBInstance } = require('@duckdb/node-api'); const path = require('path'); @@ -156,7 +156,7 @@ const runQuery = async (queryString, database, batchSize = 100000) => { // statements which should be executed before metadata queries or the main // streaming query. This allows SET/USE/CREATE statements to affect the // session. - const statements = splitSQLStatements(queryString); + const statements = splitSQLStatement(queryString); const prefixStatements = statements.slice(0, -1); const mainStatement = statements.length > 0 ? statements[statements.length - 1] : ''; diff --git a/packages/lib/db-commons/index.cjs b/packages/lib/db-commons/index.cjs index 93745dcb4b..eca0d0770d 100644 --- a/packages/lib/db-commons/index.cjs +++ b/packages/lib/db-commons/index.cjs @@ -204,7 +204,7 @@ const cleanQuery = (query) => { * @param {string} sql * @returns {string[]} */ -const splitSQLStatements = function (sql) { +const splitSQLStatement = function (sql) { const statements = []; let cur = ''; let inSingle = false; @@ -299,6 +299,6 @@ exports.asyncIterableToBatchedAsyncGenerator = asyncIterableToBatchedAsyncGenera exports.batchedAsyncGeneratorToArray = batchedAsyncGeneratorToArray; exports.cleanQuery = cleanQuery; exports.exhaustStream = exhaustStream; -exports.splitSQLStatements = splitSQLStatements; +exports.splitSQLStatement = splitSQLStatement; exports.getEnv = require('./src/getEnv.cjs').getEnv; From 0266017dac5caa1ca7dd019cb21f8afe78533992 Mon Sep 17 00:00:00 2001 From: Emanuele Bardelli Date: Thu, 13 Nov 2025 11:40:26 -0800 Subject: [PATCH 13/22] Use lastest stable releases as of 11/13/25 --- packages/datasources/duckdb/package.json | 2 +- packages/lib/universal-sql/package.json | 2 +- pnpm-lock.yaml | 52 ++++++++++++------------ 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/packages/datasources/duckdb/package.json b/packages/datasources/duckdb/package.json index dbeea12d0d..b54dcf3dea 100644 --- a/packages/datasources/duckdb/package.json +++ b/packages/datasources/duckdb/package.json @@ -10,7 +10,7 @@ }, "dependencies": { "@evidence-dev/db-commons": "workspace:*", - "@duckdb/node-api": "^1.4.1-r.5" + "@duckdb/node-api": "^1.4.2-r.1" }, "devDependencies": { "dotenv": "^16.0.1" diff --git a/packages/lib/universal-sql/package.json b/packages/lib/universal-sql/package.json index 8e2d9393ab..0abe02daa7 100644 --- a/packages/lib/universal-sql/package.json +++ b/packages/lib/universal-sql/package.json @@ -14,7 +14,7 @@ }, "type": "module", "dependencies": { - "@duckdb/duckdb-wasm": "^1.31.1-dev1.0", + "@duckdb/duckdb-wasm": "^1.30.0", "apache-arrow": "17.0.0", "chalk": "^5.2.0", "cli-progress": "^3.12.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6fbaae1c2d..4a556b6204 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -640,8 +640,8 @@ importers: packages/datasources/duckdb: dependencies: '@duckdb/node-api': - specifier: ^1.4.1-r.5 - version: 1.4.1-r.5 + specifier: ^1.4.2-r.1 + version: 1.4.2-r.1 '@evidence-dev/db-commons': specifier: workspace:* version: link:../../lib/db-commons @@ -1270,8 +1270,8 @@ importers: packages/lib/universal-sql: dependencies: '@duckdb/duckdb-wasm': - specifier: ^1.31.1-dev1.0 - version: 1.31.1-dev1.0 + specifier: ^1.30.0 + version: 1.30.0 apache-arrow: specifier: 17.0.0 version: 17.0.0 @@ -3840,66 +3840,66 @@ packages: apache-arrow: 17.0.0 dev: true - /@duckdb/duckdb-wasm@1.31.1-dev1.0: - resolution: {integrity: sha512-q2QiJ7I+D9reSSmHeI+J5nMldeH4xUtYMuFJfzEhyiddsAsH2omF7oqWjC6JyVTodsaCfFx9rtfEvBHxa5IXcg==} + /@duckdb/duckdb-wasm@1.30.0: + resolution: {integrity: sha512-9aWrm+4ayl4sTlvGtl/b+LxrUyXaac3yyVqkoJ3F7Vkd62PoS8PcQIRJ/KjXBW36LP1CnPY5jjvFyIcTFLtcXA==} dependencies: apache-arrow: 17.0.0 dev: false - /@duckdb/node-api@1.4.1-r.5: - resolution: {integrity: sha512-RBDVqlMp3/GYKF/VhHcUrIYGJoOOMmVN2Ha7FPpm0Qpbqzh5RAY+QSfq30PTkRvIav4IwhM/FOu8gJNUqYCZGw==} + /@duckdb/node-api@1.4.2-r.1: + resolution: {integrity: sha512-8Ef4R/Lq+rXTpcqZIJZe6ALfgMGxy88HmiPvRpWrRw1fUTy85x1U0NnGrqCklZsmWyZUdPwVYj90MQOF2MY/TA==} dependencies: - '@duckdb/node-bindings': 1.4.1-r.5 + '@duckdb/node-bindings': 1.4.2-r.1 dev: false - /@duckdb/node-bindings-darwin-arm64@1.4.1-r.5: - resolution: {integrity: sha512-7LhYHy98ZqkGsK5XwDT/p7tCnGBTmZuxiEeNGJ/sdqvLJeDfbJuFnkXwnKMAue8ilgpI1pdET1QmqfdiKgkJyw==} + /@duckdb/node-bindings-darwin-arm64@1.4.2-r.1: + resolution: {integrity: sha512-C4yBI3zBX7iZ9iq8zJDvPatXA+2xM22sg1kX3fx76nG+qTqKtU/dGFVYL7Fx6cBYxBO1ZZ8Y+vjgYb6/bich8A==} cpu: [arm64] os: [darwin] requiresBuild: true dev: false optional: true - /@duckdb/node-bindings-darwin-x64@1.4.1-r.5: - resolution: {integrity: sha512-LIHHNPecG5Yk5JS+CnUMvq0dUaJ9glyRyV7w9GedyjxRYqT9d4Sbxl5UCj3a6Pu6Sf8IZrWtWXQB0C/4e6ITKw==} + /@duckdb/node-bindings-darwin-x64@1.4.2-r.1: + resolution: {integrity: sha512-dgTSBuEfWyhymYovsGoRESjqcJuZWwJqla9l89SSsBDrGYcUp+EHsXUF73oUCspzTesT2lOvHrDufO8pKgtudw==} cpu: [x64] os: [darwin] requiresBuild: true dev: false optional: true - /@duckdb/node-bindings-linux-arm64@1.4.1-r.5: - resolution: {integrity: sha512-G1dp7PY7gQR0svheXVAQdrsOKv61jQxaM6jhp3FhhSNt2cHlsHld8kjnjXVCDr3OVlTP0vez/1mcbWpLZ7eB+w==} + /@duckdb/node-bindings-linux-arm64@1.4.2-r.1: + resolution: {integrity: sha512-r2Ml1LvU2vkVMx4YU04T79FjGkYg8YMVtbOz7j740SZGIi5Cch5P1/oy48jJBWRqoaXuqimpYWeTZiGVeCQiZA==} cpu: [arm64] os: [linux] requiresBuild: true dev: false optional: true - /@duckdb/node-bindings-linux-x64@1.4.1-r.5: - resolution: {integrity: sha512-BVkWzA7Lrl5VQcPNh3k8mb/83kyH+csZTdHM2sePjxCts0zvji9hnGnBIIn33xarHSd0+3igBDijeqxxiFiPYg==} + /@duckdb/node-bindings-linux-x64@1.4.2-r.1: + resolution: {integrity: sha512-ed5KiNIu1rqSva5rgo4PRVYSmBolAMVqGFWsYWLoRZ94ka2F/dHwJNkarCTmBFCEDGVZWzWzjRyWTQgYTvQxTg==} cpu: [x64] os: [linux] requiresBuild: true dev: false optional: true - /@duckdb/node-bindings-win32-x64@1.4.1-r.5: - resolution: {integrity: sha512-Y/+vsLVGXIx9UMMyYDpYCSY3m0hXC1RjSaYfb7jkim8gcWQg0eRlbjTlDnmhth/8Z+eQ2sXXy8UVBd5IZ/bqYw==} + /@duckdb/node-bindings-win32-x64@1.4.2-r.1: + resolution: {integrity: sha512-kIIT8tuoW3mzr9EPcdSoKfv9CgOhTqRryHDI10Z0nuhc9UhqOWPUM/LnSUebfo1mdD9Drm7YrKCKYxNTr5dqBQ==} cpu: [x64] os: [win32] requiresBuild: true dev: false optional: true - /@duckdb/node-bindings@1.4.1-r.5: - resolution: {integrity: sha512-HUncdmpfjT4HxbH5NO6myazzudhddPc9lly7AgsYBdQSfTr1LCaQGb45cGiS5jEafpod9VOlk0bwJVbAIaBDWw==} + /@duckdb/node-bindings@1.4.2-r.1: + resolution: {integrity: sha512-z0pJCdEnIj0Famkip6w7JZ/A17nm5VcYc6H7isOHXfEnPhBQ9PBusUTFgIzl6+3J8HOFQOv0szJq46zldbsHfQ==} optionalDependencies: - '@duckdb/node-bindings-darwin-arm64': 1.4.1-r.5 - '@duckdb/node-bindings-darwin-x64': 1.4.1-r.5 - '@duckdb/node-bindings-linux-arm64': 1.4.1-r.5 - '@duckdb/node-bindings-linux-x64': 1.4.1-r.5 - '@duckdb/node-bindings-win32-x64': 1.4.1-r.5 + '@duckdb/node-bindings-darwin-arm64': 1.4.2-r.1 + '@duckdb/node-bindings-darwin-x64': 1.4.2-r.1 + '@duckdb/node-bindings-linux-arm64': 1.4.2-r.1 + '@duckdb/node-bindings-linux-x64': 1.4.2-r.1 + '@duckdb/node-bindings-win32-x64': 1.4.2-r.1 dev: false /@esbuild/aix-ppc64@0.25.12: From 73361ad0424905bfb49cd0612780ad54accc31ff Mon Sep 17 00:00:00 2001 From: Emanuele Bardelli Date: Thu, 13 Nov 2025 12:47:10 -0800 Subject: [PATCH 14/22] Add changeset --- .changeset/tricky-snails-argue.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .changeset/tricky-snails-argue.md diff --git a/.changeset/tricky-snails-argue.md b/.changeset/tricky-snails-argue.md new file mode 100644 index 0000000000..6ad71e5294 --- /dev/null +++ b/.changeset/tricky-snails-argue.md @@ -0,0 +1,7 @@ +--- +'@evidence-dev/duckdb': major +'@evidence-dev/universal-sql': major +'@evidence-dev/db-commons': minor +--- + +Update to latest duckdb packages From 23e10c71b962a94f64e2a12fb5a67a607299d399 Mon Sep 17 00:00:00 2001 From: Emanuele Bardelli Date: Thu, 13 Nov 2025 12:54:12 -0800 Subject: [PATCH 15/22] Run pnpm run format --- packages/datasources/duckdb/index.cjs | 4 ++-- packages/datasources/duckdb/test/test.js | 4 ++-- packages/ui/core-components/CHANGELOG.md | 1 - 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/datasources/duckdb/index.cjs b/packages/datasources/duckdb/index.cjs index aa374bed3b..8bb6fd67bc 100644 --- a/packages/datasources/duckdb/index.cjs +++ b/packages/datasources/duckdb/index.cjs @@ -10,7 +10,6 @@ const { DuckDBInstance } = require('@duckdb/node-api'); const path = require('path'); const fs = require('fs/promises'); - /** * Converts BigInt values to Numbers in an object. * @param {Record} obj - The input object with potential BigInt values. @@ -237,7 +236,8 @@ const runQuery = async (queryString, database, batchSize = 100000) => { })(); const results = await asyncIterableToBatchedAsyncGenerator(rowsAsyncIterable, batchSize, { - mapResultsToEvidenceColumnTypes: column_types == null ? mapResultsToEvidenceColumnTypes : undefined, + mapResultsToEvidenceColumnTypes: + column_types == null ? mapResultsToEvidenceColumnTypes : undefined, standardizeRow, closeConnection: () => { try { diff --git a/packages/datasources/duckdb/test/test.js b/packages/datasources/duckdb/test/test.js index 3008e506dc..cec18ddc53 100644 --- a/packages/datasources/duckdb/test/test.js +++ b/packages/datasources/duckdb/test/test.js @@ -201,7 +201,7 @@ test('query runs', async () => { test('expectedRowCount present for UNION with NULL-first-row', async () => { // Simple reproduction: first row has NULL, second row has a value - const query = `select NULL as a UNION ALL select 1 as a`; + const query = `select NULL as a UNION ALL select 1 as a`; const { rows, expectedRowCount } = await runQuery(query, undefined, 2); // consume rows to ensure the stream runs const arr = []; @@ -273,7 +273,7 @@ test('semicolon inside single-quoted string should not split statements', async test('semicolon inside block comment should not split statements', async () => { try { - const query = "/* comment; still comment; */ select 1 union all select 2;"; + const query = '/* comment; still comment; */ select 1 union all select 2;'; const { rows, expectedRowCount } = await runQuery(query, undefined, 2); const arr = []; for await (const batch of rows()) { diff --git a/packages/ui/core-components/CHANGELOG.md b/packages/ui/core-components/CHANGELOG.md index 0718e0b6e2..4ad9a2d7b7 100644 --- a/packages/ui/core-components/CHANGELOG.md +++ b/packages/ui/core-components/CHANGELOG.md @@ -7,7 +7,6 @@ - 6d2782e64: Fixed base path not being applied to data file (parquet) URLs. This resolves an issue where applications deployed with a base path would fail to load data files, resulting in 404 errors. The fix restores the dependency injection pattern for the `addBasePath` function in `setParquetURLs`, ensuring that base paths are correctly applied to all data requests in both monorepo and published package environments. - - @evidence-dev/component-utilities@4.0.10 - @evidence-dev/tailwind@3.1.1 From d7ccb319de5d295e84fa1eb8b5b0e08197e6adfd Mon Sep 17 00:00:00 2001 From: Emanuele Bardelli Date: Fri, 14 Nov 2025 07:02:49 -0800 Subject: [PATCH 16/22] Run format --- packages/datasources/mssql/test/test.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/datasources/mssql/test/test.js b/packages/datasources/mssql/test/test.js index 5952074f86..ff4d12a2cb 100644 --- a/packages/datasources/mssql/test/test.js +++ b/packages/datasources/mssql/test/test.js @@ -156,7 +156,10 @@ test('runQuery returns normalized error on SQL syntax error', async () => { assert.unreachable('Expected runQuery to throw'); } catch (e) { // The function normalizes errors to strings - assert.ok(typeof e === 'string' || e instanceof String || e.message, 'error should be string or have message'); + assert.ok( + typeof e === 'string' || e instanceof String || e.message, + 'error should be string or have message' + ); const msg = typeof e === 'string' ? e : e.message || String(e); // Message should be non-empty and mention syntax/select or be an error code assert.ok(msg && msg.length > 0, 'error message should be non-empty'); From ca69d92614733ad79ad6a48d2cd2129ab6c2c0a5 Mon Sep 17 00:00:00 2001 From: Emanuele Bardelli Date: Fri, 14 Nov 2025 07:11:37 -0800 Subject: [PATCH 17/22] Clean up changeset --- .changeset/stale-mirrors-itch.md | 21 +++++++++++++++++++++ .changeset/tricky-snails-argue.md | 7 ------- .changeset/warm-drinks-collect.md | 7 ------- 3 files changed, 21 insertions(+), 14 deletions(-) create mode 100644 .changeset/stale-mirrors-itch.md delete mode 100644 .changeset/tricky-snails-argue.md delete mode 100644 .changeset/warm-drinks-collect.md diff --git a/.changeset/stale-mirrors-itch.md b/.changeset/stale-mirrors-itch.md new file mode 100644 index 0000000000..7d5ac18487 --- /dev/null +++ b/.changeset/stale-mirrors-itch.md @@ -0,0 +1,21 @@ +--- +'@evidence-dev/duckdb': major +'@evidence-dev/universal-sql': major +'@evidence-dev/mssql': minor +'@evidence-dev/db-commons': minor +'@evidence-dev/core-components': minor +--- + +Update DuckDB to latest packages + +Update DuckDB to latest packages: + +- Switch to @duckdb/node-api from duckdb-async +- Update duckdb-wasm to latest release + +This release also has small data fixes across several packages: + +- Better handling of NULL values when discovering column types +- Fix batch processing of parquet files +- Fix error with temporary parquet files when reloading data in dev environment +- Better handing of error messages for the MSSQL driver in dev environment diff --git a/.changeset/tricky-snails-argue.md b/.changeset/tricky-snails-argue.md deleted file mode 100644 index 6ad71e5294..0000000000 --- a/.changeset/tricky-snails-argue.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -'@evidence-dev/duckdb': major -'@evidence-dev/universal-sql': major -'@evidence-dev/db-commons': minor ---- - -Update to latest duckdb packages diff --git a/.changeset/warm-drinks-collect.md b/.changeset/warm-drinks-collect.md deleted file mode 100644 index 0020f198f7..0000000000 --- a/.changeset/warm-drinks-collect.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -'@evidence-dev/duckdb': minor -'@evidence-dev/universal-sql': minor -'@evidence-dev/core-components': patch ---- - -Fix NULL in first row bug and temporary parquet files bugs From 37dc4da7749ebfa210d9f195c30b8f15fd41253e Mon Sep 17 00:00:00 2001 From: Emanuele Bardelli Date: Fri, 14 Nov 2025 08:37:46 -0800 Subject: [PATCH 18/22] Fix error when query had a trailing comment --- packages/lib/db-commons/index.cjs | 65 ++++++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 2 deletions(-) diff --git a/packages/lib/db-commons/index.cjs b/packages/lib/db-commons/index.cjs index eca0d0770d..ede569e423 100644 --- a/packages/lib/db-commons/index.cjs +++ b/packages/lib/db-commons/index.cjs @@ -266,7 +266,39 @@ const splitSQLStatement = function (sql) { // semicolon splits statements only when not inside quotes or comments if (ch === ';' && !inSingle && !inDouble && !inBacktick && !inLineComment && !inBlockComment) { const s = cur.trim(); - if (s.length > 0) statements.push(s); + // When splitting, ignore segments that contain only comments/whitespace. + // If the segment starts with comment(s) but contains SQL after them, + // we should still keep it. To determine this, strip leading comments + // and whitespace and check whether anything remains. + if (s.length > 0) { + let remainder = s; + // Remove leading whitespace and comments (both line and block) iteratively + while (true) { + remainder = remainder.trimStart(); + if (remainder.startsWith('--')) { + const nl = remainder.indexOf('\n'); + if (nl === -1) { + remainder = ''; + break; + } else { + remainder = remainder.slice(nl + 1); + continue; + } + } + if (remainder.startsWith('/*')) { + const end = remainder.indexOf('*/', 2); + if (end === -1) { + remainder = ''; + break; + } else { + remainder = remainder.slice(end + 2); + continue; + } + } + break; + } + if (remainder.trim().length > 0) statements.push(s); + } cur = ''; continue; } @@ -275,7 +307,36 @@ const splitSQLStatement = function (sql) { } const last = cur.trim(); - if (last.length > 0) statements.push(last); + // Ignore trailing content that is only comments. See logic above for stripping + // leading comments from a segment before deciding whether it contains SQL. + if (last.length > 0) { + let remainder = last; + while (true) { + remainder = remainder.trimStart(); + if (remainder.startsWith('--')) { + const nl = remainder.indexOf('\n'); + if (nl === -1) { + remainder = ''; + break; + } else { + remainder = remainder.slice(nl + 1); + continue; + } + } + if (remainder.startsWith('/*')) { + const end = remainder.indexOf('*/', 2); + if (end === -1) { + remainder = ''; + break; + } else { + remainder = remainder.slice(end + 2); + continue; + } + } + break; + } + if (remainder.trim().length > 0) statements.push(last); + } return statements; }; From bd8b2b5b3abab1c8a677d22a07aa46ab08d1d29b Mon Sep 17 00:00:00 2001 From: Emanuele Bardelli Date: Fri, 14 Nov 2025 08:41:41 -0800 Subject: [PATCH 19/22] Fix linter errors --- packages/datasources/duckdb/test/test.js | 3 +-- packages/datasources/mssql/test/test.js | 4 +++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/datasources/duckdb/test/test.js b/packages/datasources/duckdb/test/test.js index cfa3922db3..cf91c670ed 100644 --- a/packages/datasources/duckdb/test/test.js +++ b/packages/datasources/duckdb/test/test.js @@ -4,7 +4,6 @@ import path from 'path'; import { fileURLToPath } from 'url'; import runQuery from '../index.cjs'; import { batchedAsyncGeneratorToArray, TypeFidelity } from '@evidence-dev/db-commons'; -import fs from 'fs/promises'; import 'dotenv/config'; test('basic select from needful_things.duckdb', async () => { @@ -14,7 +13,7 @@ test('basic select from needful_things.duckdb', async () => { const dbPath = path.join(__dirname, '..', '..', '..', '..', 'needful_things.duckdb'); // Select the rows from sqlite_master; this test database contains 7 entries const query = 'SELECT name FROM sqlite_master'; - const { rows: rowGen, expectedRowCount } = await runQuery(query, { filename: dbPath }); + const { rows: rowGen } = await runQuery(query, { filename: dbPath }); const rows = await batchedAsyncGeneratorToArray(rowGen); assert.instance(rows, Array); assert.type(rows[0], 'object'); diff --git a/packages/datasources/mssql/test/test.js b/packages/datasources/mssql/test/test.js index ff4d12a2cb..975d7a5f67 100644 --- a/packages/datasources/mssql/test/test.js +++ b/packages/datasources/mssql/test/test.js @@ -2,6 +2,8 @@ import { test } from 'uvu'; import * as assert from 'uvu/assert'; import { createRequire } from 'module'; import 'dotenv/config'; +import runQuery from '../index.cjs'; +import { batchedAsyncGeneratorToArray, TypeFidelity } from '@evidence-dev/db-commons'; const require = createRequire(import.meta.url); @@ -121,7 +123,7 @@ test('runQuery returns normalized error on SQL syntax error', async () => { const EventEmitter = require('events'); const r = new EventEmitter(); r.stream = false; - r.query = function (q) { + r.query = function (_q) { // simulate an async error emitted by the request (syntax error) process.nextTick(() => { const err = new Error("Incorrect syntax near the keyword 'select'."); From 4db0827bca1a5dac5b3574e519a7025e2414635b Mon Sep 17 00:00:00 2001 From: Emanuele Bardelli Date: Fri, 14 Nov 2025 08:46:30 -0800 Subject: [PATCH 20/22] Add test for trailing comments --- packages/datasources/duckdb/test/test.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packages/datasources/duckdb/test/test.js b/packages/datasources/duckdb/test/test.js index cf91c670ed..7285728a6b 100644 --- a/packages/datasources/duckdb/test/test.js +++ b/packages/datasources/duckdb/test/test.js @@ -398,6 +398,20 @@ test('query batches results properly', async () => { } }); +test('handles trailing comments', async () => { + // Single-line trailing comment (run in-memory) + const q1 = 'select 1 as one -- trailing comment'; + const { rows: rowGen1 } = await runQuery(q1, undefined); + const rows1 = await batchedAsyncGeneratorToArray(rowGen1); + assert.equal(rows1[0].one, 1); + + // Block trailing comment + const q2 = 'select 2 as two /* trailing block comment */'; + const { rows: rowGen2 } = await runQuery(q2, undefined); + const rows2 = await batchedAsyncGeneratorToArray(rowGen2); + assert.equal(rows2[0].two, 2); +}); + test('handles leading SET statements before main query', async () => { try { const query = ` From 642a89cdaa24d51818c2fc4c0ef161acc721fe24 Mon Sep 17 00:00:00 2001 From: Emanuele Bardelli Date: Fri, 14 Nov 2025 11:28:59 -0800 Subject: [PATCH 21/22] Revert changes to mssql --- .../mssql/.changeset/purple-readers-live.md | 8 -- packages/datasources/mssql/index.cjs | 49 ++----------- packages/datasources/mssql/test/test.js | 73 +------------------ 3 files changed, 10 insertions(+), 120 deletions(-) delete mode 100644 packages/datasources/mssql/.changeset/purple-readers-live.md diff --git a/packages/datasources/mssql/.changeset/purple-readers-live.md b/packages/datasources/mssql/.changeset/purple-readers-live.md deleted file mode 100644 index 6595225692..0000000000 --- a/packages/datasources/mssql/.changeset/purple-readers-live.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -'@evidence-dev/duckdb': minor -'@evidence-dev/mssql': minor -'@evidence-dev/universal-sql': minor -'@evidence-dev/core-components': minor ---- - -Fix various data connector bugs diff --git a/packages/datasources/mssql/index.cjs b/packages/datasources/mssql/index.cjs index fd56064495..7360078080 100755 --- a/packages/datasources/mssql/index.cjs +++ b/packages/datasources/mssql/index.cjs @@ -144,51 +144,18 @@ const runQuery = async (queryString, database = {}, batchSize = 100000) => { const request = new mssql.Request(); request.stream = true; - - // Promise that resolves when recordset is emitted or rejects on request error - const recordsetPromise = new Promise((resolve, reject) => { - request.once('recordset', resolve); - request.once('error', reject); - }); - - // Start the streaming query request.query(queryString); - try { - const columns = await recordsetPromise; - - const stream = request.toReadableStream(); - - // Ensure any stream errors are handled to avoid unhandled 'error' events - stream.on('error', async (streamErr) => { - try { - await pool.close(); - } catch (_) { - // ignore close errors - } - // Nothing else to do here; the asyncIterableToBatchedAsyncGenerator - // consumer will observe the error as a rejection when iterating. - }); - - const results = await asyncIterableToBatchedAsyncGenerator(stream, batchSize, { - closeConnection: () => pool.close() - }); - results.columnTypes = mapResultsToEvidenceColumnTypes(columns); - results.expectedRowCount = expected_row_count; + const columns = await new Promise((res) => request.once('recordset', res)); - return results; - } catch (err) { - // Close pool when we hit errors from the request/stream and normalize the error - try { - await pool.close(); - } catch (_) {} + const stream = request.toReadableStream(); + const results = await asyncIterableToBatchedAsyncGenerator(stream, batchSize, { + closeConnection: () => pool.close() + }); + results.columnTypes = mapResultsToEvidenceColumnTypes(columns); + results.expectedRowCount = expected_row_count; - if (err && err.message) { - throw err.message.replace(/\n|\r/g, ' '); - } else { - throw ('' + err).replace(/\n|\r/g, ' '); - } - } + return results; } catch (err) { if (err.message) { throw err.message.replace(/\n|\r/g, ' '); diff --git a/packages/datasources/mssql/test/test.js b/packages/datasources/mssql/test/test.js index 975d7a5f67..a5be8b17f3 100644 --- a/packages/datasources/mssql/test/test.js +++ b/packages/datasources/mssql/test/test.js @@ -1,11 +1,8 @@ import { test } from 'uvu'; import * as assert from 'uvu/assert'; -import { createRequire } from 'module'; -import 'dotenv/config'; import runQuery from '../index.cjs'; -import { batchedAsyncGeneratorToArray, TypeFidelity } from '@evidence-dev/db-commons'; - -const require = createRequire(import.meta.url); +import { TypeFidelity, batchedAsyncGeneratorToArray } from '@evidence-dev/db-commons'; +import 'dotenv/config'; test('query runs', async () => { if (process.env.MSSQL_DATABASE) { @@ -102,70 +99,4 @@ test('query batches results properly', async () => { } }); -test('runQuery returns normalized error on SQL syntax error', async () => { - // Create a fake mssql module and inject before loading index.cjs - const fakeMssql = { - TYPES: {}, - connect: async () => ({ - request: () => ({ - query: async (q) => { - // If it's the COUNT(*) wrapper, return 0 - if (typeof q === 'string' && q.trim().toUpperCase().startsWith('SELECT COUNT(*)')) { - return { recordset: [{ expected_row_count: 0 }] }; - } - // otherwise, return empty - return { recordset: [] }; - } - }), - close: async () => {} - }), - Request: function () { - const EventEmitter = require('events'); - const r = new EventEmitter(); - r.stream = false; - r.query = function (_q) { - // simulate an async error emitted by the request (syntax error) - process.nextTick(() => { - const err = new Error("Incorrect syntax near the keyword 'select'."); - err.code = 'EREQUEST'; - r.emit('error', err); - }); - }; - r.toReadableStream = function () { - const { Readable } = require('stream'); - // A readable that immediately errors when read - const s = new Readable({ objectMode: true, read() {} }); - process.nextTick(() => s.emit('error', new Error('stream error'))); - return s; - }; - return r; - } - }; - - const path = require('path'); - const mssqlModulePath = path.join(process.cwd(), 'node_modules', 'mssql', 'index.js'); - require.cache[mssqlModulePath] = { - id: mssqlModulePath, - filename: mssqlModulePath, - loaded: true, - exports: fakeMssql - }; - - const runQuery = require('../index.cjs'); - - try { - await runQuery('select * from', {}, 10); - assert.unreachable('Expected runQuery to throw'); - } catch (e) { - // The function normalizes errors to strings - assert.ok( - typeof e === 'string' || e instanceof String || e.message, - 'error should be string or have message' - ); - const msg = typeof e === 'string' ? e : e.message || String(e); - // Message should be non-empty and mention syntax/select or be an error code - assert.ok(msg && msg.length > 0, 'error message should be non-empty'); - } -}); - test.run(); From b28f63f23996124b287b9f5dec626b545c3cb8c1 Mon Sep 17 00:00:00 2001 From: Emanuele Bardelli Date: Fri, 14 Nov 2025 11:30:45 -0800 Subject: [PATCH 22/22] Update changeset --- .changeset/{stale-mirrors-itch.md => witty-bobcats-care.md} | 4 ---- 1 file changed, 4 deletions(-) rename .changeset/{stale-mirrors-itch.md => witty-bobcats-care.md} (78%) diff --git a/.changeset/stale-mirrors-itch.md b/.changeset/witty-bobcats-care.md similarity index 78% rename from .changeset/stale-mirrors-itch.md rename to .changeset/witty-bobcats-care.md index 7d5ac18487..7fc156fb02 100644 --- a/.changeset/stale-mirrors-itch.md +++ b/.changeset/witty-bobcats-care.md @@ -1,13 +1,10 @@ --- '@evidence-dev/duckdb': major '@evidence-dev/universal-sql': major -'@evidence-dev/mssql': minor '@evidence-dev/db-commons': minor '@evidence-dev/core-components': minor --- -Update DuckDB to latest packages - Update DuckDB to latest packages: - Switch to @duckdb/node-api from duckdb-async @@ -18,4 +15,3 @@ This release also has small data fixes across several packages: - Better handling of NULL values when discovering column types - Fix batch processing of parquet files - Fix error with temporary parquet files when reloading data in dev environment -- Better handing of error messages for the MSSQL driver in dev environment