From 47a22233bd8bb07d9de0863016d93ea4415163a2 Mon Sep 17 00:00:00 2001 From: BittuBarnwal7479 Date: Wed, 1 Jul 2026 00:12:06 +0530 Subject: [PATCH 1/6] feat(connectors): add Databricks connector --- README.md | 5 +- .../content/docs/cli-reference/ktx-setup.mdx | 10 +- .../content/docs/configuration/ktx-yaml.mdx | 10 +- .../docs/getting-started/quickstart.mdx | 2 +- .../docs/integrations/primary-sources.mdx | 59 +- packages/cli/package.json | 1 + packages/cli/src/commands/setup-commands.ts | 3 +- packages/cli/src/connection-drivers.ts | 1 + .../src/connectors/databricks/connector.ts | 761 ++++++++++++++++++ .../cli/src/connectors/databricks/dialect.ts | 186 +++++ .../databricks/live-database-introspection.ts | 44 + .../context/connections/connection-type.ts | 1 + .../cli/src/context/connections/dialects.ts | 2 + .../cli/src/context/connections/drivers.ts | 21 + .../connections/local-warehouse-descriptor.ts | 1 + .../cli/src/context/project/driver-schemas.ts | 2 + packages/cli/src/context/scan/local-scan.ts | 3 +- packages/cli/src/context/scan/types.ts | 1 + .../src/context/sql-analysis/dialect-notes.ts | 4 +- .../sql-analysis/dialects/databricks.md | 10 + packages/cli/src/local-adapters.ts | 19 + packages/cli/src/setup-databases.ts | 91 +++ packages/cli/src/status-project.ts | 2 +- .../connectors/databricks/connector.test.ts | 336 ++++++++ .../connectors/databricks/dialect.test.ts | 43 + .../test/context/connections/dialects.test.ts | 32 +- .../test/context/connections/drivers.test.ts | 9 + .../local-warehouse-descriptor.test.ts | 1 + .../test/context/mcp/dialect-notes.test.ts | 3 +- packages/cli/test/setup-databases.test.ts | 1 + pnpm-lock.yaml | 632 ++++++++++++++- pnpm-workspace.yaml | 1 + 32 files changed, 2276 insertions(+), 21 deletions(-) create mode 100644 packages/cli/src/connectors/databricks/connector.ts create mode 100644 packages/cli/src/connectors/databricks/dialect.ts create mode 100644 packages/cli/src/connectors/databricks/live-database-introspection.ts create mode 100644 packages/cli/src/context/sql-analysis/dialects/databricks.md create mode 100644 packages/cli/test/connectors/databricks/connector.test.ts create mode 100644 packages/cli/test/connectors/databricks/dialect.test.ts diff --git a/README.md b/README.md index 3f704f047..02da8fa75 100644 --- a/README.md +++ b/README.md @@ -103,8 +103,9 @@ upkeep and don't absorb the rest of your company's knowledge. - You don't have a SQL warehouse - **ktx** sits on top of one - You only need one ad-hoc query - `psql` or a notebook will do -Works with PostgreSQL, Snowflake, BigQuery, ClickHouse, MySQL, SQL Server, and -SQLite. Integrates with dbt, MetricFlow, LookML, Looker, Metabase, and Notion. +Works with PostgreSQL, Snowflake, Databricks, BigQuery, ClickHouse, MySQL, SQL +Server, and SQLite. Integrates with dbt, MetricFlow, LookML, Looker, Metabase, +and Notion. ## Quick Start diff --git a/docs-site/content/docs/cli-reference/ktx-setup.mdx b/docs-site/content/docs/cli-reference/ktx-setup.mdx index 10c27c160..0c7e3ed4a 100644 --- a/docs-site/content/docs/cli-reference/ktx-setup.mdx +++ b/docs-site/content/docs/cli-reference/ktx-setup.mdx @@ -120,7 +120,7 @@ runtime features are missing. | Flag | Description | |------|-------------| -| `--database ` | Database driver to configure; repeatable. Choices: `sqlite`, `postgres`, `mysql`, `clickhouse`, `sqlserver`, `bigquery`, `snowflake` | +| `--database ` | Database driver to configure; repeatable. Choices: `sqlite`, `postgres`, `mysql`, `clickhouse`, `sqlserver`, `bigquery`, `snowflake`, `databricks` | | `--database-connection-id ` | Existing selected connection id; repeatable. With `--database` or `--database-url`, connection id for the new connection. | | `--database-url ` | URL, `env:NAME`, or `file:/path` for one new URL-style database connection; also used as the SQLite path | | `--database-schema ` | Database schema or dataset to include; repeatable | @@ -131,8 +131,8 @@ context. Use `--skip-databases` only when intentionally leaving the project incomplete. `--database-schema` maps to the driver's scope field: `schemas` for PostgreSQL, -MySQL, and SQL Server; `schema_names` for Snowflake; `dataset_ids` for -BigQuery; and `databases` for ClickHouse. +MySQL, and SQL Server; `schema_names` for Snowflake and Databricks; +`dataset_ids` for BigQuery; and `databases` for ClickHouse. A BigQuery `--database-schema` value may be qualified as `project.dataset` to scan a dataset hosted in another project (such as @@ -142,8 +142,8 @@ entries explicitly. See [Primary sources → BigQuery](/docs/integrations/primary-sources#cross-project-datasets). With `--no-input`, scope for a scope-bearing driver (PostgreSQL, MySQL, -ClickHouse, SQL Server, BigQuery, Snowflake) must come from `--database-schema` -or from existing connection config in `ktx.yaml` (for example +ClickHouse, SQL Server, BigQuery, Snowflake, Databricks) must come from +`--database-schema` or from existing connection config in `ktx.yaml` (for example `connections..dataset_ids`). When neither is set, the database step fails fast and prints the missing scope flag and config key — non-interactive setup never auto-discovers and scans every schema. SQLite has no scope and is diff --git a/docs-site/content/docs/configuration/ktx-yaml.mdx b/docs-site/content/docs/configuration/ktx-yaml.mdx index 24e58e39e..524af9513 100644 --- a/docs-site/content/docs/configuration/ktx-yaml.mdx +++ b/docs-site/content/docs/configuration/ktx-yaml.mdx @@ -112,6 +112,7 @@ context-source drivers share the map. | `sqlserver` | Warehouse | `driver` | `url`, `enabled_tables` | | `bigquery` | Warehouse | `driver` | `credentials_json`, `dataset_ids`, `enabled_tables`, `historicSql` | | `snowflake` | Warehouse | `driver` | `schema_names`, `enabled_tables`, `historicSql` | +| `databricks` | Warehouse | `driver`, `server_hostname`, `http_path`, `catalog` | `schema_names`, `enabled_tables` | | `clickhouse` | Warehouse | `driver` | `url`, `database`, `databases`, `enabled_tables` | | `metabase` | Context source | `driver`, `api_url` | `api_key_ref`, `mappings` | | `looker` | Context source | `driver`, `base_url`, `client_id` | `client_secret_ref`, `mappings` | @@ -170,6 +171,12 @@ connections: credentials_json: file:./service-account.json location: US dataset_ids: [analytics, mart] + databricks-warehouse: + driver: databricks + server_hostname: dbc-example.cloud.databricks.com + http_path: /sql/1.0/warehouses/abc123 + catalog: main + schema_names: [analytics, mart] ``` A BigQuery `dataset_ids` / `dataset_id` entry may be written `project.dataset` @@ -185,7 +192,8 @@ defaults to `4`. This caps all concurrent SQL work for that connector instance, including schema introspection, table sampling, relationship profiling, relationship validation, and read-only SQL execution. BigQuery and ClickHouse do not expose `maxConnections` because their connectors don't use client-side -connection pools. +connection pools. Databricks opens short-lived SQL sessions per operation and +also does not expose `maxConnections`. For Postgres, BigQuery, and Snowflake, `historicSql` and `context.queryHistory` toggle query-history ingest. The shape is connector-specific; the setup wizard diff --git a/docs-site/content/docs/getting-started/quickstart.mdx b/docs-site/content/docs/getting-started/quickstart.mdx index 2b8cbdfb9..07ec7c1ca 100644 --- a/docs-site/content/docs/getting-started/quickstart.mdx +++ b/docs-site/content/docs/getting-started/quickstart.mdx @@ -218,7 +218,7 @@ The wizard walks you through everything **ktx** needs in one pass: 3. **Embeddings** - picks an embeddings backend. Choose OpenAI for hosted embeddings or `sentence-transformers` to run locally without an API key. 4. **Database** - adds at least one primary connection. Supported drivers: - SQLite, PostgreSQL, MySQL, SQL Server, BigQuery, and Snowflake. + SQLite, PostgreSQL, MySQL, SQL Server, BigQuery, Snowflake, and Databricks. 5. **Context sources** - optionally adds dbt, MetricFlow, LookML, Looker, Metabase, or Notion. You can skip and add them later. 6. **Build** - offers to run the first ingest so semantic sources and wiki diff --git a/docs-site/content/docs/integrations/primary-sources.mdx b/docs-site/content/docs/integrations/primary-sources.mdx index 40af30e09..bd606fd27 100644 --- a/docs-site/content/docs/integrations/primary-sources.mdx +++ b/docs-site/content/docs/integrations/primary-sources.mdx @@ -1,6 +1,6 @@ --- title: Primary Sources -description: Connect ktx to PostgreSQL, Snowflake, BigQuery, MySQL, ClickHouse, SQL Server, SQLite, or MongoDB. +description: Connect ktx to PostgreSQL, Snowflake, Databricks, BigQuery, MySQL, ClickHouse, SQL Server, SQLite, or MongoDB. --- **ktx** connects to your data warehouse or database to build schema context, @@ -26,7 +26,7 @@ Agents should prefer environment or file references over literal secrets. | Field | Required | Applies to | Description | |-------|----------|------------|-------------| -| `driver` | Yes | all connections | Connector driver such as `postgres`, `snowflake`, `bigquery`, `mysql`, `clickhouse`, `sqlserver`, `sqlite`, or `mongodb` | +| `driver` | Yes | all connections | Connector driver such as `postgres`, `snowflake`, `databricks`, `bigquery`, `mysql`, `clickhouse`, `sqlserver`, `sqlite`, or `mongodb` | | `url` | One of the connection methods | URL-style connectors | Database URL, `env:NAME`, or `file:/path/to/secret` | | `host`, `port`, `database`, `username`, `password` | One of the connection methods | PostgreSQL, MySQL, SQL Server | Field-by-field connection values | | `schema` or `schemas` | No | schema-aware warehouses | Single schema or list of schemas to scan | @@ -192,6 +192,61 @@ artifact shape as Postgres and BigQuery. --- +## Databricks + +Connects via the Databricks SQL Driver for Node.js. Supports Unity Catalog +three-level namespaces, schema introspection, table sampling, column sampling, +and read-only SQL execution. + +### Connection config + +```yaml title="ktx.yaml" +connections: + my-databricks: + driver: databricks + server_hostname: dbc-example.cloud.databricks.com + http_path: /sql/1.0/warehouses/abc123 + catalog: main + schema_names: + - sales + - finance + authMethod: pat + token: env:DATABRICKS_TOKEN +``` + +`ktx setup` discovers schemas after the connection is verified and writes the +selected list to `schema_names`. For hand-written config, prefer +`schema_names` even for a single schema. `schema_name: sales` is accepted as a +single-schema shorthand only when `schema_names` is not set. + +### Authentication + +| Method | Config | +|--------|--------| +| Personal Access Token | `authMethod: pat`, `token: env:DATABRICKS_TOKEN` | +| OAuth machine-to-machine | `authMethod: oauth-m2m`, `client_id`, `client_secret: env:DATABRICKS_CLIENT_SECRET` | + +### Features + +| Feature | Supported | Notes | +|---------|-----------|-------| +| Tables & views | Yes | Via Unity Catalog `INFORMATION_SCHEMA.TABLES` | +| Primary keys | Yes | Via table constraints when available to the service principal | +| Foreign keys | No | Not emitted by the connector | +| Row count estimates | No | - | +| Column statistics | No | - | +| Query history | No | Databricks query-history ingest is not implemented | +| Table sampling | Yes | Uses read-only `SELECT ... LIMIT` sampling | + +### Dialect notes + +- Unity Catalog table names use `catalog.schema.table` +- Identifiers are quoted with backticks +- Parameter binding uses positional `?` placeholders +- Read-only SQL is validated before execution + +--- + ## BigQuery Authenticates via GCP service account credentials. Supports multi-dataset scanning and query-history configuration for `INFORMATION_SCHEMA.JOBS_BY_PROJECT`. diff --git a/packages/cli/package.json b/packages/cli/package.json index 2c2cb5acb..135198769 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -55,6 +55,7 @@ "@clack/prompts": "1.4.0", "@clickhouse/client": "^1.18.5", "@commander-js/extra-typings": "14.0.0", + "@databricks/sql": "^1.16.0", "@duckdb/node-api": "1.5.3-r.3", "@google-cloud/bigquery": "^8.3.1", "google-auth-library": "10.6.2", diff --git a/packages/cli/src/commands/setup-commands.ts b/packages/cli/src/commands/setup-commands.ts index 5f86b9c96..fdff68aec 100644 --- a/packages/cli/src/commands/setup-commands.ts +++ b/packages/cli/src/commands/setup-commands.ts @@ -43,7 +43,8 @@ function databaseDriver(value: string): KtxSetupDatabaseDriver { value === 'clickhouse' || value === 'sqlserver' || value === 'bigquery' || - value === 'snowflake' + value === 'snowflake' || + value === 'databricks' ) { return value; } diff --git a/packages/cli/src/connection-drivers.ts b/packages/cli/src/connection-drivers.ts index 511637f5b..41a32f0c1 100644 --- a/packages/cli/src/connection-drivers.ts +++ b/packages/cli/src/connection-drivers.ts @@ -9,6 +9,7 @@ export const KTX_DATABASE_DRIVER_IDS = [ 'sqlserver', 'bigquery', 'snowflake', + 'databricks', ] as const; // mongodb is a database driver but has no SQL dialect, so it sits outside the diff --git a/packages/cli/src/connectors/databricks/connector.ts b/packages/cli/src/connectors/databricks/connector.ts new file mode 100644 index 000000000..97f12be8b --- /dev/null +++ b/packages/cli/src/connectors/databricks/connector.ts @@ -0,0 +1,761 @@ +import { DBSQLClient } from '@databricks/sql'; +import { getSqlDialectForDriver } from '../../context/connections/dialects.js'; +import { queryDeadlineExceededError, resolveQueryDeadlineMs } from '../../context/connections/query-deadline.js'; +import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js'; +import { tryConstraintQuery } from '../../context/scan/constraint-discovery.js'; +import { scopedTableNames } from '../../context/scan/table-ref.js'; +import { + connectorTestFailure, + createKtxConnectorCapabilities, + type KtxColumnSampleInput, + type KtxColumnSampleResult, + type KtxColumnStatsInput, + type KtxColumnStatsResult, + type KtxConnectorTestResult, + type KtxQueryResult, + type KtxReadOnlyQueryInput, + type KtxScanConnector, + type KtxScanContext, + type KtxScanInput, + type KtxScanWarning, + type KtxSchemaColumn, + type KtxSchemaSnapshot, + type KtxSchemaTable, + type KtxTableListEntry, + type KtxTableRef, + type KtxTableSampleInput, + type KtxTableSampleResult, +} from '../../context/scan/types.js'; +import { resolveStringReference } from '../shared/string-reference.js'; + +export interface KtxDatabricksConnectionConfig { + driver?: string; + authMethod?: 'pat' | 'oauth-m2m'; + server_hostname?: string; + http_path?: string; + catalog?: string; + schema_name?: string; + schema_names?: string[]; + token?: string; + client_id?: string; + client_secret?: string; + query_timeout_ms?: number; + [key: string]: unknown; +} + +export interface KtxDatabricksResolvedConnectionConfig { + authMethod: 'pat' | 'oauth-m2m'; + serverHostname: string; + httpPath: string; + catalog: string; + schemas: string[]; + token?: string; + clientId?: string; + clientSecret?: string; + deadlineMs: number; +} + +export interface KtxDatabricksRawColumnMetadata { + name: string; + type: string; + nullable: boolean; + comment: string | null; +} + +export interface KtxDatabricksRawTableMetadata { + name: string; + catalog: string; + db: string; + kind: 'table' | 'view'; + rowCount: number | null; + comment: string | null; + columns: KtxDatabricksRawColumnMetadata[]; +} + +export interface KtxDatabricksDriver { + test(): Promise; + query(sql: string, params?: unknown[]): Promise; + getSchemaMetadata(schemaName: string, scopedTableNames?: readonly string[] | null): Promise; + listSchemas(): Promise; + listTables(schemas?: string[]): Promise; + cleanup(): Promise; +} + +export interface KtxDatabricksDriverFactory { + createDriver(input: { resolved: KtxDatabricksResolvedConnectionConfig }): KtxDatabricksDriver; +} + +export interface KtxDatabricksScanConnectorOptions { + connectionId: string; + connection: KtxDatabricksConnectionConfig | undefined; + driverFactory?: KtxDatabricksDriverFactory; + env?: NodeJS.ProcessEnv; + now?: () => Date; +} + +export interface KtxDatabricksReadOnlyQueryInput extends KtxReadOnlyQueryInput { + params?: Record; +} + +export interface KtxDatabricksColumnDistinctValuesOptions { + maxCardinality: number; + limit: number; + sampleSize?: number; +} + +export interface KtxDatabricksColumnDistinctValuesResult { + values: string[] | null; + cardinality: number; +} + +interface DatabricksOperation { + fetchAll(options?: { maxRows?: number }): Promise>>; + getSchema(): Promise<{ columns?: Array<{ columnName?: string }> } | null>; + cancel(): Promise; + close(): Promise; +} + +interface DatabricksSession { + executeStatement( + sql: string, + options?: { + runAsync?: boolean; + maxRows?: number; + ordinalParameters?: unknown[]; + statementConf?: Record; + }, + ): Promise; + close(): Promise; +} + +interface DatabricksClient { + connect(options: Record): Promise; + openSession(request?: { initialCatalog?: string; initialSchema?: string; configuration?: Record }): Promise; + close(): Promise; +} + +function stringConfigValue( + connection: KtxDatabricksConnectionConfig | undefined, + key: keyof KtxDatabricksConnectionConfig, + env: NodeJS.ProcessEnv, +): string | undefined { + const value = connection?.[key]; + return typeof value === 'string' && value.trim().length > 0 ? resolveStringReference(value.trim(), env) : undefined; +} + +function schemaNames(connection: KtxDatabricksConnectionConfig, env: NodeJS.ProcessEnv): string[] { + if (Array.isArray(connection.schema_names) && connection.schema_names.length > 0) { + return connection.schema_names + .filter((schema) => schema.trim().length > 0) + .map((schema) => resolveStringReference(schema, env)); + } + const single = stringConfigValue(connection, 'schema_name', env); + return single ? [single] : []; +} + +function firstNumber(value: unknown): number | null { + const numberValue = Number(value); + return Number.isFinite(numberValue) ? numberValue : null; +} + +function isDeniedError(error: unknown): boolean { + if (error instanceof Error) { + return /permission denied|insufficient privileges|not authorized|access denied/i.test(error.message); + } + return false; +} + +function databricksRowsToQueryResult(rows: Array>, headers: string[]): KtxQueryResult { + const resolvedHeaders = headers.length > 0 ? headers : Object.keys(rows[0] ?? {}); + return { + headers: resolvedHeaders, + rows: rows.map((row) => resolvedHeaders.map((header) => row[header])), + totalRows: rows.length, + rowCount: rows.length, + }; +} + +function toDatabricksParams(params: unknown[] | undefined): unknown[] | undefined { + return params?.map((value) => (value instanceof Date ? value.toISOString() : value)); +} + +function isTimeoutError(error: unknown): boolean { + return error instanceof Error && /timeout|timed out|deadline/i.test(error.message); +} + +function connectionOptions(resolved: KtxDatabricksResolvedConnectionConfig): Record { + const base = { + host: resolved.serverHostname, + path: resolved.httpPath, + preserveBigNumericPrecision: true, + telemetryEnabled: false, + }; + if (resolved.authMethod === 'oauth-m2m') { + return { + ...base, + authType: 'databricks-oauth', + oauthClientId: resolved.clientId, + oauthClientSecret: resolved.clientSecret, + }; + } + return { ...base, authType: 'access-token', token: resolved.token }; +} + +/** @internal */ +export function prepareDatabricksReadOnlyQuery( + sql: string, + params?: Record, +): { sql: string; params?: unknown[] } { + if (!params || Object.keys(params).length === 0) { + return { sql, params: undefined }; + } + const values: unknown[] = []; + const used = new Set(); + let rewritten = ''; + let quote: "'" | '"' | '`' | null = null; + let lineComment = false; + let blockComment = false; + for (let index = 0; index < sql.length; index += 1) { + const char = sql[index]!; + const next = sql[index + 1]; + if (lineComment) { + rewritten += char; + if (char === '\n') lineComment = false; + continue; + } + if (blockComment) { + rewritten += char; + if (char === '*' && next === '/') { + rewritten += next; + index += 1; + blockComment = false; + } + continue; + } + if (quote) { + rewritten += char; + if (char === quote) { + if (next === quote) { + rewritten += next; + index += 1; + } else { + quote = null; + } + } + continue; + } + if (char === '-' && next === '-') { + rewritten += char + next; + index += 1; + lineComment = true; + continue; + } + if (char === '/' && next === '*') { + rewritten += char + next; + index += 1; + blockComment = true; + continue; + } + if (char === "'" || char === '"' || char === '`') { + rewritten += char; + quote = char; + continue; + } + if (char === ':' && next && /[A-Za-z_]/.test(next) && sql[index - 1] !== ':') { + let end = index + 2; + while (end < sql.length && /[A-Za-z0-9_]/.test(sql[end]!)) { + end += 1; + } + const name = sql.slice(index + 1, end); + if (!Object.prototype.hasOwnProperty.call(params, name)) { + throw new Error(`Databricks read-only SQL parameter :${name} has no supplied value`); + } + values.push(params[name]); + used.add(name); + rewritten += '?'; + index = end - 1; + continue; + } + rewritten += char; + } + if (values.length === 0) { + throw new Error('Databricks read-only SQL parameters must use named placeholders like :id'); + } + const unused = Object.keys(params).filter((name) => !used.has(name)); + if (unused.length > 0) { + throw new Error(`Databricks read-only SQL received unused parameter(s): ${unused.join(', ')}`); + } + return { sql: rewritten, params: values }; +} + +export function isKtxDatabricksConnectionConfig( + connection: KtxDatabricksConnectionConfig | undefined, +): connection is KtxDatabricksConnectionConfig { + return String(connection?.driver ?? '').toLowerCase() === 'databricks'; +} + +/** @internal */ +export function databricksConnectionConfigFromConfig(input: { + connectionId: string; + connection: KtxDatabricksConnectionConfig | undefined; + env?: NodeJS.ProcessEnv; +}): KtxDatabricksResolvedConnectionConfig { + const inputDriver = input.connection?.driver ?? 'unknown'; + if (!isKtxDatabricksConnectionConfig(input.connection)) { + throw new Error(`Native Databricks connector cannot run driver "${inputDriver}"`); + } + const env = input.env ?? process.env; + const serverHostname = stringConfigValue(input.connection, 'server_hostname', env); + const httpPath = stringConfigValue(input.connection, 'http_path', env); + const catalog = stringConfigValue(input.connection, 'catalog', env); + if (!serverHostname) { + throw new Error(`Native Databricks connector requires connections.${input.connectionId}.server_hostname`); + } + if (!httpPath) { + throw new Error(`Native Databricks connector requires connections.${input.connectionId}.http_path`); + } + if (!catalog) { + throw new Error(`Native Databricks connector requires connections.${input.connectionId}.catalog`); + } + const authMethod = input.connection.authMethod ?? 'pat'; + const resolved: KtxDatabricksResolvedConnectionConfig = { + authMethod, + serverHostname, + httpPath, + catalog, + schemas: schemaNames(input.connection, env), + deadlineMs: resolveQueryDeadlineMs(input.connection), + }; + if (authMethod === 'oauth-m2m') { + resolved.clientId = stringConfigValue(input.connection, 'client_id', env); + resolved.clientSecret = stringConfigValue(input.connection, 'client_secret', env); + if (!resolved.clientId) { + throw new Error(`Native Databricks connector requires connections.${input.connectionId}.client_id for OAuth M2M auth`); + } + if (!resolved.clientSecret) { + throw new Error(`Native Databricks connector requires connections.${input.connectionId}.client_secret for OAuth M2M auth`); + } + return resolved; + } + resolved.token = stringConfigValue(input.connection, 'token', env); + if (!resolved.token) { + throw new Error(`Native Databricks connector requires connections.${input.connectionId}.token for PAT auth`); + } + return resolved; +} + +class DefaultDatabricksDriverFactory implements KtxDatabricksDriverFactory { + createDriver(input: { resolved: KtxDatabricksResolvedConnectionConfig }): KtxDatabricksDriver { + return new DatabricksSqlDriver(input.resolved); + } +} + +class DatabricksSqlDriver implements KtxDatabricksDriver { + constructor(private readonly resolved: KtxDatabricksResolvedConnectionConfig) {} + + async test(): Promise { + try { + await this.query('SELECT 1'); + return { success: true }; + } catch (error) { + return connectorTestFailure(error); + } + } + + async query(sql: string, params?: unknown[]): Promise { + return this.withSession(async (session) => { + const operationRef: { current: DatabricksOperation | null } = { current: null }; + let timeout: ReturnType | null = null; + let deadlineExceeded = false; + try { + const query = async () => { + operationRef.current = await session.executeStatement(sql, { + runAsync: true, + maxRows: 10000, + ordinalParameters: toDatabricksParams(params), + }); + const schema = await operationRef.current.getSchema(); + const rows = await operationRef.current.fetchAll(); + return databricksRowsToQueryResult(rows, schema?.columns?.map((column) => column.columnName ?? '') ?? []); + }; + return await Promise.race([ + query(), + new Promise((_resolve, reject) => { + timeout = setTimeout(() => { + deadlineExceeded = true; + reject(queryDeadlineExceededError(this.resolved.deadlineMs)); + }, this.resolved.deadlineMs); + }), + ]); + } catch (error) { + if (isTimeoutError(error)) { + throw queryDeadlineExceededError(this.resolved.deadlineMs, { cause: error }); + } + throw error; + } finally { + if (timeout) { + clearTimeout(timeout); + } + if (operationRef.current) { + if (deadlineExceeded) { + await Promise.resolve(operationRef.current.cancel()).catch(() => undefined); + } + await Promise.resolve(operationRef.current.close()).catch(() => undefined); + } + } + }); + } + + async getSchemaMetadata( + schemaName: string, + scopedTableNames: readonly string[] | null = null, + ): Promise { + const scopeClause = + scopedTableNames && scopedTableNames.length > 0 + ? `AND TABLE_NAME IN (${scopedTableNames.map(() => '?').join(', ')})` + : ''; + const scopeParams = scopedTableNames ?? []; + const tablesResult = await this.query( + ` + SELECT TABLE_NAME, TABLE_TYPE, COMMENT + FROM ${this.informationSchemaTable('TABLES')} + WHERE TABLE_SCHEMA = ? AND TABLE_CATALOG = ? ${scopeClause} + ORDER BY TABLE_NAME + `, + [schemaName, this.resolved.catalog, ...scopeParams], + ); + const columnsResult = await this.query( + ` + SELECT TABLE_NAME, COLUMN_NAME, FULL_DATA_TYPE, IS_NULLABLE, COMMENT, ORDINAL_POSITION + FROM ${this.informationSchemaTable('COLUMNS')} + WHERE TABLE_SCHEMA = ? AND TABLE_CATALOG = ? ${scopeClause} + ORDER BY TABLE_NAME, ORDINAL_POSITION + `, + [schemaName, this.resolved.catalog, ...scopeParams], + ); + const columnsByTable = new Map(); + for (const row of columnsResult.rows) { + const tableName = String(row[0]); + const columns = columnsByTable.get(tableName) ?? []; + columns.push({ + name: String(row[1]), + type: String(row[2]), + nullable: row[3] === 'YES', + comment: row[4] ? String(row[4]) : null, + }); + columnsByTable.set(tableName, columns); + } + return tablesResult.rows.map((row) => ({ + name: String(row[0]), + catalog: this.resolved.catalog, + db: schemaName, + kind: String(row[1]).toUpperCase().includes('VIEW') ? 'view' : 'table', + rowCount: null, + comment: row[2] ? String(row[2]) : null, + columns: columnsByTable.get(String(row[0])) ?? [], + })); + } + + async listSchemas(): Promise { + const result = await this.query( + ` + SELECT SCHEMA_NAME + FROM ${this.informationSchemaTable('SCHEMATA')} + WHERE CATALOG_NAME = ? + AND SCHEMA_NAME <> 'information_schema' + ORDER BY SCHEMA_NAME + `, + [this.resolved.catalog], + ); + return result.rows.map((row) => String(row[0])); + } + + async listTables(schemas?: string[]): Promise { + const filters = schemas && schemas.length > 0 ? schemas.map(() => '?').join(', ') : null; + const result = await this.query( + ` + SELECT TABLE_SCHEMA, TABLE_NAME, TABLE_TYPE + FROM ${this.informationSchemaTable('TABLES')} + WHERE TABLE_CATALOG = ? + AND TABLE_SCHEMA <> 'information_schema' + ${filters ? `AND TABLE_SCHEMA IN (${filters})` : ''} + ORDER BY TABLE_SCHEMA, TABLE_NAME + `, + [this.resolved.catalog, ...(schemas ?? [])], + ); + return result.rows.map((row) => ({ + catalog: this.resolved.catalog, + schema: String(row[0]), + name: String(row[1]), + kind: String(row[2]).toUpperCase().includes('VIEW') ? ('view' as const) : ('table' as const), + })); + } + + async cleanup(): Promise {} + + private informationSchemaTable(name: string): string { + return [this.resolved.catalog, 'INFORMATION_SCHEMA', name].map(this.quoteIdentifier).join('.'); + } + + private quoteIdentifier(identifier: string): string { + return `\`${identifier.replace(/`/g, '``')}\``; + } + + private async withSession(fn: (session: DatabricksSession) => Promise): Promise { + const client = new DBSQLClient() as unknown as DatabricksClient; + let session: DatabricksSession | null = null; + try { + await client.connect(connectionOptions(this.resolved)); + session = await client.openSession({ + initialCatalog: this.resolved.catalog, + initialSchema: this.resolved.schemas[0], + }); + return await fn(session); + } finally { + if (session) { + await Promise.resolve(session.close()).catch(() => undefined); + } + await client.close(); + } + } +} + +export class KtxDatabricksScanConnector implements KtxScanConnector { + readonly id: string; + readonly driver = 'databricks' as const; + readonly capabilities = createKtxConnectorCapabilities({ + tableSampling: true, + columnSampling: true, + columnStats: false, + readOnlySql: true, + nestedAnalysis: true, + formalForeignKeys: false, + estimatedRowCounts: false, + }); + + private readonly resolved: KtxDatabricksResolvedConnectionConfig; + private readonly driverFactory: KtxDatabricksDriverFactory; + private readonly dialect = getSqlDialectForDriver('databricks'); + private readonly now: () => Date; + private driverInstance: KtxDatabricksDriver | null = null; + + constructor(private readonly options: KtxDatabricksScanConnectorOptions) { + this.resolved = databricksConnectionConfigFromConfig(options); + this.driverFactory = options.driverFactory ?? new DefaultDatabricksDriverFactory(); + this.now = options.now ?? (() => new Date()); + this.id = `databricks:${options.connectionId}`; + } + + async testConnection(): Promise { + return this.getDriver().test(); + } + + async introspect(input: KtxScanInput, _ctx: KtxScanContext): Promise { + this.assertConnection(input.connectionId); + const schemaScope = this.resolved.schemas.length > 0 ? this.resolved.schemas : await this.getDriver().listSchemas(); + const tables: KtxSchemaTable[] = []; + const snapshotWarnings: KtxScanWarning[] = []; + for (const schemaName of schemaScope) { + const scopedNames = input.tableScope + ? scopedTableNames(input.tableScope, { catalog: this.resolved.catalog, db: schemaName }) + : null; + if (scopedNames && scopedNames.length === 0) continue; + const rawTables = await this.getDriver().getSchemaMetadata(schemaName, scopedNames); + const primaryKeysResult = await tryConstraintQuery( + { schema: schemaName, kind: 'primary_key', isDeniedError }, + () => this.primaryKeys(rawTables.map((table) => table.name), schemaName), + ); + const primaryKeys = primaryKeysResult.ok + ? primaryKeysResult.value + : new Map(rawTables.map((table) => [table.name, new Set()])); + if (!primaryKeysResult.ok) { + snapshotWarnings.push(primaryKeysResult.warning); + } + tables.push(...rawTables.map((table) => this.toSchemaTable(table, primaryKeys))); + } + return { + connectionId: this.options.connectionId, + driver: 'databricks', + extractedAt: this.now().toISOString(), + scope: { catalogs: [this.resolved.catalog], schemas: schemaScope }, + metadata: { + server_hostname: this.resolved.serverHostname, + http_path: this.resolved.httpPath, + catalog: this.resolved.catalog, + schemas: schemaScope, + table_count: tables.length, + total_columns: tables.reduce((sum, table) => sum + table.columns.length, 0), + }, + tables, + warnings: snapshotWarnings, + }; + } + + async sampleTable(input: KtxTableSampleInput, _ctx: KtxScanContext): Promise { + this.assertConnection(input.connectionId); + const result = await this.getDriver().query( + this.dialect.generateSampleQuery(this.qTableName(input.table), input.limit, input.columns), + ); + return { headers: result.headers, rows: result.rows, totalRows: result.totalRows }; + } + + async sampleColumn(input: KtxColumnSampleInput, _ctx: KtxScanContext): Promise { + this.assertConnection(input.connectionId); + const result = await this.getDriver().query( + this.dialect.generateColumnSampleQuery(this.qTableName(input.table), input.column, input.limit), + ); + return { + values: result.rows.filter((row) => row.length > 0 && row[0] !== null).map((row) => row[0]), + nullCount: null, + distinctCount: null, + }; + } + + async columnStats(_input: KtxColumnStatsInput, _ctx: KtxScanContext): Promise { + return null; + } + + async executeReadOnly(input: KtxDatabricksReadOnlyQueryInput, _ctx: KtxScanContext): Promise { + this.assertConnection(input.connectionId); + const limitedSql = limitSqlForExecution(assertReadOnlySql(input.sql), input.maxRows); + const prepared = prepareDatabricksReadOnlyQuery(limitedSql, input.params); + return this.getDriver().query(prepared.sql, prepared.params); + } + + async getColumnDistinctValues( + table: KtxTableRef, + columnName: string, + options: KtxDatabricksColumnDistinctValuesOptions, + ): Promise { + const tableName = this.qTableName(table); + const quotedColumn = this.dialect.quoteIdentifier(columnName); + const cardinality = await this.singleNumber( + this.dialect.generateRandomizedCardinalitySampleQuery(tableName, quotedColumn, options.sampleSize ?? 10000), + 'cardinality', + ); + if (cardinality === null) { + return null; + } + if (cardinality === 0) { + return { values: [], cardinality: 0 }; + } + if (cardinality > options.maxCardinality) { + return { values: null, cardinality }; + } + const valueRows = await this.queryRaw>( + this.dialect.generateDistinctValuesQuery(tableName, quotedColumn, options.limit), + ); + return { values: valueRows.map((row) => String(row.val ?? row.VAL)).filter((value) => value !== 'null'), cardinality }; + } + + qTableName(table: Pick & Partial>): string { + return this.dialect.formatTableName(table); + } + + quoteIdentifier(identifier: string): string { + return this.dialect.quoteIdentifier(identifier); + } + + listSchemas(): Promise { + return this.getDriver().listSchemas(); + } + + listTables(schemas?: string[]): Promise { + return this.getDriver().listTables(schemas); + } + + async cleanup(): Promise { + if (this.driverInstance) { + await this.driverInstance.cleanup(); + this.driverInstance = null; + } + } + + private getDriver(): KtxDatabricksDriver { + if (!this.driverInstance) { + this.driverInstance = this.driverFactory.createDriver({ resolved: this.resolved }); + } + return this.driverInstance; + } + + private async primaryKeys(tableNames: string[], schemaName: string): Promise>> { + const grouped = new Map>(); + for (const tableName of tableNames) { + grouped.set(tableName, new Set()); + } + if (tableNames.length === 0) { + return grouped; + } + const tableNamePlaceholders = tableNames.map(() => '?').join(', '); + const result = await this.getDriver().query( + ` + SELECT tc.TABLE_NAME, kcu.COLUMN_NAME + FROM ${this.qTableName({ catalog: this.resolved.catalog, db: 'INFORMATION_SCHEMA', name: 'TABLE_CONSTRAINTS' })} tc + JOIN ${this.qTableName({ catalog: this.resolved.catalog, db: 'INFORMATION_SCHEMA', name: 'KEY_COLUMN_USAGE' })} kcu + ON tc.CONSTRAINT_NAME = kcu.CONSTRAINT_NAME + AND tc.TABLE_SCHEMA = kcu.TABLE_SCHEMA + AND tc.TABLE_CATALOG = kcu.TABLE_CATALOG + WHERE tc.CONSTRAINT_TYPE = 'PRIMARY KEY' + AND tc.TABLE_SCHEMA = ? + AND tc.TABLE_CATALOG = ? + AND tc.TABLE_NAME IN (${tableNamePlaceholders}) + ORDER BY tc.TABLE_NAME, kcu.ORDINAL_POSITION + `, + [schemaName, this.resolved.catalog, ...tableNames], + ); + for (const row of result.rows) { + const tableName = String(row[0]); + const columnName = String(row[1]); + grouped.get(tableName)?.add(columnName); + } + return grouped; + } + + private toSchemaTable( + table: KtxDatabricksRawTableMetadata, + primaryKeys: Map>, + ): KtxSchemaTable { + return { + catalog: table.catalog, + db: table.db, + name: table.name, + kind: table.kind, + comment: table.comment, + estimatedRows: table.rowCount, + columns: table.columns.map((column) => this.toSchemaColumn(table.name, column, primaryKeys)), + foreignKeys: [], + }; + } + + private toSchemaColumn( + tableName: string, + column: KtxDatabricksRawColumnMetadata, + primaryKeys: Map>, + ): KtxSchemaColumn { + return { + name: column.name, + nativeType: column.type, + normalizedType: this.dialect.mapDataType(column.type), + dimensionType: this.dialect.mapToDimensionType(column.type), + nullable: column.nullable, + primaryKey: primaryKeys.get(tableName)?.has(column.name) ?? false, + comment: column.comment, + }; + } + + private async queryRaw>(sql: string, params?: unknown[]): Promise { + const result = await this.getDriver().query(sql, params); + return result.rows.map((row) => Object.fromEntries(result.headers.map((header, index) => [header, row[index]])) as T); + } + + private async singleNumber(sql: string, header: string): Promise { + const rows = await this.queryRaw>(sql); + return firstNumber(rows[0]?.[header] ?? rows[0]?.[header.toUpperCase()]); + } + + private assertConnection(connectionId: string): void { + if (connectionId !== this.options.connectionId) { + throw new Error(`Databricks connector ${this.options.connectionId} cannot scan connection ${connectionId}`); + } + } +} diff --git a/packages/cli/src/connectors/databricks/dialect.ts b/packages/cli/src/connectors/databricks/dialect.ts new file mode 100644 index 000000000..22e95c7a6 --- /dev/null +++ b/packages/cli/src/connectors/databricks/dialect.ts @@ -0,0 +1,186 @@ +import type { KtxSqlDialect } from '../../context/connections/dialects.js'; +import { + columnDisplayPartCount, + formatDialectDisplayRef, + formatDialectTableName, + limitOffsetClause, + parseDialectDisplayRef, +} from '../../context/connections/dialect-helpers.js'; +import type { KtxSchemaDimensionType, KtxTableRef } from '../../context/scan/types.js'; + +type DatabricksTableNameRef = Pick & Partial>; + +/** @internal */ +export class KtxDatabricksDialect implements KtxSqlDialect { + readonly type = 'databricks' as const; + + private readonly typeMappings: Record = { + TIMESTAMP: 'time', + TIMESTAMP_NTZ: 'time', + DATE: 'time', + TINYINT: 'number', + SMALLINT: 'number', + INT: 'number', + INTEGER: 'number', + BIGINT: 'number', + FLOAT: 'number', + DOUBLE: 'number', + DECIMAL: 'number', + NUMERIC: 'number', + STRING: 'string', + CHAR: 'string', + VARCHAR: 'string', + BINARY: 'string', + BOOLEAN: 'boolean', + BOOL: 'boolean', + ARRAY: 'string', + MAP: 'string', + STRUCT: 'string', + VARIANT: 'string', + }; + + quoteIdentifier(identifier: string): string { + return `\`${identifier.replace(/`/g, '``')}\``; + } + + formatTableName(table: DatabricksTableNameRef): string { + return formatDialectTableName(table, this.quoteIdentifier.bind(this), 'three-part'); + } + + formatDisplayRef(table: DatabricksTableNameRef): string { + return formatDialectDisplayRef(table, 'three-part'); + } + + parseDisplayRef(display: string): KtxTableRef | null { + return parseDialectDisplayRef(display, 'three-part'); + } + + columnDisplayTablePartCount(): 1 | 2 | 3 { + return columnDisplayPartCount('three-part'); + } + + mapDataType(nativeType: string): string { + return nativeType; + } + + mapToDimensionType(nativeType: string): KtxSchemaDimensionType { + if (!nativeType) { + return 'string'; + } + const upper = nativeType.toUpperCase().trim(); + const normalized = upper.split(/[<(]/)[0]!.trim(); + if (this.typeMappings[normalized]) { + return this.typeMappings[normalized]; + } + if (normalized.includes('TIME') || normalized.includes('DATE')) { + return 'time'; + } + if ( + normalized.includes('INT') || + normalized.includes('NUM') || + normalized.includes('DEC') || + normalized.includes('FLOAT') || + normalized.includes('DOUBLE') + ) { + return 'number'; + } + if (normalized.includes('BOOL')) { + return 'boolean'; + } + return 'string'; + } + + generateSampleQuery(tableName: string, limit: number, columns?: string[]): string { + const columnList = + columns && columns.length > 0 ? columns.map((column) => this.quoteIdentifier(column)).join(', ') : '*'; + return `SELECT ${columnList} FROM ${tableName} LIMIT ${limit}`; + } + + generateColumnSampleQuery(tableName: string, columnName: string, limit: number): string { + const quotedColumn = this.quoteIdentifier(columnName); + return `SELECT ${quotedColumn} FROM ${tableName} WHERE ${quotedColumn} IS NOT NULL AND TRIM(CAST(${quotedColumn} AS STRING)) != '' LIMIT ${limit}`; + } + + getRandomSampleFilter(samplePct: number): string { + if (samplePct <= 0 || samplePct >= 1) { + return ''; + } + return `RAND() < ${samplePct}`; + } + + getTableSampleClause(samplePct: number): string { + if (samplePct <= 0 || samplePct >= 1) { + return ''; + } + return `TABLESAMPLE (${samplePct * 100} PERCENT)`; + } + + getLimitOffsetClause(limit: number, offset?: number): string { + return limitOffsetClause(limit, offset); + } + + getTopClause(_limit: number): string { + return ''; + } + + getNullCountExpression(column: string): string { + return `COUNT_IF(${column} IS NULL)`; + } + + getDistinctCountExpression(column: string): string { + return `APPROX_COUNT_DISTINCT(${column})`; + } + + textLengthExpression(columnSql: string): string { + return `LENGTH(CAST(${columnSql} AS STRING))`; + } + + castToText(columnSql: string): string { + return `CAST(${columnSql} AS STRING)`; + } + + getSampleValueAggregation(innerSql: string): string { + return `(SELECT CONCAT_WS('\\u001F', COLLECT_LIST(CAST(value AS STRING))) FROM (${innerSql}) AS relationship_profile_values)`; + } + + generateCardinalitySampleQuery(tableName: string, columnName: string, sampleSize: number): string { + return ` + WITH sampled AS ( + SELECT ${columnName} AS val + FROM ${tableName} + WHERE ${columnName} IS NOT NULL + LIMIT ${sampleSize} + ) + SELECT APPROX_COUNT_DISTINCT(val) AS cardinality + FROM sampled + `; + } + + generateDistinctValuesQuery(tableName: string, columnName: string, limit: number): string { + return ` + SELECT DISTINCT CAST(${columnName} AS STRING) AS val + FROM ${tableName} + WHERE ${columnName} IS NOT NULL + ORDER BY val + LIMIT ${limit} + `; + } + + generateColumnStatisticsQuery(_schemaName: string, _tableName: string): string | null { + return null; + } + + generateRandomizedCardinalitySampleQuery(tableName: string, columnName: string, sampleSize: number): string { + return ` + WITH sampled AS ( + SELECT ${columnName} AS val + FROM ${tableName} + WHERE ${columnName} IS NOT NULL + ORDER BY RAND() + LIMIT ${sampleSize} + ) + SELECT APPROX_COUNT_DISTINCT(val) AS cardinality + FROM sampled + `; + } +} diff --git a/packages/cli/src/connectors/databricks/live-database-introspection.ts b/packages/cli/src/connectors/databricks/live-database-introspection.ts new file mode 100644 index 000000000..5770787b2 --- /dev/null +++ b/packages/cli/src/connectors/databricks/live-database-introspection.ts @@ -0,0 +1,44 @@ +import type { + LiveDatabaseIntrospectionOptions, + LiveDatabaseIntrospectionPort, +} from '../../context/ingest/adapters/live-database/types.js'; +import type { KtxProjectConnectionConfig } from '../../context/project/config.js'; +import { + KtxDatabricksScanConnector, + type KtxDatabricksConnectionConfig, + type KtxDatabricksDriverFactory, +} from './connector.js'; + +interface CreateDatabricksLiveDatabaseIntrospectionOptions { + connections: Record; + driverFactory?: KtxDatabricksDriverFactory; + now?: () => Date; +} + +export function createDatabricksLiveDatabaseIntrospection( + options: CreateDatabricksLiveDatabaseIntrospectionOptions, +): LiveDatabaseIntrospectionPort { + return { + async extractSchema(connectionId: string, introspectionOptions?: LiveDatabaseIntrospectionOptions) { + const connection = options.connections[connectionId] as KtxDatabricksConnectionConfig | undefined; + const connector = new KtxDatabricksScanConnector({ + connectionId, + connection, + driverFactory: options.driverFactory, + now: options.now, + }); + try { + return await connector.introspect( + { + connectionId, + driver: 'databricks', + ...(introspectionOptions?.tableScope ? { tableScope: introspectionOptions.tableScope } : {}), + }, + { runId: `databricks-${connectionId}` }, + ); + } finally { + await connector.cleanup(); + } + }, + }; +} diff --git a/packages/cli/src/context/connections/connection-type.ts b/packages/cli/src/context/connections/connection-type.ts index 6cd48042f..a08faf3aa 100644 --- a/packages/cli/src/context/connections/connection-type.ts +++ b/packages/cli/src/context/connections/connection-type.ts @@ -6,6 +6,7 @@ export const connectionTypeSchema = z.enum([ 'SQLSERVER', 'BIGQUERY', 'SNOWFLAKE', + 'DATABRICKS', 'CENTRALREACH', 'EPIC', 'CERNER', diff --git a/packages/cli/src/context/connections/dialects.ts b/packages/cli/src/context/connections/dialects.ts index 30608f3e0..176a999f7 100644 --- a/packages/cli/src/context/connections/dialects.ts +++ b/packages/cli/src/context/connections/dialects.ts @@ -1,5 +1,6 @@ import { KtxBigQueryDialect } from '../../connectors/bigquery/dialect.js'; import { KtxClickHouseDialect } from '../../connectors/clickhouse/dialect.js'; +import { KtxDatabricksDialect } from '../../connectors/databricks/dialect.js'; import { KtxMongoDbDialect } from '../../connectors/mongodb/dialect.js'; import { KtxMysqlDialect } from '../../connectors/mysql/dialect.js'; import { KtxPostgresDialect } from '../../connectors/postgres/dialect.js'; @@ -55,6 +56,7 @@ type KtxSqlDriver = Exclude; const sqlDialectFactories: Record KtxSqlDialect> = { bigquery: () => new KtxBigQueryDialect(), clickhouse: () => new KtxClickHouseDialect(), + databricks: () => new KtxDatabricksDialect(), mysql: () => new KtxMysqlDialect(), postgres: () => new KtxPostgresDialect(), sqlite: () => new KtxSqliteDialect(), diff --git a/packages/cli/src/context/connections/drivers.ts b/packages/cli/src/context/connections/drivers.ts index 76ac408ef..d1abbdf08 100644 --- a/packages/cli/src/context/connections/drivers.ts +++ b/packages/cli/src/context/connections/drivers.ts @@ -173,6 +173,27 @@ export const driverRegistrations: Record { + const m = await import('../../connectors/databricks/connector.js'); + return { + isConnectionConfig: (connection) => { + const typedConnection = connection as Parameters[0]; + return m.isKtxDatabricksConnectionConfig(typedConnection); + }, + createScanConnector: ({ connectionId, connection }) => { + const typedConnection = connection as Parameters[0]; + if (!m.isKtxDatabricksConnectionConfig(typedConnection)) { + throw invalidConnectionConfig('databricks'); + } + return new m.KtxDatabricksScanConnector({ connectionId, connection: typedConnection }); + }, + }; + }, + }, sqlserver: { driver: 'sqlserver', scopeConfigKey: 'schemas', diff --git a/packages/cli/src/context/connections/local-warehouse-descriptor.ts b/packages/cli/src/context/connections/local-warehouse-descriptor.ts index 0e5d0b9de..895b81a67 100644 --- a/packages/cli/src/context/connections/local-warehouse-descriptor.ts +++ b/packages/cli/src/context/connections/local-warehouse-descriptor.ts @@ -27,6 +27,7 @@ const DRIVER_TO_CONNECTION_TYPE: Record = { mysql: 'MYSQL', clickhouse: 'CLICKHOUSE', snowflake: 'SNOWFLAKE', + databricks: 'DATABRICKS', bigquery: 'BIGQUERY', }; diff --git a/packages/cli/src/context/project/driver-schemas.ts b/packages/cli/src/context/project/driver-schemas.ts index 25fa35079..66679332c 100644 --- a/packages/cli/src/context/project/driver-schemas.ts +++ b/packages/cli/src/context/project/driver-schemas.ts @@ -9,6 +9,7 @@ const warehouseDrivers = [ 'postgres', 'mysql', 'snowflake', + 'databricks', 'bigquery', 'sqlite', 'clickhouse', @@ -50,6 +51,7 @@ const warehouseConnectionSchemas = [ warehouseConnectionSchema('postgres'), warehouseConnectionSchema('mysql'), warehouseConnectionSchema('snowflake'), + warehouseConnectionSchema('databricks'), warehouseConnectionSchema('bigquery'), warehouseConnectionSchema('sqlite'), warehouseConnectionSchema('clickhouse'), diff --git a/packages/cli/src/context/scan/local-scan.ts b/packages/cli/src/context/scan/local-scan.ts index e96447267..5762eab96 100644 --- a/packages/cli/src/context/scan/local-scan.ts +++ b/packages/cli/src/context/scan/local-scan.ts @@ -146,12 +146,13 @@ function normalizeDriver(driver: string | undefined): KtxConnectionDriver { normalized === 'sqlserver' || normalized === 'bigquery' || normalized === 'snowflake' || + normalized === 'databricks' || normalized === 'mongodb' ) { return normalized; } throw new Error( - `Standalone ktx scan supports postgres/sqlite/mysql/clickhouse/sqlserver/bigquery/snowflake/mongodb in this phase, received "${driver ?? 'unknown'}"`, + `Standalone ktx scan supports postgres/sqlite/mysql/clickhouse/sqlserver/bigquery/snowflake/databricks/mongodb in this phase, received "${driver ?? 'unknown'}"`, ); } diff --git a/packages/cli/src/context/scan/types.ts b/packages/cli/src/context/scan/types.ts index 0d269c37b..ba90d5ea0 100644 --- a/packages/cli/src/context/scan/types.ts +++ b/packages/cli/src/context/scan/types.ts @@ -6,6 +6,7 @@ export type KtxConnectionDriver = | 'sqlserver' | 'bigquery' | 'snowflake' + | 'databricks' | 'mysql' | 'clickhouse' | 'mongodb'; diff --git a/packages/cli/src/context/sql-analysis/dialect-notes.ts b/packages/cli/src/context/sql-analysis/dialect-notes.ts index d0a7c6348..63ddd2803 100644 --- a/packages/cli/src/context/sql-analysis/dialect-notes.ts +++ b/packages/cli/src/context/sql-analysis/dialect-notes.ts @@ -6,8 +6,7 @@ import type { SqlAnalysisDialect } from './ports.js'; // dialect), served by the sql_dialect_notes MCP tool. They are package-internal: // copy-runtime-assets.mjs ships them to dist, and they are never installed onto an // agent target. The set covers every dialect reachable from a configured warehouse -// driver; duckdb/databricks are intentionally absent because no connector produces -// them. +// driver; duckdb is intentionally absent because no connector produces it. /** @internal Dialects with an authored ./dialects/.md file. */ export const DIALECTS_WITH_NOTES = [ @@ -17,6 +16,7 @@ export const DIALECTS_WITH_NOTES = [ 'bigquery', 'sqlite', 'clickhouse', + 'databricks', 'tsql', ] as const; diff --git a/packages/cli/src/context/sql-analysis/dialects/databricks.md b/packages/cli/src/context/sql-analysis/dialects/databricks.md new file mode 100644 index 000000000..7544c0949 --- /dev/null +++ b/packages/cli/src/context/sql-analysis/dialects/databricks.md @@ -0,0 +1,10 @@ +**databricks** SQL conventions: +- **FQTN:** Unity Catalog uses three-part `catalog.schema.table` names, e.g. `main.sales.orders`. +- **Identifiers:** quote reserved, mixed-case, or special-character names with backticks; escape a literal backtick by doubling it. +- **Date/time:** use `date_trunc('month', ts)`, `to_date(s[, fmt])`, `dateadd(day, -7, current_date())`, and `current_timestamp()`. +- **Series:** generate date spines with `sequence(start_date, end_date, interval 1 day)` plus `explode(...)`, then left join facts onto the spine. +- **Rolling window over time:** use an interval range frame over a timestamp order key, e.g. `avg(amount) over (order by event_ts range between interval 29 days preceding and current row)`. +- **Safe cast:** `try_cast(x AS DECIMAL(12,2))` returns `NULL` instead of failing when a value does not parse. +- **Top-N / windows:** Databricks supports `QUALIFY`, so `QUALIFY row_number() OVER (PARTITION BY ... ORDER BY ...) = 1` avoids a subquery. +- **Semi-structured:** use `from_json` for typed JSON, `get_json_object` for string paths, and `explode` / `posexplode` for arrays. +- **Sampling:** `TABLESAMPLE` is available, but exact row limits are clearer with `ORDER BY rand() LIMIT n` when a small randomized sample is required. diff --git a/packages/cli/src/local-adapters.ts b/packages/cli/src/local-adapters.ts index d923a68bd..da0f63abe 100644 --- a/packages/cli/src/local-adapters.ts +++ b/packages/cli/src/local-adapters.ts @@ -44,6 +44,14 @@ function hasSnowflakeDriver(connection: unknown): boolean { ); } +function hasDatabricksDriver(connection: unknown): boolean { + return ( + typeof connection === 'object' && + connection !== null && + String((connection as { driver?: unknown }).driver ?? '').toLowerCase() === 'databricks' + ); +} + type SnowflakeConnectorModule = typeof import('./connectors/snowflake/connector.js'); function ktxCliDaemonDatabaseIntrospectionOptions( @@ -163,6 +171,17 @@ function createKtxCliLiveDatabaseIntrospection( }); return snowflake.extractSchema(connectionId, options); } + if (hasDatabricksDriver(connection)) { + const { createDatabricksLiveDatabaseIntrospection } = await import('./connectors/databricks/live-database-introspection.js'); + const { isKtxDatabricksConnectionConfig } = await import('./connectors/databricks/connector.js'); + if (!isKtxDatabricksConnectionConfig(connection)) { + return daemon.extractSchema(connectionId, options); + } + const databricks = createDatabricksLiveDatabaseIntrospection({ + connections: project.config.connections, + }); + return databricks.extractSchema(connectionId, options); + } return daemon.extractSchema(connectionId, options); }, }; diff --git a/packages/cli/src/setup-databases.ts b/packages/cli/src/setup-databases.ts index 7388f74a7..abeea6a84 100644 --- a/packages/cli/src/setup-databases.ts +++ b/packages/cli/src/setup-databases.ts @@ -70,6 +70,7 @@ export type KtxSetupDatabaseDriver = | 'sqlserver' | 'bigquery' | 'snowflake' + | 'databricks' | 'mongodb'; export interface KtxSetupDatabasesArgs { @@ -154,6 +155,7 @@ const DRIVER_OPTIONS: Array<{ value: KtxSetupDatabaseDriver; label: string }> = { value: 'postgres', label: 'PostgreSQL' }, { value: 'bigquery', label: 'BigQuery' }, { value: 'snowflake', label: 'Snowflake' }, + { value: 'databricks', label: 'Databricks' }, { value: 'mysql', label: 'MySQL' }, { value: 'clickhouse', label: 'ClickHouse' }, { value: 'sqlserver', label: 'SQL Server' }, @@ -180,6 +182,7 @@ const DEFAULT_CONNECTION_IDS: Record = { sqlserver: 'sqlserver-warehouse', bigquery: 'bigquery-warehouse', snowflake: 'snowflake-warehouse', + databricks: 'databricks-warehouse', mongodb: 'mongodb-source', }; @@ -265,6 +268,14 @@ const SCOPE_DISCOVERY_SPECS: Partial; @@ -953,6 +964,86 @@ async function buildConnectionConfig(input: { ...(role ? { role } : {}), }; } + if (driver === 'databricks') { + if (args.inputMode === 'disabled') return null; + + const serverHostname = await promptText( + prompts, + 'Databricks workspace hostname\nFor example dbc-12345678-abcd.cloud.databricks.com.', + stringConfigField(input.existingConnection, 'server_hostname'), + ); + if (serverHostname === undefined) return 'back'; + const httpPath = await promptText( + prompts, + 'Databricks SQL warehouse HTTP path\nFor example /sql/1.0/warehouses/abc123.', + stringConfigField(input.existingConnection, 'http_path'), + ); + if (httpPath === undefined) return 'back'; + const catalog = await promptText( + prompts, + 'Databricks catalog\nFor example main.', + stringConfigField(input.existingConnection, 'catalog'), + ); + if (catalog === undefined) return 'back'; + const authChoice = await prompts.select({ + message: 'Databricks authentication method', + options: [ + { value: 'pat', label: 'Personal Access Token' }, + { value: 'oauth-m2m', label: 'OAuth machine-to-machine' }, + { value: 'back', label: 'Back' }, + ], + }); + if (authChoice === 'back') return 'back'; + const authMethod: 'pat' | 'oauth-m2m' = authChoice === 'oauth-m2m' ? 'oauth-m2m' : 'pat'; + const scope = scriptedScopeConfigForDriver('databricks', args.databaseSchemas); + if (authMethod === 'pat') { + const tokenRef = await promptCredential({ + prompts, + message: 'Databricks personal access token', + projectDir: args.projectDir, + connectionId: input.connectionId, + secretName: 'token', // pragma: allowlist secret + }); + if (tokenRef === 'back') return 'back'; + const resolvedTokenRef = tokenRef ?? stringConfigField(input.existingConnection, 'token'); + if (!serverHostname || !httpPath || !catalog || !resolvedTokenRef) return null; + return { + driver: 'databricks', + authMethod: 'pat', + server_hostname: serverHostname, + http_path: httpPath, + catalog, + token: resolvedTokenRef, + ...scope, + }; + } + const clientId = await promptText( + prompts, + 'Databricks OAuth client ID', + stringConfigField(input.existingConnection, 'client_id'), + ); + if (clientId === undefined) return 'back'; + const clientSecretRef = await promptCredential({ + prompts, + message: 'Databricks OAuth client secret', + projectDir: args.projectDir, + connectionId: input.connectionId, + secretName: 'client-secret', // pragma: allowlist secret + }); + if (clientSecretRef === 'back') return 'back'; + const resolvedClientSecretRef = clientSecretRef ?? stringConfigField(input.existingConnection, 'client_secret'); + if (!serverHostname || !httpPath || !catalog || !clientId || !resolvedClientSecretRef) return null; + return { + driver: 'databricks', + authMethod: 'oauth-m2m', + server_hostname: serverHostname, + http_path: httpPath, + catalog, + client_id: clientId, + client_secret: resolvedClientSecretRef, + ...scope, + }; + } throw new Error(`Unsupported database driver: ${driver}`); } diff --git a/packages/cli/src/status-project.ts b/packages/cli/src/status-project.ts index 6d52518bd..91c0efd53 100644 --- a/packages/cli/src/status-project.ts +++ b/packages/cli/src/status-project.ts @@ -553,7 +553,7 @@ async function buildQueryHistoryStatus( } const ADAPTER_DRIVER_REQUIREMENT: Record = { - 'live-database': ['postgres', 'mysql', 'snowflake', 'bigquery', 'clickhouse', 'sqlite', 'sqlserver'], + 'live-database': ['postgres', 'mysql', 'snowflake', 'databricks', 'bigquery', 'clickhouse', 'sqlite', 'sqlserver'], dbt: ['dbt', 'dbt-core', 'dbt-cloud'], notion: ['notion'], metabase: ['metabase'], diff --git a/packages/cli/test/connectors/databricks/connector.test.ts b/packages/cli/test/connectors/databricks/connector.test.ts new file mode 100644 index 000000000..61a50a3c9 --- /dev/null +++ b/packages/cli/test/connectors/databricks/connector.test.ts @@ -0,0 +1,336 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + databricksConnectionConfigFromConfig, + isKtxDatabricksConnectionConfig, + KtxDatabricksScanConnector, + prepareDatabricksReadOnlyQuery, + type KtxDatabricksDriver, + type KtxDatabricksDriverFactory, +} from '../../../src/connectors/databricks/connector.js'; +import { createDatabricksLiveDatabaseIntrospection } from '../../../src/connectors/databricks/live-database-introspection.js'; +import { tableRefSet } from '../../../src/context/scan/table-ref.js'; + +function fakeDriverFactory(): KtxDatabricksDriverFactory { + const driver: KtxDatabricksDriver = { + test: vi.fn(async () => ({ success: true })), + query: vi.fn(async (sql: string) => { + if (sql.includes('TABLE_CONSTRAINTS')) { + return { headers: ['TABLE_NAME', 'COLUMN_NAME'], rows: [['orders', 'id']], totalRows: 1, rowCount: 1 }; + } + if (sql.includes('SELECT `id`, `status` FROM `main`.`sales`.`orders`')) { + return { headers: ['id', 'status'], rows: [[1, 'paid']], totalRows: 1, rowCount: 1 }; + } + if (sql.includes('select * from (select id, status from orders) as ktx_query_result limit 1')) { + return { headers: ['id', 'status'], rows: [[1, 'paid']], totalRows: 1, rowCount: 1 }; + } + if (sql.includes('SELECT `status` FROM `main`.`sales`.`orders`')) { + return { headers: ['status'], rows: [['paid'], ['open']], totalRows: 2, rowCount: 2 }; + } + if (sql.includes('APPROX_COUNT_DISTINCT(val)')) { + return { headers: ['cardinality'], rows: [[2]], totalRows: 1, rowCount: 1 }; + } + if (sql.includes('SELECT DISTINCT CAST(`status` AS STRING) AS val')) { + return { headers: ['val'], rows: [['open'], ['paid']], totalRows: 2, rowCount: 2 }; + } + throw new Error(`Unexpected SQL: ${sql}`); + }), + getSchemaMetadata: vi.fn(async () => [ + { + name: 'orders', + catalog: 'main', + db: 'sales', + kind: 'table' as const, + rowCount: null, + comment: 'Orders', + columns: [ + { name: 'id', type: 'BIGINT', nullable: false, comment: 'Primary key' }, + { name: 'status', type: 'STRING', nullable: true, comment: null }, + ], + }, + { + name: 'order_summary', + catalog: 'main', + db: 'sales', + kind: 'view' as const, + rowCount: null, + comment: null, + columns: [{ name: 'status', type: 'STRING', nullable: true, comment: null }], + }, + ]), + listSchemas: vi.fn(async () => ['sales', 'finance']), + listTables: vi.fn(async () => [ + { catalog: 'main', schema: 'sales', name: 'orders', kind: 'table' as const }, + { catalog: 'main', schema: 'sales', name: 'order_summary', kind: 'view' as const }, + ]), + cleanup: vi.fn(async () => undefined), + }; + return { createDriver: vi.fn(() => driver) }; +} + +describe('KtxDatabricksScanConnector', () => { + it('rewrites named read-only SQL parameters to Databricks ordinal arrays', () => { + expect( + prepareDatabricksReadOnlyQuery('SELECT * FROM orders WHERE id = :id AND status = :status OR parent_id = :id', { + status: 'paid', + id: 1, + }), + ).toEqual({ + sql: 'SELECT * FROM orders WHERE id = ? AND status = ? OR parent_id = ?', + params: [1, 'paid', 1], + }); + expect( + prepareDatabricksReadOnlyQuery("SELECT ':id' AS literal -- :status\nFROM orders WHERE id = :id", { + id: 1, + }), + ).toEqual({ + sql: "SELECT ':id' AS literal -- :status\nFROM orders WHERE id = ?", + params: [1], + }); + expect(() => + prepareDatabricksReadOnlyQuery('SELECT * FROM orders WHERE id = ? AND status = ?', { id: 1, status: 'paid' }), + ).toThrow('Databricks read-only SQL parameters must use named placeholders like :id'); + expect(() => prepareDatabricksReadOnlyQuery('SELECT * FROM orders WHERE id = :id', { id: 1, status: 'paid' })).toThrow( + 'Databricks read-only SQL received unused parameter(s): status', + ); + expect(() => prepareDatabricksReadOnlyQuery('SELECT * FROM orders WHERE id = :id', { status: 'paid' })).toThrow( + 'Databricks read-only SQL parameter :id has no supplied value', + ); + expect(prepareDatabricksReadOnlyQuery('SELECT * FROM orders WHERE id = ? AND status = ?')).toEqual({ + sql: 'SELECT * FROM orders WHERE id = ? AND status = ?', + params: undefined, + }); + expect(prepareDatabricksReadOnlyQuery('SELECT * FROM orders')).toEqual({ + sql: 'SELECT * FROM orders', + params: undefined, + }); + }); + + it('resolves PAT and OAuth M2M connection configuration', () => { + expect(isKtxDatabricksConnectionConfig({ driver: 'databricks' })).toBe(true); + expect(isKtxDatabricksConnectionConfig({ driver: 'snowflake' })).toBe(false); + expect( + databricksConnectionConfigFromConfig({ + connectionId: 'warehouse', + connection: { + driver: 'databricks', + authMethod: 'pat', + server_hostname: 'dbc-example.cloud.databricks.com', + http_path: '/sql/1.0/warehouses/abc', + catalog: 'main', + schema_name: 'sales', + token: 'fixture-token', // pragma: allowlist secret + }, + }), + ).toMatchObject({ + authMethod: 'pat', + serverHostname: 'dbc-example.cloud.databricks.com', + httpPath: '/sql/1.0/warehouses/abc', + catalog: 'main', + schemas: ['sales'], + token: 'fixture-token', // pragma: allowlist secret + }); + expect( + databricksConnectionConfigFromConfig({ + connectionId: 'warehouse', + connection: { + driver: 'databricks', + authMethod: 'oauth-m2m', + server_hostname: 'dbc-example.cloud.databricks.com', + http_path: '/sql/1.0/warehouses/abc', + catalog: 'main', + client_id: 'client-id', + client_secret: 'fixture-secret', // pragma: allowlist secret + }, + }), + ).toMatchObject({ + authMethod: 'oauth-m2m', + clientId: 'client-id', + clientSecret: 'fixture-secret', // pragma: allowlist secret + }); + }); + + it('introspects Unity Catalog schemas, primary keys, comments, and dimensions', async () => { + const connector = new KtxDatabricksScanConnector({ + connectionId: 'warehouse', + connection: { + driver: 'databricks', + server_hostname: 'dbc-example.cloud.databricks.com', + http_path: '/sql/1.0/warehouses/abc', + catalog: 'main', + schema_name: 'sales', + token: 'fixture-token', // pragma: allowlist secret + }, + driverFactory: fakeDriverFactory(), + now: () => new Date('2026-04-29T18:00:00.000Z'), + }); + + const snapshot = await connector.introspect( + { connectionId: 'warehouse', driver: 'databricks' }, + { runId: 'scan-run-1' }, + ); + + expect(snapshot).toMatchObject({ + connectionId: 'warehouse', + driver: 'databricks', + extractedAt: '2026-04-29T18:00:00.000Z', + scope: { catalogs: ['main'], schemas: ['sales'] }, + metadata: { + server_hostname: 'dbc-example.cloud.databricks.com', + http_path: '/sql/1.0/warehouses/abc', + catalog: 'main', + schemas: ['sales'], + table_count: 2, + total_columns: 3, + }, + }); + expect(snapshot.tables.find((table) => table.name === 'orders')?.columns).toEqual([ + { + name: 'id', + nativeType: 'BIGINT', + normalizedType: 'BIGINT', + dimensionType: 'number', + nullable: false, + primaryKey: true, + comment: 'Primary key', + }, + { + name: 'status', + nativeType: 'STRING', + normalizedType: 'STRING', + dimensionType: 'string', + nullable: true, + primaryKey: false, + comment: null, + }, + ]); + }); + + it('limits introspection to tables in tableScope', async () => { + const getSchemaMetadata = vi.fn(async (_schemaName?: string, scopedNames?: readonly string[] | null) => + scopedNames?.includes('orders') + ? [ + { + name: 'orders', + catalog: 'main', + db: 'sales', + kind: 'table' as const, + rowCount: null, + comment: null, + columns: [{ name: 'id', type: 'BIGINT', nullable: false, comment: null }], + }, + ] + : [], + ); + const driverFactory: KtxDatabricksDriverFactory = { + createDriver: vi.fn(() => ({ + test: vi.fn(async () => ({ success: true })), + query: vi.fn(async () => ({ headers: [], rows: [], totalRows: 0, rowCount: 0 })), + getSchemaMetadata, + listSchemas: vi.fn(async () => []), + listTables: vi.fn(async () => []), + cleanup: vi.fn(async () => undefined), + })), + }; + const connector = new KtxDatabricksScanConnector({ + connectionId: 'warehouse', + connection: { + driver: 'databricks', + server_hostname: 'dbc-example.cloud.databricks.com', + http_path: '/sql/1.0/warehouses/abc', + catalog: 'main', + schema_name: 'sales', + token: 'fixture-token', // pragma: allowlist secret + }, + driverFactory, + }); + const scope = tableRefSet([{ catalog: 'main', db: 'sales', name: 'orders' }]); + const snapshot = await connector.introspect( + { connectionId: 'warehouse', driver: 'databricks', tableScope: scope }, + { runId: 'scope-test' }, + ); + expect(snapshot.tables.map((table) => table.name)).toEqual(['orders']); + expect(getSchemaMetadata).toHaveBeenCalledWith('sales', ['orders']); + }); + + it('supports read-only query, sampling, distinct values, schema listing, and cleanup', async () => { + const driverFactory = fakeDriverFactory(); + const connector = new KtxDatabricksScanConnector({ + connectionId: 'warehouse', + connection: { + driver: 'databricks', + server_hostname: 'dbc-example.cloud.databricks.com', + http_path: '/sql/1.0/warehouses/abc', + catalog: 'main', + schema_name: 'sales', + token: 'fixture-token', // pragma: allowlist secret + }, + driverFactory, + }); + + await expect( + connector.sampleTable( + { + connectionId: 'warehouse', + table: { catalog: 'main', db: 'sales', name: 'orders' }, + limit: 1, + columns: ['id', 'status'], + }, + { runId: 'scan-run-1' }, + ), + ).resolves.toMatchObject({ headers: ['id', 'status'], rows: [[1, 'paid']], totalRows: 1 }); + await expect( + connector.executeReadOnly( + { connectionId: 'warehouse', sql: 'select id, status from orders', maxRows: 1 }, + { runId: 'scan-run-1' }, + ), + ).resolves.toMatchObject({ headers: ['id', 'status'], rows: [[1, 'paid']], rowCount: 1 }); + await expect( + connector.sampleColumn( + { + connectionId: 'warehouse', + table: { catalog: 'main', db: 'sales', name: 'orders' }, + column: 'status', + limit: 2, + }, + { runId: 'scan-run-1' }, + ), + ).resolves.toEqual({ values: ['paid', 'open'], nullCount: null, distinctCount: null }); + await expect( + connector.getColumnDistinctValues({ catalog: 'main', db: 'sales', name: 'orders' }, 'status', { + maxCardinality: 10, + limit: 5, + }), + ).resolves.toEqual({ values: ['open', 'paid'], cardinality: 2 }); + await expect(connector.listSchemas()).resolves.toEqual(['sales', 'finance']); + await expect(connector.listTables(['sales'])).resolves.toEqual([ + { catalog: 'main', schema: 'sales', name: 'orders', kind: 'table' }, + { catalog: 'main', schema: 'sales', name: 'order_summary', kind: 'view' }, + ]); + await connector.cleanup(); + const driver = (driverFactory.createDriver as ReturnType).mock.results[0]?.value as KtxDatabricksDriver; + expect(driver.cleanup).toHaveBeenCalledTimes(1); + }); + + it('converts a native snapshot into a live-database introspection snapshot', async () => { + const introspection = createDatabricksLiveDatabaseIntrospection({ + connections: { + warehouse: { + driver: 'databricks', + server_hostname: 'dbc-example.cloud.databricks.com', + http_path: '/sql/1.0/warehouses/abc', + catalog: 'main', + schema_name: 'sales', + token: 'fixture-token', // pragma: allowlist secret + }, + }, + driverFactory: fakeDriverFactory(), + now: () => new Date('2026-04-29T18:00:00.000Z'), + }); + + await expect(introspection.extractSchema('warehouse')).resolves.toMatchObject({ + connectionId: 'warehouse', + metadata: { catalog: 'main', schemas: ['sales'] }, + tables: expect.arrayContaining([expect.objectContaining({ catalog: 'main', db: 'sales', name: 'orders' })]), + }); + }); +}); diff --git a/packages/cli/test/connectors/databricks/dialect.test.ts b/packages/cli/test/connectors/databricks/dialect.test.ts new file mode 100644 index 000000000..94c005243 --- /dev/null +++ b/packages/cli/test/connectors/databricks/dialect.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from 'vitest'; +import { KtxDatabricksDialect } from '../../../src/connectors/databricks/dialect.js'; + +describe('KtxDatabricksDialect', () => { + const dialect = new KtxDatabricksDialect(); + + it('quotes identifiers and formats Unity Catalog names', () => { + expect(dialect.quoteIdentifier('order`items')).toBe('`order``items`'); + expect(dialect.formatTableName({ catalog: 'main', db: 'sales', name: 'orders' })).toBe( + '`main`.`sales`.`orders`', + ); + expect(dialect.formatDisplayRef({ catalog: 'main', db: 'sales', name: 'orders' })).toBe('main.sales.orders'); + expect(dialect.parseDisplayRef('main.sales.orders')).toEqual({ catalog: 'main', db: 'sales', name: 'orders' }); + expect(dialect.parseDisplayRef('sales.orders')).toBeNull(); + }); + + it('maps Databricks native types to scan dimensions', () => { + expect(dialect.mapDataType('DECIMAL(12,2)')).toBe('DECIMAL(12,2)'); + expect(dialect.mapToDimensionType('TIMESTAMP_NTZ')).toBe('time'); + expect(dialect.mapToDimensionType('BIGINT')).toBe('number'); + expect(dialect.mapToDimensionType('BOOLEAN')).toBe('boolean'); + expect(dialect.mapToDimensionType('STRUCT')).toBe('string'); + }); + + it('generates sampling and dictionary SQL', () => { + expect(dialect.generateSampleQuery('`main`.`sales`.`orders`', 5, ['id', 'status'])).toBe( + 'SELECT `id`, `status` FROM `main`.`sales`.`orders` LIMIT 5', + ); + expect(dialect.generateColumnSampleQuery('`main`.`sales`.`orders`', 'status', 10)).toBe( + "SELECT `status` FROM `main`.`sales`.`orders` WHERE `status` IS NOT NULL AND TRIM(CAST(`status` AS STRING)) != '' LIMIT 10", + ); + expect(dialect.generateCardinalitySampleQuery('`main`.`sales`.`orders`', '`status`', 100)).toContain( + 'SELECT APPROX_COUNT_DISTINCT(val) AS cardinality', + ); + expect(dialect.generateDistinctValuesQuery('`main`.`sales`.`orders`', '`status`', 20)).toContain( + 'SELECT DISTINCT CAST(`status` AS STRING) AS val', + ); + }); + + it('keeps unsupported statistics explicit', () => { + expect(dialect.generateColumnStatisticsQuery('sales', 'orders')).toBeNull(); + }); +}); diff --git a/packages/cli/test/context/connections/dialects.test.ts b/packages/cli/test/context/connections/dialects.test.ts index cc7eaa59e..c00c20656 100644 --- a/packages/cli/test/context/connections/dialects.test.ts +++ b/packages/cli/test/context/connections/dialects.test.ts @@ -185,6 +185,36 @@ const fixtures: DialectFixture[] = [ nativeTypeInput: 'NUMBER(38,0)', normalizedType: 'NUMBER(38,0)', }, + { + driver: 'databricks', + table: { catalog: 'main', db: 'sales', name: 'orders' }, + quoteInput: 'order`items', + quotedIdentifier: '`order``items`', + formattedTable: '`main`.`sales`.`orders`', + display: 'main.sales.orders', + invalidDisplay: 'sales.orders', + columnDisplayTablePartCount: 3, + limitClause: 'LIMIT 25 OFFSET 5', + topClause: '', + randomFilter: 'RAND() < 0.25', + tableSampleClause: 'TABLESAMPLE (25 PERCENT)', + sampleQuery: 'SELECT `id`, `status` FROM `main`.`sales`.`orders` LIMIT 5', + columnSampleContains: "TRIM(CAST(`status` AS STRING)) != ''", + nullCountExpression: 'COUNT_IF(`status` IS NULL)', + distinctCountExpression: 'APPROX_COUNT_DISTINCT(`status`)', + textLengthExpression: 'LENGTH(CAST(`status` AS STRING))', + castToText: 'CAST(`status` AS STRING)', + sampleValueAggregation: + "(SELECT CONCAT_WS('\\u001F', COLLECT_LIST(CAST(value AS STRING))) FROM (SELECT status AS value FROM orders) AS relationship_profile_values)", + cardinalityContains: 'SELECT APPROX_COUNT_DISTINCT(val) AS cardinality', + randomizedCardinalityContains: 'ORDER BY RAND()', + distinctValuesContains: 'SELECT DISTINCT CAST(`status` AS STRING) AS val', + statisticsContains: null, + dimensionInput: 'TIMESTAMP_NTZ', + dimensionType: 'time', + nativeTypeInput: 'DECIMAL(12,2)', + normalizedType: 'DECIMAL(12,2)', + }, { driver: 'bigquery', table: { catalog: 'analytics-project', db: 'warehouse', name: 'orders' }, @@ -305,7 +335,7 @@ describe('getDialectForDriver', () => { it('throws with a supported-driver list for unknown drivers', () => { expect(() => getDialectForDriver('oracle')).toThrow( - 'Unsupported driver "oracle". Supported drivers: bigquery, clickhouse, mongodb, mysql, postgres, snowflake, sqlite, sqlserver', + 'Unsupported driver "oracle". Supported drivers: bigquery, clickhouse, databricks, mongodb, mysql, postgres, snowflake, sqlite, sqlserver', ); }); diff --git a/packages/cli/test/context/connections/drivers.test.ts b/packages/cli/test/context/connections/drivers.test.ts index 65bbca4b8..9536fc7c8 100644 --- a/packages/cli/test/context/connections/drivers.test.ts +++ b/packages/cli/test/context/connections/drivers.test.ts @@ -69,6 +69,14 @@ const connectionFixtures: Record = { database: 'ANALYTICS', schema: 'PUBLIC', }), + databricks: () => ({ + driver: 'databricks', + server_hostname: 'dbc-example.cloud.databricks.com', + http_path: '/sql/1.0/warehouses/abc', + catalog: 'main', + schema_name: 'sales', + token: 'secret', // pragma: allowlist secret + }), }; const allowedScopeKeys = new Set(['dataset_ids', 'databases', 'schemas', 'schema_names']); @@ -101,6 +109,7 @@ describe('driverRegistrations', () => { expect(listSupportedDrivers()).toEqual([ 'bigquery', 'clickhouse', + 'databricks', 'mongodb', 'mysql', 'postgres', diff --git a/packages/cli/test/context/connections/local-warehouse-descriptor.test.ts b/packages/cli/test/context/connections/local-warehouse-descriptor.test.ts index e0a285a97..aadeef8fa 100644 --- a/packages/cli/test/context/connections/local-warehouse-descriptor.test.ts +++ b/packages/cli/test/context/connections/local-warehouse-descriptor.test.ts @@ -51,6 +51,7 @@ describe('local connection info helpers', () => { expect(localConnectionTypeForConfig('warehouse', { driver: 'postgres' })).toBe('POSTGRESQL'); expect(localConnectionTypeForConfig('bq', { driver: 'bigquery', project_id: 'acme' })).toBe('BIGQUERY'); expect(localConnectionTypeForConfig('snowflake', { driver: 'snowflake' })).toBe('SNOWFLAKE'); + expect(localConnectionTypeForConfig('databricks', { driver: 'databricks' })).toBe('DATABRICKS'); }); it('keeps removed driver aliases as display-only labels', () => { diff --git a/packages/cli/test/context/mcp/dialect-notes.test.ts b/packages/cli/test/context/mcp/dialect-notes.test.ts index 27e9d9226..61859a175 100644 --- a/packages/cli/test/context/mcp/dialect-notes.test.ts +++ b/packages/cli/test/context/mcp/dialect-notes.test.ts @@ -33,9 +33,8 @@ describe('per-dialect SQL notes', () => { }); it('does not author notes for unreachable dialects', () => { - // duckdb/databricks appear in the resolver map but no connector produces them. + // duckdb appears in the resolver map but no connector produces it. expect(DIALECTS_WITH_NOTES).not.toContain('duckdb'); - expect(DIALECTS_WITH_NOTES).not.toContain('databricks'); }); it('answers the full rubric for every dialect', () => { diff --git a/packages/cli/test/setup-databases.test.ts b/packages/cli/test/setup-databases.test.ts index 3f3638052..3afca4a90 100644 --- a/packages/cli/test/setup-databases.test.ts +++ b/packages/cli/test/setup-databases.test.ts @@ -240,6 +240,7 @@ describe('setup databases step', () => { { value: 'postgres', label: 'PostgreSQL' }, { value: 'bigquery', label: 'BigQuery' }, { value: 'snowflake', label: 'Snowflake' }, + { value: 'databricks', label: 'Databricks' }, { value: 'mysql', label: 'MySQL' }, { value: 'clickhouse', label: 'ClickHouse' }, { value: 'sqlserver', label: 'SQL Server' }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5521fa08f..fa9425dec 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -143,6 +143,9 @@ importers: '@commander-js/extra-typings': specifier: 14.0.0 version: 14.0.0(commander@14.0.3) + '@databricks/sql': + specifier: ^1.16.0 + version: 1.16.0 '@duckdb/node-api': specifier: 1.5.3-r.3 version: 1.5.3-r.3 @@ -196,7 +199,7 @@ importers: version: 10.2.5 mongodb: specifier: ^6.12.0 - version: 6.21.0 + version: 6.21.0(socks@2.8.9) mssql: specifier: ^12.5.4 version: 12.5.4(@azure/core-client@1.10.1) @@ -282,6 +285,15 @@ importers: packages: + '@75lb/deep-merge@1.1.4': + resolution: {integrity: sha512-Fjmi8VSxoGF80wn8HSHTPfUKLSJ+aiQE7rXyrFV6apP8Xyj8v4bqdC7BBoklxjt671gUqP0yREwpDlUP5R/VyQ==} + engines: {node: '>=12.17'} + peerDependencies: + '@75lb/nature': latest + peerDependenciesMeta: + '@75lb/nature': + optional: true + '@actions/core@3.0.1': resolution: {integrity: sha512-a6d/Nwahm9fliVGRhdhofo40HjHQasUPusmc7vBfyky+7Z+P2A1J68zyFVaNcEclc/Se+eO595oAr5nwEIoIUA==} @@ -762,6 +774,62 @@ packages: '@dabh/diagnostics@2.0.8': resolution: {integrity: sha512-R4MSXTVnuMzGD7bzHdW2ZhhdPC/igELENcq5IjEverBvq5hn1SXCWcsi6eSsdWP0/Ur+SItRRjAktmdoX/8R/Q==} + '@databricks/databricks-sql-kernel-darwin-arm64@0.2.0': + resolution: {integrity: sha512-dSZJD1uileOqRDfs5KsW6m43PAl3GqzQMcETbRT1Zjw2FRLQDtLKTA3+VWmlUUHhIz583WeTps91Sjee79cXbA==} + engines: {node: '>=18.0.0'} + cpu: [arm64] + os: [darwin] + + '@databricks/databricks-sql-kernel-darwin-x64@0.2.0': + resolution: {integrity: sha512-JEe7TqLrXrAoN3SWsi7NgjAo6V+7jEvBwLswwhl0qUp1MoPrZT+y5Kf49vuHPlh61vFgi2F6NlCNfdB/8wpihg==} + engines: {node: '>=18.0.0'} + cpu: [x64] + os: [darwin] + + '@databricks/databricks-sql-kernel-linux-arm64-gnu@0.2.0': + resolution: {integrity: sha512-meJ2JF5w4qEiaiI9TNwg2iMgasadqeKSDAw3r3hWTEdzKwUY7saMTFNcjumwR8R3cQMCFJfp3YBOwKS5/hvN9A==} + engines: {node: '>=18.0.0'} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@databricks/databricks-sql-kernel-linux-arm64-musl@0.2.0': + resolution: {integrity: sha512-YTCPs11w6purXCQwJBCh99oHBwTEW4q46OWxO9Rnddo1wJd8EPBa3Misw08URoUVuVuEfYGy5YtSvuPfkkj8Vw==} + engines: {node: '>=18.0.0'} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@databricks/databricks-sql-kernel-linux-x64-gnu@0.2.0': + resolution: {integrity: sha512-wIEJX2mtoCc/KGOzzbxXwOR5aUB5dBSUG1Ypp+JLI28XsZB2eCLcP4jyAQ+Uklxn+SU1hICuok+30t/7wSvKUg==} + engines: {node: '>=18.0.0'} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@databricks/databricks-sql-kernel-linux-x64-musl@0.2.0': + resolution: {integrity: sha512-0wtWdOxYh7BfeklDpI0tpTk/ljdjJmCQsNFPLy2C7EnK60J3sroYzTaFqgn8Qqc7T03VHZl/6GG1pXC3zPuU1g==} + engines: {node: '>=18.0.0'} + cpu: [x64] + os: [linux] + libc: [musl] + + '@databricks/databricks-sql-kernel-win32-arm64-msvc@0.2.0': + resolution: {integrity: sha512-gS4y1hIIDitr1d9bW6yWEkn4wMW7abinDwFFVUJtYsWkFS+rMq/8CwAYioqNprXQP+XEQS7fLiLYq/BYsQB3Aw==} + engines: {node: '>=18.0.0'} + cpu: [arm64] + os: [win32] + + '@databricks/databricks-sql-kernel-win32-x64-msvc@0.2.0': + resolution: {integrity: sha512-zl6KtfB02eVsBNZEQ/kf/dhmesUqbVGwhSFb5oKlmtyj/1kah9R1FRKBVKTLt3v5n9LnjBnomV2s+DRxRSh4rA==} + engines: {node: '>=18.0.0'} + cpu: [x64] + os: [win32] + + '@databricks/sql@1.16.0': + resolution: {integrity: sha512-kaSBjYUT0rDYsrouvPakwRjs030KKvImOTHRG95EsYOLkNN+/YvJ8JgQ0vQjvgT3NUREoGys+SQ4ZzqwHuwoCA==} + engines: {node: '>=14.0.0'} + '@duckdb/node-api@1.5.3-r.3': resolution: {integrity: sha512-FzuL6sevuFfEFwkgiUMRMUAj4TaVqV//L0oo2FVZ9s9oYpLpALF9qZyQv2ucclTNQZwDCkm8+e6yLMc6t8IjlA==} @@ -2504,6 +2572,9 @@ packages: '@tediousjs/connection-string@1.1.0': resolution: {integrity: sha512-z9ZBWEG+8pIB5V1zYzlRPXx0oRJ5H7coPnMQK8EZOw03UTPI9Umn6viL36f5w+CuqkKsnCM50RVStpjZmR0Bng==} + '@tootallnate/quickjs-emscripten@0.23.0': + resolution: {integrity: sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==} + '@tybys/wasm-util@0.10.2': resolution: {integrity: sha512-RoBvJ2X0wuKlWFIjrwffGw1IqZHKQqzIchKaadZZfnNpsAYp2mM0h36JtPCjNDAHGgYez/15uMBpfGwchhiMgg==} @@ -2513,6 +2584,12 @@ packages: '@types/chai@5.2.3': resolution: {integrity: sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==} + '@types/command-line-args@5.2.0': + resolution: {integrity: sha512-UuKzKpJJ/Ief6ufIaIzr3A/0XnluX7RvFgwkV89Yzvm77wCh1kFaFmqN8XEnGcN62EuHdedQjEMb8mYxFLGPyA==} + + '@types/command-line-usage@5.0.2': + resolution: {integrity: sha512-n7RlEEJ+4x4TS7ZQddTmNSxP+zziEG0TNsMfiRIxcIVXt71ENJ9ojeXmGO3wPoTdn7pJcU2xc3CJYMktNT6DPg==} + '@types/d3-color@3.1.3': resolution: {integrity: sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A==} @@ -2564,6 +2641,9 @@ packages: '@types/normalize-package-data@2.4.4': resolution: {integrity: sha512-37i+OaWTh9qeK4LSHPsyRC7NahnGotNuZvjLSgcPzblpHB3rrCJxAOgI5gCdKm7coonsaX1Of0ILiTcnZjbfxA==} + '@types/pad-left@2.1.1': + resolution: {integrity: sha512-Xd22WCRBydkGSApl5Bw0PhAOHKSVjNL3E3AwzKaps96IMraPqy5BvZIsBVK6JLwdybUzjHnuWVwpDd0JjTfHXA==} + '@types/pg@8.20.0': resolution: {integrity: sha512-bEPFOaMAHTEP1EzpvHTbmwR8UsFyHSKsRisLIHVMXnpNefSbGA1bD6CVy+qKjGSqmZqNqBDV2azOBo8TgkcVow==} @@ -2736,6 +2816,10 @@ packages: any-promise@1.3.0: resolution: {integrity: sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==} + apache-arrow@13.0.0: + resolution: {integrity: sha512-3gvCX0GDawWz6KFNC28p65U+zGh/LZ6ZNKWNu74N6CQlKzxeoWHpi4CgEQsgRSEMuyrIIXi1Ea2syja7dwcHvw==} + hasBin: true + argparse@2.0.1: resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} @@ -2746,6 +2830,14 @@ packages: resolution: {integrity: sha512-ik3ZgC9dY/lYVVM++OISsaYDeg1tb0VtP5uL3ouh1koGOaUMDPpbFIei4JkFimWUFPn90sbMNMXQAIVOlnYKJA==} engines: {node: '>=10'} + array-back@3.1.0: + resolution: {integrity: sha512-TkuxA4UCOvxuDK6NZYXCalszEzj+TLszyASooky+i742l9TqsOdYCMJJupxRic61hwquNtppB3hgcuq9SVSH1Q==} + engines: {node: '>=6'} + + array-back@6.2.3: + resolution: {integrity: sha512-SGDvmg6QTYiTxCBkYVmThcoa67uLl35pyzRHdpCGBOcqFy6BtwnphoFPk7LhJshD+Yk1Kt35WGWeZPTgwR4Fhw==} + engines: {node: '>=12.17'} + array-ify@1.0.0: resolution: {integrity: sha512-c5AMf34bKdvPhQ7tBGhqkgKNUzMr4WUs+WDtC2ZUGOUncbxKMTvqxYctiseW3+L4bA8ec+GcZ6/A/FW4m8ukng==} @@ -2772,6 +2864,10 @@ packages: resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} engines: {node: '>=12'} + ast-types@0.13.4: + resolution: {integrity: sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==} + engines: {node: '>=4'} + ast-v8-to-istanbul@1.0.0: resolution: {integrity: sha512-1fSfIwuDICFA4LKkCzRPO7F0hzFf0B7+Xqrl27ynQaa+Rh0e1Es0v6kWHPott3lU10AyAr7oKHa65OppjLn3Rg==} @@ -2818,6 +2914,10 @@ packages: engines: {node: '>=6.0.0'} hasBin: true + basic-ftp@5.3.1: + resolution: {integrity: sha512-bopVNp6ugyA150DDuZfPFdt1KZ5a94ZDiwX4hMgZDzF+GttD80lEy8kj98kbyhLXnPvhtIo93mdnLIjpCAeeOw==} + engines: {node: '>=10.0.0'} + before-after-hook@4.0.0: resolution: {integrity: sha512-q6tR3RPqIB1pMiTRMFcZwuG5T8vwp+vUvEG0vuI6B+Rikh5BfPp2fQ82c925FOs+b0lcFQ8CFrL+KbilfZFhOQ==} @@ -2871,6 +2971,9 @@ packages: resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} engines: {node: '>=8'} + browser-or-node@1.3.0: + resolution: {integrity: sha512-0F2z/VSnLbmEeBcUrSuDH5l0HxTXdQQzLjkmBR4cYfvg1zJrKSlmIZFqyFR8oX0NrwPhy3c3HQ6i3OxMbew4Tg==} + bson@6.10.4: resolution: {integrity: sha512-WIsKqkSC0ABoBJuT1LEX+2HEvNmNKKgnTAyd0fL8qzK4SH2i9NXg+t08YtdZp/V9IZ33cxe3iV4yM0qg8lMQng==} engines: {node: '>=16.20.1'} @@ -2914,6 +3017,10 @@ packages: resolution: {integrity: sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==} engines: {node: '>=18'} + chalk-template@0.4.0: + resolution: {integrity: sha512-/ghrgmhfY8RaSdeo43hNXxpoHAtxdbskUHjPpfqUWGttFgycUhYPGx3YZBCnUCvOa7Doivn1IZec3DEGFoMgLg==} + engines: {node: '>=12'} + chalk@2.4.2: resolution: {integrity: sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==} engines: {node: '>=4'} @@ -3044,6 +3151,14 @@ packages: comma-separated-tokens@2.0.3: resolution: {integrity: sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==} + command-line-args@5.2.1: + resolution: {integrity: sha512-H4UfQhZyakIjC74I9d34fGYDwk3XpSr17QhEd0Q3I9Xq1CETHo4Hcuo87WyWHpAF1aSLjLRf5lD9ZGX2qStUvg==} + engines: {node: '>=4.0.0'} + + command-line-usage@7.0.1: + resolution: {integrity: sha512-NCyznE//MuTjwi3y84QVUGEOT+P5oto1e1Pk/jFPVdPPfsG03qpTIl3yw6etR+v73d0lXsoojRpvbru2sqePxQ==} + engines: {node: '>=12.20.0'} + commander@11.1.0: resolution: {integrity: sha512-yPVavfyCcRhmorC7rWlkHn15b4wDVgVmBA7kV4QVBsF7kv/9TKJAbAXVTxvTnwP8HHKjRCJDClKbciiYS7p0DQ==} engines: {node: '>=16'} @@ -3052,6 +3167,10 @@ packages: resolution: {integrity: sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw==} engines: {node: '>=20'} + commander@9.5.0: + resolution: {integrity: sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ==} + engines: {node: ^12.20.0 || >=14} + compare-func@2.0.0: resolution: {integrity: sha512-zHig5N+tPWARooBnb0Zx1MFcdfpyJrfTJ3Y5L+IFvUm8rM74hHz66z0gw0x4tijh5CorKkKUCnW82R2vmpeCRA==} @@ -3144,6 +3263,9 @@ packages: csstype@3.2.3: resolution: {integrity: sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==} + cuint@0.2.2: + resolution: {integrity: sha512-d4ZVpCW31eWwCMe1YT3ur7mUDnTXbgwyzaL320DrcRT45rfjYxkt5QWLrmOJ+/UEAI2+fQgKe/fCjR8l4TpRgw==} + d3-color@3.1.0: resolution: {integrity: sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==} engines: {node: '>=12'} @@ -3186,6 +3308,10 @@ packages: resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==} engines: {node: '>= 12'} + data-uri-to-buffer@6.0.2: + resolution: {integrity: sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw==} + engines: {node: '>= 14'} + dateformat@4.6.3: resolution: {integrity: sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA==} @@ -3217,10 +3343,18 @@ packages: resolution: {integrity: sha512-H9LMLr5zwIbSxrmvikGuI/5KGhZ8E2zH3stkMgM5LpOWDutGM2JZaj460Udnf1a+946zc7YBgrqEWwbk7zHvGw==} engines: {node: '>=18'} + define-lazy-prop@2.0.0: + resolution: {integrity: sha512-Ds09qNh8yw3khSjiJjiUInaGX9xlqZDY7JVryGxdxV7NPeuqQfplOpQ66yJFZut3jLa5zOwkXw1g9EI2uKh4Og==} + engines: {node: '>=8'} + define-lazy-prop@3.0.0: resolution: {integrity: sha512-N+MeXYoqr3pOgn8xfyRPREN7gHakLYjhsHhWGT3fWAiL4IkAt0iDw14QiiEm2bE30c5XX5q0FtAA3CK5f9/BUg==} engines: {node: '>=12'} + degenerator@5.0.1: + resolution: {integrity: sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ==} + engines: {node: '>= 14'} + delayed-stream@1.0.0: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} engines: {node: '>=0.4.0'} @@ -3365,6 +3499,20 @@ packages: resolution: {integrity: sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==} engines: {node: '>=12'} + escodegen@2.1.0: + resolution: {integrity: sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==} + engines: {node: '>=6.0'} + hasBin: true + + esprima@4.0.1: + resolution: {integrity: sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==} + engines: {node: '>=4'} + hasBin: true + + estraverse@5.3.0: + resolution: {integrity: sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==} + engines: {node: '>=4.0'} + estree-util-attach-comments@3.0.0: resolution: {integrity: sha512-cKUwm/HUcTDsYh/9FgnuFqpfquUbwIqwKM26BVCGDPVgvaCl/nDCCjUfiLlx6lsEZ3Z4RFxNbOQ60pkaEwFxGw==} @@ -3389,6 +3537,10 @@ packages: estree-walker@3.0.3: resolution: {integrity: sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==} + esutils@2.0.3: + resolution: {integrity: sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==} + engines: {node: '>=0.10.0'} + etag@1.8.1: resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==} engines: {node: '>= 0.6'} @@ -3529,6 +3681,10 @@ packages: resolution: {integrity: sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==} engines: {node: '>= 18.0.0'} + find-replace@3.0.0: + resolution: {integrity: sha512-6Tb2myMioCAgv5kfvP5/PkZZ/ntTpVK39fHY7WkWBgvbeE+VHd/tZuZ4mrC+bxh4cfOZeYKVPaJIZtZXV7GNCQ==} + engines: {node: '>=4.0.0'} + find-up-simple@1.0.1: resolution: {integrity: sha512-afd4O7zpqHeRyg4PfDQsXmlDe2PfdHtJt6Akt8jOWaApLOZk5JXs6VMR29lz03pRe9mpykrRCYIYxaJYcfpncQ==} engines: {node: '>=18'} @@ -3541,6 +3697,9 @@ packages: resolution: {integrity: sha512-2kCCtc+JvcZ86IGAz3Z2Y0A1baIz9fL31pH/0S1IqZr9Iwnjq8izfPtrCyQKO6TLMPELLsQMre7VDqeIKCsHkA==} engines: {node: '>=18'} + flatbuffers@23.5.26: + resolution: {integrity: sha512-vE+SI9vrJDwi1oETtTIFldC/o9GsVKRM+s6EL0nQgxXlYV1Vc4Tk30hj4xGICftInKQKj1F3up2n8UbIVobISQ==} + fn.name@1.1.0: resolution: {integrity: sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==} @@ -3770,6 +3929,10 @@ packages: get-tsconfig@4.14.0: resolution: {integrity: sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA==} + get-uri@6.0.5: + resolution: {integrity: sha512-b1O07XYq8eRuVzBNgJLstU6FYc1tS6wnMtF1I1D9lE8LxZSOGZ7LhxN54yPP6mGw5f2CkXY2BQUL9Fx41qvcIg==} + engines: {node: '>= 14'} + git-log-parser@1.2.1: resolution: {integrity: sha512-PI+sPDvHXNPl5WNOErAK05s3j0lgwUzMN6o8cyQrDaKfT3qd7TmNJKeXX+SknI5I0QhG5fVPAEwSY4tRGDtYoQ==} @@ -4095,6 +4258,11 @@ packages: isexe@2.0.0: resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} + isomorphic-ws@4.0.1: + resolution: {integrity: sha512-BhBvN2MBpWTaSHdWRb/bwdZJ1WaehQ2L1KngkCkfLUGF0mAWAT1sQUQacEmQ0jXkFw/czDXPNQSL5u2/Krsz1w==} + peerDependencies: + ws: 8.20.1 + issue-parser@7.0.2: resolution: {integrity: sha512-7atWPjhGEIX3JEtMrOYd8TKzboYlq+5sNbdl9POiLYOI14G5HZiQbZP0Xj5EZdrufQVXfJlpTV0hys0CuxwxZw==} engines: {node: ^18.17 || >=20.6.1} @@ -4119,6 +4287,9 @@ packages: resolution: {integrity: sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ==} hasBin: true + jose@4.15.9: + resolution: {integrity: sha512-1vUQX+IdDMVPj4k8kOxgUqlcK518yluMuGZwqlr44FS1ppZB/5GWh4rZG89erpOBOJjU/OBsnCVFfapsRz6nEA==} + jose@6.2.3: resolution: {integrity: sha512-YYVDInQKFJfR/xa3ojUTl8c2KoTwiL1R5Wg9YCydwH0x0B9grbzlg5HC7mMjCtUJjbQ/YnGEZIhI5tCgfTb4Hw==} @@ -4142,6 +4313,10 @@ packages: json-bigint@1.0.0: resolution: {integrity: sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==} + json-bignum@0.0.3: + resolution: {integrity: sha512-2WHyXj3OfHSgNyuzDbSxI1w2jgw5gkWSWhS7Qg4bWXx1nLk3jnbwfUeS0PSba3IzpTUWdHxBieELUzXRjQB2zg==} + engines: {node: '>=0.8'} + json-parse-better-errors@1.0.2: resolution: {integrity: sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==} @@ -4273,6 +4448,9 @@ packages: lodash-es@4.18.1: resolution: {integrity: sha512-J8xewKD/Gk22OZbhpOVSwcs60zhd95ESDwezOFuA3/099925PdHJ7OFHNTGtajL3AlZkykD32HykiMo+BIBI8A==} + lodash.camelcase@4.3.0: + resolution: {integrity: sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==} + lodash.capitalize@4.2.1: resolution: {integrity: sha512-kZzYOKspf8XVX5AvmQF94gQW0lejFVgb80G85bU4ZWzoJ6C03PQg3coYAUpSTpQWelrZELd3XWgHzw4Ck5kaIw==} @@ -4332,6 +4510,14 @@ packages: resolution: {integrity: sha512-5YgH9UJd7wVb9hIouI2adWpgqrrICkt070Dnj8EUY1+B4B2P9eRLPAkAAo6NICA7CEhOIeBHl46u9zSNpNu7zA==} engines: {node: 20 || >=22} + lru-cache@6.0.0: + resolution: {integrity: sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==} + engines: {node: '>=10'} + + lru-cache@7.18.3: + resolution: {integrity: sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==} + engines: {node: '>=12'} + lru.min@1.1.4: resolution: {integrity: sha512-DqC6n3QQ77zdFpCMASA1a3Jlb64Hv2N2DciFGkO/4L9+q/IpIAuRlKOvCXabtRW6cQf8usbmM6BE/TOPysCdIA==} engines: {bun: '>=1.0.0', deno: '>=1.30.0', node: '>=8.0.0'} @@ -4341,6 +4527,10 @@ packages: peerDependencies: react: ^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0 + lz4@0.6.5: + resolution: {integrity: sha512-KSZcJU49QZOlJSItaeIU3p8WoAvkTmD9fJqeahQXNu1iQ/kR0/mQLdbrK8JY9MY8f6AhJoMrihp1nu1xDbscSQ==} + engines: {node: '>= 0.10'} + magic-string@0.30.21: resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} @@ -4678,6 +4868,9 @@ packages: resolution: {integrity: sha512-Tz09sEL2EEuv5fFowm419c1+a/jSMiBjI9gHxVLrVdbUkkNUUfjsVYs9pVZu5oCon/kmRh9TfLEObFtkVxmY0w==} engines: {node: '>=8.0.0'} + nan@2.27.0: + resolution: {integrity: sha512-hC+0LidcL3XE4rp1C4H54KujgXKzbfyTngZTwBByQxsOxCEKZT0MPQ4hOKUH2jU1OYstqdDH4onyHPDzcV0XdQ==} + nanoid@3.3.12: resolution: {integrity: sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} @@ -4699,6 +4892,10 @@ packages: nerf-dart@1.0.0: resolution: {integrity: sha512-EZSPZB70jiVsivaBLYDCyntd5eH8NTSMOn3rB+HxwdmKThGELLdYv8qVIMWvZEFy9w8ZZpW9h9OB32l1rGtj7g==} + netmask@2.1.1: + resolution: {integrity: sha512-eonl3sLUha+S1GzTPxychyhnUzKyeQkZ7jLjKrBagJgPla13F+uQ71HgpFefyHgqrjEbCPkDArxYsjY8/+gLKA==} + engines: {node: '>= 0.4.0'} + next-themes@0.4.6: resolution: {integrity: sha512-pZvgD5L0IEvX5/9GWyHMf3m8BKiVQwsCMHfoFosXtXBMnaS0ZnIJ9ST4b4NqLVKDEm8QBxoNNGNaBv2JNF6XNA==} peerDependencies: @@ -4739,10 +4936,22 @@ packages: resolution: {integrity: sha512-Z3lTE9pLaJF47NyMhd4ww1yFTAP8YhYI8SleJiHzM46Fgpm5cnNzSl9XfzFNqbaz+VlJrIj3fXQ4DeN1Rjm6cw==} engines: {node: '>=18'} + node-fetch@2.7.0: + resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==} + engines: {node: 4.x || >=6.0.0} + peerDependencies: + encoding: ^0.1.0 + peerDependenciesMeta: + encoding: + optional: true + node-fetch@3.3.2: resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + node-int64@0.4.0: + resolution: {integrity: sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==} + normalize-package-data@6.0.2: resolution: {integrity: sha512-V6gygoYb/5EmNI+MEGrWkC+e6+Rr7mTmfHrxDbLzxQogBkgzo76rkok0Am6thgSF7Mv2nLOajAJj5vDJZEFn7g==} engines: {node: ^16.14.0 || >=18.0.0} @@ -4845,6 +5054,10 @@ packages: resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} engines: {node: '>=0.10.0'} + object-hash@2.2.0: + resolution: {integrity: sha512-gScRMn0bS5fH+IuwyIFgnh9zBdo4DV+6GhygmWM9HyNJSgS0hScp1f5vjtm7oIIOiT9trXrShAkLFSc2IqKNgw==} + engines: {node: '>= 6'} + object-inspect@1.13.4: resolution: {integrity: sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==} engines: {node: '>= 0.4'} @@ -4852,6 +5065,10 @@ packages: obug@2.1.1: resolution: {integrity: sha512-uTqF9MuPraAQ+IsnPf366RG4cP9RtUi7MLO1N3KEc+wb0a6yKpeL0lmk2IB1jY5KHPAlTc6T/JRdC/YqxHNwkQ==} + oidc-token-hash@5.2.0: + resolution: {integrity: sha512-6gj2m8cJZ+iSW8bm0FXdGF0YhIQbKrfP4yWTNzxc31U6MOjfEmB1rHvlYvxI1B7t7BCi1F2vYTT6YhtQRG4hxw==} + engines: {node: ^10.13.0 || >=12.0.0} + on-exit-leak-free@2.1.2: resolution: {integrity: sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==} engines: {node: '>=14.0.0'} @@ -4888,6 +5105,10 @@ packages: resolution: {integrity: sha512-MVHddDVweXZF3awtlAS+6pgKLlm/JgxZ90+/NBurBoQctVOOB/zDdVjcyPzQ+0laDGbsWgrRkflI65sQeOgT9Q==} engines: {node: '>=8'} + open@8.4.2: + resolution: {integrity: sha512-7x81NCL719oNbsq/3mh+hVrAWmFuEYUqrq/Iw3kUzH8ReypT9QQ0BLoJS7/G9k6N81XjW4qHWtjWwe/9eLy1EQ==} + engines: {node: '>=12'} + openai@6.38.0: resolution: {integrity: sha512-AoMplt2UalrpgUDMh3L09QWjNRlgJPipclQvA6sYAaeF6nHNBMgmikAZGmcYLn8on4d9sQY9Q8bOLfrBS7Lc8g==} hasBin: true @@ -4900,6 +5121,9 @@ packages: zod: optional: true + openid-client@5.7.1: + resolution: {integrity: sha512-jDBPgSVfTnkIh71Hg9pRvtJc6wTwqjRkN88+gCFtYWrlP4Yx2Dsrow8uPi3qLr/aeymPF3o2+dS+wOpglK04ew==} + oxc-parser@0.130.0: resolution: {integrity: sha512-X0PJ+NmOok8qP3vK9uaW431ngkdM9UPEK7KG466urtIL2+EYTEgbZK2yqe2MWKJKBjRlFweP/pJPx0x9muMEVw==} engines: {node: ^20.19.0 || >=22.12.0} @@ -4951,6 +5175,18 @@ packages: resolution: {integrity: sha512-U1etNYuMJoIz3ZXSrrySFjsXQTWOx2/jdi86L+2pRvph/qMKL6sbcCYdH23fqsbm8TH2Gn0OybpT4eSFlCVHww==} engines: {node: '>=4'} + pac-proxy-agent@7.2.0: + resolution: {integrity: sha512-TEB8ESquiLMc0lV8vcd5Ql/JAKAoyzHFXaStwjkzpOpC5Yv+pIzLfHvjTSdf3vpa2bMiUQrg9i6276yn8666aA==} + engines: {node: '>= 14'} + + pac-resolver@7.0.1: + resolution: {integrity: sha512-5NPgf87AT2STgwa2ntRMr45jTKrYBGkVU36yT0ig/n/GMAa3oPqhZfIQ2kMEimReg0+t9kZViDVZ83qfVUlckg==} + engines: {node: '>= 14'} + + pad-left@2.1.0: + resolution: {integrity: sha512-HJxs9K9AztdIQIAIa/OIazRAUW/L6B9hbQDxO4X07roW3eo9XqZc2ur9bn1StH9CnbbI9EgvejHQX7CBpCF1QA==} + engines: {node: '>=0.10.0'} + parent-module@1.0.1: resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==} engines: {node: '>=6'} @@ -5163,6 +5399,13 @@ packages: resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} engines: {node: '>= 0.10'} + proxy-agent@6.5.0: + resolution: {integrity: sha512-TmatMXdr2KlRiA2CyDu8GqR8EjahTG3aY3nXjdzFyoZbmB8hrBsTyMezhULIXKnC0jpfjlmiZ3+EaCzoInSu/A==} + engines: {node: '>= 14'} + + proxy-from-env@1.1.0: + resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} + proxy-from-env@2.1.0: resolution: {integrity: sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA==} engines: {node: '>=10'} @@ -5174,6 +5417,14 @@ packages: resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} engines: {node: '>=6'} + q@1.5.1: + resolution: {integrity: sha512-kV/CThkXo6xyFEZUugw/+pIOywXcDbFYgSct5cT3gqlbkBE1SJdwy6UQoZvodiWF/ckQLZyDE/Bu1M6gVu5lVw==} + engines: {node: '>=0.6.0', teleport: '>=0.2.0'} + deprecated: |- + You or someone you depend on is using Q, the JavaScript Promise library that gave JavaScript developers strong feelings about promises. They can almost certainly migrate to the native JavaScript promise now. Thank you literally everyone for joining me in this bet against the odds. Be excellent to each other. + + (For a CapTP with native promises, see @endo/eventual-send and @endo/captp) + qs@6.15.2: resolution: {integrity: sha512-Rzq0KEyX/w/tEybncDgdkZrJgVUsUMk3xjh3t5bv3S1HTAtg+uOYt72+ZfwiQwKdysThkTBdL/rTi6HDmX9Ddw==} engines: {node: '>=0.6'} @@ -5327,6 +5578,10 @@ packages: remark@15.0.1: resolution: {integrity: sha512-Eht5w30ruCXgFmxVUSlNWQ9iiimq07URKeFS3hNc8cUWy1llX4KDWfyEDZRycMc+znsN9Ux5/tJ/BFdgdOwA3A==} + repeat-string@1.6.1: + resolution: {integrity: sha512-PV0dzCYDNfRi1jCDbJzpW7jNNDRuCOG/jI5ctQcGKt/clZD+YcPS3yIlWuTJMmESC8aevCFmWJy5wjAFgNqN6w==} + engines: {node: '>=0.10'} + require-directory@2.1.1: resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==} engines: {node: '>=0.10.0'} @@ -5488,6 +5743,10 @@ packages: resolution: {integrity: sha512-SO/3iYL5S3W57LLEniscOGPZgOqZUPCx6d3dB+52B80yJ0XstzsC/eV8gnA4tM3MHDrKz+OCFSLNjswdSC+/bA==} engines: {node: '>=22'} + smart-buffer@4.2.0: + resolution: {integrity: sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==} + engines: {node: '>= 6.0.0', npm: '>= 3.0.0'} + smol-toml@1.6.1: resolution: {integrity: sha512-dWUG8F5sIIARXih1DTaQAX4SsiTXhInKf1buxdY9DIg4ZYPZK5nGM1VRIYmEbDbsHt7USo99xSLFu5Q1IqTmsg==} engines: {node: '>= 18'} @@ -5498,6 +5757,14 @@ packages: peerDependencies: asn1.js: ^5.4.1 + socks-proxy-agent@8.0.5: + resolution: {integrity: sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==} + engines: {node: '>= 14'} + + socks@2.8.9: + resolution: {integrity: sha512-LJhUYUvItdQ0LkJTmPeaEObWXAqFyfmP85x0tch/ez9cahmhlBBLbIqDFnvBnUJGagb0JbIQrkBs1wJ+yRYpEw==} + engines: {node: '>= 10.0.0', npm: '>= 3.0.0'} + sonic-boom@4.2.1: resolution: {integrity: sha512-w6AxtubXa2wTXAUsZMMWERrsIRAdrK0Sc+FUytWvYAhBJLyuI4llrMIC1DtlNSdI99EI86KZum2MMq3EAZlF9Q==} @@ -5574,6 +5841,10 @@ packages: stream-events@1.0.5: resolution: {integrity: sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==} + stream-read-all@3.0.1: + resolution: {integrity: sha512-EWZT9XOceBPlVJRrYcykW8jyRSZYbkb/0ZK36uLEmoWVO5gxBOnntNTseNzfREsqxqdfEGQrD8SXQ3QWbBmq8A==} + engines: {node: '>=10'} + stream-shift@1.0.3: resolution: {integrity: sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==} @@ -5671,6 +5942,11 @@ packages: resolution: {integrity: sha512-zFObLMyZeEwzAoKCyu1B91U79K2t7ApXuQfo8OuxwXLDgcKxuwM+YvcbIhm6QWqz7mHUH1TVytR1PwVVjEuMig==} engines: {node: '>=14.18'} + table-layout@3.0.2: + resolution: {integrity: sha512-rpyNZYRw+/C+dYkcQ3Pr+rLxW4CfHpXjPDnG7lYhdRoUcZTUt+KEsX+94RGp/aVp/MQU35JCITv2T/beY4m+hw==} + engines: {node: '>=12.17'} + hasBin: true + tagged-tag@1.0.0: resolution: {integrity: sha512-yEFYrVhod+hdNyx7g5Bnkkb0G6si8HJurOoOEgC8B/O0uXLHlaey/65KRv6cuWBNhBgHKAROVpc7QyYqE5gFng==} engines: {node: '>=20'} @@ -5730,6 +6006,10 @@ packages: resolution: {integrity: sha512-e2zZ96wSChazBsbENf/Pcm/4swHt2cEKQ92rhUjkL9GCKiTDJIaTBenjE/m9DXi0QBmTMDkFDdOomUy20A1tDQ==} engines: {node: '>=20'} + thrift@0.16.0: + resolution: {integrity: sha512-W8DpGyTPlIaK3f+e1XOCLxefaUWXtrOXAaVIDbfYhmVyriYeAKgsBVFNJUV1F9SQ2SPt2sG44AZQxSGwGj/3VA==} + engines: {node: '>= 10.18.0'} + through2@2.0.5: resolution: {integrity: sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==} @@ -5763,6 +6043,9 @@ packages: toml@3.0.0: resolution: {integrity: sha512-y/mWCZinnvxjTKYhJ+pYxwD0mRLVvOtdS2Awbgxln6iEnt4rk0yBxeSBHkGJcPucRiG0e55mwWp+g/05rsrd6w==} + tr46@0.0.3: + resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} + tr46@5.1.1: resolution: {integrity: sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw==} engines: {node: '>=18'} @@ -5819,6 +6102,14 @@ packages: engines: {node: '>=14.17'} hasBin: true + typical@4.0.0: + resolution: {integrity: sha512-VAH4IvQ7BDFYglMd7BPRDfLgxZZX4O4TFcRDA6EN5X7erNJJq+McIEp8np9aVtxrCJ6qx4GTYVfOWNjcqwZgRw==} + engines: {node: '>=8'} + + typical@7.3.0: + resolution: {integrity: sha512-ya4mg/30vm+DOWfBg4YK3j2WD6TWtRkCbasOJr40CseYENzCUby/7rIvXA99JGsQHeNxLbnXdyLLxKSv3tauFw==} + engines: {node: '>=12.17'} + uglify-js@3.19.3: resolution: {integrity: sha512-v3Xu+yuwBXisp6QYTcH4UbH+xYJXqnq2m/LtQVWKWzYc1iehYnLixoQDN9FH6/j9/oybfd6W9Ghwkl8+UMKTKQ==} engines: {node: '>=0.8.0'} @@ -5926,6 +6217,11 @@ packages: util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} + uuid@9.0.1: + resolution: {integrity: sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==} + deprecated: uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028). + hasBin: true + validate-npm-package-license@3.0.4: resolution: {integrity: sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==} @@ -6040,6 +6336,9 @@ packages: web-worker@1.5.0: resolution: {integrity: sha512-RiMReJrTAiA+mBjGONMnjVDP2u3p9R1vkcGz6gDIrOMT3oGuYwX2WRMYI9ipkphSuE5XKEhydbhNEJh4NY9mlw==} + webidl-conversions@3.0.1: + resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} + webidl-conversions@7.0.0: resolution: {integrity: sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==} engines: {node: '>=12'} @@ -6048,6 +6347,9 @@ packages: resolution: {integrity: sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw==} engines: {node: '>=18'} + whatwg-url@5.0.0: + resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==} + which@2.0.2: resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} engines: {node: '>= 8'} @@ -6073,6 +6375,10 @@ packages: wordwrap@1.0.0: resolution: {integrity: sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==} + wordwrapjs@5.1.1: + resolution: {integrity: sha512-0yweIbkINJodk27gX9LBGMzyQdBDan3s/dEAiwBOj+Mf0PPyWL6/rikalkv8EeD0E8jm4o5RXEOrFTP3NXbhJg==} + engines: {node: '>=12.17'} + wrap-ansi@10.0.0: resolution: {integrity: sha512-SGcvg80f0wUy2/fXES19feHMz8E0JoXv2uNgHOu4Dgi2OrCy1lqwFYEJz1BLbDI0exjPMe/ZdzZ/YpGECBG/aQ==} engines: {node: '>=20'} @@ -6112,10 +6418,16 @@ packages: resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} engines: {node: '>=0.4'} + xxhashjs@0.2.2: + resolution: {integrity: sha512-AkTuIuVTET12tpsVIQo+ZU6f/qDmKuRUcjaqR+OIvm+aCBsZ95i7UVY5WJ9TMsSaZ0DA2WxoZ4acu0sPH+OKAw==} + y18n@5.0.8: resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} engines: {node: '>=10'} + yallist@4.0.0: + resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} + yaml@2.9.0: resolution: {integrity: sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==} engines: {node: '>= 14.6'} @@ -6176,6 +6488,11 @@ packages: snapshots: + '@75lb/deep-merge@1.1.4': + dependencies: + lodash: 4.18.1 + typical: 7.3.0 + '@actions/core@3.0.1': dependencies: '@actions/exec': 3.0.0 @@ -6952,6 +7269,60 @@ snapshots: enabled: 2.0.0 kuler: 2.0.0 + '@databricks/databricks-sql-kernel-darwin-arm64@0.2.0': + optional: true + + '@databricks/databricks-sql-kernel-darwin-x64@0.2.0': + optional: true + + '@databricks/databricks-sql-kernel-linux-arm64-gnu@0.2.0': + optional: true + + '@databricks/databricks-sql-kernel-linux-arm64-musl@0.2.0': + optional: true + + '@databricks/databricks-sql-kernel-linux-x64-gnu@0.2.0': + optional: true + + '@databricks/databricks-sql-kernel-linux-x64-musl@0.2.0': + optional: true + + '@databricks/databricks-sql-kernel-win32-arm64-msvc@0.2.0': + optional: true + + '@databricks/databricks-sql-kernel-win32-x64-msvc@0.2.0': + optional: true + + '@databricks/sql@1.16.0': + dependencies: + apache-arrow: 13.0.0 + commander: 9.5.0 + flatbuffers: 23.5.26 + node-fetch: 2.7.0 + node-int64: 0.4.0 + open: 8.4.2 + openid-client: 5.7.1 + proxy-agent: 6.5.0 + thrift: 0.16.0 + uuid: 9.0.1 + winston: 3.19.0 + optionalDependencies: + '@databricks/databricks-sql-kernel-darwin-arm64': 0.2.0 + '@databricks/databricks-sql-kernel-darwin-x64': 0.2.0 + '@databricks/databricks-sql-kernel-linux-arm64-gnu': 0.2.0 + '@databricks/databricks-sql-kernel-linux-arm64-musl': 0.2.0 + '@databricks/databricks-sql-kernel-linux-x64-gnu': 0.2.0 + '@databricks/databricks-sql-kernel-linux-x64-musl': 0.2.0 + '@databricks/databricks-sql-kernel-win32-arm64-msvc': 0.2.0 + '@databricks/databricks-sql-kernel-win32-x64-msvc': 0.2.0 + lz4: 0.6.5 + transitivePeerDependencies: + - '@75lb/nature' + - bufferutil + - encoding + - supports-color + - utf-8-validate + '@duckdb/node-api@1.5.3-r.3': dependencies: '@duckdb/node-bindings': 1.5.3-r.3 @@ -8487,6 +8858,8 @@ snapshots: '@tediousjs/connection-string@1.1.0': {} + '@tootallnate/quickjs-emscripten@0.23.0': {} + '@tybys/wasm-util@0.10.2': dependencies: tslib: 2.8.1 @@ -8501,6 +8874,10 @@ snapshots: '@types/deep-eql': 4.0.2 assertion-error: 2.0.1 + '@types/command-line-args@5.2.0': {} + + '@types/command-line-usage@5.0.2': {} + '@types/d3-color@3.1.3': {} '@types/d3-drag@3.0.7': @@ -8561,6 +8938,8 @@ snapshots: '@types/normalize-package-data@2.4.4': {} + '@types/pad-left@2.1.1': {} + '@types/pg@8.20.0': dependencies: '@types/node': 24.12.4 @@ -8757,6 +9136,21 @@ snapshots: any-promise@1.3.0: {} + apache-arrow@13.0.0: + dependencies: + '@types/command-line-args': 5.2.0 + '@types/command-line-usage': 5.0.2 + '@types/node': 24.12.4 + '@types/pad-left': 2.1.1 + command-line-args: 5.2.1 + command-line-usage: 7.0.1 + flatbuffers: 23.5.26 + json-bignum: 0.0.3 + pad-left: 2.1.0 + tslib: 2.8.1 + transitivePeerDependencies: + - '@75lb/nature' + argparse@2.0.1: {} argv-formatter@1.0.0: {} @@ -8765,6 +9159,10 @@ snapshots: dependencies: tslib: 2.8.1 + array-back@3.1.0: {} + + array-back@6.2.3: {} + array-ify@1.0.0: {} arrify@2.0.1: {} @@ -8789,6 +9187,10 @@ snapshots: assertion-error@2.0.1: {} + ast-types@0.13.4: + dependencies: + tslib: 2.8.1 + ast-v8-to-istanbul@1.0.0: dependencies: '@jridgewell/trace-mapping': 0.3.31 @@ -8827,6 +9229,8 @@ snapshots: baseline-browser-mapping@2.10.31: {} + basic-ftp@5.3.1: {} + before-after-hook@4.0.0: {} better-sqlite3@12.10.0: @@ -8892,6 +9296,8 @@ snapshots: dependencies: fill-range: 7.1.1 + browser-or-node@1.3.0: {} + bson@6.10.4: {} buffer-equal-constant-time@1.0.1: {} @@ -8930,6 +9336,10 @@ snapshots: chai@6.2.2: {} + chalk-template@0.4.0: + dependencies: + chalk: 4.1.2 + chalk@2.4.2: dependencies: ansi-styles: 3.2.1 @@ -9054,10 +9464,28 @@ snapshots: comma-separated-tokens@2.0.3: {} + command-line-args@5.2.1: + dependencies: + array-back: 3.1.0 + find-replace: 3.0.0 + lodash.camelcase: 4.3.0 + typical: 4.0.0 + + command-line-usage@7.0.1: + dependencies: + array-back: 6.2.3 + chalk-template: 0.4.0 + table-layout: 3.0.2 + typical: 7.3.0 + transitivePeerDependencies: + - '@75lb/nature' + commander@11.1.0: {} commander@14.0.3: {} + commander@9.5.0: {} + compare-func@2.0.0: dependencies: array-ify: 1.0.0 @@ -9139,6 +9567,9 @@ snapshots: csstype@3.2.3: {} + cuint@0.2.2: + optional: true + d3-color@3.1.0: {} d3-dispatch@3.0.1: {} @@ -9177,6 +9608,8 @@ snapshots: data-uri-to-buffer@4.0.1: {} + data-uri-to-buffer@6.0.2: {} + dateformat@4.6.3: {} debug@4.4.3: @@ -9200,8 +9633,16 @@ snapshots: bundle-name: 4.1.0 default-browser-id: 5.0.1 + define-lazy-prop@2.0.0: {} + define-lazy-prop@3.0.0: {} + degenerator@5.0.1: + dependencies: + ast-types: 0.13.4 + escodegen: 2.1.0 + esprima: 4.0.1 + delayed-stream@1.0.0: {} denque@2.1.0: {} @@ -9355,6 +9796,18 @@ snapshots: escape-string-regexp@5.0.0: {} + escodegen@2.1.0: + dependencies: + esprima: 4.0.1 + estraverse: 5.3.0 + esutils: 2.0.3 + optionalDependencies: + source-map: 0.6.1 + + esprima@4.0.1: {} + + estraverse@5.3.0: {} + estree-util-attach-comments@3.0.0: dependencies: '@types/estree': 1.0.9 @@ -9392,6 +9845,8 @@ snapshots: dependencies: '@types/estree': 1.0.9 + esutils@2.0.3: {} + etag@1.8.1: {} event-target-shim@5.0.1: {} @@ -9576,6 +10031,10 @@ snapshots: transitivePeerDependencies: - supports-color + find-replace@3.0.0: + dependencies: + array-back: 3.1.0 + find-up-simple@1.0.1: {} find-up@2.1.0: @@ -9587,6 +10046,8 @@ snapshots: semver-regex: 4.0.5 super-regex: 1.1.0 + flatbuffers@23.5.26: {} + fn.name@1.1.0: {} follow-redirects@1.16.0: {} @@ -9795,6 +10256,14 @@ snapshots: dependencies: resolve-pkg-maps: 1.0.0 + get-uri@6.0.5: + dependencies: + basic-ftp: 5.3.1 + data-uri-to-buffer: 6.0.2 + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + git-log-parser@1.2.1: dependencies: argv-formatter: 1.0.0 @@ -10184,6 +10653,10 @@ snapshots: isexe@2.0.0: {} + isomorphic-ws@4.0.1(ws@8.20.1): + dependencies: + ws: 8.20.1 + issue-parser@7.0.2: dependencies: lodash.capitalize: 4.2.1 @@ -10209,6 +10682,8 @@ snapshots: jiti@2.7.0: {} + jose@4.15.9: {} + jose@6.2.3: {} joycon@3.1.1: {} @@ -10227,6 +10702,8 @@ snapshots: dependencies: bignumber.js: 9.3.1 + json-bignum@0.0.3: {} + json-parse-better-errors@1.0.2: {} json-parse-even-better-errors@2.3.1: {} @@ -10361,6 +10838,8 @@ snapshots: lodash-es@4.18.1: {} + lodash.camelcase@4.3.0: {} + lodash.capitalize@4.2.1: {} lodash.escaperegexp@4.1.2: {} @@ -10409,12 +10888,26 @@ snapshots: lru-cache@11.5.0: {} + lru-cache@6.0.0: + dependencies: + yallist: 4.0.0 + + lru-cache@7.18.3: {} + lru.min@1.1.4: {} lucide-react@1.16.0(react@19.2.6): dependencies: react: 19.2.6 + lz4@0.6.5: + dependencies: + buffer: 5.7.1 + cuint: 0.2.2 + nan: 2.27.0 + xxhashjs: 0.2.2 + optional: true + magic-string@0.30.21: dependencies: '@jridgewell/sourcemap-codec': 1.5.5 @@ -10941,11 +11434,13 @@ snapshots: '@types/whatwg-url': 11.0.5 whatwg-url: 14.2.0 - mongodb@6.21.0: + mongodb@6.21.0(socks@2.8.9): dependencies: '@mongodb-js/saslprep': 1.4.11 bson: 6.10.4 mongodb-connection-string-url: 3.0.2 + optionalDependencies: + socks: 2.8.9 motion-dom@12.40.0: dependencies: @@ -10996,6 +11491,9 @@ snapshots: dependencies: lru.min: 1.1.4 + nan@2.27.0: + optional: true + nanoid@3.3.12: {} napi-build-utils@2.0.0: {} @@ -11008,6 +11506,8 @@ snapshots: nerf-dart@1.0.0: {} + netmask@2.1.1: {} + next-themes@0.4.6(react-dom@19.2.6(react@19.2.6))(react@19.2.6): dependencies: react: 19.2.6 @@ -11051,12 +11551,18 @@ snapshots: emojilib: 2.4.0 skin-tone: 2.0.0 + node-fetch@2.7.0: + dependencies: + whatwg-url: 5.0.0 + node-fetch@3.3.2: dependencies: data-uri-to-buffer: 4.0.1 fetch-blob: 3.2.0 formdata-polyfill: 4.0.10 + node-int64@0.4.0: {} + normalize-package-data@6.0.2: dependencies: hosted-git-info: 7.0.2 @@ -11090,10 +11596,14 @@ snapshots: object-assign@4.1.1: {} + object-hash@2.2.0: {} + object-inspect@1.13.4: {} obug@2.1.1: {} + oidc-token-hash@5.2.0: {} + on-exit-leak-free@2.1.2: {} on-finished@2.4.1: @@ -11136,11 +11646,24 @@ snapshots: is-docker: 2.2.1 is-wsl: 2.2.0 + open@8.4.2: + dependencies: + define-lazy-prop: 2.0.0 + is-docker: 2.2.1 + is-wsl: 2.2.0 + openai@6.38.0(ws@8.20.1)(zod@4.4.3): optionalDependencies: ws: 8.20.1 zod: 4.4.3 + openid-client@5.7.1: + dependencies: + jose: 4.15.9 + lru-cache: 6.0.0 + object-hash: 2.2.0 + oidc-token-hash: 5.2.0 + oxc-parser@0.130.0: dependencies: '@oxc-project/types': 0.130.0 @@ -11224,6 +11747,28 @@ snapshots: p-try@1.0.0: {} + pac-proxy-agent@7.2.0: + dependencies: + '@tootallnate/quickjs-emscripten': 0.23.0 + agent-base: 7.1.4 + debug: 4.4.3 + get-uri: 6.0.5 + http-proxy-agent: 7.0.2 + https-proxy-agent: 7.0.6 + pac-resolver: 7.0.1 + socks-proxy-agent: 8.0.5 + transitivePeerDependencies: + - supports-color + + pac-resolver@7.0.1: + dependencies: + degenerator: 5.0.1 + netmask: 2.1.1 + + pad-left@2.1.0: + dependencies: + repeat-string: 1.6.1 + parent-module@1.0.1: dependencies: callsites: 3.1.0 @@ -11434,6 +11979,21 @@ snapshots: forwarded: 0.2.0 ipaddr.js: 1.9.1 + proxy-agent@6.5.0: + dependencies: + agent-base: 7.1.4 + debug: 4.4.3 + http-proxy-agent: 7.0.2 + https-proxy-agent: 7.0.6 + lru-cache: 7.18.3 + pac-proxy-agent: 7.2.0 + proxy-from-env: 1.1.0 + socks-proxy-agent: 8.0.5 + transitivePeerDependencies: + - supports-color + + proxy-from-env@1.1.0: {} + proxy-from-env@2.1.0: {} pump@3.0.4: @@ -11443,6 +12003,8 @@ snapshots: punycode@2.3.1: {} + q@1.5.1: {} + qs@6.15.2: dependencies: side-channel: 1.1.0 @@ -11669,6 +12231,8 @@ snapshots: transitivePeerDependencies: - supports-color + repeat-string@1.6.1: {} + require-directory@2.1.1: {} require-from-string@2.0.2: {} @@ -11927,6 +12491,8 @@ snapshots: ansi-styles: 6.2.3 is-fullwidth-code-point: 5.1.0 + smart-buffer@4.2.0: {} + smol-toml@1.6.1: {} snowflake-sdk@2.4.2(asn1.js@5.4.1): @@ -11967,6 +12533,19 @@ snapshots: - debug - supports-color + socks-proxy-agent@8.0.5: + dependencies: + agent-base: 7.1.4 + debug: 4.4.3 + socks: 2.8.9 + transitivePeerDependencies: + - supports-color + + socks@2.8.9: + dependencies: + ip-address: 10.1.1 + smart-buffer: 4.2.0 + sonic-boom@4.2.1: dependencies: atomic-sleep: 1.0.0 @@ -12035,6 +12614,8 @@ snapshots: dependencies: stubs: 3.0.0 + stream-read-all@3.0.1: {} + stream-shift@1.0.3: {} string-width@4.2.3: @@ -12123,6 +12704,18 @@ snapshots: has-flag: 4.0.0 supports-color: 7.2.0 + table-layout@3.0.2: + dependencies: + '@75lb/deep-merge': 1.1.4 + array-back: 6.2.3 + command-line-args: 5.2.1 + command-line-usage: 7.0.1 + stream-read-all: 3.0.1 + typical: 7.3.0 + wordwrapjs: 5.1.1 + transitivePeerDependencies: + - '@75lb/nature' + tagged-tag@1.0.0: {} tailwind-merge@3.6.0: {} @@ -12198,6 +12791,17 @@ snapshots: dependencies: real-require: 1.0.0 + thrift@0.16.0: + dependencies: + browser-or-node: 1.3.0 + isomorphic-ws: 4.0.1(ws@8.20.1) + node-int64: 0.4.0 + q: 1.5.1 + ws: 8.20.1 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + through2@2.0.5: dependencies: readable-stream: 2.3.8 @@ -12226,6 +12830,8 @@ snapshots: toml@3.0.0: {} + tr46@0.0.3: {} + tr46@5.1.1: dependencies: punycode: 2.3.1 @@ -12266,6 +12872,10 @@ snapshots: typescript@6.0.3: {} + typical@4.0.0: {} + + typical@7.3.0: {} + uglify-js@3.19.3: optional: true @@ -12360,6 +12970,8 @@ snapshots: util-deprecate@1.0.2: {} + uuid@9.0.1: {} + validate-npm-package-license@3.0.4: dependencies: spdx-correct: 3.2.0 @@ -12433,6 +13045,8 @@ snapshots: web-worker@1.5.0: {} + webidl-conversions@3.0.1: {} + webidl-conversions@7.0.0: {} whatwg-url@14.2.0: @@ -12440,6 +13054,11 @@ snapshots: tr46: 5.1.1 webidl-conversions: 7.0.0 + whatwg-url@5.0.0: + dependencies: + tr46: 0.0.3 + webidl-conversions: 3.0.1 + which@2.0.2: dependencies: isexe: 2.0.0 @@ -12475,6 +13094,8 @@ snapshots: wordwrap@1.0.0: {} + wordwrapjs@5.1.1: {} + wrap-ansi@10.0.0: dependencies: ansi-styles: 6.2.3 @@ -12505,8 +13126,15 @@ snapshots: xtend@4.0.2: {} + xxhashjs@0.2.2: + dependencies: + cuint: 0.2.2 + optional: true + y18n@5.0.8: {} + yallist@4.0.0: {} + yaml@2.9.0: {} yargs-parser@20.2.9: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index c66de723d..b45c2fedf 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -22,5 +22,6 @@ verifyDepsBeforeRun: false allowBuilds: better-sqlite3: true esbuild: true + lz4: false sharp: true minimumReleaseAge: 10080 From 647a520d57ff51a0b33522ee6af8fc4b38c7b969 Mon Sep 17 00:00:00 2001 From: BittuBarnwal7479 Date: Wed, 1 Jul 2026 00:25:54 +0530 Subject: [PATCH 2/6] fix: CI failing --- packages/cli/src/setup-databases.ts | 4 ++-- pnpm-workspace.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/cli/src/setup-databases.ts b/packages/cli/src/setup-databases.ts index abeea6a84..bdf62b70e 100644 --- a/packages/cli/src/setup-databases.ts +++ b/packages/cli/src/setup-databases.ts @@ -1031,7 +1031,7 @@ async function buildConnectionConfig(input: { secretName: 'client-secret', // pragma: allowlist secret }); if (clientSecretRef === 'back') return 'back'; - const resolvedClientSecretRef = clientSecretRef ?? stringConfigField(input.existingConnection, 'client_secret'); + const resolvedClientSecretRef = clientSecretRef ?? stringConfigField(input.existingConnection, 'client_secret'); // pragma: allowlist secret if (!serverHostname || !httpPath || !catalog || !clientId || !resolvedClientSecretRef) return null; return { driver: 'databricks', @@ -1040,7 +1040,7 @@ async function buildConnectionConfig(input: { http_path: httpPath, catalog, client_id: clientId, - client_secret: resolvedClientSecretRef, + client_secret: resolvedClientSecretRef, // pragma: allowlist secret ...scope, }; } diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index b45c2fedf..febf82f85 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -22,6 +22,6 @@ verifyDepsBeforeRun: false allowBuilds: better-sqlite3: true esbuild: true - lz4: false + lz4: true sharp: true minimumReleaseAge: 10080 From c1daa82a3a1fd4f5abbfc2d200d8e727599a8cd3 Mon Sep 17 00:00:00 2001 From: BittuBarnwal7479 Date: Wed, 1 Jul 2026 00:57:35 +0530 Subject: [PATCH 3/6] fix: CI failling --- packages/cli/src/setup-databases.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/src/setup-databases.ts b/packages/cli/src/setup-databases.ts index bdf62b70e..50abaf3b1 100644 --- a/packages/cli/src/setup-databases.ts +++ b/packages/cli/src/setup-databases.ts @@ -1030,7 +1030,7 @@ async function buildConnectionConfig(input: { connectionId: input.connectionId, secretName: 'client-secret', // pragma: allowlist secret }); - if (clientSecretRef === 'back') return 'back'; + if (clientSecretRef === 'back') return 'back'; // pragma: allowlist secret const resolvedClientSecretRef = clientSecretRef ?? stringConfigField(input.existingConnection, 'client_secret'); // pragma: allowlist secret if (!serverHostname || !httpPath || !catalog || !clientId || !resolvedClientSecretRef) return null; return { From 11031f9264a8b82888fc1a5f42d37224abeedee6 Mon Sep 17 00:00:00 2001 From: BittuBarnwal7479 Date: Wed, 1 Jul 2026 01:14:14 +0530 Subject: [PATCH 4/6] Fix Databricks connector CI checks --- scripts/package-artifacts.mjs | 2 +- scripts/package-artifacts.test.mjs | 2 +- scripts/published-package-smoke.mjs | 2 +- scripts/published-package-smoke.test.mjs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/package-artifacts.mjs b/scripts/package-artifacts.mjs index 14567c317..4e39e2cc4 100644 --- a/scripts/package-artifacts.mjs +++ b/scripts/package-artifacts.mjs @@ -447,7 +447,7 @@ export function npmSmokePackageJson(layout) { } export function npmSmokePnpmWorkspaceYaml() { - return ['packages:', ' - "."', 'allowBuilds:', ' better-sqlite3: true', ''].join('\n'); + return ['packages:', ' - "."', 'allowBuilds:', ' better-sqlite3: true', ' lz4: true', ''].join('\n'); } export function npmVerifySource() { diff --git a/scripts/package-artifacts.test.mjs b/scripts/package-artifacts.test.mjs index f307dcec2..4c973c590 100644 --- a/scripts/package-artifacts.test.mjs +++ b/scripts/package-artifacts.test.mjs @@ -452,7 +452,7 @@ describe('verification snippets', () => { assert.equal(packageJson.devDependencies, undefined); assert.equal( npmSmokePnpmWorkspaceYaml(), - ['packages:', ' - "."', 'allowBuilds:', ' better-sqlite3: true', ''].join('\n'), + ['packages:', ' - "."', 'allowBuilds:', ' better-sqlite3: true', ' lz4: true', ''].join('\n'), ); }); diff --git a/scripts/published-package-smoke.mjs b/scripts/published-package-smoke.mjs index 53e8d565c..ed9f011b0 100644 --- a/scripts/published-package-smoke.mjs +++ b/scripts/published-package-smoke.mjs @@ -36,7 +36,7 @@ export function isPublishedPackageVersionLabel(label) { } export function publishedPackageSmokePnpmWorkspaceYaml() { - return ['packages:', ' - "."', 'allowBuilds:', ' better-sqlite3: true', ''].join('\n'); + return ['packages:', ' - "."', 'allowBuilds:', ' better-sqlite3: true', ' lz4: true', ''].join('\n'); } export function isTransientPublishedPackageLookupFailure(result) { diff --git a/scripts/published-package-smoke.test.mjs b/scripts/published-package-smoke.test.mjs index 09c54934b..119bac728 100644 --- a/scripts/published-package-smoke.test.mjs +++ b/scripts/published-package-smoke.test.mjs @@ -276,7 +276,7 @@ describe('published package smoke command construction', () => { it('allows native dependency build scripts in clean pnpm smoke installs', () => { assert.equal( publishedPackageSmokePnpmWorkspaceYaml(), - ['packages:', ' - "."', 'allowBuilds:', ' better-sqlite3: true', ''].join('\n'), + ['packages:', ' - "."', 'allowBuilds:', ' better-sqlite3: true', ' lz4: true', ''].join('\n'), ); }); From 56413a48f25bce7f61ac9cce19cbdce7c12ff2f2 Mon Sep 17 00:00:00 2001 From: BittuBarnwal7479 Date: Wed, 1 Jul 2026 01:21:55 +0530 Subject: [PATCH 5/6] fix: CI failling --- packages/cli/src/setup-databases.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/src/setup-databases.ts b/packages/cli/src/setup-databases.ts index 50abaf3b1..d72f640be 100644 --- a/packages/cli/src/setup-databases.ts +++ b/packages/cli/src/setup-databases.ts @@ -966,7 +966,7 @@ async function buildConnectionConfig(input: { } if (driver === 'databricks') { if (args.inputMode === 'disabled') return null; - + const serverHostname = await promptText( prompts, 'Databricks workspace hostname\nFor example dbc-12345678-abcd.cloud.databricks.com.', From 5419ce5663ae4d9ff148f5b6398e688bfb513516 Mon Sep 17 00:00:00 2001 From: BittuBarnwal7479 Date: Fri, 3 Jul 2026 20:21:40 +0530 Subject: [PATCH 6/6] Fix semantic layer query test expectation --- packages/cli/test/sl.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/test/sl.test.ts b/packages/cli/test/sl.test.ts index ca37747ea..e7a47a51e 100644 --- a/packages/cli/test/sl.test.ts +++ b/packages/cli/test/sl.test.ts @@ -717,7 +717,7 @@ joins: [] expect(query).toHaveBeenCalledWith( expect.objectContaining({ - query: { measures: ['orders.order_count'], dimensions: [] }, + query: expect.objectContaining({ measures: ['orders.order_count'], dimensions: [] }), }), ); expect(JSON.parse(String(stdout.write.mock.calls[0][0]))).toMatchObject({