Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 30 additions & 30 deletions packages/cli/src/connection.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import { stat } from 'node:fs/promises';
import { resolve } from 'node:path';
import { DEFAULT_METABASE_CLIENT_CONFIG, DefaultMetabaseConnectionClientFactory } from './context/ingest/adapters/metabase/client.js';
import { DefaultLookerConnectionClientFactory } from './context/ingest/adapters/looker/factory.js';
import type { LookerClient } from './context/ingest/adapters/looker/client.js';
Expand All @@ -10,6 +12,7 @@ import { federatedConnectionListing } from './context/connections/federation.js'
import { getDriverRegistration } from './context/connections/drivers.js';
import { parseNotionConnectionConfig, resolveNotionConnectionAuthToken } from './context/connections/notion-config.js';
import { resolveKtxConfigReference } from './context/core/config-reference.js';
import { readGitRepoConnectionFields } from './context/project/git-connection-fields.js';
import { type KtxLocalProject, loadKtxProject } from './context/project/project.js';
import type { KtxScanConnector } from './context/scan/types.js';
import type { KtxCliIo } from './index.js';
Expand Down Expand Up @@ -183,49 +186,46 @@ async function testNotionConnection(
return { bot: describeNotionBot(bot) };
}

interface GitConnectionFields {
repoUrl: string;
authToken: string | null;
async function checkLocalDbtSourceDir(projectDir: string, sourceDir: string): Promise<void> {
const stats = await stat(resolve(projectDir, sourceDir)).catch(() => null);
if (!stats) {
throw new Error(`dbt source_dir "${sourceDir}" does not exist`);
}
if (!stats.isDirectory()) {
throw new Error(`dbt source_dir "${sourceDir}" is not a directory`);
}
}

function extractGitConnectionFields(
async function testGitRepoConnection(
project: KtxLocalProject,
connectionId: string,
driver: string,
): GitConnectionFields {
runTest: TestRepoConnection,
): Promise<{ location: string }> {
const connection = project.config.connections[connectionId];
if (!connection) {
throw new Error(`Connection "${connectionId}" is not configured in ktx.yaml`);
}
const stringField = (value: unknown): string | null =>
typeof value === 'string' && value.trim().length > 0 ? value.trim() : null;
const record =
driver === 'metricflow' && typeof connection.metricflow === 'object' && connection.metricflow !== null
? (connection.metricflow as Record<string, unknown>)
: (connection as Record<string, unknown>);
const repoUrl = driver === 'dbt' ? stringField(record.repo_url) : stringField(record.repoUrl);
if (!repoUrl) {
const field = driver === 'dbt' ? 'repo_url' : 'repoUrl';
throw new Error(`Connection "${connectionId}" (driver: ${driver}) is missing ${field}`);
const fields = readGitRepoConnectionFields(connection, driver);

// dbt setup writes source_dir for a local project and ingest consumes it, so a
// local source check must stand in for the remote repo reachability check.
if (fields.sourceDir) {
await checkLocalDbtSourceDir(project.projectDir, fields.sourceDir);
return { location: fields.sourceDir };
}
const literalToken = stringField(record.auth_token);
const ref = stringField(record.auth_token_ref);
const resolvedRef = ref ? resolveKtxConfigReference(ref, process.env) : null;
return { repoUrl, authToken: literalToken ?? resolvedRef ?? null };
}

async function testGitRepoConnection(
project: KtxLocalProject,
connectionId: string,
driver: string,
runTest: TestRepoConnection,
): Promise<{ repoUrl: string }> {
const { repoUrl, authToken } = extractGitConnectionFields(project, connectionId, driver);
const result = await runTest({ repoUrl, authToken });
if (!fields.repoUrl) {
const field = driver === 'dbt' ? 'repo_url or source_dir' : 'repoUrl';
throw new Error(`Connection "${connectionId}" (driver: ${driver}) is missing ${field}`);
}
const resolvedRef = fields.authTokenRef ? resolveKtxConfigReference(fields.authTokenRef, process.env) : null;
const authToken = fields.authTokenLiteral ?? resolvedRef ?? null;
const result = await runTest({ repoUrl: fields.repoUrl, authToken });
if (!result.ok) {
throw new Error(`${driver} repository check failed: ${result.error}`);
}
return { repoUrl };
return { location: fields.repoUrl };
}

interface DriverTestOutcome {
Expand Down Expand Up @@ -278,7 +278,7 @@ async function testConnectionByDriver(
driver,
deps.testRepoConnection ?? testRepoConnection,
);
return { driver, detailKey: 'Repo', detailValue: result.repoUrl };
return { driver, detailKey: 'Repo', detailValue: result.location };
}

if (getDriverRegistration(driver)) {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { resolveKtxConfigReference } from '../../../core/config-reference.js';
import type { KtxLocalProject } from '../../../../context/project/project.js';
import type { KtxProjectConnectionConfig } from '../../../../context/project/config.js';
import {
Expand All @@ -8,13 +9,6 @@ function stringField(value: unknown): string | null {
return typeof value === 'string' && value.trim().length > 0 ? value.trim() : null;
}

function resolveEnvReference(ref: string, env: NodeJS.ProcessEnv): string | null {
if (!ref.startsWith('env:')) {
return null;
}
return stringField(env[ref.slice('env:'.length)]);
}

export function lookerCredentialsFromLocalConnection(
connectionId: string,
connection: KtxProjectConnectionConfig | undefined,
Expand All @@ -27,7 +21,8 @@ export function lookerCredentialsFromLocalConnection(
const clientId = stringField(connection.client_id);
const clientSecret =
stringField(connection.client_secret) ??
(stringField(connection.client_secret_ref) ? resolveEnvReference(String(connection.client_secret_ref), env) : null);
resolveKtxConfigReference(stringField(connection.client_secret_ref) ?? undefined, env) ??
null;

if (!baseUrl) {
throw new Error(`Connection "${connectionId}" is missing Looker base_url`);
Expand Down
47 changes: 47 additions & 0 deletions packages/cli/src/context/project/git-connection-fields.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
import type { KtxProjectConnectionConfig } from './config.js';

export interface GitRepoConnectionFields {
repoUrl: string | null;
sourceDir: string | null;
branch: string | null;
path: string | null;
authTokenLiteral: string | null;
authTokenRef: string | null;
}

function trimmedString(value: unknown): string | null {
return typeof value === 'string' && value.trim().length > 0 ? value.trim() : null;
}

// metricflow nests its repo fields under `metricflow.*`; dbt/lookml keep them top-level.
function fieldRecord(connection: KtxProjectConnectionConfig, driver: string): Record<string, unknown> {
if (driver === 'metricflow') {
const nested = (connection as Record<string, unknown>).metricflow;
if (typeof nested === 'object' && nested !== null && !Array.isArray(nested)) {
return nested as Record<string, unknown>;
}
return {};
}
return connection as Record<string, unknown>;
}

/**
* Single source of truth for where the git-repo context-source drivers
* (dbt, metricflow, lookml) store their repository fields: dbt uses snake_case
* `repo_url`/`source_dir`, lookml uses camelCase `repoUrl`, metricflow nests
* camelCase fields under `metricflow.*`.
*/
export function readGitRepoConnectionFields(
connection: KtxProjectConnectionConfig,
driver: string,
): GitRepoConnectionFields {
const record = fieldRecord(connection, driver);
return {
repoUrl: trimmedString(driver === 'dbt' ? record.repo_url : record.repoUrl),
sourceDir: driver === 'dbt' ? trimmedString(record.source_dir) : null,
branch: trimmedString(record.branch),
path: trimmedString(record.path),
authTokenLiteral: trimmedString(record.auth_token),
authTokenRef: trimmedString(record.auth_token_ref),
};
}
15 changes: 6 additions & 9 deletions packages/cli/src/local-adapters.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,9 @@ import { createHttpSqlAnalysisPort } from './context/sql-analysis/http-sql-analy
import type { SqlAnalysisPort } from './context/sql-analysis/ports.js';
import {
createManagedDaemonLookerTableIdentifierParser,
createManagedDaemonSqlAnalysisPort,
externalDaemonSqlAnalysisBaseUrl,
managedDaemonDatabaseIntrospectionOptions,
resolveSqlAnalysisPort,
type ManagedPythonDaemonHttpOptions,
} from './managed-python-http.js';
import type { KtxOperationalLogger } from './io/logger.js';
Expand Down Expand Up @@ -78,16 +79,12 @@ export function resolveKtxCliSqlAnalysis(options: KtxCliLocalIngestAdaptersOptio
if (options.sqlAnalysisUrl) {
return createHttpSqlAnalysisPort({ baseUrl: options.sqlAnalysisUrl });
}
if (process.env.KTX_SQL_ANALYSIS_URL) {
return createHttpSqlAnalysisPort({ baseUrl: process.env.KTX_SQL_ANALYSIS_URL });
}
if (process.env.KTX_DAEMON_URL) {
return createHttpSqlAnalysisPort({ baseUrl: process.env.KTX_DAEMON_URL });
}
if (options.managedDaemon) {
return createManagedDaemonSqlAnalysisPort(options.managedDaemon);
return resolveSqlAnalysisPort(options.managedDaemon);
}
return createHttpSqlAnalysisPort({ baseUrl: 'http://127.0.0.1:8765' });
return createHttpSqlAnalysisPort({
baseUrl: externalDaemonSqlAnalysisBaseUrl(process.env) ?? 'http://127.0.0.1:8765',
});
}

function createKtxCliLiveDatabaseIntrospection(
Expand Down
26 changes: 26 additions & 0 deletions packages/cli/src/managed-python-http.ts
Original file line number Diff line number Diff line change
Expand Up @@ -178,13 +178,39 @@ export function createManagedDaemonLookerTableIdentifierParser(
});
}

/** @internal */
export function createManagedDaemonSqlAnalysisPort(options: ManagedPythonDaemonHttpOptions): SqlAnalysisPort {
return createHttpSqlAnalysisPort({
baseUrl: 'http://127.0.0.1:0',
requestJson: createManagedDaemonHttpJsonRunner(options) as KtxSqlAnalysisHttpJsonRunner,
});
}

export function externalDaemonSqlAnalysisBaseUrl(
env: NodeJS.ProcessEnv | Record<string, string | undefined>,
): string | undefined {
const sqlAnalysisUrl = env.KTX_SQL_ANALYSIS_URL?.trim();
if (sqlAnalysisUrl) {
return sqlAnalysisUrl;
}
const daemonUrl = env.KTX_DAEMON_URL?.trim();
if (daemonUrl) {
return daemonUrl;
}
return undefined;
}

export function resolveSqlAnalysisPort(
managedDaemon: ManagedPythonDaemonHttpOptions,
env: NodeJS.ProcessEnv | Record<string, string | undefined> = process.env,
): SqlAnalysisPort {
const externalBaseUrl = externalDaemonSqlAnalysisBaseUrl(env);
if (externalBaseUrl) {
return createHttpSqlAnalysisPort({ baseUrl: externalBaseUrl });
}
return createManagedDaemonSqlAnalysisPort(managedDaemon);
}

export function managedDaemonDatabaseIntrospectionOptions(
options: ManagedPythonDaemonHttpOptions,
): Pick<DaemonLiveDatabaseIntrospectionOptions, 'requestJson'> {
Expand Down
4 changes: 2 additions & 2 deletions packages/cli/src/mcp-server-factory.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import { resolveProjectEmbeddingProvider } from './embedding-resolution.js';
import { createKtxCliIngestQueryExecutor } from './ingest-query-executor.js';
import { createKtxCliScanConnector } from './local-scan-connectors.js';
import { createLazyManagedPythonSemanticLayerComputePort } from './managed-python-command.js';
import { createManagedDaemonSqlAnalysisPort } from './managed-python-http.js';
import { resolveSqlAnalysisPort } from './managed-python-http.js';

function noopMcpIo(): KtxCliIo {
return {
Expand All @@ -31,7 +31,7 @@ export async function createKtxMcpServerFactory(input: {
installPolicy: 'auto',
io,
});
const sqlAnalysis = createManagedDaemonSqlAnalysisPort({
const sqlAnalysis = resolveSqlAnalysisPort({
cliVersion: input.cliVersion,
projectDir: input.projectDir,
installPolicy: 'auto',
Expand Down
4 changes: 2 additions & 2 deletions packages/cli/src/sql.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import type { SqlAnalysisDialect, SqlAnalysisPort } from './context/sql-analysis
import type { KtxCliIo } from './cli-runtime.js';
import { type KtxOutputMode, resolveOutputMode } from './io/mode.js';
import { createKtxCliScanConnector } from './local-scan-connectors.js';
import { createManagedDaemonSqlAnalysisPort } from './managed-python-http.js';
import { resolveSqlAnalysisPort } from './managed-python-http.js';
import { profileMark } from './startup-profile.js';
import { isDemoConnection } from './telemetry/demo-detect.js';
import { emitTelemetryEvent, reportException } from './telemetry/index.js';
Expand Down Expand Up @@ -140,7 +140,7 @@ export async function runKtxSql(args: KtxSqlArgs, io: KtxCliIo = process, deps:
const createSqlAnalysis =
deps.createSqlAnalysis ??
(() =>
createManagedDaemonSqlAnalysisPort({
resolveSqlAnalysisPort({
cliVersion: args.cliVersion,
projectDir: args.projectDir,
installPolicy: 'auto',
Expand Down
41 changes: 23 additions & 18 deletions packages/cli/src/status-project.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,10 @@ import {
CODEX_ISOLATION_WARNING_FIX,
} from './context/llm/codex-isolation.js';
import { runCodexAuthProbe } from './context/llm/codex-runtime.js';
import { resolveKtxConfigReference } from './context/core/config-reference.js';
import type { KtxConfigIssue, KtxProjectConfig, KtxProjectConnectionConfig, KtxProjectEmbeddingConfig, KtxProjectLlmConfig } from './context/project/config.js';
import type { KtxLocalProject } from './context/project/project.js';
import { readGitRepoConnectionFields } from './context/project/git-connection-fields.js';
import { ktxLocalStateDbPath } from './context/project/local-state-db.js';
import {
isQueryHistoryEnabled,
Expand Down Expand Up @@ -167,19 +169,21 @@ export interface ProjectStatus {
};
}

// Delegates to the canonical resolver so status reports what the runtime resolves;
// it reads file: refs and throws on a missing/empty file, so the catch maps that
// to "missing" rather than letting status crash.
function resolveRef(value: unknown, env: NodeJS.ProcessEnv): { resolved: string; via: 'literal' | 'env' | 'file' | 'missing' } {
if (typeof value !== 'string') return { resolved: '', via: 'missing' };
const trimmed = value.trim();
if (trimmed.length === 0) return { resolved: '', via: 'missing' };
if (trimmed.startsWith('env:')) {
const name = trimmed.slice(4).trim();
const v = env[name];
return v && v.trim().length > 0 ? { resolved: v, via: 'env' } : { resolved: '', via: 'missing' };
}
if (trimmed.startsWith('file:')) {
return { resolved: trimmed.slice(5), via: 'file' };
const via = trimmed.startsWith('env:') ? 'env' : trimmed.startsWith('file:') ? 'file' : 'literal';
let resolved: string | undefined;
try {
resolved = resolveKtxConfigReference(trimmed, env);
} catch {
resolved = undefined;
}
return { resolved: trimmed, via: 'literal' };
return resolved && resolved.length > 0 ? { resolved, via } : { resolved: '', via: 'missing' };
}

function failureDetail(error: unknown): string {
Expand Down Expand Up @@ -350,7 +354,9 @@ function buildEmbeddingsStatus(config: KtxProjectEmbeddingConfig, env: NodeJS.Pr
if (backend === 'openai') {
const ref = config.openai?.api_key;
const resolved = resolveRef(ref, env);
if (resolved.resolved.length > 0 || (env.OPENAI_API_KEY && env.OPENAI_API_KEY.trim().length > 0)) {
// resolveLocalKtxEmbeddingConfig requires a resolved openai.api_key and never
// falls back to a bare OPENAI_API_KEY env var, so status must not either.
if (resolved.resolved.length > 0) {
return { backend, model, dimensions, status: 'ok', detail: 'key set' };
}
const hint = envHint(ref);
Expand Down Expand Up @@ -427,26 +433,25 @@ function buildConnectionStatus(
case 'dbt':
case 'dbt-core':
case 'dbt-cloud': {
const repoUrl =
(conn as Record<string, unknown>).repoUrl ??
(conn as Record<string, unknown>).repo_url;
if (typeof repoUrl === 'string' && repoUrl.length > 0) return ok(`repo: ${repoUrl}`);
return warn('repoUrl not set', 'Rerun `ktx setup`');
const { repoUrl, sourceDir } = readGitRepoConnectionFields(conn, 'dbt');
const source = repoUrl ?? sourceDir;
if (source) return ok(`repo: ${source}`);
return warn('repo_url not set', 'Rerun `ktx setup`');
}
case 'metabase': {
const url = (conn as Record<string, unknown>).api_url;
if (typeof url === 'string' && url.length > 0) return ok(`url: ${url}`);
return warn('api_url not set', 'Rerun `ktx setup`');
}
case 'looker':
case 'lookml': {
case 'looker': {
const url = (conn as Record<string, unknown>).base_url ?? (conn as Record<string, unknown>).url;
if (typeof url === 'string' && url.length > 0) return ok(`url: ${url}`);
return warn('base_url not set', 'Rerun `ktx setup`');
}
case 'lookml':
case 'metricflow': {
const repoUrl = (conn as Record<string, unknown>).repoUrl ?? (conn as Record<string, unknown>).repo_url;
if (typeof repoUrl === 'string' && repoUrl.length > 0) return ok(`repo: ${repoUrl}`);
const { repoUrl } = readGitRepoConnectionFields(conn, driver);
if (repoUrl) return ok(`repo: ${repoUrl}`);
return warn('repoUrl not set', 'Rerun `ktx setup`');
}
default:
Expand Down
Loading
Loading