From 6d8d5cafd511690130d3dc8973f8de06447a825a Mon Sep 17 00:00:00 2001 From: saksham Date: Mon, 6 Jul 2026 00:22:17 +0530 Subject: [PATCH 1/4] feat(connectors): add Tableau context-source adapter (scaffolding, client, fetch, tests) Adds a new `tableau` context source under packages/cli/src/context/ingest/adapters/tableau/, modeled on the Sigma adapter's scaffolding and Looker's graph-shaped semantic model. This first commit covers the read path: PAT sign-in (REST) + published data sources, calculated fields, and workbook metadata via the Metadata API (GraphQL), staged to disk and chunked into work units. - client.ts: PAT auth + Metadata GraphQL, retry + 401 re-auth, offset paging - fetch.ts: stages datasources/.json + workbooks/.json + manifest - detect.ts / chunk.ts: bundle detection + data-source / workbook work units - driver-schemas.ts: `tableau` connection (host, site, PAT via env:/file: ref) - wired into local-adapters, setup-sources, setup-commands, public-ingest - Vitest suite (client, fetch, detect, chunk, adapter, client-boundary) with fixtures under test/fixtures/tableau/ Note: the issue's path packages/context/src/ingest/adapters/tableau/ is stale; the real location is packages/cli/src/context/ingest/adapters/tableau/. Wiki output (tableau_ingest skill) and deterministic semantic-layer projection follow in subsequent commits. Refs #166. --- packages/cli/src/commands/setup-commands.ts | 1 + .../context/ingest/adapters/tableau/chunk.ts | 148 +++++++ .../ingest/adapters/tableau/client-port.ts | 62 +++ .../context/ingest/adapters/tableau/client.ts | 379 ++++++++++++++++++ .../context/ingest/adapters/tableau/detect.ts | 21 + .../context/ingest/adapters/tableau/fetch.ts | 204 ++++++++++ .../adapters/tableau/local-tableau.adapter.ts | 87 ++++ .../adapters/tableau/tableau.adapter.ts | 37 ++ .../context/ingest/adapters/tableau/types.ts | 131 ++++++ .../cli/src/context/ingest/local-adapters.ts | 24 +- .../cli/src/context/project/driver-schemas.ts | 51 +++ packages/cli/src/public-ingest.ts | 1 + packages/cli/src/setup-sources.ts | 114 +++++- .../ingest/adapters/tableau/chunk.test.ts | 36 ++ .../adapters/tableau/client-boundary.test.ts | 19 + .../ingest/adapters/tableau/client.test.ts | 202 ++++++++++ .../ingest/adapters/tableau/detect.test.ts | 41 ++ .../ingest/adapters/tableau/fetch.test.ts | 113 ++++++ .../adapters/tableau/tableau.adapter.test.ts | 64 +++ .../tableau/single/datasources/ds-1.json | 22 + .../tableau/single/tableau-manifest.json | 6 + .../tableau/single/workbooks/wb-1.json | 7 + 22 files changed, 1768 insertions(+), 2 deletions(-) create mode 100644 packages/cli/src/context/ingest/adapters/tableau/chunk.ts create mode 100644 packages/cli/src/context/ingest/adapters/tableau/client-port.ts create mode 100644 packages/cli/src/context/ingest/adapters/tableau/client.ts create mode 100644 packages/cli/src/context/ingest/adapters/tableau/detect.ts create mode 100644 packages/cli/src/context/ingest/adapters/tableau/fetch.ts create mode 100644 packages/cli/src/context/ingest/adapters/tableau/local-tableau.adapter.ts create mode 100644 packages/cli/src/context/ingest/adapters/tableau/tableau.adapter.ts create mode 100644 packages/cli/src/context/ingest/adapters/tableau/types.ts create mode 100644 packages/cli/test/context/ingest/adapters/tableau/chunk.test.ts create mode 100644 packages/cli/test/context/ingest/adapters/tableau/client-boundary.test.ts create mode 100644 packages/cli/test/context/ingest/adapters/tableau/client.test.ts create mode 100644 packages/cli/test/context/ingest/adapters/tableau/detect.test.ts create mode 100644 packages/cli/test/context/ingest/adapters/tableau/fetch.test.ts create mode 100644 packages/cli/test/context/ingest/adapters/tableau/tableau.adapter.test.ts create mode 100644 packages/cli/test/fixtures/tableau/single/datasources/ds-1.json create mode 100644 packages/cli/test/fixtures/tableau/single/tableau-manifest.json create mode 100644 packages/cli/test/fixtures/tableau/single/workbooks/wb-1.json diff --git a/packages/cli/src/commands/setup-commands.ts b/packages/cli/src/commands/setup-commands.ts index d838e472..f3c4ae7a 100644 --- a/packages/cli/src/commands/setup-commands.ts +++ b/packages/cli/src/commands/setup-commands.ts @@ -60,6 +60,7 @@ function sourceType(value: string): KtxSetupSourceType { value === 'lookml' || value === 'notion' || value === 'sigma' || + value === 'tableau' || value === 'gdrive' ) { return value; diff --git a/packages/cli/src/context/ingest/adapters/tableau/chunk.ts b/packages/cli/src/context/ingest/adapters/tableau/chunk.ts new file mode 100644 index 00000000..4a843bd2 --- /dev/null +++ b/packages/cli/src/context/ingest/adapters/tableau/chunk.ts @@ -0,0 +1,148 @@ +import { readdir, readFile } from 'node:fs/promises'; +import { join, relative } from 'node:path'; +import type { ChunkResult, DiffSet, WorkUnit } from '../../types.js'; +import { + type StagedDatasourceFile, + type StagedWorkbookFile, + type TableauManifest, + stagedDatasourceFileSchema, + stagedWorkbookFileSchema, + tableauManifestSchema, + STAGED_FILES, +} from './types.js'; + +interface LoadedBundle { + manifest: TableauManifest | null; + datasourcesByPath: Map; + workbooksByPath: Map; + allPaths: string[]; +} + +async function walkStagedDir(stagedDir: string): Promise { + let entries; + try { + entries = await readdir(stagedDir, { withFileTypes: true, recursive: true }); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') return []; + throw err; + } + const paths: string[] = []; + for (const entry of entries) { + if (!entry.isFile()) continue; + const abs = join(entry.parentPath, entry.name); + paths.push(relative(stagedDir, abs).replace(/\\/g, '/')); + } + paths.sort(); + return paths; +} + +async function loadTyped( + stagedDir: string, + allPaths: string[], + prefix: string, + parse: (raw: unknown) => T, +): Promise> { + const byPath = new Map(); + for (const path of allPaths) { + if (!path.startsWith(prefix) || !path.endsWith('.json')) continue; + try { + const body = await readFile(join(stagedDir, path), 'utf-8'); + byPath.set(path, parse(JSON.parse(body))); + } catch { + // Malformed file — skip. + } + } + return byPath; +} + +async function loadBundle(stagedDir: string): Promise { + const allPaths = await walkStagedDir(stagedDir); + let manifest: TableauManifest | null = null; + try { + const body = await readFile(join(stagedDir, STAGED_FILES.manifest), 'utf-8'); + manifest = tableauManifestSchema.parse(JSON.parse(body)); + } catch { + manifest = null; + } + + const datasourcesByPath = await loadTyped(stagedDir, allPaths, `${STAGED_FILES.datasourcesDir}/`, (raw) => + stagedDatasourceFileSchema.parse(raw), + ); + const workbooksByPath = await loadTyped(stagedDir, allPaths, `${STAGED_FILES.workbooksDir}/`, (raw) => + stagedWorkbookFileSchema.parse(raw), + ); + + return { manifest, datasourcesByPath, workbooksByPath, allPaths }; +} + +/** Max data sources per LLM work unit. Controls parallel processing granularity. */ +const DATASOURCES_PER_UNIT = 50; +/** Max workbooks per LLM work unit. Controls incremental re-sync granularity. */ +const WORKBOOKS_PER_UNIT = 2000; + +function emitBatches( + paths: string[], + perUnit: number, + unitKeyBase: string, + labelBase: string, + noun: string, + allPaths: string[], +): WorkUnit[] { + const batches = Math.ceil(paths.length / perUnit) || 0; + const units: WorkUnit[] = []; + for (let i = 0; i < batches; i++) { + const batch = paths.slice(i * perUnit, (i + 1) * perUnit); + const rawFiles = [...batch].sort(); + const rawFilesSet = new Set(rawFiles); + const suffix = batches > 1 ? `-${i}` : ''; + units.push({ + unitKey: `${unitKeyBase}${suffix}`, + displayLabel: batches > 1 ? `${labelBase} (${i + 1}/${batches})` : labelBase, + rawFiles, + peerFileIndex: allPaths.filter((p) => !rawFilesSet.has(p)).sort(), + dependencyPaths: [], + notes: `${batch.length} ${noun}${batch.length === 1 ? '' : 's'}`, + }); + } + return units; +} + +function emitWorkUnits(bundle: LoadedBundle): WorkUnit[] { + if (!bundle.manifest) return []; + const dsPaths = [...bundle.datasourcesByPath.keys()].sort(); + const wbPaths = [...bundle.workbooksByPath.keys()].sort(); + return [ + ...emitBatches(dsPaths, DATASOURCES_PER_UNIT, 'tableau-datasources', 'Tableau: data sources', 'data source', bundle.allPaths), + ...emitBatches(wbPaths, WORKBOOKS_PER_UNIT, 'tableau-workbooks', 'Tableau: workbooks', 'workbook', bundle.allPaths), + ]; +} + +interface ChunkOptions { + diffSet?: DiffSet; +} + +export async function chunkTableauStagedDir(stagedDir: string, opts: ChunkOptions = {}): Promise { + const bundle = await loadBundle(stagedDir); + if (!bundle.manifest) { + return { workUnits: [] }; + } + + const firstRunUnits = emitWorkUnits(bundle); + if (!opts.diffSet) { + return { workUnits: firstRunUnits }; + } + + const touched = new Set([...opts.diffSet.added, ...opts.diffSet.modified]); + const kept: WorkUnit[] = []; + for (const wu of firstRunUnits) { + const anyTouched = wu.rawFiles.some((p) => touched.has(p)); + if (!anyTouched) continue; + const changedFiles = wu.rawFiles.filter((p) => touched.has(p)); + const unchangedFiles = wu.rawFiles.filter((p) => !touched.has(p)); + const deps = new Set([...wu.dependencyPaths, ...unchangedFiles]); + kept.push({ ...wu, rawFiles: changedFiles.sort(), dependencyPaths: [...deps].sort() }); + } + const eviction = + opts.diffSet.deleted.length > 0 ? { deletedRawPaths: [...opts.diffSet.deleted].sort() } : undefined; + return { workUnits: kept, eviction }; +} diff --git a/packages/cli/src/context/ingest/adapters/tableau/client-port.ts b/packages/cli/src/context/ingest/adapters/tableau/client-port.ts new file mode 100644 index 00000000..41e6fc60 --- /dev/null +++ b/packages/cli/src/context/ingest/adapters/tableau/client-port.ts @@ -0,0 +1,62 @@ +import type { FetchContext } from '../../types.js'; +import type { DatasourceFilterInput, TableauPullConfig, WorkbookFilterInput } from './types.js'; + +export interface TableauTestConnectionResult { + success: boolean; + message?: string; + error?: string; +} + +/** A field record as returned by the Metadata API for a published data source. */ +export interface TableauFieldRecord { + name: string; + role?: string; + dataType?: string; + /** Present for a CalculatedField; absent for a plain ColumnField. */ + formula?: string; + description?: string; +} + +/** An upstream (physical) table record from the Metadata API lineage. */ +export interface TableauUpstreamTableRecord { + luid?: string; + name: string; + schema?: string; + fullName?: string; +} + +/** A published data source with its fields and upstream tables, from the Metadata API. */ +export interface TableauDatasourceRecord { + luid: string; + name: string; + projectName?: string; + updatedAt?: string; + hasExtracts?: boolean; + description?: string; + fields: TableauFieldRecord[]; + upstreamTables: TableauUpstreamTableRecord[]; +} + +/** Workbook summary metadata from the Metadata API. */ +export interface TableauWorkbookRecord { + luid: string; + name: string; + projectName?: string; + description?: string; + updatedAt?: string; +} + +/** Re-exported so callers can reference the option types without importing from types.ts directly. */ +export type { DatasourceFilterInput as ListDatasourcesOptions } from './types.js'; +export type { WorkbookFilterInput as ListWorkbooksOptions } from './types.js'; + +export interface TableauRuntimeClient { + testConnection(): Promise; + listDatasources(opts?: DatasourceFilterInput): Promise; + listWorkbooks(opts?: WorkbookFilterInput): Promise; + cleanup(): Promise; +} + +export interface TableauClientFactory { + createClient(config: TableauPullConfig, ctx: FetchContext): Promise | TableauRuntimeClient; +} diff --git a/packages/cli/src/context/ingest/adapters/tableau/client.ts b/packages/cli/src/context/ingest/adapters/tableau/client.ts new file mode 100644 index 00000000..891c384a --- /dev/null +++ b/packages/cli/src/context/ingest/adapters/tableau/client.ts @@ -0,0 +1,379 @@ +import type { + TableauDatasourceRecord, + TableauFieldRecord, + TableauRuntimeClient, + TableauTestConnectionResult, + TableauUpstreamTableRecord, + TableauWorkbookRecord, +} from './client-port.js'; +import type { DatasourceFilterInput, WorkbookFilterInput } from './types.js'; + +export interface TableauClientRuntimeConfig { + /** Base host URL of the Tableau Cloud pod or Tableau Server, e.g. https://us-west-2b.online.tableau.com */ + host: string; + /** Site content URL (the site subpath). Empty string targets the Default site on Tableau Server. */ + siteContentUrl: string; + /** REST API version, e.g. "3.29". */ + apiVersion: string; + /** Personal Access Token name. */ + patName: string; + /** Personal Access Token secret. */ + patSecret: string; +} + +export interface TableauClientConfig { + maxRetries: number; + baseDelayMs: number; + maxDelayMs: number; + timeoutMs: number; + /** Page size for Metadata API `*Connection(first, offset)` pagination. */ + pageSize: number; +} + +export const DEFAULT_TABLEAU_CLIENT_CONFIG: TableauClientConfig = { + maxRetries: 3, + baseDelayMs: 500, + maxDelayMs: 10_000, + timeoutMs: 30_000, + pageSize: 100, +}; + +interface SignInResponse { + credentials: { + token: string; + site: { id: string; contentUrl: string }; + user: { id: string }; + }; +} + +interface GraphQLResponse { + data?: T; + errors?: Array<{ message: string }>; +} + +/** + * A field as returned by the Metadata API. `fields` is polymorphic — a `CalculatedField` + * carries `formula`, a `ColumnField` does not — so we read `formula` optionally off either. + */ +interface RawField { + __typename?: string; + name?: string; + description?: string | null; + dataType?: string | null; + role?: string | null; + formula?: string | null; +} + +interface RawUpstreamTable { + luid?: string | null; + name?: string | null; + schema?: string | null; + fullName?: string | null; +} + +interface RawDatasource { + luid?: string | null; + name?: string | null; + projectName?: string | null; + updatedAt?: string | null; + hasExtracts?: boolean | null; + description?: string | null; + fields?: RawField[] | null; + upstreamTables?: RawUpstreamTable[] | null; +} + +interface RawWorkbook { + luid?: string | null; + name?: string | null; + projectName?: string | null; + description?: string | null; + updatedAt?: string | null; +} + +interface ConnectionPage { + totalCount?: number; + pageInfo?: { hasNextPage?: boolean }; + nodes: T[]; +} + +const DATASOURCES_QUERY = ` +query Datasources($first: Int!, $offset: Int!) { + publishedDatasourcesConnection(first: $first, offset: $offset) { + totalCount + pageInfo { hasNextPage } + nodes { + luid + name + projectName + updatedAt + hasExtracts + description + upstreamTables { luid name schema fullName } + fields { + __typename + name + description + ... on ColumnField { dataType role } + ... on CalculatedField { formula dataType role } + } + } + } +}`; + +const WORKBOOKS_QUERY = ` +query Workbooks($first: Int!, $offset: Int!) { + workbooksConnection(first: $first, offset: $offset) { + totalCount + pageInfo { hasNextPage } + nodes { + luid + name + projectName + description + updatedAt + } + } +}`; + +function nonEmpty(value: string | null | undefined): string | undefined { + return typeof value === 'string' && value.trim().length > 0 ? value : undefined; +} + +function mapField(raw: RawField): TableauFieldRecord | null { + const name = nonEmpty(raw.name); + if (!name) return null; + const field: TableauFieldRecord = { name }; + const role = nonEmpty(raw.role); + if (role) field.role = role; + const dataType = nonEmpty(raw.dataType); + if (dataType) field.dataType = dataType; + const formula = nonEmpty(raw.formula); + if (formula) field.formula = formula; + const description = nonEmpty(raw.description); + if (description) field.description = description; + return field; +} + +function mapUpstreamTable(raw: RawUpstreamTable): TableauUpstreamTableRecord | null { + const name = nonEmpty(raw.name); + if (!name) return null; + const table: TableauUpstreamTableRecord = { name }; + const luid = nonEmpty(raw.luid); + if (luid) table.luid = luid; + const schema = nonEmpty(raw.schema); + if (schema) table.schema = schema; + const fullName = nonEmpty(raw.fullName); + if (fullName) table.fullName = fullName; + return table; +} + +function mapDatasource(raw: RawDatasource): TableauDatasourceRecord | null { + const luid = nonEmpty(raw.luid); + const name = nonEmpty(raw.name); + if (!luid || !name) return null; + const record: TableauDatasourceRecord = { + luid, + name, + hasExtracts: raw.hasExtracts ?? false, + fields: (raw.fields ?? []).map(mapField).filter((f): f is TableauFieldRecord => f !== null), + upstreamTables: (raw.upstreamTables ?? []) + .map(mapUpstreamTable) + .filter((t): t is TableauUpstreamTableRecord => t !== null), + }; + const projectName = nonEmpty(raw.projectName); + if (projectName) record.projectName = projectName; + const updatedAt = nonEmpty(raw.updatedAt); + if (updatedAt) record.updatedAt = updatedAt; + const description = nonEmpty(raw.description); + if (description) record.description = description; + return record; +} + +function mapWorkbook(raw: RawWorkbook): TableauWorkbookRecord | null { + const luid = nonEmpty(raw.luid); + const name = nonEmpty(raw.name); + if (!luid || !name) return null; + const record: TableauWorkbookRecord = { luid, name }; + const projectName = nonEmpty(raw.projectName); + if (projectName) record.projectName = projectName; + const description = nonEmpty(raw.description); + if (description) record.description = description; + const updatedAt = nonEmpty(raw.updatedAt); + if (updatedAt) record.updatedAt = updatedAt; + return record; +} + +function isAfter(updatedAt: string | undefined, since: number): boolean { + if (!updatedAt) return true; // keep records with no timestamp rather than silently dropping them + const ts = new Date(updatedAt).getTime(); + return Number.isNaN(ts) ? true : ts >= since; +} + +export class DefaultTableauClient implements TableauRuntimeClient { + private authToken: string | null = null; + private authInflight: Promise | null = null; + + constructor( + private readonly runtimeConfig: TableauClientRuntimeConfig, + private readonly clientConfig: TableauClientConfig = DEFAULT_TABLEAU_CLIENT_CONFIG, + ) {} + + private get host(): string { + return this.runtimeConfig.host.replace(/\/$/, ''); + } + + private async signIn(): Promise { + const url = `${this.host}/api/${this.runtimeConfig.apiVersion}/auth/signin`; + const res = await fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json', Accept: 'application/json' }, + body: JSON.stringify({ + credentials: { + personalAccessTokenName: this.runtimeConfig.patName, + personalAccessTokenSecret: this.runtimeConfig.patSecret, + site: { contentUrl: this.runtimeConfig.siteContentUrl }, + }, + }), + signal: AbortSignal.timeout(this.clientConfig.timeoutMs), + }); + if (!res.ok) { + const text = await res.text().catch(() => ''); + throw new Error(`Tableau sign-in failed (${res.status}): ${text}`); + } + const body = (await res.json()) as SignInResponse; + const token = body.credentials?.token; + if (!token) { + throw new Error('Tableau sign-in succeeded but returned no auth token'); + } + this.authToken = token; + } + + private async ensureToken(): Promise { + if (this.authToken) return; + if (this.authInflight) return this.authInflight; + this.authInflight = this.signIn().finally(() => { + this.authInflight = null; + }); + return this.authInflight; + } + + /** Executes a Metadata API GraphQL query with retry + one re-auth on 401. */ + private async metadataQuery(query: string, variables: Record): Promise { + await this.ensureToken(); + const url = `${this.host}/api/metadata/graphql`; + + let lastError: Error | null = null; + for (let attempt = 0; attempt <= this.clientConfig.maxRetries; attempt++) { + if (attempt > 0) { + const delay = Math.min(this.clientConfig.baseDelayMs * 2 ** (attempt - 1), this.clientConfig.maxDelayMs); + await new Promise((resolve) => setTimeout(resolve, delay)); + } + + const res = await fetch(url, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Accept: 'application/json', + 'X-Tableau-Auth': this.authToken ?? '', + }, + body: JSON.stringify({ query, variables }), + signal: AbortSignal.timeout(this.clientConfig.timeoutMs), + }); + + if (res.status === 401) { + // Token rejected/expired — force a fresh sign-in and retry once. + this.authToken = null; + await this.ensureToken(); + continue; + } + + if (res.status === 429 || res.status >= 500) { + const text = await res.text().catch(() => ''); + lastError = new Error(`Tableau Metadata API error (${res.status}): ${text}`); + continue; + } + + if (!res.ok) { + const text = await res.text().catch(() => ''); + throw new Error(`Tableau Metadata API error (${res.status}): ${text}`); + } + + const body = (await res.json()) as GraphQLResponse; + if (body.errors && body.errors.length > 0) { + throw new Error(`Tableau Metadata API returned errors: ${body.errors.map((e) => e.message).join('; ')}`); + } + if (body.data === undefined) { + throw new Error('Tableau Metadata API returned no data'); + } + return body.data; + } + + throw lastError ?? new Error('Tableau Metadata API request failed after retries'); + } + + private async paginate( + query: string, + connectionField: string, + ): Promise { + const all: TNode[] = []; + let offset = 0; + for (;;) { + const data = await this.metadataQuery>>(query, { + first: this.clientConfig.pageSize, + offset, + }); + const page = data[connectionField]; + const nodes = page?.nodes ?? []; + all.push(...nodes); + const hasNext = page?.pageInfo?.hasNextPage ?? false; + if (!hasNext || nodes.length === 0) break; + offset += this.clientConfig.pageSize; + } + return all; + } + + async testConnection(): Promise { + try { + await this.ensureToken(); + return { success: true }; + } catch (err) { + return { success: false, error: err instanceof Error ? err.message : String(err) }; + } + } + + async listDatasources(opts: DatasourceFilterInput = {}): Promise { + const raw = await this.paginate(DATASOURCES_QUERY, 'publishedDatasourcesConnection'); + let records = raw.map(mapDatasource).filter((d): d is TableauDatasourceRecord => d !== null); + if (opts.updatedSince) { + const since = new Date(opts.updatedSince).getTime(); + if (!Number.isNaN(since)) records = records.filter((d) => isAfter(d.updatedAt, since)); + } + return records; + } + + async listWorkbooks(opts: WorkbookFilterInput = {}): Promise { + const raw = await this.paginate(WORKBOOKS_QUERY, 'workbooksConnection'); + let records = raw.map(mapWorkbook).filter((w): w is TableauWorkbookRecord => w !== null); + if (opts.updatedSince) { + const since = new Date(opts.updatedSince).getTime(); + if (!Number.isNaN(since)) records = records.filter((w) => isAfter(w.updatedAt, since)); + } + return records; + } + + async cleanup(): Promise { + // Best-effort sign-out so the session token is invalidated server-side. + if (this.authToken) { + try { + await fetch(`${this.host}/api/${this.runtimeConfig.apiVersion}/auth/signout`, { + method: 'POST', + headers: { 'X-Tableau-Auth': this.authToken }, + signal: AbortSignal.timeout(this.clientConfig.timeoutMs), + }); + } catch { + // Ignore sign-out failures — the token will expire on its own. + } + } + this.authToken = null; + } +} diff --git a/packages/cli/src/context/ingest/adapters/tableau/detect.ts b/packages/cli/src/context/ingest/adapters/tableau/detect.ts new file mode 100644 index 00000000..9a40a24e --- /dev/null +++ b/packages/cli/src/context/ingest/adapters/tableau/detect.ts @@ -0,0 +1,21 @@ +import { readdir, stat } from 'node:fs/promises'; +import { join } from 'node:path'; +import { STAGED_FILES } from './types.js'; + +export async function detectTableauStagedDir(stagedDir: string): Promise { + try { + await stat(join(stagedDir, STAGED_FILES.manifest)); + } catch { + return false; + } + for (const subdir of [STAGED_FILES.datasourcesDir, STAGED_FILES.workbooksDir]) { + let entries: string[]; + try { + entries = await readdir(join(stagedDir, subdir)); + } catch { + continue; + } + if (entries.some((name) => name.endsWith('.json'))) return true; + } + return false; +} diff --git a/packages/cli/src/context/ingest/adapters/tableau/fetch.ts b/packages/cli/src/context/ingest/adapters/tableau/fetch.ts new file mode 100644 index 00000000..d45f2b84 --- /dev/null +++ b/packages/cli/src/context/ingest/adapters/tableau/fetch.ts @@ -0,0 +1,204 @@ +import { mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import type { FetchContext } from '../../types.js'; +import type { TableauClientFactory } from './client-port.js'; +import { + type StagedDatasourceFile, + type StagedWorkbookFile, + type TableauManifest, + type TableauProjectionConfig, + parseTableauPullConfig, + stagedDatasourceFileSchema, + stagedWorkbookFileSchema, + STAGED_FILES, +} from './types.js'; + +export interface TableauFetchLogger { + log(message: string): void; + warn(message: string): void; +} + +const noopLogger: TableauFetchLogger = { log: () => undefined, warn: () => undefined }; + +export interface FetchTableauBundleParams { + pullConfig: unknown; + stagedDir: string; + ctx: FetchContext; + clientFactory: TableauClientFactory; + logger?: TableauFetchLogger; + /** Injectable clock for deterministic manifest timestamps in tests. */ + now?: () => Date; +} + +async function loadExistingStaged( + stagedDir: string, + subdir: string, + parse: (raw: unknown) => T, + keyOf: (parsed: T) => string, +): Promise> { + const existing = new Map(); + let entries: string[]; + try { + entries = await readdir(join(stagedDir, subdir)); + } catch { + return existing; + } + for (const entry of entries) { + if (!entry.endsWith('.json')) continue; + try { + const body = await readFile(join(stagedDir, subdir, entry), 'utf-8'); + const parsed = parse(JSON.parse(body)); + existing.set(keyOf(parsed), parsed); + } catch { + // Skip malformed files. + } + } + return existing; +} + +export async function fetchTableauBundle({ + pullConfig, + stagedDir, + ctx, + clientFactory, + logger = noopLogger, + now = () => new Date(), +}: FetchTableauBundleParams): Promise { + const config = parseTableauPullConfig(pullConfig); + const client = await clientFactory.createClient(config, ctx); + + try { + await mkdir(join(stagedDir, STAGED_FILES.datasourcesDir), { recursive: true }); + await mkdir(join(stagedDir, STAGED_FILES.workbooksDir), { recursive: true }); + + const existingDatasources = await loadExistingStaged( + stagedDir, + STAGED_FILES.datasourcesDir, + (raw) => stagedDatasourceFileSchema.parse(raw), + (d) => d.luid, + ); + const existingWorkbooks = await loadExistingStaged( + stagedDir, + STAGED_FILES.workbooksDir, + (raw) => stagedWorkbookFileSchema.parse(raw), + (w) => w.luid, + ); + + // --- Published data sources (the graph: fields incl. calculated formulas + upstream tables) --- + logger.log('Listing Tableau published data sources...'); + const datasources = await client.listDatasources(config.datasourceFilter); + const datasourceLuids = new Set(datasources.map((d) => d.luid)); + logger.log(`Found ${datasources.length} published data source(s).`); + + let datasourcesFetched = 0; + let datasourcesSkipped = 0; + for (const ds of datasources) { + const existing = existingDatasources.get(ds.luid); + if (existing && ds.updatedAt && existing.updatedAt === ds.updatedAt) { + datasourcesSkipped++; + continue; + } + const staged: StagedDatasourceFile = { + luid: ds.luid, + name: ds.name, + ...(ds.projectName ? { projectName: ds.projectName } : {}), + ...(ds.updatedAt ? { updatedAt: ds.updatedAt } : {}), + hasExtracts: ds.hasExtracts ?? false, + ...(ds.description ? { description: ds.description } : {}), + fields: ds.fields.map((f) => ({ + name: f.name, + ...(f.role ? { role: f.role } : {}), + ...(f.dataType ? { dataType: f.dataType } : {}), + ...(f.formula ? { formula: f.formula } : {}), + ...(f.description ? { description: f.description } : {}), + isCalculated: f.formula != null && f.formula.length > 0, + })), + upstreamTables: ds.upstreamTables.map((t) => ({ + name: t.name, + ...(t.luid ? { luid: t.luid } : {}), + ...(t.schema ? { schema: t.schema } : {}), + ...(t.fullName ? { fullName: t.fullName } : {}), + })), + }; + await writeFile( + join(stagedDir, STAGED_FILES.datasourcesDir, `${ds.luid}.json`), + JSON.stringify(staged, null, 2), + 'utf-8', + ); + logger.log(`Staged data source: ${ds.name}`); + datasourcesFetched++; + } + + // Evict staged files for data sources that no longer exist — but keep those merely outside + // the updatedSince window (they are still present in the workspace, just not re-fetched). + if (!config.datasourceFilter?.updatedSince) { + for (const [luid] of existingDatasources) { + if (datasourceLuids.has(luid)) continue; + await rm(join(stagedDir, STAGED_FILES.datasourcesDir, `${luid}.json`)).catch(() => undefined); + logger.log(`Removed stale staged data source ${luid}.`); + } + } + + // --- Workbooks (summary metadata; the durable signal is the name) --- + logger.log('Listing Tableau workbooks...'); + const workbooks = await client.listWorkbooks(config.workbookFilter); + const workbookLuids = new Set(workbooks.map((w) => w.luid)); + logger.log(`Found ${workbooks.length} workbook(s).`); + + let workbooksFetched = 0; + let workbooksSkipped = 0; + for (const wb of workbooks) { + const existing = existingWorkbooks.get(wb.luid); + if (existing && wb.updatedAt && existing.updatedAt === wb.updatedAt) { + workbooksSkipped++; + continue; + } + const staged: StagedWorkbookFile = { + luid: wb.luid, + name: wb.name, + ...(wb.projectName ? { projectName: wb.projectName } : {}), + ...(wb.description ? { description: wb.description } : {}), + ...(wb.updatedAt ? { updatedAt: wb.updatedAt } : {}), + }; + await writeFile( + join(stagedDir, STAGED_FILES.workbooksDir, `${wb.luid}.json`), + JSON.stringify(staged, null, 2), + 'utf-8', + ); + logger.log(`Staged workbook: ${wb.name}`); + workbooksFetched++; + } + + if (!config.workbookFilter?.updatedSince) { + for (const [luid] of existingWorkbooks) { + if (workbookLuids.has(luid)) continue; + await rm(join(stagedDir, STAGED_FILES.workbooksDir, `${luid}.json`)).catch(() => undefined); + logger.log(`Removed stale staged workbook ${luid}.`); + } + } + + const projectionConfig: TableauProjectionConfig = { + ...(config.datasourceFilter ? { datasourceFilter: config.datasourceFilter } : {}), + ...(config.workbookFilter ? { workbookFilter: config.workbookFilter } : {}), + }; + await writeFile( + join(stagedDir, STAGED_FILES.projectionConfig), + JSON.stringify(projectionConfig, null, 2), + 'utf-8', + ); + + const manifest: TableauManifest = { + tableauConnectionId: config.tableauConnectionId, + fetchedAt: now().toISOString(), + datasourceCount: datasources.length, + workbookCount: workbooks.length, + }; + await writeFile(join(stagedDir, STAGED_FILES.manifest), JSON.stringify(manifest, null, 2), 'utf-8'); + logger.log( + `Tableau fetch complete. Data sources: ${datasourcesFetched} fetched, ${datasourcesSkipped} unchanged. ` + + `Workbooks: ${workbooksFetched} fetched, ${workbooksSkipped} unchanged.`, + ); + } finally { + await client.cleanup(); + } +} diff --git a/packages/cli/src/context/ingest/adapters/tableau/local-tableau.adapter.ts b/packages/cli/src/context/ingest/adapters/tableau/local-tableau.adapter.ts new file mode 100644 index 00000000..bba30fa8 --- /dev/null +++ b/packages/cli/src/context/ingest/adapters/tableau/local-tableau.adapter.ts @@ -0,0 +1,87 @@ +import type { KtxProjectConnectionConfig } from '../../../../context/project/config.js'; +import type { KtxLocalProject } from '../../../../context/project/project.js'; +import { resolveKtxConfigReference } from '../../../core/config-reference.js'; +import type { FetchContext } from '../../types.js'; +import { + DEFAULT_TABLEAU_CLIENT_CONFIG, + DefaultTableauClient, + type TableauClientConfig, + type TableauClientRuntimeConfig, +} from './client.js'; +import type { TableauClientFactory, TableauRuntimeClient } from './client-port.js'; +import type { TableauFetchLogger } from './fetch.js'; +import { TableauSourceAdapter } from './tableau.adapter.js'; +import type { TableauPullConfig } from './types.js'; + +/** REST API version used when a connection does not pin one. */ +const DEFAULT_TABLEAU_API_VERSION = '3.29'; + +function stringField(value: unknown): string | null { + return typeof value === 'string' && value.trim().length > 0 ? value.trim() : null; +} + +export function tableauRuntimeConfigFromLocalConnection( + connectionId: string, + connection: KtxProjectConnectionConfig | undefined, + env: NodeJS.ProcessEnv = process.env, +): TableauClientRuntimeConfig { + if (!connection || String(connection.driver).toLowerCase() !== 'tableau') { + throw new Error(`Connection "${connectionId}" is not a Tableau connection`); + } + + const host = stringField(connection.host); + const siteContentUrl = stringField(connection.site_content_url) ?? ''; + const apiVersion = stringField(connection.api_version) ?? DEFAULT_TABLEAU_API_VERSION; + const patName = stringField(connection.personal_access_token_name); + const literalSecret = stringField(connection.personal_access_token_secret); + const secretRef = stringField(connection.personal_access_token_secret_ref); + const patSecret = literalSecret ?? (secretRef ? (resolveKtxConfigReference(secretRef, env) ?? null) : null); + + if (!host) { + throw new Error(`Connection "${connectionId}" is missing Tableau host`); + } + if (!patName) { + throw new Error(`Connection "${connectionId}" is missing Tableau personal_access_token_name`); + } + if (!patSecret) { + throw new Error( + `Connection "${connectionId}" is missing Tableau personal_access_token_secret or personal_access_token_secret_ref`, + ); + } + + return { host, siteContentUrl, apiVersion, patName, patSecret }; +} + +interface CreateLocalTableauSourceAdapterOptions { + env?: NodeJS.ProcessEnv; + defaultClientConfig?: TableauClientConfig; + logger?: TableauFetchLogger; + now?: () => Date; +} + +class LocalTableauClientFactory implements TableauClientFactory { + constructor( + private readonly project: KtxLocalProject, + private readonly options: CreateLocalTableauSourceAdapterOptions, + ) {} + + createClient(config: TableauPullConfig, _ctx: FetchContext): TableauRuntimeClient { + const runtimeConfig = tableauRuntimeConfigFromLocalConnection( + config.tableauConnectionId, + this.project.config.connections[config.tableauConnectionId], + this.options.env, + ); + return new DefaultTableauClient(runtimeConfig, this.options.defaultClientConfig ?? DEFAULT_TABLEAU_CLIENT_CONFIG); + } +} + +export function createLocalTableauSourceAdapter( + project: KtxLocalProject, + options: CreateLocalTableauSourceAdapterOptions = {}, +): TableauSourceAdapter { + return new TableauSourceAdapter({ + clientFactory: new LocalTableauClientFactory(project, options), + ...(options.logger ? { logger: options.logger } : {}), + ...(options.now ? { now: options.now } : {}), + }); +} diff --git a/packages/cli/src/context/ingest/adapters/tableau/tableau.adapter.ts b/packages/cli/src/context/ingest/adapters/tableau/tableau.adapter.ts new file mode 100644 index 00000000..dc78ee30 --- /dev/null +++ b/packages/cli/src/context/ingest/adapters/tableau/tableau.adapter.ts @@ -0,0 +1,37 @@ +import type { ChunkResult, DiffSet, FetchContext, SourceAdapter } from '../../types.js'; +import { chunkTableauStagedDir } from './chunk.js'; +import type { TableauClientFactory } from './client-port.js'; +import { detectTableauStagedDir } from './detect.js'; +import { fetchTableauBundle, type TableauFetchLogger } from './fetch.js'; + +export interface TableauSourceAdapterDeps { + clientFactory: TableauClientFactory; + logger?: TableauFetchLogger; + now?: () => Date; +} + +export class TableauSourceAdapter implements SourceAdapter { + readonly source = 'tableau'; + readonly skillNames: string[] = ['tableau_ingest']; + + constructor(private readonly deps: TableauSourceAdapterDeps) {} + + detect(stagedDir: string): Promise { + return detectTableauStagedDir(stagedDir); + } + + async fetch(pullConfig: unknown, stagedDir: string, ctx: FetchContext): Promise { + await fetchTableauBundle({ + pullConfig, + stagedDir, + ctx, + clientFactory: this.deps.clientFactory, + ...(this.deps.logger ? { logger: this.deps.logger } : {}), + ...(this.deps.now ? { now: this.deps.now } : {}), + }); + } + + chunk(stagedDir: string, diffSet?: DiffSet): Promise { + return chunkTableauStagedDir(stagedDir, { diffSet }); + } +} diff --git a/packages/cli/src/context/ingest/adapters/tableau/types.ts b/packages/cli/src/context/ingest/adapters/tableau/types.ts new file mode 100644 index 00000000..7339651f --- /dev/null +++ b/packages/cli/src/context/ingest/adapters/tableau/types.ts @@ -0,0 +1,131 @@ +import { z } from 'zod'; + +const tableauLocalConnectionIdSchema = z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/); + +/** Filters applied when listing published data sources. */ +const datasourceFilterSchema = z.object({ + /** ISO 8601 date string. Only data sources updated on or after this date are fetched. */ + updatedSince: z.string().optional(), +}); + +/** Input shape for datasource filtering — all fields optional. */ +export type DatasourceFilterInput = z.input; + +/** Filters applied when listing workbooks. */ +const workbookFilterSchema = z.object({ + /** ISO 8601 date string. Only workbooks updated on or after this date are ingested. */ + updatedSince: z.string().optional(), +}); + +/** Input shape for workbook filtering — all fields optional. */ +export type WorkbookFilterInput = z.input; + +/** + * The lean config the adapter needs at `fetch()` time, stored in the ingest job's `bundleRef.config`. + */ +const tableauPullConfigSchema = z.object({ + /** The ktx connection ID for the Tableau instance being swept. */ + tableauConnectionId: tableauLocalConnectionIdSchema, + /** Filters applied when listing data sources during ingest. */ + datasourceFilter: datasourceFilterSchema.optional(), + /** Filters applied when listing workbooks during ingest. */ + workbookFilter: workbookFilterSchema.optional(), +}); + +export type TableauPullConfig = z.infer; + +export function parseTableauPullConfig(raw: unknown): TableauPullConfig { + return tableauPullConfigSchema.parse(raw); +} + +/** Written to stagedDir during fetch() and read back by chunk(), project(), and the tableau_ingest skill. */ +export const tableauProjectionConfigSchema = z.object({ + /** Filters that were active when data sources were last fetched. Tells the skill what the staged set covers. */ + datasourceFilter: datasourceFilterSchema.optional(), + /** Filters that were active when workbooks were last fetched. */ + workbookFilter: workbookFilterSchema.optional(), +}); + +export type TableauProjectionConfig = z.infer; + +/** + * A field on a published data source. A `formula` marks it as a calculated field; a plain + * column field carries `dataType`/`role` only. `role` is Tableau's dimension/measure classification. + */ +export const stagedFieldSchema = z.object({ + name: z.string(), + /** Tableau field role, e.g. "DIMENSION" | "MEASURE". */ + role: z.string().optional(), + /** Tableau data type, e.g. "STRING" | "INTEGER" | "REAL" | "DATE". */ + dataType: z.string().optional(), + /** The calculation expression for a CalculatedField; absent for a plain ColumnField. */ + formula: z.string().optional(), + description: z.string().optional(), + /** True when this field is a Tableau CalculatedField (has a formula). */ + isCalculated: z.boolean().default(false), +}); + +export type StagedField = z.infer; + +/** An upstream (physical) table feeding a data source — the lineage edge to the warehouse. */ +export const stagedUpstreamTableSchema = z.object({ + luid: z.string().optional(), + name: z.string(), + /** Schema/dataset the table lives in, when Tableau reports it. */ + schema: z.string().optional(), + /** Fully-qualified name as Tableau reports it, e.g. "DATABASE.SCHEMA.TABLE". */ + fullName: z.string().optional(), +}); + +export type StagedUpstreamTable = z.infer; + +/** + * A staged published-data-source file, one per `datasources/.json`. + * This document IS the graph: its `fields` (including calculated-field formulas) plus its + * `upstreamTables` (lineage to physical warehouse tables). + */ +export const stagedDatasourceFileSchema = z.object({ + luid: z.string(), + name: z.string(), + /** Tableau project (folder) the data source lives in. */ + projectName: z.string().optional(), + updatedAt: z.string().optional(), + hasExtracts: z.boolean().default(false), + description: z.string().optional(), + fields: z.array(stagedFieldSchema).default([]), + upstreamTables: z.array(stagedUpstreamTableSchema).default([]), +}); + +export type StagedDatasourceFile = z.infer; + +/** + * A staged workbook file, one per `workbooks/.json`. + * Summary metadata only (name, project, description) — the durable business signal is the name. + */ +export const stagedWorkbookFileSchema = z.object({ + luid: z.string(), + name: z.string(), + projectName: z.string().optional(), + description: z.string().optional(), + updatedAt: z.string().optional(), +}); + +export type StagedWorkbookFile = z.infer; + +/** The manifest written once per `fetch()`. Presence acts as the detect() sentinel. */ +export const tableauManifestSchema = z.object({ + tableauConnectionId: tableauLocalConnectionIdSchema, + fetchedAt: z.string(), + datasourceCount: z.number().int(), + workbookCount: z.number().int().default(0), +}); + +export type TableauManifest = z.infer; + +/** Filenames inside stagedDir. Centralized so chunk() + fetch() + detect() all agree. */ +export const STAGED_FILES = { + manifest: 'tableau-manifest.json', + projectionConfig: 'tableau-projection-config.json', + datasourcesDir: 'datasources', + workbooksDir: 'workbooks', +} as const; diff --git a/packages/cli/src/context/ingest/local-adapters.ts b/packages/cli/src/context/ingest/local-adapters.ts index 75e54a33..f8557fb6 100644 --- a/packages/cli/src/context/ingest/local-adapters.ts +++ b/packages/cli/src/context/ingest/local-adapters.ts @@ -42,6 +42,8 @@ import type { MetabaseClientLogger } from './adapters/metabase/client.js'; import type { MetabaseFetchLogger } from './adapters/metabase/fetch.js'; import { createLocalSigmaSourceAdapter } from './adapters/sigma/local-sigma.adapter.js'; import type { SigmaFetchLogger } from './adapters/sigma/fetch.js'; +import { createLocalTableauSourceAdapter } from './adapters/tableau/local-tableau.adapter.js'; +import type { TableauFetchLogger } from './adapters/tableau/fetch.js'; import { MetricflowSourceAdapter } from './adapters/metricflow/metricflow.adapter.js'; import { pullConfigFromMetricflowIntegration } from './adapters/metricflow/pull-config.js'; import { LocalNotionRuntimeStore } from './adapters/notion/local-state-store.js'; @@ -75,7 +77,8 @@ type LocalIngestOperationalLogger = MetabaseClientLogger & MetabaseFetchLogger & LookerClientLogger & NotionFetchLogger & - SigmaFetchLogger; + SigmaFetchLogger & + TableauFetchLogger; export function createDefaultLocalIngestAdapters( project: KtxLocalProject, @@ -111,6 +114,9 @@ export function createDefaultLocalIngestAdapters( createLocalSigmaSourceAdapter(project, { ...(options.logger ? { logger: options.logger } : {}), }), + createLocalTableauSourceAdapter(project, { + ...(options.logger ? { logger: options.logger } : {}), + }), new GdriveSourceAdapter(), new LookerSourceAdapter({ clientFactory: { @@ -298,6 +304,22 @@ export async function localPullConfigForAdapter( ...(dataModelFilter ? { dataModelFilter } : {}), }; } + if (adapter.source === 'tableau') { + const tableauConn = project.config.connections[connectionId]; + const datasourceFilter = + tableauConn && 'datasourceFilter' in tableauConn && tableauConn.datasourceFilter != null + ? (tableauConn.datasourceFilter as { updatedSince?: string }) + : undefined; + const workbookFilter = + tableauConn && 'workbookFilter' in tableauConn && tableauConn.workbookFilter != null + ? (tableauConn.workbookFilter as { updatedSince?: string }) + : undefined; + return { + tableauConnectionId: connectionId, + ...(datasourceFilter ? { datasourceFilter } : {}), + ...(workbookFilter ? { workbookFilter } : {}), + }; + } const connection = project.config.connections[connectionId]; if (adapter.source === HISTORIC_SQL_SOURCE_KEY) { if (options.historicSqlPullConfigOverride) { diff --git a/packages/cli/src/context/project/driver-schemas.ts b/packages/cli/src/context/project/driver-schemas.ts index 72d1a133..6f573530 100644 --- a/packages/cli/src/context/project/driver-schemas.ts +++ b/packages/cli/src/context/project/driver-schemas.ts @@ -302,6 +302,56 @@ const sigmaConnectionSchema = z }) .describe('Sigma Computing API connection for ingesting data models.'); +const tableauConnectionSchema = z + .looseObject({ + driver: z.literal('tableau'), + host: z + .string() + .url() + .describe( + 'Base host URL of the Tableau Cloud pod or Tableau Server, ' + + 'e.g. https://us-west-2b.online.tableau.com or https://tableau.mycompany.com.', + ), + site_content_url: z + .string() + .default('') + .describe('Tableau site content URL (the site subpath). Empty string targets the Default site on Tableau Server.'), + api_version: z + .string() + .optional() + .describe('Tableau REST API version used for sign-in, e.g. "3.29". Defaults to 3.29 when omitted.'), + personal_access_token_name: z.string().min(1).describe('Tableau Personal Access Token name.'), + personal_access_token_secret: z + .string() + .min(1) + .optional() + .describe('Literal Tableau PAT secret. Prefer personal_access_token_secret_ref.'), + personal_access_token_secret_ref: z + .string() + .min(1) + .optional() + .describe('Reference to the Tableau PAT secret (e.g. env:TABLEAU_PAT_SECRET or file:secrets/tableau-pat).'), + datasourceFilter: z + .object({ + updatedSince: z + .string() + .optional() + .describe('ISO 8601 date string. Only data sources updated on or after this date are ingested.'), + }) + .optional() + .describe('Filters applied when listing published data sources during ingest.'), + workbookFilter: z + .object({ + updatedSince: z + .string() + .optional() + .describe('ISO 8601 date string. Only workbooks updated on or after this date are ingested.'), + }) + .optional() + .describe('Filters applied when listing workbooks during ingest.'), + }) + .describe('Tableau Cloud / Server connection for ingesting published data sources and workbooks.'); + export const connectionConfigSchema = z.discriminatedUnion('driver', [ ...warehouseConnectionSchemas, mongodbConnectionSchema, @@ -313,4 +363,5 @@ export const connectionConfigSchema = z.discriminatedUnion('driver', [ dbtConnectionSchema, metricflowConnectionSchema, sigmaConnectionSchema, + tableauConnectionSchema, ]); diff --git a/packages/cli/src/public-ingest.ts b/packages/cli/src/public-ingest.ts index beab6dca..997a4e3b 100644 --- a/packages/cli/src/public-ingest.ts +++ b/packages/cli/src/public-ingest.ts @@ -140,6 +140,7 @@ const sourceAdapterByDriver = new Map([ ['dbt', 'dbt'], ['lookml', 'lookml'], ['sigma', 'sigma'], + ['tableau', 'tableau'], ]); export function publicProgressMessage(message: string, target: KtxPublicIngestPlanTarget): string { diff --git a/packages/cli/src/setup-sources.ts b/packages/cli/src/setup-sources.ts index e13ec6f3..1f1b0a4b 100644 --- a/packages/cli/src/setup-sources.ts +++ b/packages/cli/src/setup-sources.ts @@ -18,6 +18,8 @@ import { cloneOrPull, testRepoConnection } from './context/ingest/repo-fetch.js' import { DEFAULT_METABASE_CLIENT_CONFIG, MetabaseClient } from './context/ingest/adapters/metabase/client.js'; import { DEFAULT_SIGMA_CLIENT_CONFIG, DefaultSigmaClient } from './context/ingest/adapters/sigma/client.js'; import { sigmaRuntimeConfigFromLocalConnection } from './context/ingest/adapters/sigma/local-sigma.adapter.js'; +import { DEFAULT_TABLEAU_CLIENT_CONFIG, DefaultTableauClient } from './context/ingest/adapters/tableau/client.js'; +import { tableauRuntimeConfigFromLocalConnection } from './context/ingest/adapters/tableau/local-tableau.adapter.js'; import { discoverMetabaseDatabases, type DiscoveredMetabaseDatabase } from './context/ingest/adapters/metabase/mapping.js'; import { loadDbtSchemaFiles } from './context/ingest/dbt-shared/schema-files.js'; import { loadProjectInfo } from './context/ingest/dbt-shared/project-vars.js'; @@ -48,7 +50,16 @@ import { type KtxSetupPromptOption, } from './setup-prompts.js'; -export type KtxSetupSourceType = 'dbt' | 'metricflow' | 'metabase' | 'looker' | 'lookml' | 'notion' | 'sigma' | 'gdrive'; +export type KtxSetupSourceType = + | 'dbt' + | 'metricflow' + | 'metabase' + | 'looker' + | 'lookml' + | 'notion' + | 'sigma' + | 'tableau' + | 'gdrive'; const DEFAULT_NOTION_MAX_KNOWLEDGE_CREATES_PER_RUN = 25; @@ -66,6 +77,10 @@ export interface KtxSetupSourcesArgs { sourceApiKeyRef?: string; sourceClientId?: string; sourceClientSecretRef?: string; + /** Tableau Personal Access Token name. */ + sourcePatName?: string; + /** Tableau site content URL (the site subpath); empty string targets the Default site. */ + sourceSiteContentUrl?: string; sourceWarehouseConnectionId?: string; sourceProjectName?: string; sourceProfilesPath?: string; @@ -118,6 +133,7 @@ export interface KtxSetupSourcesDeps { validateLookml?: (connection: KtxProjectConnectionConfig) => Promise; validateNotion?: (connection: KtxProjectConnectionConfig) => Promise; validateSigma?: (connection: KtxProjectConnectionConfig) => Promise; + validateTableau?: (connection: KtxProjectConnectionConfig) => Promise; validateGdrive?: (connection: KtxProjectConnectionConfig) => Promise; pickNotionRootPages?: typeof pickNotionRootPages; discoverMetabaseDatabases?: (args: { @@ -142,6 +158,7 @@ const SOURCE_OPTIONS: Array<{ value: KtxSetupSourceType; label: string }> = [ { value: 'looker', label: 'Looker' }, { value: 'lookml', label: 'LookML' }, { value: 'sigma', label: 'Sigma Computing' }, + { value: 'tableau', label: 'Tableau' }, { value: 'gdrive', label: 'Google Drive' }, ]; @@ -253,6 +270,7 @@ const SOURCE_CREDENTIAL_FLAG: Record = metabase: { field: 'sourceApiKeyRef', flag: '--source-api-key-ref' }, looker: { field: 'sourceClientSecretRef', flag: '--source-client-secret-ref' }, sigma: { field: 'sourceClientSecretRef', flag: '--source-client-secret-ref' }, + tableau: { field: 'sourceClientSecretRef', flag: '--source-client-secret-ref' }, gdrive: { field: null, flag: '--gdrive-service-account-key-ref' }, }; @@ -594,6 +612,22 @@ function buildSigmaConnection(args: KtxSetupSourcesArgs): KtxProjectConnectionCo }; } +function buildTableauConnection(args: KtxSetupSourcesArgs): KtxProjectConnectionConfig { + if (!args.sourceUrl) { + throw new Error('Missing Tableau host: pass --source-url (e.g. https://us-west-2b.online.tableau.com).'); + } + if (!args.sourcePatName) { + throw new Error('Missing Tableau personal access token name: pass --source-pat-name.'); + } + return { + driver: 'tableau', + host: args.sourceUrl, + site_content_url: args.sourceSiteContentUrl ?? '', + personal_access_token_name: args.sourcePatName, + personal_access_token_secret_ref: credentialRef(args.sourceClientSecretRef, 'Tableau PAT secret ref'), + }; +} + function buildGdriveConnection(args: KtxSetupSourcesArgs): KtxProjectConnectionConfig { const folderId = args.gdriveFolderId?.trim(); if (!folderId) { @@ -747,6 +781,23 @@ async function defaultValidateSigma(connection: KtxProjectConnectionConfig): Pro } } +async function defaultValidateTableau(connection: KtxProjectConnectionConfig): Promise { + try { + const runtimeConfig = tableauRuntimeConfigFromLocalConnection('tableau-main', connection); + const client = new DefaultTableauClient(runtimeConfig, DEFAULT_TABLEAU_CLIENT_CONFIG); + try { + const result = await client.testConnection(); + return result.success + ? { ok: true, detail: 'Tableau API connection verified' } + : { ok: false, message: result.error ?? 'Tableau connection test failed' }; + } finally { + await client.cleanup(); + } + } catch (err) { + return { ok: false, message: err instanceof Error ? err.message : String(err) }; + } +} + async function defaultValidateGdrive(connection: KtxProjectConnectionConfig): Promise { const config = parseGdriveConnectionConfig(connection); const keyText = await resolveGdriveServiceAccountKey(config.service_account_key_ref); @@ -1441,6 +1492,52 @@ async function promptForInteractiveSource( ]); } + if (source === 'tableau') { + return await runSourcePromptSteps(initialState, () => [ + ...connectionSteps, + async (state) => { + const sourceUrl = await promptText(prompts, { + message: 'Tableau host URL (Cloud pod or Server), e.g. https://us-west-2b.online.tableau.com', + ...(state.sourceUrl ? { initialValue: state.sourceUrl } : {}), + }); + if (sourceUrl === undefined) return 'back'; + state.sourceUrl = sourceUrl; + return 'next'; + }, + async (state) => { + const siteContentUrl = await promptText(prompts, { + message: 'Tableau site content URL (leave empty for the Default site)', + initialValue: state.sourceSiteContentUrl ?? '', + }); + if (siteContentUrl === undefined) return 'back'; + state.sourceSiteContentUrl = siteContentUrl; + return 'next'; + }, + async (state) => { + const patName = await promptText(prompts, { + message: 'Tableau personal access token name', + ...(state.sourcePatName ? { initialValue: state.sourcePatName } : {}), + }); + if (patName === undefined) return 'back'; + state.sourcePatName = patName; + return 'next'; + }, + async (state) => { + const ref = await chooseSourceCredentialRef({ + prompts, + projectDir: args.projectDir, + label: 'Tableau personal access token secret', + envName: 'TABLEAU_PAT_SECRET', + secretFileName: `${state.sourceConnectionId ?? 'tableau-main'}-pat-secret`, + existingRef: state.sourceClientSecretRef, + }); + if (ref === 'back') return 'back'; + state.sourceClientSecretRef = ref; + return 'next'; + }, + ]); + } + if (source === 'notion') { return await runSourcePromptSteps(initialState, (state) => [ ...connectionSteps, @@ -1716,6 +1813,15 @@ function sourceArgsFromExistingConnection(input: { return sourceArgs; } + if (input.source === 'tableau') { + sourceArgs.sourceUrl = stringField(input.connection.host) ?? undefined; + sourceArgs.sourceSiteContentUrl = stringField(input.connection.site_content_url) ?? undefined; + sourceArgs.sourcePatName = stringField(input.connection.personal_access_token_name) ?? undefined; + sourceArgs.sourceClientSecretRef = + stringField(input.connection.personal_access_token_secret_ref) ?? undefined; + return sourceArgs; + } + if (input.source === 'gdrive') { sourceArgs.gdriveServiceAccountKeyRef = stringField(input.connection.service_account_key_ref); sourceArgs.gdriveFolderId = stringField(input.connection.folder_id); @@ -1907,6 +2013,9 @@ function buildConnection(source: KtxSetupSourceType, args: KtxSetupSourcesArgs): if (source === 'sigma') { return buildSigmaConnection(args); } + if (source === 'tableau') { + return buildTableauConnection(args); + } if (source === 'notion') { return buildNotionConnection(args); } @@ -1938,6 +2047,9 @@ async function validateSource( if (source === 'sigma') { return await (deps.validateSigma ?? defaultValidateSigma)(args.connection); } + if (source === 'tableau') { + return await (deps.validateTableau ?? defaultValidateTableau)(args.connection); + } if (source === 'notion') { return await (deps.validateNotion ?? defaultValidateNotion)(args.connection); } diff --git a/packages/cli/test/context/ingest/adapters/tableau/chunk.test.ts b/packages/cli/test/context/ingest/adapters/tableau/chunk.test.ts new file mode 100644 index 00000000..350c6a46 --- /dev/null +++ b/packages/cli/test/context/ingest/adapters/tableau/chunk.test.ts @@ -0,0 +1,36 @@ +import { resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { chunkTableauStagedDir } from '../../../../../src/context/ingest/adapters/tableau/chunk.js'; + +const FIXTURES = resolve(import.meta.dirname, '../../../../fixtures/tableau'); + +describe('chunkTableauStagedDir', () => { + it('emits one data-sources unit and one workbooks unit', async () => { + const result = await chunkTableauStagedDir(resolve(FIXTURES, 'single')); + const keys = result.workUnits.map((u) => u.unitKey).sort(); + expect(keys).toEqual(['tableau-datasources', 'tableau-workbooks']); + + const dsUnit = result.workUnits.find((u) => u.unitKey === 'tableau-datasources')!; + expect(dsUnit.rawFiles).toEqual(['datasources/ds-1.json']); + // The workbook file is available as a peer, not a raw file, for the data-sources unit. + expect(dsUnit.peerFileIndex).toContain('workbooks/wb-1.json'); + }); + + it('returns no work units when the manifest is absent', async () => { + const result = await chunkTableauStagedDir(resolve(FIXTURES, 'single', 'datasources')); + expect(result.workUnits).toEqual([]); + }); + + it('is deterministic across runs', async () => { + const a = await chunkTableauStagedDir(resolve(FIXTURES, 'single')); + const b = await chunkTableauStagedDir(resolve(FIXTURES, 'single')); + expect(JSON.stringify(a)).toBe(JSON.stringify(b)); + }); + + it('keeps only touched units when a diffSet is supplied', async () => { + const result = await chunkTableauStagedDir(resolve(FIXTURES, 'single'), { + diffSet: { added: [], modified: ['datasources/ds-1.json'], deleted: [], unchanged: ['workbooks/wb-1.json'] }, + }); + expect(result.workUnits.map((u) => u.unitKey)).toEqual(['tableau-datasources']); + }); +}); diff --git a/packages/cli/test/context/ingest/adapters/tableau/client-boundary.test.ts b/packages/cli/test/context/ingest/adapters/tableau/client-boundary.test.ts new file mode 100644 index 00000000..8381bb9a --- /dev/null +++ b/packages/cli/test/context/ingest/adapters/tableau/client-boundary.test.ts @@ -0,0 +1,19 @@ +import { readFile } from 'node:fs/promises'; +import { describe, expect, it } from 'vitest'; + +describe('TableauClient boundary', () => { + it('does not import server, NestJS, or ingest-internal modules', async () => { + const source = await readFile( + new URL('../../../../../src/context/ingest/adapters/tableau/client.ts', import.meta.url), + 'utf-8', + ); + + expect(source).not.toMatch(/@nestjs\/common/); + expect(source).not.toMatch(/DataSourceClient/); + expect(source).not.toMatch(/\.\.\/interfaces/); + // The client must only depend on its own adapter folder (client-port / types), + // never on the shared ingest types or anything above the adapter directory. + expect(source).not.toMatch(/from '\.\.\//); + expect(source).not.toMatch(/server\/src/); + }); +}); diff --git a/packages/cli/test/context/ingest/adapters/tableau/client.test.ts b/packages/cli/test/context/ingest/adapters/tableau/client.test.ts new file mode 100644 index 00000000..6872b511 --- /dev/null +++ b/packages/cli/test/context/ingest/adapters/tableau/client.test.ts @@ -0,0 +1,202 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { DefaultTableauClient } from '../../../../../src/context/ingest/adapters/tableau/client.js'; + +const HOST = 'https://us-west-2b.online.tableau.com'; + +const SIGNIN_RESPONSE = { + credentials: { + token: 'test-auth-token', + site: { id: 'site-1', contentUrl: 'mysite' }, + user: { id: 'user-1' }, + }, +}; + +function makeResponse(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'Content-Type': 'application/json' }, + }); +} + +function datasourcesPage(nodes: unknown[], hasNextPage = false) { + return makeResponse({ + data: { publishedDatasourcesConnection: { totalCount: nodes.length, pageInfo: { hasNextPage }, nodes } }, + }); +} + +function workbooksPage(nodes: unknown[], hasNextPage = false) { + return makeResponse({ + data: { workbooksConnection: { totalCount: nodes.length, pageInfo: { hasNextPage }, nodes } }, + }); +} + +function makeClient(): DefaultTableauClient { + return new DefaultTableauClient( + { host: HOST, siteContentUrl: 'mysite', apiVersion: '3.29', patName: 'ci', patSecret: 'secret' }, // pragma: allowlist secret + { maxRetries: 1, baseDelayMs: 0, maxDelayMs: 0, timeoutMs: 5000, pageSize: 2 }, + ); +} + +beforeEach(() => { + globalThis.fetch = vi.fn(); +}); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe('DefaultTableauClient.testConnection', () => { + it('signs in with the PAT and returns success', async () => { + vi.mocked(fetch).mockResolvedValueOnce(makeResponse(SIGNIN_RESPONSE)); + const client = makeClient(); + const result = await client.testConnection(); + expect(result.success).toBe(true); + + const [url, init] = vi.mocked(fetch).mock.calls[0]!; + expect(String(url)).toBe(`${HOST}/api/3.29/auth/signin`); + const body = JSON.parse(String((init as RequestInit).body)); + expect(body.credentials.personalAccessTokenName).toBe('ci'); + expect(body.credentials.personalAccessTokenSecret).toBe('secret'); + expect(body.credentials.site.contentUrl).toBe('mysite'); + }); + + it('returns success:false with the error when sign-in fails', async () => { + vi.mocked(fetch).mockResolvedValueOnce(makeResponse({ error: 'unauthorized' }, 401)); + const client = makeClient(); + const result = await client.testConnection(); + expect(result.success).toBe(false); + expect(result.error).toMatch(/401/); + }); +}); + +describe('DefaultTableauClient.listDatasources', () => { + it('sends the auth token as X-Tableau-Auth on metadata calls', async () => { + vi.mocked(fetch).mockResolvedValueOnce(makeResponse(SIGNIN_RESPONSE)).mockResolvedValueOnce(datasourcesPage([])); + const client = makeClient(); + await client.listDatasources(); + const [url, init] = vi.mocked(fetch).mock.calls[1]!; + expect(String(url)).toBe(`${HOST}/api/metadata/graphql`); + expect((init as RequestInit).headers).toMatchObject({ 'X-Tableau-Auth': 'test-auth-token' }); + }); + + it('maps column fields and calculated fields (with formula)', async () => { + vi.mocked(fetch) + .mockResolvedValueOnce(makeResponse(SIGNIN_RESPONSE)) + .mockResolvedValueOnce( + datasourcesPage([ + { + luid: 'ds-1', + name: 'Sales', + projectName: 'Finance', + updatedAt: '2026-01-01T00:00:00Z', + hasExtracts: true, + upstreamTables: [{ luid: 't-1', name: 'ORDERS', schema: 'PUBLIC', fullName: 'DB.PUBLIC.ORDERS' }], + fields: [ + { __typename: 'ColumnField', name: 'amount', dataType: 'INTEGER', role: 'MEASURE' }, + { __typename: 'CalculatedField', name: 'Profit', formula: '[Revenue] - [Cost]', dataType: 'REAL', role: 'MEASURE' }, + ], + }, + ]), + ); + const client = makeClient(); + const datasources = await client.listDatasources(); + expect(datasources).toHaveLength(1); + const ds = datasources[0]!; + expect(ds.name).toBe('Sales'); + expect(ds.upstreamTables[0]!.fullName).toBe('DB.PUBLIC.ORDERS'); + const calc = ds.fields.find((f) => f.name === 'Profit'); + expect(calc?.formula).toBe('[Revenue] - [Cost]'); + const col = ds.fields.find((f) => f.name === 'amount'); + expect(col?.formula).toBeUndefined(); + }); + + it('paginates using offset until hasNextPage is false', async () => { + vi.mocked(fetch) + .mockResolvedValueOnce(makeResponse(SIGNIN_RESPONSE)) + .mockResolvedValueOnce(datasourcesPage([{ luid: 'ds-1', name: 'A', fields: [], upstreamTables: [] }], true)) + .mockResolvedValueOnce(datasourcesPage([{ luid: 'ds-2', name: 'B', fields: [], upstreamTables: [] }], false)); + const client = makeClient(); + const datasources = await client.listDatasources(); + expect(datasources.map((d) => d.name)).toEqual(['A', 'B']); + const secondQuery = JSON.parse(String((vi.mocked(fetch).mock.calls[2]![1] as RequestInit).body)); + expect(secondQuery.variables.offset).toBe(2); + }); + + it('filters by updatedSince', async () => { + vi.mocked(fetch) + .mockResolvedValueOnce(makeResponse(SIGNIN_RESPONSE)) + .mockResolvedValueOnce( + datasourcesPage([ + { luid: 'ds-1', name: 'Old', updatedAt: '2026-01-10T00:00:00Z', fields: [], upstreamTables: [] }, + { luid: 'ds-2', name: 'New', updatedAt: '2026-01-20T00:00:00Z', fields: [], upstreamTables: [] }, + ]), + ); + const client = makeClient(); + const datasources = await client.listDatasources({ updatedSince: '2026-01-15T00:00:00Z' }); + expect(datasources.map((d) => d.name)).toEqual(['New']); + }); +}); + +describe('DefaultTableauClient.listWorkbooks', () => { + it('returns workbook summaries', async () => { + vi.mocked(fetch) + .mockResolvedValueOnce(makeResponse(SIGNIN_RESPONSE)) + .mockResolvedValueOnce( + workbooksPage([{ luid: 'wb-1', name: 'ARR Tracker', projectName: 'Finance', description: 'ARR by segment' }]), + ); + const client = makeClient(); + const workbooks = await client.listWorkbooks(); + expect(workbooks).toHaveLength(1); + expect(workbooks[0]!.name).toBe('ARR Tracker'); + }); +}); + +describe('DefaultTableauClient — error handling', () => { + it('retries on 500 and succeeds on retry', async () => { + vi.mocked(fetch) + .mockResolvedValueOnce(makeResponse(SIGNIN_RESPONSE)) + .mockResolvedValueOnce(makeResponse({ error: 'server error' }, 500)) + .mockResolvedValueOnce(datasourcesPage([])); + const client = makeClient(); + const datasources = await client.listDatasources(); + expect(datasources).toHaveLength(0); + }); + + it('surfaces GraphQL-level errors', async () => { + vi.mocked(fetch) + .mockResolvedValueOnce(makeResponse(SIGNIN_RESPONSE)) + .mockResolvedValueOnce(makeResponse({ errors: [{ message: 'Field "bogus" not found' }] })); + const client = makeClient(); + await expect(client.listDatasources()).rejects.toThrow(/bogus/); + }); + + it('re-authenticates and retries on 401', async () => { + vi.mocked(fetch) + .mockResolvedValueOnce(makeResponse(SIGNIN_RESPONSE)) // initial sign-in + .mockResolvedValueOnce(makeResponse({ error: 'expired' }, 401)) // 401 on first query + .mockResolvedValueOnce(makeResponse(SIGNIN_RESPONSE)) // re-sign-in + .mockResolvedValueOnce(datasourcesPage([])); // retried query + const client = makeClient(); + const datasources = await client.listDatasources(); + expect(datasources).toHaveLength(0); + }); +}); + +describe('DefaultTableauClient.cleanup', () => { + it('signs out and clears the token so the next call re-authenticates', async () => { + vi.mocked(fetch) + .mockResolvedValueOnce(makeResponse(SIGNIN_RESPONSE)) // first sign-in + .mockResolvedValueOnce(datasourcesPage([])) // first list + .mockResolvedValueOnce(makeResponse({})) // signout + .mockResolvedValueOnce(makeResponse(SIGNIN_RESPONSE)) // second sign-in + .mockResolvedValueOnce(datasourcesPage([])); // second list + const client = makeClient(); + await client.listDatasources(); + await client.cleanup(); + await client.listDatasources(); + + const signoutCall = vi.mocked(fetch).mock.calls[2]!; + expect(String(signoutCall[0])).toBe(`${HOST}/api/3.29/auth/signout`); + expect(vi.mocked(fetch)).toHaveBeenCalledTimes(5); + }); +}); diff --git a/packages/cli/test/context/ingest/adapters/tableau/detect.test.ts b/packages/cli/test/context/ingest/adapters/tableau/detect.test.ts new file mode 100644 index 00000000..0dffc53d --- /dev/null +++ b/packages/cli/test/context/ingest/adapters/tableau/detect.test.ts @@ -0,0 +1,41 @@ +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { detectTableauStagedDir } from '../../../../../src/context/ingest/adapters/tableau/detect.js'; + +const FIXTURES = resolve(import.meta.dirname, '../../../../fixtures/tableau'); + +describe('detectTableauStagedDir', () => { + let dir: string; + + beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), 'tableau-detect-')); + }); + + afterEach(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + it('returns true for a valid staged dir with a manifest and a datasource', async () => { + expect(await detectTableauStagedDir(join(FIXTURES, 'single'))).toBe(true); + }); + + it('returns false when the manifest is missing', async () => { + await mkdir(join(dir, 'datasources'), { recursive: true }); + await writeFile(join(dir, 'datasources', 'ds-1.json'), '{}', 'utf-8'); + expect(await detectTableauStagedDir(dir)).toBe(false); + }); + + it('returns false when the manifest exists but no datasource/workbook json is present', async () => { + await writeFile(join(dir, 'tableau-manifest.json'), '{}', 'utf-8'); + expect(await detectTableauStagedDir(dir)).toBe(false); + }); + + it('returns true when only a workbook json is present alongside the manifest', async () => { + await writeFile(join(dir, 'tableau-manifest.json'), '{}', 'utf-8'); + await mkdir(join(dir, 'workbooks'), { recursive: true }); + await writeFile(join(dir, 'workbooks', 'wb-1.json'), '{}', 'utf-8'); + expect(await detectTableauStagedDir(dir)).toBe(true); + }); +}); diff --git a/packages/cli/test/context/ingest/adapters/tableau/fetch.test.ts b/packages/cli/test/context/ingest/adapters/tableau/fetch.test.ts new file mode 100644 index 00000000..38d4f031 --- /dev/null +++ b/packages/cli/test/context/ingest/adapters/tableau/fetch.test.ts @@ -0,0 +1,113 @@ +import { mkdtemp, readFile, readdir, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { + TableauClientFactory, + TableauDatasourceRecord, + TableauRuntimeClient, + TableauWorkbookRecord, +} from '../../../../../src/context/ingest/adapters/tableau/client-port.js'; +import { fetchTableauBundle } from '../../../../../src/context/ingest/adapters/tableau/fetch.js'; + +const CTX = { connectionId: 'tableau-main', sourceKey: 'tableau' }; + +function fakeClient(overrides: Partial = {}): TableauRuntimeClient { + return { + testConnection: vi.fn().mockResolvedValue({ success: true }), + listDatasources: vi.fn().mockResolvedValue([] as TableauDatasourceRecord[]), + listWorkbooks: vi.fn().mockResolvedValue([] as TableauWorkbookRecord[]), + cleanup: vi.fn().mockResolvedValue(undefined), + ...overrides, + }; +} + +function factoryFor(client: TableauRuntimeClient): TableauClientFactory { + return { createClient: vi.fn().mockResolvedValue(client) }; +} + +const NOW = () => new Date('2026-04-30T12:30:00.000Z'); + +describe('fetchTableauBundle', () => { + let dir: string; + + beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), 'tableau-fetch-')); + }); + + afterEach(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + it('stages data sources (with calculated fields + upstream tables) and workbooks', async () => { + const client = fakeClient({ + listDatasources: vi.fn().mockResolvedValue([ + { + luid: 'ds-1', + name: 'Revenue Model', + projectName: 'Finance', + updatedAt: '2026-04-01T00:00:00Z', + hasExtracts: true, + fields: [ + { name: 'amount', role: 'MEASURE', dataType: 'INTEGER' }, + { name: 'Net Revenue', role: 'MEASURE', dataType: 'REAL', formula: 'SUM([g]) - SUM([r])' }, + ], + upstreamTables: [{ name: 'ORDERS', schema: 'PUBLIC', fullName: 'DEMO.PUBLIC.ORDERS' }], + } satisfies TableauDatasourceRecord, + ]), + listWorkbooks: vi.fn().mockResolvedValue([ + { luid: 'wb-1', name: 'ARR Tracker', projectName: 'Finance', description: 'ARR by segment' } satisfies TableauWorkbookRecord, + ]), + }); + + await fetchTableauBundle({ + pullConfig: { tableauConnectionId: 'tableau-main' }, + stagedDir: dir, + ctx: CTX, + clientFactory: factoryFor(client), + now: NOW, + }); + + expect((await readdir(join(dir, 'datasources'))).sort()).toEqual(['ds-1.json']); + expect((await readdir(join(dir, 'workbooks'))).sort()).toEqual(['wb-1.json']); + + const ds = JSON.parse(await readFile(join(dir, 'datasources', 'ds-1.json'), 'utf-8')); + expect(ds.name).toBe('Revenue Model'); + expect(ds.upstreamTables[0].fullName).toBe('DEMO.PUBLIC.ORDERS'); + const calc = ds.fields.find((f: { name: string }) => f.name === 'Net Revenue'); + expect(calc.isCalculated).toBe(true); + expect(calc.formula).toBe('SUM([g]) - SUM([r])'); + const col = ds.fields.find((f: { name: string }) => f.name === 'amount'); + expect(col.isCalculated).toBe(false); + + const manifest = JSON.parse(await readFile(join(dir, 'tableau-manifest.json'), 'utf-8')); + expect(manifest).toMatchObject({ + tableauConnectionId: 'tableau-main', + fetchedAt: '2026-04-30T12:30:00.000Z', + datasourceCount: 1, + workbookCount: 1, + }); + + // A projection config is always written for the chunk step / skill to read. + const projection = JSON.parse(await readFile(join(dir, 'tableau-projection-config.json'), 'utf-8')); + expect(projection).toBeTypeOf('object'); + + expect(client.cleanup).toHaveBeenCalledTimes(1); + }); + + it('calls cleanup even when listing throws', async () => { + const client = fakeClient({ + listDatasources: vi.fn().mockRejectedValue(new Error('boom')), + }); + await expect( + fetchTableauBundle({ + pullConfig: { tableauConnectionId: 'tableau-main' }, + stagedDir: dir, + ctx: CTX, + clientFactory: factoryFor(client), + now: NOW, + }), + ).rejects.toThrow(/boom/); + expect(client.cleanup).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/cli/test/context/ingest/adapters/tableau/tableau.adapter.test.ts b/packages/cli/test/context/ingest/adapters/tableau/tableau.adapter.test.ts new file mode 100644 index 00000000..c66bdd08 --- /dev/null +++ b/packages/cli/test/context/ingest/adapters/tableau/tableau.adapter.test.ts @@ -0,0 +1,64 @@ +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { + TableauClientFactory, + TableauRuntimeClient, +} from '../../../../../src/context/ingest/adapters/tableau/client-port.js'; +import { TableauSourceAdapter } from '../../../../../src/context/ingest/adapters/tableau/tableau.adapter.js'; + +const FIXTURES = resolve(import.meta.dirname, '../../../../fixtures/tableau'); +const CTX = { connectionId: 'tableau-main', sourceKey: 'tableau' }; + +function fakeClient(): TableauRuntimeClient { + return { + testConnection: vi.fn().mockResolvedValue({ success: true }), + listDatasources: vi.fn().mockResolvedValue([]), + listWorkbooks: vi.fn().mockResolvedValue([]), + cleanup: vi.fn().mockResolvedValue(undefined), + }; +} + +function adapter(client: TableauRuntimeClient): TableauSourceAdapter { + const clientFactory: TableauClientFactory = { createClient: vi.fn().mockResolvedValue(client) }; + return new TableauSourceAdapter({ clientFactory, now: () => new Date('2026-04-30T12:30:00.000Z') }); +} + +describe('TableauSourceAdapter', () => { + let dir: string; + + beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), 'tableau-adapter-')); + }); + + afterEach(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + it('exposes the tableau source key and ingest skill', () => { + const a = adapter(fakeClient()); + expect(a.source).toBe('tableau'); + expect(a.skillNames).toEqual(['tableau_ingest']); + }); + + it('detects a staged fixture bundle', async () => { + const a = adapter(fakeClient()); + expect(await a.detect(resolve(FIXTURES, 'single'))).toBe(true); + }); + + it('fetch() drives the client and produces a detectable, chunkable bundle', async () => { + const client = fakeClient(); + vi.mocked(client.listDatasources).mockResolvedValue([ + { luid: 'ds-1', name: 'Sales', fields: [], upstreamTables: [] }, + ]); + const a = adapter(client); + + await a.fetch({ tableauConnectionId: 'tableau-main' }, dir, CTX); + + expect(client.listDatasources).toHaveBeenCalledTimes(1); + expect(await a.detect(dir)).toBe(true); + const chunked = await a.chunk(dir); + expect(chunked.workUnits.map((u) => u.unitKey)).toContain('tableau-datasources'); + }); +}); diff --git a/packages/cli/test/fixtures/tableau/single/datasources/ds-1.json b/packages/cli/test/fixtures/tableau/single/datasources/ds-1.json new file mode 100644 index 00000000..bd7dcaaf --- /dev/null +++ b/packages/cli/test/fixtures/tableau/single/datasources/ds-1.json @@ -0,0 +1,22 @@ +{ + "luid": "ds-1", + "name": "Revenue Model", + "projectName": "Finance", + "updatedAt": "2026-04-01T00:00:00Z", + "hasExtracts": true, + "description": "Published revenue data source", + "fields": [ + { "name": "Order Amount", "role": "MEASURE", "dataType": "INTEGER", "isCalculated": false }, + { + "name": "Net Revenue", + "role": "MEASURE", + "dataType": "REAL", + "formula": "SUM([Gross Revenue]) - SUM([Refunds])", + "description": "Gross revenue minus refunds", + "isCalculated": true + } + ], + "upstreamTables": [ + { "luid": "t-1", "name": "ORDERS", "schema": "PUBLIC", "fullName": "DEMO.PUBLIC.ORDERS" } + ] +} diff --git a/packages/cli/test/fixtures/tableau/single/tableau-manifest.json b/packages/cli/test/fixtures/tableau/single/tableau-manifest.json new file mode 100644 index 00000000..6e2f5f8a --- /dev/null +++ b/packages/cli/test/fixtures/tableau/single/tableau-manifest.json @@ -0,0 +1,6 @@ +{ + "tableauConnectionId": "tableau-main", + "fetchedAt": "2026-04-30T12:30:00.000Z", + "datasourceCount": 1, + "workbookCount": 1 +} diff --git a/packages/cli/test/fixtures/tableau/single/workbooks/wb-1.json b/packages/cli/test/fixtures/tableau/single/workbooks/wb-1.json new file mode 100644 index 00000000..40516a4f --- /dev/null +++ b/packages/cli/test/fixtures/tableau/single/workbooks/wb-1.json @@ -0,0 +1,7 @@ +{ + "luid": "wb-1", + "name": "ARR Tracker", + "projectName": "Finance", + "description": "Tracks ARR by segment and cohort for the finance team", + "updatedAt": "2026-04-02T00:00:00Z" +} From 8013642ac54b98ef800998226e3b3d5b5230b42d Mon Sep 17 00:00:00 2001 From: saksham Date: Mon, 6 Jul 2026 00:25:26 +0530 Subject: [PATCH 2/4] feat(connectors): add tableau_ingest skill for wiki knowledge capture Adds packages/cli/src/skills/tableau_ingest/SKILL.md, the memory-agent skill loaded for tableau-datasources / tableau-workbooks work units. It extracts durable ktx wiki knowledge from published data sources (field names, calculated-field formulas as metric definitions) and workbook summaries via context_candidate_write, with the standard identifier-verification protocol. v1 is wiki-only: the skill states that no semantic-layer YAML is written for Tableau yet (deterministic project() follows in the next commit). Registers tableau_ingest in the memory-runtime-assets adapter-skill and verification-writer checks. Refs #166. --- .../cli/src/skills/tableau_ingest/SKILL.md | 146 ++++++++++++++++++ .../memory/memory-runtime-assets.test.ts | 2 + 2 files changed, 148 insertions(+) create mode 100644 packages/cli/src/skills/tableau_ingest/SKILL.md diff --git a/packages/cli/src/skills/tableau_ingest/SKILL.md b/packages/cli/src/skills/tableau_ingest/SKILL.md new file mode 100644 index 00000000..db73dadb --- /dev/null +++ b/packages/cli/src/skills/tableau_ingest/SKILL.md @@ -0,0 +1,146 @@ +--- +name: tableau_ingest +description: Extract durable ktx wiki knowledge from staged Tableau published data sources (fields, calculated fields) and workbook summaries. Load for WorkUnits with unitKey tableau-datasources or tableau-workbooks. +callers: [memory_agent] +--- + +# Tableau Ingest + +Tableau ingest turns staged published-data-source definitions and workbook summaries into durable +ktx wiki knowledge. Each published data source is a graph-shaped semantic model: its fields +(including calculated fields with formulas) plus the upstream physical tables it draws from. + +## Work unit structure + +Tableau produces at minimum two work units per ingest run: + +- `tableau-datasources` or `tableau-datasources-N` + - `rawFiles`: `datasources/.json` files (one per published data source in this batch) + - `peerFileIndex`: `workbooks/.json` files + `tableau-manifest.json` + `tableau-projection-config.json` + - When the site has more than 50 data sources, split into batches: `tableau-datasources-0`, + `tableau-datasources-1`, … with `displayLabel` like `"Tableau: data sources (1/8)"`. When ≤50, + the unitKey is simply `tableau-datasources` with no suffix. +- `tableau-workbooks` or `tableau-workbooks-N` + - `rawFiles`: `workbooks/.json` files (one per workbook in this batch) + - `peerFileIndex`: `datasources/.json` files + `tableau-manifest.json` + `tableau-projection-config.json` + - When the site has more than 2000 workbooks, split into batches: `tableau-workbooks-0`, … with + `displayLabel` like `"Tableau: workbooks (1/4)"`. When ≤2000, the unitKey is simply + `tableau-workbooks` with no suffix. + +`tableau-manifest.json` and `tableau-projection-config.json` are never in `rawFiles`. They live at +the staged dir root and always appear in `peerFileIndex`. + +## Staged file shapes + +**`datasources/.json`** — one per published data source (in `rawFiles` for data-source units): +```json +{ + "luid": "ds-1", + "name": "Revenue Model", + "projectName": "Finance", + "updatedAt": "2026-04-01T00:00:00Z", + "hasExtracts": true, + "description": "Published revenue data source", + "fields": [ + { "name": "Order Amount", "role": "MEASURE", "dataType": "INTEGER", "isCalculated": false }, + { + "name": "Net Revenue", + "role": "MEASURE", + "dataType": "REAL", + "formula": "SUM([Gross Revenue]) - SUM([Refunds])", + "description": "Gross revenue minus refunds", + "isCalculated": true + } + ], + "upstreamTables": [ + { "luid": "t-1", "name": "ORDERS", "schema": "PUBLIC", "fullName": "DEMO.PUBLIC.ORDERS" } + ] +} +``` + +- `fields[]` — every field on the data source. `isCalculated: true` (equivalently, a non-empty + `formula`) marks a Tableau **calculated field**; the `formula` is the business logic to capture. + `role` is `DIMENSION` or `MEASURE`. +- `upstreamTables[]` — the physical warehouse tables the data source draws from (lineage). Use + `fullName` / `schema` when verifying identifiers. + +**`workbooks/.json`** — one per workbook (in `rawFiles` for workbook units; summary only): +```json +{ + "luid": "wb-1", + "name": "ARR Tracker", + "projectName": "Finance", + "description": "Tracks ARR by segment and cohort for the finance team", + "updatedAt": "2026-04-02T00:00:00Z" +} +``` + +**Peer files (available via `peerFileIndex`, not `rawFiles`):** + +**`tableau-manifest.json`** — fetch summary; use for provenance only. + +**`tableau-projection-config.json`** — written by `fetch()`; records the `datasourceFilter` / +`workbookFilter` that were active during the last fetch. When `updatedSince` was set, the staged set +is a recent-changes slice, not the full site — do not infer that absent data sources or workbooks +were deleted. + +## Required workflow + +1. Read every `rawFiles` entry for the WorkUnit. +2. Read `tableau-projection-config.json` from the staged dir to understand the fetch window. +3. For each data source file: extract business semantics from the data source name/description, + field names, calculated-field formulas, and field descriptions. A calculated field's `formula` + often encodes a governed metric definition worth capturing. +4. For each workbook file: extract business domain knowledge from the name and description. When a + filter `updatedSince` is set, treat the staged set as a recent-changes slice. +5. Use `discover_data` before writing to find existing wiki pages on the same topic. +6. Write wiki candidates with `context_candidate_write`. Do not call `wiki_write` directly from a + Tableau WorkUnit; Stage 4 reconciliation promotes candidates. + +## Identifier Verification Protocol + +Before writing a wiki page on any topic: + +1. `discover_data({query: ""})` — see what wikis, SL sources, and raw tables already exist. + Prefer updating existing pages over creating new ones. + +Before emitting any `schema.table` or `schema.table.column` into a wiki body, `tables:` frontmatter, +`sl_refs`, or `emit_unmapped_fallback`: + +2. `entity_details({connectionId, targets: [{display: ""}]})` — confirm the identifier + resolves; inspect native types, FK/PK, and sampleValues. Use a warehouse `connectionId` that + covers the data source's `upstreamTables[].fullName`. If no warehouse connection covers it, skip + `entity_details` and wrap any identifier references with `[unverified - from ]`. +3. For literal values from a formula (status codes, plan tiers), check whether they appear in + `entity_details` sampleValues for the relevant column. If sampleValues is short or may have missed + real values, run a probe with the same warehouse connection id: + `sql_execution({connectionId, sql: "SELECT DISTINCT FROM LIMIT 50"})`. +4. If the candidate identifier still does not resolve, do one of: + - Use `sql_execution({connectionId, sql: "SELECT 1 FROM LIMIT 0"})`. If it errors, the + identifier is fictional. + - Wrap the identifier in `[unverified - from ]` in the wiki body, citing the exact raw + path that mentioned it. +5. Never copy `.` placeholder strings from these instructions into output. + +## Capture rules + +Write wiki candidates for: +- Metric definitions encoded in calculated-field names and formulas (e.g. "Net Revenue", + "Churned ARR") — capture the business meaning, and the formula when it clarifies the definition. +- Domain conventions revealed by field names/descriptions (segment taxonomies, cohort or fiscal + rules). +- Business concepts a workbook name/description reveals (e.g. "ARR Tracker" → ARR tracking). Write + one candidate per distinct concept, not one per workbook. + +Skip: +- Visualization settings, layout, colors, chart types. +- Owner names, project/folder paths, and version numbers as wiki narrative. +- Hidden or purely technical fields with no business meaning. +- Workbooks whose name/description carries no durable semantics (e.g. "Untitled", "Test Dashboard"). + +## Note on semantic-layer output + +In this version the Tableau adapter does not run a deterministic `project()` step, so **no +semantic-layer YAML is written for Tableau** — all durable output is wiki knowledge. Do not claim an +SL source exists for a data source; reference upstream warehouse tables only via the verification +protocol above. diff --git a/packages/cli/test/context/memory/memory-runtime-assets.test.ts b/packages/cli/test/context/memory/memory-runtime-assets.test.ts index 92f72f38..48daa253 100644 --- a/packages/cli/test/context/memory/memory-runtime-assets.test.ts +++ b/packages/cli/test/context/memory/memory-runtime-assets.test.ts @@ -25,6 +25,7 @@ const expectedAdapterSkillHeadings: Record = { metabase_ingest: '# Metabase to ktx Semantic Layer', metricflow_ingest: '# MetricFlow to ktx Semantic Layer', sigma_ingest: '# Sigma Ingest', + tableau_ingest: '# Tableau Ingest', }; const verificationWriterSkills = [ 'gdrive_synthesize', @@ -35,6 +36,7 @@ const verificationWriterSkills = [ 'metabase_ingest', 'metricflow_ingest', 'sigma_ingest', + 'tableau_ingest', 'live_database_ingest', 'historic_sql_table_digest', 'historic_sql_patterns', From fd4f4f95b03de02cde1a293d7213e70df396060a Mon Sep 17 00:00:00 2001 From: saksham Date: Mon, 6 Jul 2026 00:31:37 +0530 Subject: [PATCH 3/4] docs(connectors): document the Tableau context source Adds a Tableau section to the context-sources integration guide (connection config, PAT auth via env:/file: reference, Cloud vs Server host, data source / workbook filters, and a pointer to the free Tableau Developer Program sandbox) and lists `tableau` in the driver table, page description, and README connector list. Notes that this release is wiki-only; deterministic semantic-layer projection is planned for a follow-up. Refs #166. --- README.md | 2 +- .../docs/integrations/context-sources.mdx | 80 ++++++++++++++++++- 2 files changed, 79 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 618e9c81..0391fb9f 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ upkeep and don't absorb the rest of your company's knowledge. Works with PostgreSQL, Snowflake, BigQuery, ClickHouse, MySQL, SQL Server, SQLite, DuckDB, Amazon Athena, and MongoDB. Integrates with dbt, MetricFlow, -LookML, Looker, Metabase, Sigma, Notion, and Google Drive. +LookML, Looker, Metabase, Sigma, Tableau, Notion, and Google Drive. ## Quick Start diff --git a/docs-site/content/docs/integrations/context-sources.mdx b/docs-site/content/docs/integrations/context-sources.mdx index 3bfc3ff3..718693f5 100644 --- a/docs-site/content/docs/integrations/context-sources.mdx +++ b/docs-site/content/docs/integrations/context-sources.mdx @@ -1,6 +1,6 @@ --- title: Context Sources -description: Ingest semantic context from dbt, MetricFlow, LookML, Metabase, Looker, Notion, Sigma, and Google Drive. +description: Ingest semantic context from dbt, MetricFlow, LookML, Metabase, Looker, Notion, Sigma, Tableau, and Google Drive. --- Context sources feed your existing analytics tooling into **ktx**. During ingestion, **ktx** extracts metadata from each source and uses a reconciliation agent to reconcile it with your existing semantic layer and knowledge base - preserving accepted edits rather than overwriting. @@ -27,7 +27,7 @@ LookML uses top-level `repoUrl`, and MetricFlow uses nested | Field | Required | Description | |-------|----------|-------------| -| `driver` | Yes | Source connector: `dbt`, `metricflow`, `lookml`, `metabase`, `looker`, `notion`, `sigma`, or `gdrive` | +| `driver` | Yes | Source connector: `dbt`, `metricflow`, `lookml`, `metabase`, `looker`, `notion`, `sigma`, `tableau`, or `gdrive` | | `source_dir` | For local file sources | Absolute or project-relative source directory | | `repo_url` | For Git-hosted dbt sources | Git repository URL | | `repoUrl` | For Git-hosted LookML sources | Git repository URL | @@ -473,6 +473,82 @@ connections: --- +## Tableau + +Ingests published data sources, calculated fields, and workbook metadata from **Tableau Cloud** or **Tableau Server** as semantic context. Uses the Tableau REST API for authentication (Personal Access Token) and the Tableau **Metadata API** (GraphQL) to fetch data source definitions and lineage. + +### What it provides + +- Published data source names, projects, and descriptions +- Field definitions — dimensions and measures, with data types +- **Calculated fields** with their formulas (captured as candidate metric definitions) +- Upstream (physical) warehouse tables each data source draws from +- Workbook names, projects, and descriptions + +### Connection config + +```yaml title="ktx.yaml" +connections: + tableau-main: + driver: tableau + host: https://us-west-2b.online.tableau.com # Tableau Cloud pod, or your Server host + site_content_url: "your-site" # site subpath; "" for the Default site on Server + personal_access_token_name: "ktx-ingest" + personal_access_token_secret_ref: env:TABLEAU_PAT_SECRET +``` + +**Tableau Cloud** uses your pod hostname (e.g. `https://10ax.online.tableau.com`); **Tableau Server** uses your own host (e.g. `https://tableau.mycompany.com`). On Cloud a non-default `site_content_url` is almost always required. Override the REST API version with `api_version` if needed (defaults to `3.29`). + +### Authentication + +| Method | Config | +|--------|--------| +| Personal Access Token | `personal_access_token_name` + `personal_access_token_secret_ref: env:TABLEAU_PAT_SECRET` | + +Create a PAT in Tableau: **My Account Settings → Personal Access Tokens → Create Token**. The secret is shown only once at creation. Provide it to ktx via an `env:` or `file:` reference — never inline the literal secret in `ktx.yaml`: + +```yaml title="ktx.yaml" + personal_access_token_secret_ref: file:secrets/tableau-pat # or env:TABLEAU_PAT_SECRET +``` + +Don't have an instance to test with? Join the free [Tableau Developer Program](https://www.tableau.com/developer) for a personal Tableau Cloud sandbox (the Metadata API is enabled by default) and create a PAT there. + +### What gets ingested + +- Published data sources, organized into work units, each with its fields (including calculated-field formulas) and upstream tables +- Workbook metadata (name, project, description) +- Ingest is incremental: items whose `updatedAt` timestamp is unchanged since the last run are skipped + +### Data source and workbook filters + +At large scale, limit which items are fetched during ingest with `datasourceFilter` / `workbookFilter`: + +```yaml title="ktx.yaml" +connections: + tableau-main: + driver: tableau + host: https://us-west-2b.online.tableau.com + site_content_url: "your-site" + personal_access_token_name: "ktx-ingest" + personal_access_token_secret_ref: env:TABLEAU_PAT_SECRET + datasourceFilter: + updatedSince: "2026-01-01T00:00:00Z" # only recently updated data sources + workbookFilter: + updatedSince: "2026-01-01T00:00:00Z" # only recently updated workbooks +``` + +| Field | Default | Description | +|-------|---------|-------------| +| `updatedSince` | — | ISO 8601 date; only items updated on or after this date are fetched | + +### Notes + +- This release is **wiki-only**: Tableau ingest produces ktx wiki knowledge (metric definitions from calculated fields, business concepts from data source and workbook names). Deterministic semantic-layer sources under `semantic-layer//` are not written yet — warehouse-table projection is planned for a follow-up release. +- Context ingest (`ktx ingest tableau-main`) fetches from the Tableau APIs directly. +- The PAT secret must be provided via an `env:` or `file:` reference; the token authenticates both the REST sign-in and the Metadata API calls. + +--- + ## Google Drive Ingests Google Docs from a shared Google Drive folder as wiki-ready knowledge content. This v1 implementation is knowledge-only and ingests Google Docs MIME types only. From 56a5a92463b65f13d37c33db591051d1b3e8e8e6 Mon Sep 17 00:00:00 2001 From: saksham Date: Mon, 6 Jul 2026 01:56:09 +0530 Subject: [PATCH 4/4] test(connectors): add live-captured Tableau fixture and real-data regression test Captured a real staged bundle by running the adapter's fetch() against a live Tableau Cloud (Developer) site's built-in "Superstore" sample data source, and committed it under test/fixtures/tableau/superstore-live/. Adds a regression test that detects, chunks, and parses the real bundle and asserts the "Profit Ratio" calculated field (formula SUM([Profit])/SUM([Sales])) and the upstream lineage are surfaced. This grounds the adapter in genuine Metadata API output: confirms the GraphQL field selection (luid, name, role, dataType, CalculatedField.formula, upstreamTables) matches the live schema. Refs #166. --- .../ingest/adapters/tableau/real-data.test.ts | 37 +++ .../18ad5ef7-9b61-4c3a-9ecd-01fcd2fd079f.json | 221 ++++++++++++++++++ .../superstore-live/tableau-manifest.json | 6 + .../tableau-projection-config.json | 1 + .../5c6c38a0-ebe7-4ac9-af19-110fb152c41b.json | 6 + .../79b0faa6-75a8-4fc5-abf2-4ab340cb2812.json | 6 + .../d62dca4b-2611-4049-96ba-75b7fa594274.json | 6 + 7 files changed, 283 insertions(+) create mode 100644 packages/cli/test/context/ingest/adapters/tableau/real-data.test.ts create mode 100644 packages/cli/test/fixtures/tableau/superstore-live/datasources/18ad5ef7-9b61-4c3a-9ecd-01fcd2fd079f.json create mode 100644 packages/cli/test/fixtures/tableau/superstore-live/tableau-manifest.json create mode 100644 packages/cli/test/fixtures/tableau/superstore-live/tableau-projection-config.json create mode 100644 packages/cli/test/fixtures/tableau/superstore-live/workbooks/5c6c38a0-ebe7-4ac9-af19-110fb152c41b.json create mode 100644 packages/cli/test/fixtures/tableau/superstore-live/workbooks/79b0faa6-75a8-4fc5-abf2-4ab340cb2812.json create mode 100644 packages/cli/test/fixtures/tableau/superstore-live/workbooks/d62dca4b-2611-4049-96ba-75b7fa594274.json diff --git a/packages/cli/test/context/ingest/adapters/tableau/real-data.test.ts b/packages/cli/test/context/ingest/adapters/tableau/real-data.test.ts new file mode 100644 index 00000000..4802b197 --- /dev/null +++ b/packages/cli/test/context/ingest/adapters/tableau/real-data.test.ts @@ -0,0 +1,37 @@ +import { readFile } from 'node:fs/promises'; +import { resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { chunkTableauStagedDir } from '../../../../../src/context/ingest/adapters/tableau/chunk.js'; +import { detectTableauStagedDir } from '../../../../../src/context/ingest/adapters/tableau/detect.js'; +import { stagedDatasourceFileSchema } from '../../../../../src/context/ingest/adapters/tableau/types.js'; + +// Bundle captured from a live Tableau Cloud (Developer) site by running the adapter's fetch() +// against the built-in "Superstore" sample data source. Grounds the adapter in real Metadata API +// output rather than hand-authored shapes. +const LIVE = resolve(import.meta.dirname, '../../../../fixtures/tableau/superstore-live'); + +describe('Tableau adapter — live-captured Superstore bundle', () => { + it('detects the real staged bundle', async () => { + expect(await detectTableauStagedDir(LIVE)).toBe(true); + }); + + it('chunks into a data-sources unit and a workbooks unit', async () => { + const result = await chunkTableauStagedDir(LIVE); + const keys = result.workUnits.map((u) => u.unitKey).sort(); + expect(keys).toEqual(['tableau-datasources', 'tableau-workbooks']); + }); + + it('parses the Superstore data source and surfaces the "Profit Ratio" calculated field', async () => { + const body = await readFile(resolve(LIVE, 'datasources/18ad5ef7-9b61-4c3a-9ecd-01fcd2fd079f.json'), 'utf-8'); + const ds = stagedDatasourceFileSchema.parse(JSON.parse(body)); + + expect(ds.name).toBe('Superstore Datasource'); + const calc = ds.fields.find((f) => f.isCalculated); + expect(calc?.name).toBe('Profit Ratio'); + expect(calc?.formula).toBe('SUM([Profit])/SUM([Sales])'); + // Plain column fields carry no formula. + expect(ds.fields.some((f) => f.name === 'Sales' && !f.isCalculated)).toBe(true); + // Upstream lineage is captured (Excel-backed sample → sheet-style table names). + expect(ds.upstreamTables.map((t) => t.name).sort()).toEqual(['Orders', 'People', 'Returns']); + }); +}); diff --git a/packages/cli/test/fixtures/tableau/superstore-live/datasources/18ad5ef7-9b61-4c3a-9ecd-01fcd2fd079f.json b/packages/cli/test/fixtures/tableau/superstore-live/datasources/18ad5ef7-9b61-4c3a-9ecd-01fcd2fd079f.json new file mode 100644 index 00000000..76a6f200 --- /dev/null +++ b/packages/cli/test/fixtures/tableau/superstore-live/datasources/18ad5ef7-9b61-4c3a-9ecd-01fcd2fd079f.json @@ -0,0 +1,221 @@ +{ + "luid": "18ad5ef7-9b61-4c3a-9ecd-01fcd2fd079f", + "name": "Superstore Datasource", + "projectName": "Samples", + "updatedAt": "2026-07-05T19:19:30Z", + "hasExtracts": false, + "fields": [ + { + "name": "Top Customers by Profit", + "isCalculated": false + }, + { + "name": "Ship Mode", + "role": "DIMENSION", + "dataType": "STRING", + "isCalculated": false + }, + { + "name": "State/Province", + "role": "DIMENSION", + "dataType": "STRING", + "isCalculated": false + }, + { + "name": "Category", + "role": "DIMENSION", + "dataType": "STRING", + "isCalculated": false + }, + { + "name": "Profit", + "role": "MEASURE", + "dataType": "REAL", + "isCalculated": false + }, + { + "name": "Sales", + "role": "MEASURE", + "dataType": "REAL", + "isCalculated": false + }, + { + "name": "Order Date", + "role": "DIMENSION", + "dataType": "DATE", + "isCalculated": false + }, + { + "name": "Product ID", + "role": "DIMENSION", + "dataType": "STRING", + "isCalculated": false + }, + { + "name": "Order ID", + "role": "DIMENSION", + "dataType": "STRING", + "isCalculated": false + }, + { + "name": "Ship Date", + "role": "DIMENSION", + "dataType": "DATE", + "isCalculated": false + }, + { + "name": "Profit (bin)", + "isCalculated": false + }, + { + "name": "Region", + "role": "DIMENSION", + "dataType": "STRING", + "isCalculated": false + }, + { + "name": "People", + "role": "MEASURE", + "dataType": "TABLE", + "isCalculated": false + }, + { + "name": "Returned", + "role": "DIMENSION", + "dataType": "STRING", + "isCalculated": false + }, + { + "name": "Postal Code", + "role": "DIMENSION", + "dataType": "INTEGER", + "isCalculated": false + }, + { + "name": "Order ID (Returns)", + "role": "DIMENSION", + "dataType": "STRING", + "isCalculated": false + }, + { + "name": "Sub-Category", + "role": "DIMENSION", + "dataType": "STRING", + "isCalculated": false + }, + { + "name": "Person", + "role": "DIMENSION", + "dataType": "STRING", + "isCalculated": false + }, + { + "name": "Segment", + "role": "DIMENSION", + "dataType": "STRING", + "isCalculated": false + }, + { + "name": "Customer Name", + "role": "DIMENSION", + "dataType": "STRING", + "isCalculated": false + }, + { + "name": "Quantity", + "role": "MEASURE", + "dataType": "INTEGER", + "isCalculated": false + }, + { + "name": "Profit Ratio", + "role": "MEASURE", + "dataType": "REAL", + "formula": "SUM([Profit])/SUM([Sales])", + "isCalculated": true + }, + { + "name": "Discount", + "role": "MEASURE", + "dataType": "REAL", + "isCalculated": false + }, + { + "name": "Product Name", + "role": "DIMENSION", + "dataType": "STRING", + "isCalculated": false + }, + { + "name": "Returns", + "role": "MEASURE", + "dataType": "TABLE", + "isCalculated": false + }, + { + "name": "Orders", + "role": "MEASURE", + "dataType": "TABLE", + "isCalculated": false + }, + { + "name": "Country/Region", + "role": "DIMENSION", + "dataType": "STRING", + "isCalculated": false + }, + { + "name": "Customer ID", + "role": "DIMENSION", + "dataType": "STRING", + "isCalculated": false + }, + { + "name": "Row ID", + "role": "DIMENSION", + "dataType": "INTEGER", + "isCalculated": false + }, + { + "name": "Product", + "isCalculated": false + }, + { + "name": "City", + "role": "DIMENSION", + "dataType": "STRING", + "isCalculated": false + }, + { + "name": "Manufacturer", + "isCalculated": false + }, + { + "name": "Region (People)", + "role": "DIMENSION", + "dataType": "STRING", + "isCalculated": false + }, + { + "name": "Location", + "isCalculated": false + } + ], + "upstreamTables": [ + { + "name": "Orders", + "luid": "e6504e84-4d76-487a-8900-b9a8c004c7d5", + "fullName": "[Orders$]" + }, + { + "name": "Returns", + "luid": "ceabf8a4-27dd-40d8-9532-8b7bc151efa1", + "fullName": "[Returns$]" + }, + { + "name": "People", + "luid": "d559f216-8673-47d1-96c2-ff90db965e49", + "fullName": "[People$]" + } + ] +} \ No newline at end of file diff --git a/packages/cli/test/fixtures/tableau/superstore-live/tableau-manifest.json b/packages/cli/test/fixtures/tableau/superstore-live/tableau-manifest.json new file mode 100644 index 00000000..5e1a9bc8 --- /dev/null +++ b/packages/cli/test/fixtures/tableau/superstore-live/tableau-manifest.json @@ -0,0 +1,6 @@ +{ + "tableauConnectionId": "tableau-main", + "fetchedAt": "2026-07-05T20:23:46.142Z", + "datasourceCount": 1, + "workbookCount": 3 +} \ No newline at end of file diff --git a/packages/cli/test/fixtures/tableau/superstore-live/tableau-projection-config.json b/packages/cli/test/fixtures/tableau/superstore-live/tableau-projection-config.json new file mode 100644 index 00000000..9e26dfee --- /dev/null +++ b/packages/cli/test/fixtures/tableau/superstore-live/tableau-projection-config.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/packages/cli/test/fixtures/tableau/superstore-live/workbooks/5c6c38a0-ebe7-4ac9-af19-110fb152c41b.json b/packages/cli/test/fixtures/tableau/superstore-live/workbooks/5c6c38a0-ebe7-4ac9-af19-110fb152c41b.json new file mode 100644 index 00000000..6560beb6 --- /dev/null +++ b/packages/cli/test/fixtures/tableau/superstore-live/workbooks/5c6c38a0-ebe7-4ac9-af19-110fb152c41b.json @@ -0,0 +1,6 @@ +{ + "luid": "5c6c38a0-ebe7-4ac9-af19-110fb152c41b", + "name": "Superstore", + "projectName": "Samples", + "updatedAt": "2026-07-05T19:19:26Z" +} \ No newline at end of file diff --git a/packages/cli/test/fixtures/tableau/superstore-live/workbooks/79b0faa6-75a8-4fc5-abf2-4ab340cb2812.json b/packages/cli/test/fixtures/tableau/superstore-live/workbooks/79b0faa6-75a8-4fc5-abf2-4ab340cb2812.json new file mode 100644 index 00000000..fd520e85 --- /dev/null +++ b/packages/cli/test/fixtures/tableau/superstore-live/workbooks/79b0faa6-75a8-4fc5-abf2-4ab340cb2812.json @@ -0,0 +1,6 @@ +{ + "luid": "79b0faa6-75a8-4fc5-abf2-4ab340cb2812", + "name": "World Indicators", + "projectName": "Samples", + "updatedAt": "2026-07-05T19:19:28Z" +} \ No newline at end of file diff --git a/packages/cli/test/fixtures/tableau/superstore-live/workbooks/d62dca4b-2611-4049-96ba-75b7fa594274.json b/packages/cli/test/fixtures/tableau/superstore-live/workbooks/d62dca4b-2611-4049-96ba-75b7fa594274.json new file mode 100644 index 00000000..0e382bfb --- /dev/null +++ b/packages/cli/test/fixtures/tableau/superstore-live/workbooks/d62dca4b-2611-4049-96ba-75b7fa594274.json @@ -0,0 +1,6 @@ +{ + "luid": "d62dca4b-2611-4049-96ba-75b7fa594274", + "name": "sales overview", + "projectName": "Personal Space", + "updatedAt": "2026-07-05T20:09:08Z" +} \ No newline at end of file