Add Slack context source adapter

docs-site/content/docs/integrations/context-sources.mdx

@@ -1,6 +1,6 @@
 ---
 title: Context Sources
-description: Ingest semantic context from dbt, MetricFlow, LookML, Metabase, Looker, and Notion.
+description: Ingest semantic context from dbt, MetricFlow, LookML, Metabase, Looker, Notion, and Slack.
 ---
 
 Context sources feed your existing analytics tooling into **ktx**. During ingestion, **ktx** extracts metadata from each source and uses a reconciliation agent to reconcile it with your existing semantic layer and knowledge base - preserving accepted edits rather than overwriting.
@@ -27,7 +27,7 @@ LookML uses top-level `repoUrl`, and MetricFlow uses nested
 
 | Field | Required | Description |
 |-------|----------|-------------|
-| `driver` | Yes | Source connector: `dbt`, `metricflow`, `lookml`, `metabase`, `looker`, or `notion` |
+| `driver` | Yes | Source connector: `dbt`, `metricflow`, `lookml`, `metabase`, `looker`, `notion`, or `slack` |
 | `source_dir` | For local file sources | Absolute or project-relative source directory |
 | `repo_url` | For Git-hosted dbt sources | Git repository URL |
 | `repoUrl` | For Git-hosted LookML sources | Git repository URL |
@@ -374,6 +374,47 @@ Create an integration at [notion.so/my-integrations](https://www.notion.so/my-in
 - Incremental sync cursors are stored in `.ktx/db.sqlite`; don't add
   `last_successful_cursor` to `ktx.yaml`
 
+---
+
+## Slack
+
+Ingests messages from allowlisted Slack channels as wiki context. Slack is knowledge-only and does not produce semantic layer sources.
+
+### What it provides
+
+- Markdown pages for messages in configured channels
+- Channel allowlisting so ingest never scans full workspace history
+
+### Connection config
+
+```yaml title="ktx.yaml"
+connections:
+  team-slack:
+    driver: slack
+    bot_token_ref: env:SLACK_BOT_TOKEN
+    channel_ids:
+      - C0123456789
+    max_messages_per_channel: 1000
+```
+
+### Authentication
+
+| Method | Config |
+|--------|--------|
+| Bot token | `bot_token_ref: env:SLACK_BOT_TOKEN` |
+
+The Slack app needs `channels:history` for public channels and `groups:history` for private channels that the app is added to.
+
+### What gets ingested
+
+- `wiki/global/slack/<channel>/<message-ts>.md` for channel messages
+
+### Notes
+
+- `channel_ids` is required and must not be empty
+- Slack ingest reads only the configured `channel_ids`, never every workspace channel
+- `max_messages_per_channel` caps each channel fetch in one ingest run
+
 ## Common errors
 
 | Error or symptom | Likely cause | Recovery |
@@ -382,4 +423,5 @@ Create an integration at [notion.so/my-integrations](https://www.notion.so/my-in
 | Private repo/API authentication fails | Token env var or secret file is missing | Export the env var or update `auth_token_ref` to a readable file |
 | Ingest creates duplicate context | Existing source names or wiki pages do not match imported terminology | Review the diff, rename duplicates, and add wiki pages with canonical names |
 | Notion ingest skips pages | Integration lacks access or root ids are missing | Share pages with the Notion integration and set `root_page_ids` or use `all_accessible` carefully |
+| Slack ingest reads no messages | The app is not in the channel or the channel has no readable messages | Invite the app to the configured channel and verify the token has the matching history scope |
 | Generated semantic sources fail validation | Tool metadata does not match the live warehouse schema | Map BI/source databases to primary warehouse connections and rerun validation |

packages/cli/src/context/connections/slack-config.test.ts

@@ -0,0 +1,71 @@
+import { mkdtemp, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import { parseSlackConnectionConfig, resolveSlackBotToken, slackConnectionToPullConfig } from './slack-config.js';
+
+describe('standalone Slack connection config', () => {
+  let tempDir: string;
+
+  beforeEach(async () => {
+    tempDir = await mkdtemp(join(tmpdir(), 'ktx-slack-config-'));
+  });
+
+  afterEach(async () => {
+    await rm(tempDir, { recursive: true, force: true });
+  });
+
+  it('parses Slack config with a required channel allowlist', () => {
+    expect(
+      parseSlackConnectionConfig({
+        driver: 'slack',
+        bot_token_ref: 'env:SLACK_BOT_TOKEN',
+        channel_ids: ['C2', 'C1', 'C1'],
+        max_messages_per_channel: 25,
+      }),
+    ).toEqual({
+      driver: 'slack',
+      bot_token: null,
+      bot_token_ref: 'env:SLACK_BOT_TOKEN',
+      channel_ids: ['C1', 'C2'],
+      max_messages_per_channel: 25,
+    });
+  });
+
+  it('rejects Slack config without channels', () => {
+    expect(() =>
+      parseSlackConnectionConfig({
+        driver: 'slack',
+        bot_token_ref: 'env:SLACK_BOT_TOKEN',
+        channel_ids: [],
+      }),
+    ).toThrow('Slack connection config requires at least one channel_id');
+  });
+
+  it('resolves Slack env and file token references', async () => {
+    const tokenPath = join(tempDir, 'slack-token.txt');
+    await writeFile(tokenPath, 'xoxb-file-token\n', 'utf-8');
+
+    await expect(resolveSlackBotToken('env:SLACK_BOT_TOKEN', { env: { SLACK_BOT_TOKEN: 'xoxb-env-token' } })).resolves.toBe(
+      'xoxb-env-token',
+    );
+    await expect(resolveSlackBotToken(`file:${tokenPath}`)).resolves.toBe('xoxb-file-token');
+  });
+
+  it('converts Slack config into adapter pull config', async () => {
+    await expect(
+      slackConnectionToPullConfig(
+        parseSlackConnectionConfig({
+          driver: 'slack',
+          bot_token_ref: 'env:SLACK_BOT_TOKEN',
+          channel_ids: ['C1'],
+        }),
+        { env: { SLACK_BOT_TOKEN: 'xoxb-env-token' } },
+      ),
+    ).resolves.toEqual({
+      authToken: 'xoxb-env-token',
+      channelIds: ['C1'],
+      maxMessagesPerChannel: 1000,
+    });
+  });
+});

packages/cli/src/context/connections/slack-config.ts

@@ -0,0 +1,138 @@
+import { readFile } from 'node:fs/promises';
+import { homedir } from 'node:os';
+import { resolve } from 'node:path';
+import { slackPullConfigSchema, type SlackPullConfig } from '../ingest/adapters/slack/types.js';
+import type { KtxProjectConnectionConfig } from '../project/config.js';
+
+type RawKtxSlackConnectionConfig = Extract<KtxProjectConnectionConfig, { driver: 'slack' }>;
+
+export type KtxSlackConnectionConfig = Omit<
+  RawKtxSlackConnectionConfig,
+  'bot_token' | 'bot_token_ref' | 'channel_ids' | 'max_messages_per_channel'
+> & {
+  driver: 'slack';
+  bot_token: string | null;
+  bot_token_ref: string | null;
+  channel_ids: string[];
+  max_messages_per_channel: number;
+};
+
+interface ResolveSlackTokenOptions {
+  env?: Record<string, string | undefined>;
+  readTextFile?: (path: string) => Promise<string>;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+function optionalString(value: unknown): string | null {
+  return typeof value === 'string' && value.trim().length > 0 ? value.trim() : null;
+}
+
+function stringArray(value: unknown): string[] {
+  if (!Array.isArray(value)) {
+    return [];
+  }
+  return value.flatMap((item): string[] => {
+    if (typeof item !== 'string') {
+      return [];
+    }
+    const trimmed = item.trim();
+    return trimmed ? [trimmed] : [];
+  });
+}
+
+function integerWithFallback(value: unknown, fallback: number, name: string): number {
+  if (value === undefined || value === null) {
+    return fallback;
+  }
+  if (typeof value !== 'number' || !Number.isInteger(value)) {
+    throw new Error(`${name} must be an integer`);
+  }
+  return value;
+}
+
+function boundedInteger(value: unknown, fallback: number, name: string, min: number, max: number): number {
+  const parsed = integerWithFallback(value, fallback, name);
+  if (parsed < min || parsed > max) {
+    throw new Error(`${name} must be between ${min} and ${max}`);
+  }
+  return parsed;
+}
+
+function expandHome(path: string): string {
+  return path === '~' || path.startsWith('~/') ? resolve(homedir(), path.slice(2)) : path;
+}
+
+export function parseSlackConnectionConfig(raw: unknown): KtxSlackConnectionConfig {
+  if (!isRecord(raw)) {
+    throw new Error('Slack connection config must be an object');
+  }
+  if (raw.driver !== 'slack') {
+    throw new Error('Slack connection config requires driver: slack');
+  }
+  const botToken = optionalString(raw.bot_token);
+  const botTokenRef = optionalString(raw.bot_token_ref);
+  if (!botToken && !botTokenRef) {
+    throw new Error('Slack connection config requires bot_token or bot_token_ref');
+  }
+  if (botTokenRef && !botTokenRef.startsWith('env:') && !botTokenRef.startsWith('file:')) {
+    throw new Error('Slack bot_token_ref must use env:NAME or file:/path');
+  }
+  const channelIds = stringArray(raw.channel_ids);
+  if (channelIds.length === 0) {
+    throw new Error('Slack connection config requires at least one channel_id');
+  }
+
+  return {
+    driver: 'slack',
+    bot_token: botToken,
+    bot_token_ref: botTokenRef,
+    channel_ids: [...new Set(channelIds)].sort((left, right) => left.localeCompare(right)),
+    max_messages_per_channel: boundedInteger(
+      raw.max_messages_per_channel,
+      1000,
+      'max_messages_per_channel',
+      1,
+      10_000,
+    ),
+  };
+}
+
+/** @internal */
+export async function resolveSlackBotToken(
+  botTokenRef: string,
+  options: ResolveSlackTokenOptions = {},
+): Promise<string> {
+  if (botTokenRef.startsWith('env:')) {
+    const envName = botTokenRef.slice('env:'.length);
+    const value = (options.env ?? process.env)[envName];
+    if (!value) {
+      throw new Error(`Slack token environment variable ${envName} is not set`);
+    }
+    return value.trim();
+  }
+  if (botTokenRef.startsWith('file:')) {
+    const path = expandHome(botTokenRef.slice('file:'.length));
+    const readTextFile = options.readTextFile ?? ((filePath: string) => readFile(filePath, 'utf-8'));
+    const value = (await readTextFile(path)).trim();
+    if (!value) {
+      throw new Error(`Slack token file is empty: ${path}`);
+    }
+    return value;
+  }
+  throw new Error('Slack bot_token_ref must use env:NAME or file:/path');
+}
+
+export async function slackConnectionToPullConfig(
+  config: KtxSlackConnectionConfig,
+  options: ResolveSlackTokenOptions = {},
+): Promise<SlackPullConfig> {
+  const authToken = config.bot_token ?? (await resolveSlackBotToken(config.bot_token_ref ?? '', options));
+  return slackPullConfigSchema.parse({
+    authToken,
+    channelIds: config.channel_ids,
+    maxMessagesPerChannel: config.max_messages_per_channel,
+  });
+}

packages/cli/src/context/ingest/adapters/slack/chunk.ts

@@ -0,0 +1,80 @@
+import { createHash } from 'node:crypto';
+import { readdir, readFile } from 'node:fs/promises';
+import { join, relative } from 'node:path';
+import type { ChunkResult, DiffSet, ScopeDescriptor, WorkUnit } from '../../types.js';
+import { slackManifestSchema } from './types.js';
+
+async function walk(root: string): Promise<string[]> {
+  const entries = await readdir(root, { withFileTypes: true, recursive: true });
+  return entries
+    .filter((entry) => entry.isFile())
+    .map((entry) => relative(root, join(entry.parentPath, entry.name)).replace(/\\/g, '/'))
+    .sort();
+}
+
+function safeUnitKey(path: string): string {
+  return `slack-${path
+    .replace(/^wiki\/global\//, '')
+    .replace(/\.md$/, '')
+    .replace(/[^a-zA-Z0-9]+/g, '-')
+    .replace(/^-+|-+$/g, '')}`;
+}
+
+async function readManifest(stagedDir: string) {
+  try {
+    return slackManifestSchema.parse(JSON.parse(await readFile(join(stagedDir, 'manifest.json'), 'utf-8')));
+  } catch (error) {
+    throw new Error(`Invalid Slack manifest: ${error instanceof Error ? error.message : String(error)}`);
+  }
+}
+
+export async function chunkSlackStagedDir(stagedDir: string, diffSet?: DiffSet): Promise<ChunkResult> {
+  const files = await walk(stagedDir);
+  const manifest = await readManifest(stagedDir);
+  const touched = diffSet ? new Set([...diffSet.added, ...diffSet.modified]) : null;
+  const markdownFiles = files.filter((path) => path.startsWith('wiki/global/') && path.endsWith('.md'));
+  const workUnits: WorkUnit[] = [];
+
+  for (const markdownPath of markdownFiles) {
+    if (touched && !touched.has(markdownPath)) {
+      continue;
+    }
+    const dependencyPaths = ['manifest.json'].filter((path) => path !== markdownPath);
+    workUnits.push({
+      unitKey: safeUnitKey(markdownPath),
+      displayLabel: markdownPath.replace(/^wiki\/global\//, ''),
+      rawFiles: [markdownPath],
+      dependencyPaths,
+      peerFileIndex: markdownFiles.filter((path) => path !== markdownPath).sort(),
+      notes:
+        'Synthesize durable wiki knowledge from this allowlisted Slack source. Treat Slack as wiki-only context unless another mapped source is confirmed with discover_data.',
+    });
+  }
+
+  return {
+    workUnits,
+    eviction: diffSet && diffSet.deleted.length > 0 ? { deletedRawPaths: [...diffSet.deleted].sort() } : undefined,
+    reconcileNotes: [
+      `Slack channels fetched: ${manifest.channelIds.join(', ')}`,
+      `Slack maxMessagesPerChannel=${manifest.maxMessagesPerChannel}`,
+      'Slack ingest reads only configured allowlisted channels.',
+    ],
+    contextReport: {
+      warnings: manifest.warnings,
+    },
+  };
+}
+
+export async function describeSlackScope(stagedDir: string): Promise<ScopeDescriptor> {
+  const manifest = await readManifest(stagedDir);
+  const scopeKey = JSON.stringify({
+    channelIds: [...manifest.channelIds].sort(),
+    maxMessagesPerChannel: manifest.maxMessagesPerChannel,
+  });
+  const fingerprint = createHash('sha256').update(scopeKey).digest('hex');
+  return {
+    fingerprint,
+    isPathInScope: (rawPath) =>
+      rawPath === 'manifest.json' || rawPath.startsWith('wiki/global/slack/'),
+  };
+}

packages/cli/src/context/ingest/adapters/slack/detect.ts

@@ -0,0 +1,21 @@
+import { readFile, readdir } from 'node:fs/promises';
+import { join } from 'node:path';
+import { SLACK_SOURCE_KEY } from './types.js';
+
+export async function detectSlackStagedDir(stagedDir: string): Promise<boolean> {
+  try {
+    const manifest = JSON.parse(await readFile(join(stagedDir, 'manifest.json'), 'utf-8')) as { source?: unknown };
+    if (manifest.source === SLACK_SOURCE_KEY) {
+      return true;
+    }
+  } catch {
+    // Fall through to structural detection for manually staged dirs.
+  }
+
+  try {
+    const entries = await readdir(join(stagedDir, 'wiki', 'global'), { withFileTypes: true, recursive: true });
+    return entries.some((entry) => entry.isFile() && entry.name.endsWith('.md'));
+  } catch {
+    return false;
+  }
+}