diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ded99603..6a23fae8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -181,15 +181,6 @@ jobs: run: npm run test:unit - name: Run integration tests - env: - CI_CODEMIE_USERNAME: ${{ secrets.CI_CODEMIE_USERNAME }} - CI_CODEMIE_PASSWORD: ${{ secrets.CI_CODEMIE_PASSWORD }} - CI_AGENT_MAX_WORKERS: ${{ vars.CI_AGENT_MAX_WORKERS }} - CI_CODEMIE_AUTH_CLIENT_ID: ${{ vars.CI_CODEMIE_AUTH_CLIENT_ID }} - CI_CODEMIE_AUTH_URL: ${{ vars.CI_CODEMIE_AUTH_URL }} - CI_CODEMIE_MODEL: ${{ vars.CI_CODEMIE_MODEL }} - CI_CODEMIE_URL: ${{ vars.CI_CODEMIE_URL }} - CI_IS_LOCAL_RUN: ${{ vars.CI_IS_LOCAL_RUN }} run: npm run test:integration test-windows: @@ -223,13 +214,4 @@ jobs: run: npm run test:unit - name: Run integration tests - env: - CI_CODEMIE_USERNAME: ${{ secrets.CI_CODEMIE_USERNAME }} - CI_CODEMIE_PASSWORD: ${{ secrets.CI_CODEMIE_PASSWORD }} - CI_AGENT_MAX_WORKERS: ${{ vars.CI_AGENT_MAX_WORKERS }} - CI_CODEMIE_AUTH_CLIENT_ID: ${{ vars.CI_CODEMIE_AUTH_CLIENT_ID }} - CI_CODEMIE_AUTH_URL: ${{ vars.CI_CODEMIE_AUTH_URL }} - CI_CODEMIE_MODEL: ${{ vars.CI_CODEMIE_MODEL }} - CI_CODEMIE_URL: ${{ vars.CI_CODEMIE_URL }} - CI_IS_LOCAL_RUN: ${{ vars.CI_IS_LOCAL_RUN }} - run: npm run test:integration + run: npm run test:integration \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md index 5907a52e..9bfec415 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -164,6 +164,7 @@ Detailed patterns for architecture, error handling, logging, security, project c |---|---| | `require()` and `__dirname` | ES modules and `getDirname(import.meta.url)` | | Imports without `.js` | Always include `.js` extension | +| Deep relative imports (`../../..`) | `@/` alias (e.g. `@/env/types.js`) | | Writing tests by default | Tests only on explicit request | | `child_process.exec` directly | `exec()` from `src/utils/processes.ts` | | `console.log()` debug output | `logger.debug()` | diff --git a/docs/specs/2026-05-27-cli-integration-tests-design.md b/docs/specs/2026-05-27-cli-integration-tests-design.md new file mode 100644 index 00000000..38efe354 --- /dev/null +++ b/docs/specs/2026-05-27-cli-integration-tests-design.md @@ -0,0 +1,297 @@ +# CLI Integration Test Design + +**Last updated:** 2026-06-22 +**Branch:** `test/cli-integration-tests` + +--- + +## Overview + +Integration tests for the `codemie-claude` CLI binary. Tests are end-to-end: they spawn the compiled binary as a child process (or PTY), drive it with real environment variables, and assert on exit codes, stdout, and session/metrics artefacts written to `CODEMIE_HOME`. + +Tests are split into two Vitest configurations: + +| Project | Includes | GlobalSetup | +|---|---|---| +| `unit` | `src/**/*.test.ts` | none | +| `cli` | `tests/integration/**/*.test.ts` (excl. `agent-*`) | none | +| `agent` | `tests/integration/agent-*.test.ts` | `tests/setup/agent-build-setup.ts` (build + SSO auth) | + +All three projects are defined in a single `vitest.config.ts` using `defineWorkspace`. + +CLI-commands tests (`cli-commands/*.test.ts`) exercise commands that need no network auth (health, help, version, doctor, etc.). +Agent tests (`agent-*.test.ts`) exercise commands that require authentication and make real network calls. + +--- + +## Auth Model + +### `CI_IS_LOCAL_RUN` dual-mode + +Tests gate on the `CI_IS_LOCAL_RUN` flag (read via `getTestEnvFlagOrDefault('CI_IS_LOCAL_RUN', true)`): + +| Value | Mode | Auth mechanism | +|---|---|---| +| `true` (default) | SSO / local dev | Existing `sso-autotest` profile in `~/.codemie` | +| `false` | JWT / CI pipeline | Bearer token fetched from `CI_CODEMIE_AUTH_URL` | + +This replaces the old `INCLUDE_JWT_TESTS=true` gate that ran JWT tests only. Tests that can exercise both modes are now **dual-mode** and run in every environment. Tests that are logically specific to the JWT mechanism are marked **JWT-only** and wrapped in `describe.runIf(!CI_IS_LOCAL_RUN)`. + +### JWT-only describe convention + +```ts +describe.runIf(!CI_IS_LOCAL_RUN)( + 'TC-NNN — description [JWT-only, skipped when CI_IS_LOCAL_RUN=true]', + () => { ... }, +); +``` + +The skip reason is embedded in the describe name so it appears in test output when the suite is skipped. + +--- + +## Environment Variables + +| Variable | Required | Description | +|---|---|---| +| `CI_IS_LOCAL_RUN` | optional | `true` = SSO mode (default), `false` = JWT mode | +| `CI_CODEMIE_URL` | both modes | CodeMie frontend URL (API base derived as `CI_CODEMIE_URL/code-assistant-api`) | +| `CI_CODEMIE_MODEL` | optional | Model override (default: `claude-sonnet-4-6`) | +| `CI_CODEMIE_USERNAME` | JWT mode | Username for token fetch | +| `CI_CODEMIE_PASSWORD` | JWT mode | Password for token fetch | +| `CI_CODEMIE_AUTH_URL` | JWT mode | Keycloak/auth server base URL | +| `CI_AGENT_MAX_WORKERS` | optional | `maxWorkers` for agent test runner (default: 2) | +| `DEFAULT_TIMEOUT` | optional | Command timeout in seconds (default: 60) | +| `CODEMIE_HOME` | set per-test | Isolated temp dir; overrides `~/.codemie` for the test run | + +Local dev: set these in `.env.test.local` at the repo root (gitignored). + +--- + +## Directory Layout + +``` +tests/ + helpers/ + index.ts # Re-exports all helpers + cli-runner.ts # CLIRunner, createCLIRunner, createAgentRunner + jwt-auth.ts # fetchJwtToken, writeJwtProfile, jwtCleanEnv + sso-auth.ts # writeSsoProfile, ssoCleanEnv, copySsoCredentials, + # setupSsoAutotestProfile, teardownSsoAutotestProfile + pty-session.ts # spawnPty, PtySession + metrics.ts # getLatestMetricsRecord + temp-workspace.ts # TempWorkspace, createTempWorkspace, getTempDir, resolveLongPath + interactive-helpers.ts # waitForOutput, cleanKill (legacy; prefer spawnPty) + test-env.ts # getTestEnvFlag, getTestEnvFlagOrDefault + session-poll.ts # pollForSession + setup/ + agent-build-setup.ts # Vitest globalSetup: npm run build + SSO credential auth + load-test-env.ts # Imports .env.test.local at the top of each test file + integration/ + agent-task.test.ts # TC-016 dual-mode + agent-task-session.test.ts # Session/metrics artefact validation + agent-model.test.ts # TC-020, TC-021, TC-022, TC-024 + agent-skills.test.ts # TC-025 + agent-assistant.test.ts # TC-014, TC-015, TC-026 + agent-jwt-token.test.ts # TC-017, TC-027 [JWT-only] + agent-negative.test.ts # TC-018 [JWT-only], TC-019 [dual-mode] + agent-setup.test.ts # TC-029 [SSO-only] + agent-shortcuts.test.ts # Slash command smoke tests + cli-commands/ + doctor.test.ts + help.test.ts + version.test.ts + list.test.ts + profile.test.ts + skills.test.ts + workflow.test.ts + self-update.test.ts + error-handling.test.ts +``` + +--- + +## Helper Layer + +### JWT helpers (`helpers/jwt-auth.ts`) + +| Helper | Purpose | +|---|---| +| `fetchJwtToken()` | Fetches bearer token from auth server using `CI_CODEMIE_*` env vars | +| `writeJwtProfile(home, { jwtToken })` | Writes a `bearer-auth` profile config to `/codemie-cli.config.json` | +| `jwtCleanEnv()` | Returns a minimal env object (no `CODEMIE_HOME`, no inherited profile) for JWT runs | + +### SSO helpers (`helpers/sso-auth.ts`) + +| Helper | Purpose | +|---|---| +| `writeSsoProfile(home)` | Writes an `ai-run-sso` profile config to `/codemie-cli.config.json` | +| `ssoCleanEnv()` | Returns a minimal env object for SSO runs (strips inherited auth env vars) | +| `copySsoCredentials(home)` | Copies SSO credential files from `~/.codemie` into the test's isolated `home` | +| `setupSsoAutotestProfile()` | Sets `sso-autotest` as the active profile in `~/.codemie`; returns the original active profile name | +| `teardownSsoAutotestProfile(original)` | Restores the original active profile after the test | + +### PTY helper (`helpers/pty-session.ts`) + +| Helper | Purpose | +|---|---| +| `spawnPty(args, env, cwd)` → `PtySession` | Spawns `codemie-claude` in a node-pty PTY; returns `{ send(text), waitFor(pattern), close() }` | + +Used by interactive tests (TC-024, TC-025) that need to drive a running session with slash commands. + +### Metrics helper (`helpers/metrics.ts`) + +| Helper | Purpose | +|---|---| +| `getLatestMetricsRecord(sessionsDir)` | Reads `_metrics.jsonl` in `sessionsDir` and returns the latest record as a parsed object | + +### Other helpers + +| Helper | File | Purpose | +|---|---|---| +| `getTempDir()` | `temp-workspace.ts` | Returns system temp dir, platform-aware | +| `resolveLongPath(p)` | `temp-workspace.ts` | Resolves Windows long path prefix | +| `getTestEnvFlagOrDefault(name, def)` | `test-env.ts` | Reads an env flag as boolean with fallback | +| `pollForSession(dir, opts)` | `session-poll.ts` | Polls for session file creation with timeout | + +--- + +## TC Map + +| TC | File | Mode | Description | +|---|---|---|---| +| TC-014 | `agent-assistant.test.ts` | dual | Setup assistants wizard registers assistant as skill | +| TC-015 | `agent-assistant.test.ts` | dual | Assistants chat with invalid ID returns error | +| TC-016 | `agent-task.test.ts` | dual | `--task` exits 0 and agent response appears in stdout | +| TC-017 | `agent-jwt-token.test.ts` | JWT-only | `--profile` + `--jwt-token` overrides active SSO profile; session records `bearer-auth` | +| TC-018 | `agent-negative.test.ts` | JWT-only | Invalid JWT token exits non-zero with auth error | +| TC-019 | `agent-negative.test.ts` | dual | No profile and no token exits non-zero with config error | +| TC-020 | `agent-model.test.ts` | dual | Session records model configured in profile | +| TC-021 | `agent-model.test.ts` | dual | Metrics records configured model in models array | +| TC-022 | `agent-model.test.ts` | dual | `codemie models list` returns the configured model name | +| TC-024 | `agent-model.test.ts` | dual | In-session `/model` slash command records new model in metrics (PTY) | +| TC-025 | `agent-skills.test.ts` | dual | Skill slash command invocation inside running session (PTY) | +| TC-026 | `agent-assistant.test.ts` | dual | Assistants chat non-interactive (random number round-trip) | +| TC-027 | `agent-jwt-token.test.ts` | JWT-only | `--jwt-token` with no profile (empty CODEMIE_HOME) exits 0 and prints agent response | +| TC-029 | `agent-setup.test.ts` | SSO-only | Setup wizard creates SSO profile; config has correct provider, URL, project, model | +| TC-030 | `cli-commands/self-update.test.ts` | no-auth | `self-update --check` exits 0 and outputs current version from package.json | + +--- + +## Gating Patterns + +### Dual-mode test (runs in both SSO and JWT) + +```ts +const CI_IS_LOCAL_RUN = getTestEnvFlagOrDefault('CI_IS_LOCAL_RUN', true); + +beforeAll(async () => { + if (!CI_IS_LOCAL_RUN) { + jwtToken = await fetchJwtToken(); + } else { + originalActiveProfile = setupSsoAutotestProfile(); + } +}, 30_000); + +afterAll(() => { + if (CI_IS_LOCAL_RUN) teardownSsoAutotestProfile(originalActiveProfile); +}); + +// Inner beforeAll (per-describe): +beforeAll(() => { + testHome = mkdtempSync(join(getTempDir(), 'codemie-test-')); + if (!CI_IS_LOCAL_RUN) { + writeJwtProfile(testHome, { jwtToken }); + } else { + writeSsoProfile(testHome); + copySsoCredentials(testHome); + } + result = spawnSync( + process.execPath, + CI_IS_LOCAL_RUN + ? [CLAUDE_BIN, '--task', 'Say READY'] + : [CLAUDE_BIN, '--task', 'Say READY', '--jwt-token', jwtToken], + { + env: { ...(CI_IS_LOCAL_RUN ? ssoCleanEnv() : jwtCleanEnv()), CODEMIE_HOME: testHome }, + encoding: 'utf-8', + timeout: 120_000, + }, + ); +}, 180_000); +``` + +### JWT-only test + +```ts +describe.runIf(!CI_IS_LOCAL_RUN)( + 'TC-NNN — description [JWT-only, skipped when CI_IS_LOCAL_RUN=true]', + () => { + let jwtToken: string; + beforeAll(async () => { jwtToken = await fetchJwtToken(); }, 30_000); + // ... tests using jwtToken + }, +); +``` + +### Interactive PTY test (dual-mode) + +```ts +// TC-024, TC-025 — interactive sessions driven via node-pty +const session = await spawnPty( + [CLAUDE_BIN, ...(CI_IS_LOCAL_RUN ? [] : ['--jwt-token', jwtToken])], + { ...(CI_IS_LOCAL_RUN ? ssoCleanEnv() : jwtCleanEnv()), CODEMIE_HOME: testHome }, + testHome, +); +await session.waitFor(/\$/); // prompt +session.send('/model claude-haiku-4-5\n'); +await session.waitFor(/switched|model/i); +await session.close(); +``` + +--- + +## Global Setup (`tests/setup/agent-build-setup.ts`) + +Runs once per agent test session (`vitest.config.ts` agent project → `globalSetup`): + +1. Loads `.env.test.local`. +2. Runs `npm run build` to produce `dist/`. +3. Ensures `~/.local/bin` is in `PATH` (needed on some Windows CI runners). +4. Installs or skips the native `claude` binary. +5. If `CI_IS_LOCAL_RUN=true`, authenticates the `sso-autotest` profile so SSO credentials are valid for the test session. + +--- + +## Multi-profile Override Pattern (TC-017) + +TC-017 exercises the `--profile` + `--jwt-token` runtime override. The test writes a two-profile config where the **active** profile is SSO and the **non-active** profile is `bearer-auth`. Running with `--profile profile-jwt-override --jwt-token ` must: + +1. Select the non-active profile (not the active SSO one). +2. Use the supplied token for auth. + +The observable proof is the `provider` field in the session file. Because `--jwt-token` does **not** mutate the config's `provider` key, the session will record `bearer-auth` only if the non-active JWT profile was actually selected. If the active SSO profile were used instead, `provider` would be `ai-run-sso`. + +Config shape written by `writeTwoProfileConfig(testHome)`: + +```json +{ + "version": 2, + "activeProfile": "profile-sso-active", + "profiles": { + "profile-sso-active": { "provider": "ai-run-sso", "authMethod": "sso" }, + "profile-jwt-override": { "provider": "bearer-auth", "authMethod": "jwt" } + } +} +``` + +Assertion: `session.provider` matches `/bearer-auth/i`. + +--- + +## Conventions + +- Every test writes to an isolated `mkdtempSync(...)` temp dir, set as `CODEMIE_HOME`. +- `afterAll` always `rmSync(testHome, { recursive: true, force: true })`. +- `spawnSync` is used for non-interactive `--task` invocations; `spawnPty` for interactive sessions. +- `testTimeout: 180_000` and `hookTimeout: 300_000` in the `agent` project of `vitest.config.ts`. +- TC numbers appear in the `describe` name so they are visible in test output. diff --git a/docs/superpowers/plans/2026-05-27-cli-integration-tests.md b/docs/superpowers/plans/2026-05-27-cli-integration-tests.md new file mode 100644 index 00000000..9a01ab16 --- /dev/null +++ b/docs/superpowers/plans/2026-05-27-cli-integration-tests.md @@ -0,0 +1,1592 @@ +# CLI Integration Tests Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Implement 34 CLI integration test cases (TC-001–TC-034) covering CLI management commands, JWT-authenticated agent sessions, interactive stdin/stdout session control, and budget/project configuration. + +**Architecture:** Tests live in `tests/integration/` and `tests/integration/cli-commands/`. A shared helper layer (`tests/helpers/jwt-auth.ts`, `tests/helpers/interactive-helpers.ts`) provides JWT token fetching, profile config writing, and async process interaction. Agent session tests (those that spawn `bin/codemie-claude.js`) use a dedicated `vitest.agent.config.ts` with a `globalSetup` that runs `npm run build` once per session before any test file executes. + +**Tech Stack:** Vitest 4.x, Node.js child_process (`spawnSync`, `spawn`), TypeScript ESM, node:readline for stdout streaming, Keycloak password grant for JWT. + +--- + +## File Map + +| Action | Path | Responsibility | +|---|---|---| +| Create | `tests/helpers/jwt-auth.ts` | `fetchJwtToken()`, `writeJwtProfile()` | +| Create | `tests/helpers/interactive-helpers.ts` | `waitForOutput()`, `cleanKill()` | +| Modify | `tests/helpers/index.ts` | Re-export new helpers | +| Create | `tests/setup/agent-build-setup.ts` | Vitest globalSetup — runs `npm run build` once | +| Create | `vitest.agent.config.ts` | Agent-only vitest config with globalSetup + long timeouts | +| Modify | `package.json` | Add `test:integration:agent` and `test:integration:cli` scripts | +| Modify | `tests/integration/cli-commands/doctor.test.ts` | Add TC-002 (--verbose), TC-003 (JWT profile) | +| Modify | `tests/integration/cli-commands/profile.test.ts` | Add TC-004..TC-010, TC-032, TC-033 | +| Modify | `tests/integration/cli-commands/skills.test.ts` | Add TC-012 (JWT lifecycle), TC-013 (invalid source) | +| Create | `tests/integration/cli-commands/assistants.test.ts` | TC-014, TC-015 | +| Create | `tests/integration/cli-commands/models.test.ts` | TC-022 | +| Create | `tests/integration/agent-jwt-basic.test.ts` | TC-016..TC-019, TC-031 | +| Create | `tests/integration/agent-jwt-models.test.ts` | TC-020, TC-021 | +| Create | `tests/integration/agent-interactive-session.test.ts` | TC-024..TC-026 | + +**TC-023/TC-034 (`claude-cli-task.test.ts`) are deferred** — noted in a comment in `agent-jwt-basic.test.ts`. + +--- + +## Task 1: Helper Foundation + +**Files:** +- Create: `tests/helpers/jwt-auth.ts` +- Create: `tests/helpers/interactive-helpers.ts` +- Modify: `tests/helpers/index.ts` + +- [ ] **Step 1: Create `tests/helpers/jwt-auth.ts`** + +```typescript +import { mkdirSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; + +/** + * Fetch a fresh JWT token via Keycloak password grant. + * Requires CI_CODEMIE_USERNAME and CI_CODEMIE_PASSWORD env vars. + */ +export async function fetchJwtToken(): Promise { + const resp = await fetch( + 'https://auth.codemie.lab.epam.com/realms/codemie-prod/protocol/openid-connect/token', + { + method: 'POST', + headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, + body: new URLSearchParams({ + grant_type: 'password', + client_id: 'codemie-sdk', + username: process.env.CI_CODEMIE_USERNAME!, + password: process.env.CI_CODEMIE_PASSWORD!, + }), + } + ); + const data = (await resp.json()) as Record; + if (!data.access_token) throw new Error(`JWT token fetch failed: ${JSON.stringify(data)}`); + return data.access_token as string; +} + +export interface JwtProfileOverrides { + profileName?: string; + model?: string; + codeMieUrl?: string; + baseUrl?: string; + jwtToken?: string; + codeMieProject?: string; +} + +/** + * Write a bearer-auth profile to ${codemieHome}/codemie-cli.config.json. + * The config location matches getCodemiePath() which uses CODEMIE_HOME as the + * base directory (not ~/.codemie/.codemie). + */ +export function writeJwtProfile(codemieHome: string, overrides: JwtProfileOverrides = {}): void { + const profileName = overrides.profileName ?? 'jwt-autotest'; + const profile: Record = { + name: profileName, + provider: 'bearer-auth', + authMethod: 'jwt', + codeMieUrl: overrides.codeMieUrl ?? process.env.CI_CODEMIE_URL ?? '', + baseUrl: overrides.baseUrl ?? process.env.CI_CODEMIE_API_DOMAIN ?? '', + model: overrides.model ?? process.env.CI_CODEMIE_MODEL ?? 'claude-sonnet-4-6', + }; + if (overrides.jwtToken) profile.jwtToken = overrides.jwtToken; + if (overrides.codeMieProject) profile.codeMieProject = overrides.codeMieProject; + + const config = { version: 2, activeProfile: profileName, profiles: { [profileName]: profile } }; + mkdirSync(codemieHome, { recursive: true }); + writeFileSync(join(codemieHome, 'codemie-cli.config.json'), JSON.stringify(config, null, 2), 'utf-8'); +} +``` + +- [ ] **Step 2: Create `tests/helpers/interactive-helpers.ts`** + +```typescript +import { createInterface } from 'node:readline'; +import type { ChildProcess } from 'node:child_process'; + +/** + * Resolves with the matching line when stdout matches pattern. + * Rejects on timeout or process exit before match. + */ +export function waitForOutput( + proc: ChildProcess, + pattern: RegExp, + timeoutMs: number +): Promise { + return new Promise((resolve, reject) => { + const lines: string[] = []; + const rl = createInterface({ input: proc.stdout! }); + + const timer = setTimeout(() => { + rl.close(); + reject(new Error(`Timeout (${timeoutMs}ms) waiting for ${pattern}.\nGot:\n${lines.join('\n')}`)); + }, timeoutMs); + + rl.on('line', (line) => { + lines.push(line); + if (pattern.test(line)) { + clearTimeout(timer); + rl.close(); + resolve(line); + } + }); + + proc.on('close', (code) => { + clearTimeout(timer); + rl.close(); + if (code !== 0) { + reject(new Error(`Process exited with code ${code} before matching ${pattern}`)); + } + }); + }); +} + +/** + * Send SIGTERM and wait for the process to exit. + * Falls back to SIGKILL after 5 seconds. + */ +export function cleanKill(proc: ChildProcess): Promise { + return new Promise((resolve) => { + const fallback = setTimeout(() => proc.kill('SIGKILL'), 5000); + proc.on('close', () => { clearTimeout(fallback); resolve(); }); + proc.kill('SIGTERM'); + }); +} +``` + +- [ ] **Step 3: Add re-exports to `tests/helpers/index.ts`** + +Append to the existing file: +```typescript +export { fetchJwtToken, writeJwtProfile, type JwtProfileOverrides } from './jwt-auth.js'; +export { waitForOutput, cleanKill } from './interactive-helpers.js'; +``` + +- [ ] **Step 4: Verify helpers compile** + +```bash +npx tsc --noEmit +``` + +Expected: no TypeScript errors. + +- [ ] **Step 5: Commit** + +```bash +git add tests/helpers/jwt-auth.ts tests/helpers/interactive-helpers.ts tests/helpers/index.ts +git commit -m "test(helpers): add JWT auth and interactive process helpers" +``` + +--- + +## Task 2: Vitest Agent Config + Build Setup + npm Scripts + +**Files:** +- Create: `tests/setup/agent-build-setup.ts` +- Create: `vitest.agent.config.ts` +- Modify: `package.json` + +- [ ] **Step 1: Create `tests/setup/agent-build-setup.ts`** + +```typescript +import { execSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { dirname, resolve } from 'node:path'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); + +/** + * Vitest globalSetup — runs once per test session before any test file. + * Equivalent to pytest scope="session" fixture. + * Ensures dist/ exists so agent session tests can spawn bin/codemie-claude.js. + */ +export async function setup(): Promise { + const root = resolve(__dirname, '../..'); + console.log('\n[agent-integration] Building dist/ (runs once per session)...'); + execSync('npm run build', { cwd: root, stdio: 'inherit' }); + console.log('[agent-integration] Build complete.\n'); +} +``` + +- [ ] **Step 2: Create `vitest.agent.config.ts`** + +```typescript +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + // Picks up all agent-*.test.ts files (agent-jwt-basic, agent-jwt-models, + // agent-interactive-session) + include: ['tests/integration/agent-*.test.ts'], + globalSetup: ['tests/setup/agent-build-setup.ts'], + testTimeout: 180_000, // 3 min — real agent calls over the network + hookTimeout: 300_000, // 5 min — covers build + token fetch in beforeAll + reporters: ['verbose'], + env: { + FORCE_COLOR: '1', + NODE_ENV: 'test', + }, + pool: 'threads', + poolOptions: { + threads: { maxThreads: 4, minThreads: 1 }, + }, + isolate: true, + }, +}); +``` + +- [ ] **Step 3: Add scripts to `package.json`** + +In the `"scripts"` section, after `"test:integration"`, add: +```json +"test:integration:agent": "vitest run --config vitest.agent.config.ts", +"test:integration:cli": "vitest run tests/integration/cli-commands/", +``` + +- [ ] **Step 4: Verify config is valid** + +```bash +npx vitest --config vitest.agent.config.ts --reporter=verbose 2>&1 | head -20 +``` + +Expected: Vitest starts, finds no tests to run (INCLUDE_JWT_TESTS not set), exits 0 or prints "No test files found". + +- [ ] **Step 5: Commit** + +```bash +git add tests/setup/agent-build-setup.ts vitest.agent.config.ts package.json +git commit -m "test(config): add vitest.agent.config.ts with session-scoped build fixture" +``` + +--- + +## Task 3: doctor.test.ts — TC-002 and TC-003 + +**Files:** +- Modify: `tests/integration/cli-commands/doctor.test.ts` + +TC-001 is already covered by the existing `Doctor Command` describe block. TC-002 adds `--verbose`, TC-003 adds a JWT profile check (gated). + +- [ ] **Step 1: Read the current file** + +Read `tests/integration/cli-commands/doctor.test.ts` to find the end of the existing describe block (currently ends at line ~58). + +- [ ] **Step 2: Append TC-002 and TC-003 to the file** + +Add after the closing `});` of the existing `Doctor Command` describe block: + +```typescript +import { describe, it, expect, beforeAll } from 'vitest'; +import { fetchJwtToken, writeJwtProfile } from '../../helpers/index.js'; +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +``` + +> **Note:** The existing file already imports `describe`, `it`, `expect`, `beforeAll` — add only the new imports that are missing. Add them at the top of the file alongside existing imports. + +Append these two describe blocks after the existing one: + +```typescript +describe('Doctor Command — verbose (TC-002)', () => { + setupTestIsolation(); + + let verboseResult: CommandResult; + let baseResult: CommandResult; + + beforeAll(() => { + verboseResult = cli.runSilent('doctor --verbose'); + baseResult = cli.runSilent('doctor'); + }, 120_000); + + it('should not crash with --verbose', () => { + expect(verboseResult).toBeDefined(); + expect(verboseResult.output).toBeDefined(); + }); + + it('should produce output at least as long as non-verbose (or contain extra info)', () => { + // --verbose should either add more lines or include a debug path/indicator + const verboseLen = (verboseResult.output + (verboseResult.error ?? '')).length; + const baseLen = (baseResult.output + (baseResult.error ?? '')).length; + expect(verboseLen).toBeGreaterThanOrEqual(baseLen); + }); +}); + +const INCLUDE_JWT_TESTS = process.env.INCLUDE_JWT_TESTS === 'true'; + +describe.runIf(INCLUDE_JWT_TESTS)('Doctor Command — JWT profile (TC-003)', () => { + const REPO_ROOT = resolve(__dirname, '..', '..', '..'); + const CLI_BIN = join(REPO_ROOT, 'bin', 'codemie.js'); + + let testHome: string; + + beforeAll(async () => { + testHome = mkdtempSync(join(tmpdir(), 'codemie-jwt-doctor-')); + const token = await fetchJwtToken(); + writeJwtProfile(testHome, { profileName: 'jwt-autotest', jwtToken: token }); + }, 30_000); + + afterAll(() => { + rmSync(testHome, { recursive: true, force: true }); + }); + + it('should show JWT profile name in doctor output', () => { + const result = spawnSync(process.execPath, [CLI_BIN, 'doctor'], { + env: { ...process.env, CODEMIE_HOME: testHome, CI: '1' }, + encoding: 'utf-8', + timeout: 120_000, + }); + const combined = result.stdout + (result.stderr ?? ''); + expect(combined).toMatch(/jwt-autotest/i); + }); + + it('should not crash with JWT profile', () => { + const result = spawnSync(process.execPath, [CLI_BIN, 'doctor'], { + env: { ...process.env, CODEMIE_HOME: testHome, CI: '1' }, + encoding: 'utf-8', + timeout: 120_000, + }); + expect(result.status === 0 || result.status === 1).toBe(true); + }); +}); +``` + +Also add `afterAll` to the imports if not already there. + +- [ ] **Step 3: Run TC-001/TC-002 (non-JWT) to confirm no regressions** + +```bash +npx vitest run tests/integration/cli-commands/doctor.test.ts +``` + +Expected: existing TC-001 tests pass, TC-002 tests pass, TC-003 suite is skipped (INCLUDE_JWT_TESTS not set). + +- [ ] **Step 4: Commit** + +```bash +git add tests/integration/cli-commands/doctor.test.ts +git commit -m "test(cli): add TC-002 (doctor --verbose) and TC-003 (doctor JWT profile)" +``` + +--- + +## Task 4: profile.test.ts — TC-004..TC-010, TC-032, TC-033 + +**Files:** +- Modify: `tests/integration/cli-commands/profile.test.ts` + +The existing file has a basic two-test describe block. Replace it entirely with the full test suite below (the two existing tests become part of a broader suite). + +- [ ] **Step 1: Rewrite `tests/integration/cli-commands/profile.test.ts`** + +```typescript +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, writeFileSync, mkdirSync, readFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { fetchJwtToken, writeJwtProfile } from '../../helpers/index.js'; + +const REPO_ROOT = resolve(__dirname, '..', '..', '..'); +const CLI_BIN = join(REPO_ROOT, 'bin', 'codemie.js'); +const INCLUDE_JWT_TESTS = process.env.INCLUDE_JWT_TESTS === 'true'; + +/** Write a raw multi-profile config to CODEMIE_HOME */ +function writeConfig(codemieHome: string, config: object): void { + mkdirSync(codemieHome, { recursive: true }); + writeFileSync(join(codemieHome, 'codemie-cli.config.json'), JSON.stringify(config, null, 2), 'utf-8'); +} + +/** Read the current config from CODEMIE_HOME */ +function readConfig(codemieHome: string): Record { + return JSON.parse(readFileSync(join(codemieHome, 'codemie-cli.config.json'), 'utf-8')); +} + +/** Minimal profile shape — no real credentials needed for management tests */ +function fakeProfile(name: string) { + return { name, provider: 'bearer-auth', authMethod: 'jwt', codeMieUrl: 'https://test.example.com', baseUrl: 'https://test.example.com/api', model: 'test-model' }; +} + +function runCLI(args: string[], codemieHome: string) { + return spawnSync(process.execPath, [CLI_BIN, ...args], { + env: { ...process.env, CODEMIE_HOME: codemieHome, CI: '1' }, + encoding: 'utf-8', + timeout: 30_000, + }); +} + +// ─── TC-005: List profiles ──────────────────────────────────────────────────── +describe('Profile list — two profiles (TC-005)', () => { + let testHome: string; + + beforeAll(() => { + testHome = mkdtempSync(join(tmpdir(), 'codemie-prof-list-')); + writeConfig(testHome, { + version: 2, activeProfile: 'jwt-autotest', + profiles: { 'jwt-autotest': fakeProfile('jwt-autotest'), 'jwt-secondary': fakeProfile('jwt-secondary') }, + }); + }); + afterAll(() => rmSync(testHome, { recursive: true, force: true })); + + it('lists both profiles', () => { + const r = runCLI(['profile'], testHome); + const out = r.stdout + r.stderr; + expect(out).toMatch(/jwt-autotest/); + expect(out).toMatch(/jwt-secondary/); + }); +}); + +// ─── TC-006: Switch profile ─────────────────────────────────────────────────── +describe('Profile switch (TC-006)', () => { + let testHome: string; + + beforeAll(() => { + testHome = mkdtempSync(join(tmpdir(), 'codemie-prof-switch-')); + writeConfig(testHome, { + version: 2, activeProfile: 'jwt-autotest', + profiles: { 'jwt-autotest': fakeProfile('jwt-autotest'), 'jwt-secondary': fakeProfile('jwt-secondary') }, + }); + }); + afterAll(() => rmSync(testHome, { recursive: true, force: true })); + + it('exits 0 when switching to an existing profile', () => { + const r = runCLI(['profile', 'switch', 'jwt-secondary'], testHome); + expect(r.status).toBe(0); + }); + + it('updates activeProfile in the config file', () => { + const cfg = readConfig(testHome); + expect(cfg.activeProfile).toBe('jwt-secondary'); + }); + + it('profile status shows jwt-secondary as active', () => { + const r = runCLI(['profile', 'status'], testHome); + const out = r.stdout + r.stderr; + expect(out).toMatch(/jwt-secondary/); + }); +}); + +// ─── TC-007: Delete inactive profile ───────────────────────────────────────── +describe('Profile delete inactive (TC-007)', () => { + let testHome: string; + + beforeAll(() => { + testHome = mkdtempSync(join(tmpdir(), 'codemie-prof-del-')); + writeConfig(testHome, { + version: 2, activeProfile: 'jwt-autotest', + profiles: { 'jwt-autotest': fakeProfile('jwt-autotest'), 'jwt-secondary': fakeProfile('jwt-secondary') }, + }); + }); + afterAll(() => rmSync(testHome, { recursive: true, force: true })); + + it('exits 0 when deleting an inactive profile', () => { + const r = runCLI(['profile', 'delete', 'jwt-secondary', '-y'], testHome); + expect(r.status).toBe(0); + }); + + it('removed profile no longer appears in listing', () => { + const r = runCLI(['profile'], testHome); + expect(r.stdout + r.stderr).not.toMatch(/jwt-secondary/); + }); + + it('active profile jwt-autotest still exists', () => { + const r = runCLI(['profile'], testHome); + expect(r.stdout + r.stderr).toMatch(/jwt-autotest/); + }); +}); + +// ─── TC-008: Delete active profile (negative) ──────────────────────────────── +describe('Profile delete active — negative (TC-008)', () => { + let testHome: string; + + beforeAll(() => { + testHome = mkdtempSync(join(tmpdir(), 'codemie-prof-del-active-')); + writeConfig(testHome, { + version: 2, activeProfile: 'jwt-autotest', + profiles: { 'jwt-autotest': fakeProfile('jwt-autotest') }, + }); + }); + afterAll(() => rmSync(testHome, { recursive: true, force: true })); + + it('returns non-zero exit or warns when deleting active profile', () => { + const r = runCLI(['profile', 'delete', 'jwt-autotest', '-y'], testHome); + const out = r.stdout + r.stderr; + const isError = r.status !== 0 || /cannot|active|warning/i.test(out); + expect(isError).toBe(true); + }); + + it('profile still exists after failed delete', () => { + const cfg = readConfig(testHome); + const profiles = cfg.profiles as Record; + expect(profiles['jwt-autotest']).toBeDefined(); + }); +}); + +// ─── TC-009: Profile rename ─────────────────────────────────────────────────── +describe('Profile rename (TC-009)', () => { + let testHome: string; + + beforeAll(() => { + testHome = mkdtempSync(join(tmpdir(), 'codemie-prof-rename-')); + writeConfig(testHome, { + version: 2, activeProfile: 'jwt-autotest', + profiles: { 'jwt-autotest': fakeProfile('jwt-autotest') }, + }); + }); + afterAll(() => rmSync(testHome, { recursive: true, force: true })); + + it('exits 0 when renaming to a new name', () => { + const r = runCLI(['profile', 'rename', 'jwt-autotest', 'jwt-renamed'], testHome); + expect(r.status).toBe(0); + }); + + it('new name appears in profile listing', () => { + const r = runCLI(['profile'], testHome); + expect(r.stdout + r.stderr).toMatch(/jwt-renamed/); + }); + + it('old name no longer appears in profile listing', () => { + const r = runCLI(['profile'], testHome); + expect(r.stdout + r.stderr).not.toMatch(/jwt-autotest/); + }); +}); + +// ─── TC-010: Profile status with no profiles (negative) ────────────────────── +describe('Profile status — no profiles (TC-010)', () => { + let testHome: string; + + beforeAll(() => { + testHome = mkdtempSync(join(tmpdir(), 'codemie-prof-empty-')); + // Leave CODEMIE_HOME empty — no config file + }); + afterAll(() => rmSync(testHome, { recursive: true, force: true })); + + it('does not crash when no profiles configured', () => { + const r = runCLI(['profile', 'status'], testHome); + expect(r.status === 0 || r.status === 1).toBe(true); + }); + + it('produces non-empty output', () => { + const r = runCLI(['profile', 'status'], testHome); + expect((r.stdout + r.stderr).trim().length).toBeGreaterThan(0); + }); +}); + +// ─── TC-032: Switch to non-existent profile (negative) ─────────────────────── +describe('Profile switch — non-existent (TC-032)', () => { + let testHome: string; + + beforeAll(() => { + testHome = mkdtempSync(join(tmpdir(), 'codemie-prof-switch-neg-')); + writeConfig(testHome, { + version: 2, activeProfile: 'jwt-autotest', + profiles: { 'jwt-autotest': fakeProfile('jwt-autotest') }, + }); + }); + afterAll(() => rmSync(testHome, { recursive: true, force: true })); + + it('exits non-zero when switching to a non-existent profile', () => { + const r = runCLI(['profile', 'switch', 'does-not-exist'], testHome); + expect(r.status).not.toBe(0); + }); + + it('shows a not-found error message', () => { + const r = runCLI(['profile', 'switch', 'does-not-exist'], testHome); + const out = r.stdout + r.stderr; + expect(out).toMatch(/not found|does not exist|no profile/i); + }); +}); + +// ─── TC-033: Rename to existing name (negative) ────────────────────────────── +describe('Profile rename — to existing name (TC-033)', () => { + let testHome: string; + + beforeAll(() => { + testHome = mkdtempSync(join(tmpdir(), 'codemie-prof-rename-neg-')); + writeConfig(testHome, { + version: 2, activeProfile: 'profile-a', + profiles: { 'profile-a': fakeProfile('profile-a'), 'profile-b': fakeProfile('profile-b') }, + }); + }); + afterAll(() => rmSync(testHome, { recursive: true, force: true })); + + it('exits non-zero or shows error when renaming to existing name', () => { + const r = runCLI(['profile', 'rename', 'profile-a', 'profile-b'], testHome); + const out = r.stdout + r.stderr; + const isError = r.status !== 0 || /already exists|conflict|cannot/i.test(out); + expect(isError).toBe(true); + }); + + it('neither profile is corrupted after failed rename', () => { + const cfg = readConfig(testHome); + const profiles = cfg.profiles as Record; + expect(profiles['profile-a']).toBeDefined(); + expect(profiles['profile-b']).toBeDefined(); + }); +}); + +// ─── TC-004: Create profile via config write — JWT-gated ───────────────────── +describe.runIf(INCLUDE_JWT_TESTS)('Profile create via config (TC-004)', () => { + let testHome: string; + + beforeAll(async () => { + testHome = mkdtempSync(join(tmpdir(), 'codemie-prof-jwt-')); + const token = await fetchJwtToken(); + writeJwtProfile(testHome, { jwtToken: token }); + }, 30_000); + afterAll(() => rmSync(testHome, { recursive: true, force: true })); + + it('profile list shows jwt-autotest', () => { + const r = runCLI(['profile'], testHome); + expect(r.stdout + r.stderr).toMatch(/jwt-autotest/); + }); + + it('profile status shows provider and profile name', () => { + const r = runCLI(['profile', 'status'], testHome); + const out = r.stdout + r.stderr; + expect(out).toMatch(/jwt-autotest/); + expect(out).toMatch(/bearer-auth|jwt/i); + }); +}); +``` + +- [ ] **Step 2: Run profile tests to verify** + +```bash +npx vitest run tests/integration/cli-commands/profile.test.ts +``` + +Expected: TC-005 through TC-010, TC-032, TC-033 pass. TC-004 suite is skipped (INCLUDE_JWT_TESTS not set). + +- [ ] **Step 3: Commit** + +```bash +git add tests/integration/cli-commands/profile.test.ts +git commit -m "test(cli): add profile management tests TC-004..TC-010, TC-032, TC-033" +``` + +--- + +## Task 5: skills.test.ts — TC-012 and TC-013 + +**Files:** +- Modify: `tests/integration/cli-commands/skills.test.ts` + +TC-011 (unauthenticated block) is already covered by `'blocks every subcommand on unauthenticated invocation'`. Append two new JWT-gated describe blocks after the existing `describe.runIf(HAS_LOCAL_SSO)` block. + +- [ ] **Step 1: Append JWT-gated blocks to `tests/integration/cli-commands/skills.test.ts`** + +Add these imports at the top of the file (alongside existing imports): +```typescript +import { fetchJwtToken, writeJwtProfile } from '../../helpers/index.js'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +``` + +Append at the end of the file: + +```typescript +const INCLUDE_JWT_TESTS = process.env.INCLUDE_JWT_TESTS === 'true'; + +describe.runIf(INCLUDE_JWT_TESTS)('codemie skills — JWT lifecycle (TC-012)', () => { + let testHome: string; + let jwtToken: string; + let skillSource: string; + let skillName: string; + + beforeAll(async () => { + testHome = mkdtempSync(path.join(tmpdir(), 'codemie-skills-jwt-')); + jwtToken = await fetchJwtToken(); + writeJwtProfile(testHome, { jwtToken }); + + // Discover first available skill from the marketplace using skills find --json + const findResult = spawnSync(process.execPath, [CLI_BIN, 'skills', 'find', '--json', '--limit', '1'], { + cwd: workspace, + env: { ...process.env, CODEMIE_HOME: testHome, CI: '1' }, + encoding: 'utf-8', + timeout: 30_000, + }); + const found = JSON.parse(findResult.stdout) as Array<{ source: string; name: string }>; + if (!found.length) throw new Error('No skills found in marketplace — cannot run TC-012'); + skillSource = found[0].source; + skillName = found[0].name; + }, 60_000); + + afterAll(() => { + if (testHome) rmSync(testHome, { recursive: true, force: true }); + }); + + it('skills add exits 0 for a valid marketplace source', () => { + const r = spawnSync(process.execPath, [CLI_BIN, 'skills', 'add', skillSource, '-a', 'claude-code', '-y'], { + cwd: workspace, + env: { ...process.env, CODEMIE_HOME: testHome, CI: '1' }, + encoding: 'utf-8', + timeout: 60_000, + }); + expect(r.status).toBe(0); + }); + + it('skills list shows the installed skill', () => { + const r = spawnSync(process.execPath, [CLI_BIN, 'skills', 'list', '-a', 'claude-code'], { + cwd: workspace, + env: { ...process.env, CODEMIE_HOME: testHome, CI: '1' }, + encoding: 'utf-8', + timeout: 30_000, + }); + expect(r.stdout + r.stderr).toMatch(new RegExp(skillName, 'i')); + }); + + it('skills remove exits 0', () => { + const r = spawnSync(process.execPath, [CLI_BIN, 'skills', 'remove', '-s', skillName, '-a', 'claude-code', '-y'], { + cwd: workspace, + env: { ...process.env, CODEMIE_HOME: testHome, CI: '1' }, + encoding: 'utf-8', + timeout: 30_000, + }); + expect(r.status).toBe(0); + }); + + it('skills list no longer shows the removed skill', () => { + const r = spawnSync(process.execPath, [CLI_BIN, 'skills', 'list', '-a', 'claude-code'], { + cwd: workspace, + env: { ...process.env, CODEMIE_HOME: testHome, CI: '1' }, + encoding: 'utf-8', + timeout: 30_000, + }); + expect(r.stdout + r.stderr).not.toMatch(new RegExp(skillName, 'i')); + }); +}); + +describe.runIf(INCLUDE_JWT_TESTS)('codemie skills add — invalid source (TC-013)', () => { + let testHome: string; + + beforeAll(async () => { + testHome = mkdtempSync(path.join(tmpdir(), 'codemie-skills-invalid-')); + const token = await fetchJwtToken(); + writeJwtProfile(testHome, { jwtToken: token }); + }, 30_000); + + afterAll(() => { + if (testHome) rmSync(testHome, { recursive: true, force: true }); + }); + + it('exits non-zero for a nonexistent skill source', () => { + const r = spawnSync( + process.execPath, + [CLI_BIN, 'skills', 'add', 'nonexistent-owner/nonexistent-repo-xyz-99999', '-y'], + { + cwd: workspace, + env: { ...process.env, CODEMIE_HOME: testHome, CI: '1' }, + encoding: 'utf-8', + timeout: 30_000, + } + ); + expect(r.status).not.toBe(0); + }); + + it('shows an error message about not found or invalid source', () => { + const r = spawnSync( + process.execPath, + [CLI_BIN, 'skills', 'add', 'nonexistent-owner/nonexistent-repo-xyz-99999', '-y'], + { + cwd: workspace, + env: { ...process.env, CODEMIE_HOME: testHome, CI: '1' }, + encoding: 'utf-8', + timeout: 30_000, + } + ); + const out = r.stdout + r.stderr; + expect(out).toMatch(/not found|invalid|error|failed/i); + }); +}); +``` + +- [ ] **Step 2: Run skills tests to check for regressions** + +```bash +npx vitest run tests/integration/cli-commands/skills.test.ts +``` + +Expected: existing tests pass, TC-012/TC-013 suites skipped. + +- [ ] **Step 3: Commit** + +```bash +git add tests/integration/cli-commands/skills.test.ts +git commit -m "test(cli): add JWT skills lifecycle tests TC-012 and TC-013" +``` + +--- + +## Task 6: assistants.test.ts — TC-014 and TC-015 + +**Files:** +- Create: `tests/integration/cli-commands/assistants.test.ts` + +TC-014 writes a config entry + a mock `.claude/agents/.md` file directly (no interactive wizard). It overrides `HOME`/`USERPROFILE` in the subprocess env so the agent file lookup uses the temp dir. + +- [ ] **Step 1: Create `tests/integration/cli-commands/assistants.test.ts`** + +```typescript +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from 'node:fs'; +import { tmpdir, platform } from 'node:os'; +import { join, resolve } from 'node:path'; +import { fetchJwtToken, writeJwtProfile } from '../../helpers/index.js'; + +const REPO_ROOT = resolve(__dirname, '..', '..', '..'); +const CLI_BIN = join(REPO_ROOT, 'bin', 'codemie.js'); +const INCLUDE_JWT_TESTS = process.env.INCLUDE_JWT_TESTS === 'true'; + +const ASSISTANT_ID = process.env.CI_CODEMIE_ASSISTANT_ID ?? ''; + +function makeEnv(codemieHome: string, fakeHome: string): NodeJS.ProcessEnv { + const env: NodeJS.ProcessEnv = { ...process.env, CODEMIE_HOME: codemieHome, CI: '1' }; + // Override home so loadAssistantsByScope uses fakeHome for .claude/agents/ lookup + if (platform() === 'win32') { + env.USERPROFILE = fakeHome; + env.HOMEDRIVE = fakeHome.slice(0, 2); + env.HOMEPATH = fakeHome.slice(2); + } else { + env.HOME = fakeHome; + } + return env; +} + +describe.runIf(INCLUDE_JWT_TESTS)('Assistants — setup and chat (TC-014)', () => { + let testHome: string; // CODEMIE_HOME + let fakeHome: string; // fake os.homedir() for .claude/agents/ lookup + const assistantSlug = 'test-assistant'; + + beforeAll(async () => { + fakeHome = mkdtempSync(join(tmpdir(), 'codemie-asst-home-')); + testHome = join(fakeHome, '.codemie'); + + const token = await fetchJwtToken(); + // Write a config that includes the assistant registration + const profile = { + name: 'jwt-autotest', + provider: 'bearer-auth', + authMethod: 'jwt', + codeMieUrl: process.env.CI_CODEMIE_URL ?? '', + baseUrl: process.env.CI_CODEMIE_API_DOMAIN ?? '', + model: process.env.CI_CODEMIE_MODEL ?? 'claude-sonnet-4-6', + jwtToken: token, + }; + const assistant = { + id: ASSISTANT_ID, + name: 'Test Assistant', + slug: assistantSlug, + description: 'Integration test assistant', + registrationMode: 'agent', + }; + const config = { + version: 2, + activeProfile: 'jwt-autotest', + profiles: { 'jwt-autotest': profile }, + codemieAssistants: [assistant], + }; + mkdirSync(testHome, { recursive: true }); + writeFileSync(join(testHome, 'codemie-cli.config.json'), JSON.stringify(config, null, 2), 'utf-8'); + + // Write the required .claude/agents/.md file that loadAssistantsByScope checks + const agentsDir = join(fakeHome, '.claude', 'agents'); + mkdirSync(agentsDir, { recursive: true }); + writeFileSync(join(agentsDir, `${assistantSlug}.md`), `# ${assistantSlug}\n`, 'utf-8'); + }, 30_000); + + afterAll(() => rmSync(fakeHome, { recursive: true, force: true })); + + it('assistants chat returns a response for a registered assistant', () => { + const r = spawnSync(process.execPath, [CLI_BIN, 'assistants', 'chat', ASSISTANT_ID, 'Say PONG'], { + env: makeEnv(testHome, fakeHome), + encoding: 'utf-8', + timeout: 60_000, + }); + const out = r.stdout + r.stderr; + expect(r.status).toBe(0); + expect(out.length).toBeGreaterThan(0); + }); +}); + +describe.runIf(INCLUDE_JWT_TESTS)('Assistants chat — invalid ID (TC-015)', () => { + let testHome: string; + let fakeHome: string; + + beforeAll(async () => { + fakeHome = mkdtempSync(join(tmpdir(), 'codemie-asst-invalid-')); + testHome = join(fakeHome, '.codemie'); + const token = await fetchJwtToken(); + writeJwtProfile(testHome, { jwtToken: token }); + }, 30_000); + + afterAll(() => rmSync(fakeHome, { recursive: true, force: true })); + + it('exits non-zero for a nonexistent assistant ID', () => { + const r = spawnSync( + process.execPath, + [CLI_BIN, 'assistants', 'chat', 'nonexistent-assistant-id-xyz', 'hello'], + { + env: makeEnv(testHome, fakeHome), + encoding: 'utf-8', + timeout: 30_000, + } + ); + expect(r.status).not.toBe(0); + }); + + it('shows a not-found or error message', () => { + const r = spawnSync( + process.execPath, + [CLI_BIN, 'assistants', 'chat', 'nonexistent-assistant-id-xyz', 'hello'], + { + env: makeEnv(testHome, fakeHome), + encoding: 'utf-8', + timeout: 30_000, + } + ); + const out = r.stdout + r.stderr; + expect(out).toMatch(/not found|error|invalid|no assistant/i); + }); +}); +``` + +- [ ] **Step 2: Run the file to confirm it compiles and skips cleanly** + +```bash +npx vitest run tests/integration/cli-commands/assistants.test.ts +``` + +Expected: both suites skipped (INCLUDE_JWT_TESTS not set), exit 0. + +- [ ] **Step 3: Commit** + +```bash +git add tests/integration/cli-commands/assistants.test.ts +git commit -m "test(cli): add assistants chat tests TC-014 and TC-015" +``` + +--- + +## Task 7: models.test.ts — TC-022 + +**Files:** +- Create: `tests/integration/cli-commands/models.test.ts` + +- [ ] **Step 1: Create `tests/integration/cli-commands/models.test.ts`** + +```typescript +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { fetchJwtToken, writeJwtProfile } from '../../helpers/index.js'; + +const REPO_ROOT = resolve(__dirname, '..', '..', '..'); +const CLI_BIN = join(REPO_ROOT, 'bin', 'codemie.js'); +const INCLUDE_JWT_TESTS = process.env.INCLUDE_JWT_TESTS === 'true'; + +describe.runIf(INCLUDE_JWT_TESTS)('codemie models list (TC-022)', () => { + let testHome: string; + + beforeAll(async () => { + testHome = mkdtempSync(join(tmpdir(), 'codemie-models-')); + const token = await fetchJwtToken(); + writeJwtProfile(testHome, { jwtToken: token }); + }, 30_000); + + afterAll(() => rmSync(testHome, { recursive: true, force: true })); + + it('exits 0', () => { + const r = spawnSync(process.execPath, [CLI_BIN, 'models', 'list'], { + env: { ...process.env, CODEMIE_HOME: testHome, CI: '1' }, + encoding: 'utf-8', + timeout: 30_000, + }); + expect(r.status).toBe(0); + }); + + it('output contains at least one known model name', () => { + const r = spawnSync(process.execPath, [CLI_BIN, 'models', 'list'], { + env: { ...process.env, CODEMIE_HOME: testHome, CI: '1' }, + encoding: 'utf-8', + timeout: 30_000, + }); + expect(r.stdout + r.stderr).toMatch(/claude|gpt|gemini/i); + }); +}); +``` + +- [ ] **Step 2: Run to verify it skips cleanly** + +```bash +npx vitest run tests/integration/cli-commands/models.test.ts +``` + +Expected: suite skipped, exit 0. + +- [ ] **Step 3: Commit** + +```bash +git add tests/integration/cli-commands/models.test.ts +git commit -m "test(cli): add models list test TC-022" +``` + +--- + +## Task 8: agent-jwt-basic.test.ts — TC-016..TC-019, TC-031 + +**Files:** +- Create: `tests/integration/agent-jwt-basic.test.ts` + +These tests spawn `bin/codemie-claude.js`. Use `vitest.agent.config.ts` (via `npm run test:integration:agent`) — `dist/` is guaranteed by the globalSetup. + +- [ ] **Step 1: Create `tests/integration/agent-jwt-basic.test.ts`** + +```typescript +/** + * Agent JWT Basic Tests — TC-016..TC-019, TC-031 + * + * Run with: npm run test:integration:agent + * Requires: INCLUDE_JWT_TESTS=true, CI_CODEMIE_* env vars + * + * TC-023 / TC-034 (claude-cli-task.test.ts JWT migration) are deferred — + * that file does not yet exist in the repo. + */ + +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, readdirSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { fetchJwtToken, writeJwtProfile } from '../helpers/index.js'; + +const REPO_ROOT = resolve(__dirname, '..', '..'); +const CLAUDE_BIN = join(REPO_ROOT, 'bin', 'codemie-claude.js'); +const INCLUDE_JWT_TESTS = process.env.INCLUDE_JWT_TESTS === 'true'; + +/** Strip CodeMie tokens from env to prevent credential leakage into subprocesses */ +function cleanEnv(): NodeJS.ProcessEnv { + const env = { ...process.env }; + delete env.CODEMIE_SSO_TOKEN; + delete env.CODEMIE_JWT_TOKEN; + return env; +} + +describe.runIf(INCLUDE_JWT_TESTS)('Agent — JWT basic (TC-016..TC-019, TC-031)', () => { + let jwtToken: string; + + beforeAll(async () => { + jwtToken = await fetchJwtToken(); + }, 30_000); + + // ── TC-016: Agent runs successfully with JWT token ────────────────────────── + describe('TC-016 — agent runs with JWT token', () => { + let testHome: string; + + beforeAll(() => { + testHome = mkdtempSync(join(tmpdir(), 'codemie-jwt-basic-')); + }); + afterAll(() => rmSync(testHome, { recursive: true, force: true })); + + it('exits 0 and prints agent output', () => { + const r = spawnSync( + process.execPath, + [CLAUDE_BIN, '--task', 'Say the word READY and nothing else', '--jwt-token', jwtToken], + { env: { ...cleanEnv(), CODEMIE_HOME: testHome }, encoding: 'utf-8', timeout: 120_000 } + ); + expect(r.status).toBe(0); + expect(r.stdout).toMatch(/READY/i); + }); + + it('writes a session file to CODEMIE_HOME/sessions/', () => { + const sessionsDir = join(testHome, 'sessions'); + const files = readdirSync(sessionsDir).filter((f) => f.endsWith('.json')); + expect(files.length).toBeGreaterThan(0); + }); + }); + + // ── TC-017: Agent with profile + JWT override ─────────────────────────────── + describe('TC-017 — agent with profile and JWT token override', () => { + let testHome: string; + + beforeAll(() => { + testHome = mkdtempSync(join(tmpdir(), 'codemie-jwt-profile-')); + writeJwtProfile(testHome, { profileName: 'jwt-autotest' }); + }); + afterAll(() => rmSync(testHome, { recursive: true, force: true })); + + it('exits 0 when using --profile + --jwt-token', () => { + const r = spawnSync( + process.execPath, + [CLAUDE_BIN, '--profile', 'jwt-autotest', '--jwt-token', jwtToken, '--task', 'Say READY'], + { env: { ...cleanEnv(), CODEMIE_HOME: testHome }, encoding: 'utf-8', timeout: 120_000 } + ); + expect(r.status).toBe(0); + }); + + it('session file shows bearer-auth provider', () => { + const sessionsDir = join(testHome, 'sessions'); + const files = readdirSync(sessionsDir).filter((f) => f.endsWith('.json')); + expect(files.length).toBeGreaterThan(0); + const session = JSON.parse( + readdirSync(sessionsDir).map((f) => join(sessionsDir, f)).reduce((a, b) => + // Pick the most recently modified session file + require('node:fs').statSync(a).mtimeMs > require('node:fs').statSync(b).mtimeMs ? a : b + ) + ); + expect(session.provider ?? session.providerName ?? '').toMatch(/bearer-auth/i); + }); + }); + + // ── TC-018: Invalid JWT token (negative) ──────────────────────────────────── + describe('TC-018 — invalid JWT token (negative)', () => { + it('exits non-zero with an invalid JWT token', () => { + const testHome = mkdtempSync(join(tmpdir(), 'codemie-jwt-invalid-')); + try { + const r = spawnSync( + process.execPath, + [CLAUDE_BIN, '--task', 'Say hello', '--jwt-token', 'INVALID_TOKEN_VALUE'], + { env: { ...cleanEnv(), CODEMIE_HOME: testHome }, encoding: 'utf-8', timeout: 60_000 } + ); + expect(r.status).not.toBe(0); + expect(r.stdout + r.stderr).toMatch(/auth|unauthorized|401|invalid|token/i); + } finally { + rmSync(testHome, { recursive: true, force: true }); + } + }); + }); + + // ── TC-019: No profile, no JWT (negative) ─────────────────────────────────── + describe('TC-019 — no profile and no JWT (negative)', () => { + it('exits non-zero with empty CODEMIE_HOME and no --jwt-token', () => { + const testHome = mkdtempSync(join(tmpdir(), 'codemie-jwt-none-')); + try { + const r = spawnSync( + process.execPath, + [CLAUDE_BIN, '--task', 'Say hello'], + { env: { ...cleanEnv(), CODEMIE_HOME: testHome }, encoding: 'utf-8', timeout: 30_000 } + ); + expect(r.status).not.toBe(0); + expect(r.stdout + r.stderr).toMatch(/no profile|not configured|setup|profile/i); + } finally { + rmSync(testHome, { recursive: true, force: true }); + } + }); + }); + + // ── TC-031: Agent health check ────────────────────────────────────────────── + describe('TC-031 — agent health check', () => { + it('codemie-claude health exits 0', () => { + const testHome = mkdtempSync(join(tmpdir(), 'codemie-health-')); + try { + const r = spawnSync( + process.execPath, + [CLAUDE_BIN, 'health'], + { env: { ...cleanEnv(), CODEMIE_HOME: testHome }, encoding: 'utf-8', timeout: 15_000 } + ); + expect(r.status).toBe(0); + expect(r.stdout + r.stderr).toMatch(/install|binary|health/i); + } finally { + rmSync(testHome, { recursive: true, force: true }); + } + }); + }); +}); +``` + +- [ ] **Step 2: Verify the file compiles** + +```bash +npx tsc --noEmit +``` + +Expected: no errors. + +- [ ] **Step 3: Verify skip works (no JWT env)** + +```bash +npx vitest run --config vitest.agent.config.ts tests/integration/agent-jwt-basic.test.ts 2>&1 | tail -5 +``` + +Expected: suite skipped, exit 0. + +- [ ] **Step 4: Commit** + +```bash +git add tests/integration/agent-jwt-basic.test.ts +git commit -m "test(agent): add JWT basic agent tests TC-016..TC-019 and TC-031" +``` + +--- + +## Task 9: agent-jwt-models.test.ts — TC-020 and TC-021 + +**Files:** +- Create: `tests/integration/agent-jwt-models.test.ts` + +- [ ] **Step 1: Create `tests/integration/agent-jwt-models.test.ts`** + +```typescript +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync, readdirSync, readFileSync, mkdirSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { fetchJwtToken } from '../helpers/index.js'; + +const REPO_ROOT = resolve(__dirname, '..', '..'); +const CLAUDE_BIN = join(REPO_ROOT, 'bin', 'codemie-claude.js'); +const INCLUDE_JWT_TESTS = process.env.INCLUDE_JWT_TESTS === 'true'; + +function cleanEnv(): NodeJS.ProcessEnv { + const env = { ...process.env }; + delete env.CODEMIE_SSO_TOKEN; + delete env.CODEMIE_JWT_TOKEN; + return env; +} + +function writeModelProfile(codemieHome: string, profileName: string, model: string): void { + const config = { + version: 2, + activeProfile: profileName, + profiles: { + [profileName]: { + name: profileName, + provider: 'bearer-auth', + authMethod: 'jwt', + codeMieUrl: process.env.CI_CODEMIE_URL ?? '', + baseUrl: process.env.CI_CODEMIE_API_DOMAIN ?? '', + model, + }, + }, + }; + mkdirSync(codemieHome, { recursive: true }); + writeFileSync(join(codemieHome, 'codemie-cli.config.json'), JSON.stringify(config, null, 2), 'utf-8'); +} + +function getLatestSessionFile(sessionsDir: string): Record { + const files = readdirSync(sessionsDir) + .filter((f) => f.endsWith('.json')) + .map((f) => join(sessionsDir, f)) + .sort((a, b) => { + const { statSync } = require('node:fs'); + return statSync(b).mtimeMs - statSync(a).mtimeMs; + }); + if (!files.length) throw new Error('No session files found in ' + sessionsDir); + return JSON.parse(readFileSync(files[0], 'utf-8')); +} + +describe.runIf(INCLUDE_JWT_TESTS)('Agent — model selection (TC-020, TC-021)', () => { + let jwtToken: string; + + beforeAll(async () => { + jwtToken = await fetchJwtToken(); + }, 30_000); + + // ── TC-020: Session model field matches profile ────────────────────────────── + describe('TC-020 — session uses model from profile', () => { + let testHome: string; + + beforeAll(() => { + testHome = mkdtempSync(join(tmpdir(), 'codemie-model-match-')); + }); + afterAll(() => rmSync(testHome, { recursive: true, force: true })); + + it('session file model matches claude-sonnet-4-6 profile', () => { + writeModelProfile(testHome, 'profile-sonnet', 'claude-sonnet-4-6'); + spawnSync( + process.execPath, + [CLAUDE_BIN, '--profile', 'profile-sonnet', '--jwt-token', jwtToken, '--task', 'Say READY'], + { env: { ...cleanEnv(), CODEMIE_HOME: testHome }, encoding: 'utf-8', timeout: 120_000 } + ); + const session = getLatestSessionFile(join(testHome, 'sessions')); + expect(String(session.model ?? session.sonnetModel ?? '')).toMatch(/sonnet/i); + }); + + it('session file model matches claude-haiku-4-5-20251001 profile', () => { + writeModelProfile(testHome, 'profile-haiku', 'claude-haiku-4-5-20251001'); + spawnSync( + process.execPath, + [CLAUDE_BIN, '--profile', 'profile-haiku', '--jwt-token', jwtToken, '--task', 'Say READY'], + { env: { ...cleanEnv(), CODEMIE_HOME: testHome }, encoding: 'utf-8', timeout: 120_000 } + ); + const session = getLatestSessionFile(join(testHome, 'sessions')); + expect(String(session.model ?? session.haikuModel ?? '')).toMatch(/haiku/i); + }); + }); + + // ── TC-021: Haiku/Sonnet/Opus tiers all populated ────────────────────────── + describe('TC-021 — model tiers assigned correctly', () => { + let testHome: string; + + beforeAll(() => { + testHome = mkdtempSync(join(tmpdir(), 'codemie-tiers-')); + writeModelProfile(testHome, 'profile-tiers', 'claude-sonnet-4-6'); + }); + afterAll(() => rmSync(testHome, { recursive: true, force: true })); + + it('session file has haikuModel, sonnetModel, opusModel all set', () => { + spawnSync( + process.execPath, + [CLAUDE_BIN, '--profile', 'profile-tiers', '--jwt-token', jwtToken, '--task', 'Say READY'], + { env: { ...cleanEnv(), CODEMIE_HOME: testHome }, encoding: 'utf-8', timeout: 120_000 } + ); + const session = getLatestSessionFile(join(testHome, 'sessions')); + expect(session.haikuModel).toBeTruthy(); + expect(session.sonnetModel).toBeTruthy(); + expect(session.opusModel).toBeTruthy(); + expect(session.haikuModel).not.toBe(session.sonnetModel); + expect(session.sonnetModel).not.toBe(session.opusModel); + }); + }); +}); +``` + +- [ ] **Step 2: Verify compiles and skips** + +```bash +npx tsc --noEmit && npx vitest run --config vitest.agent.config.ts tests/integration/agent-jwt-models.test.ts 2>&1 | tail -5 +``` + +Expected: no TS errors, suite skipped. + +- [ ] **Step 3: Commit** + +```bash +git add tests/integration/agent-jwt-models.test.ts +git commit -m "test(agent): add model selection tests TC-020 and TC-021" +``` + +--- + +## Task 10: agent-interactive-session.test.ts — TC-024..TC-026 + +**Files:** +- Create: `tests/integration/agent-interactive-session.test.ts` + +TC-025 (skill invocation) and TC-026 (assistant chat) require live skill/assistant setup, making them the most complex. TC-024 (in-session model switch) is the baseline interactive test. + +- [ ] **Step 1: Create `tests/integration/agent-interactive-session.test.ts`** + +```typescript +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { spawn } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { fetchJwtToken, writeJwtProfile, waitForOutput, cleanKill } from '../helpers/index.js'; + +const REPO_ROOT = resolve(__dirname, '..', '..'); +const CLAUDE_BIN = join(REPO_ROOT, 'bin', 'codemie-claude.js'); +const CLI_BIN = join(REPO_ROOT, 'bin', 'codemie.js'); +const INCLUDE_JWT_TESTS = process.env.INCLUDE_JWT_TESTS === 'true'; + +function cleanEnv(): NodeJS.ProcessEnv { + const env = { ...process.env }; + delete env.CODEMIE_SSO_TOKEN; + delete env.CODEMIE_JWT_TOKEN; + return env; +} + +describe.runIf(INCLUDE_JWT_TESTS)('Interactive session tests', () => { + let jwtToken: string; + + beforeAll(async () => { + jwtToken = await fetchJwtToken(); + }, 30_000); + + // ── TC-024: Change model via /model slash command ─────────────────────────── + describe('TC-024 — in-session model switch via /model', () => { + let testHome: string; + + beforeAll(() => { + testHome = mkdtempSync(join(tmpdir(), 'codemie-interactive-model-')); + writeJwtProfile(testHome, { jwtToken }); + }); + afterAll(() => rmSync(testHome, { recursive: true, force: true })); + + it('agent acknowledges /model switch and responds with new model', async () => { + const proc = spawn( + process.execPath, + [CLAUDE_BIN, '--profile', 'jwt-autotest', '--jwt-token', jwtToken], + { + env: { ...cleanEnv(), CODEMIE_HOME: testHome }, + stdio: ['pipe', 'pipe', 'pipe'], + } + ); + + try { + // Wait for agent interactive ready prompt + await waitForOutput(proc, />\s*$|human:|ready/i, 60_000); + + // Send model switch command + proc.stdin!.write('/model claude-haiku-4-5-20251001\n'); + await waitForOutput(proc, /haiku|model.*switch|changed/i, 30_000); + + // Confirm new model responds + proc.stdin!.write('Say the word CONFIRMED and nothing else\n'); + const line = await waitForOutput(proc, /CONFIRMED/i, 60_000); + expect(line).toMatch(/CONFIRMED/i); + } finally { + await cleanKill(proc); + } + }, 180_000); + }); + + // ── TC-025: Skill invocation inside running session ───────────────────────── + describe('TC-025 — skill slash command in running session', () => { + let testHome: string; + let skillSource: string; + let skillSlashCommand: string; + + beforeAll(async () => { + testHome = mkdtempSync(join(tmpdir(), 'codemie-interactive-skill-')); + writeJwtProfile(testHome, { jwtToken }); + + // Discover a skill from the marketplace + const { spawnSync } = await import('node:child_process'); + const findResult = spawnSync(process.execPath, [CLI_BIN, 'skills', 'find', '--json', '--limit', '1'], { + env: { ...process.env, CODEMIE_HOME: testHome, CI: '1' }, + encoding: 'utf-8', + timeout: 30_000, + }); + const found = JSON.parse(findResult.stdout) as Array<{ source: string; name: string }>; + if (!found.length) throw new Error('No skills in marketplace — cannot run TC-025'); + skillSource = found[0].source; + skillSlashCommand = `/${found[0].name.replace(/[^a-z0-9-]/gi, '-').toLowerCase()}`; + + // Install the skill for claude-code agent + spawnSync(process.execPath, [CLI_BIN, 'skills', 'add', skillSource, '-a', 'claude-code', '-y'], { + env: { ...process.env, CODEMIE_HOME: testHome, CI: '1' }, + encoding: 'utf-8', + timeout: 60_000, + }); + }, 90_000); + + afterAll(async () => { + // Clean up installed skill + const { spawnSync } = await import('node:child_process'); + spawnSync(process.execPath, [CLI_BIN, 'skills', 'remove', '-s', skillSource, '-a', 'claude-code', '-y'], { + env: { ...process.env, CODEMIE_HOME: testHome, CI: '1' }, + encoding: 'utf-8', + timeout: 30_000, + }); + rmSync(testHome, { recursive: true, force: true }); + }); + + it('agent responds to skill slash command invocation', async () => { + const proc = spawn( + process.execPath, + [CLAUDE_BIN, '--profile', 'jwt-autotest', '--jwt-token', jwtToken], + { + env: { ...cleanEnv(), CODEMIE_HOME: testHome }, + stdio: ['pipe', 'pipe', 'pipe'], + } + ); + + try { + await waitForOutput(proc, />\s*$|human:|ready/i, 60_000); + proc.stdin!.write(`${skillSlashCommand}\n`); + // Skill produces some output — any non-empty response is sufficient + const line = await waitForOutput(proc, /.+/, 60_000); + expect(line.length).toBeGreaterThan(0); + } finally { + await cleanKill(proc); + } + }, 180_000); + }); + + // ── TC-026: Assistant chat (non-interactive via CLI) ──────────────────────── + describe('TC-026 — assistants chat non-interactive (PONG test)', () => { + let testHome: string; + const assistantId = process.env.CI_CODEMIE_ASSISTANT_ID ?? ''; + + beforeAll(() => { + testHome = mkdtempSync(join(tmpdir(), 'codemie-asst-chat-')); + writeJwtProfile(testHome, { jwtToken }); + }); + afterAll(() => rmSync(testHome, { recursive: true, force: true })); + + it('exits 0 and returns a non-empty response', () => { + const { spawnSync } = require('node:child_process'); + const r = spawnSync( + process.execPath, + [CLI_BIN, 'assistants', 'chat', assistantId, 'Say PONG and nothing else'], + { + env: { ...cleanEnv(), CODEMIE_HOME: testHome, CODEMIE_JWT_TOKEN: jwtToken, CI: '1' }, + encoding: 'utf-8', + timeout: 60_000, + } + ); + expect(r.status).toBe(0); + expect(r.stdout + r.stderr).toMatch(/PONG/i); + }); + }); +}); +``` + +- [ ] **Step 2: Verify compiles and skips** + +```bash +npx tsc --noEmit && npx vitest run --config vitest.agent.config.ts tests/integration/agent-interactive-session.test.ts 2>&1 | tail -5 +``` + +Expected: suite skipped, exit 0. + +- [ ] **Step 3: Commit** + +```bash +git add tests/integration/agent-interactive-session.test.ts +git commit -m "test(agent): add interactive session tests TC-024, TC-025, TC-026" +``` + +--- + +## Task 12: Final Validation + +- [ ] **Step 1: Run all CLI integration tests (no JWT)** + +```bash +npm run test:integration:cli +``` + +Expected: all non-JWT tests pass, JWT suites show as skipped. + +- [ ] **Step 2: Run full integration suite to check no regressions** + +```bash +npm run test:integration +``` + +Expected: all pre-existing tests still pass. + +- [ ] **Step 3: TypeScript full check** + +```bash +npx tsc --noEmit +``` + +Expected: no errors. + +- [ ] **Step 4: Lint** + +```bash +npm run lint +``` + +Expected: zero warnings. + +- [ ] **Step 5: Final commit** + +```bash +git add . +git commit -m "test(integration): complete CLI integration test suite TC-001..TC-034" +``` diff --git a/docs/superpowers/plans/2026-06-22-hybrid-auth-test-analysis.md b/docs/superpowers/plans/2026-06-22-hybrid-auth-test-analysis.md new file mode 100644 index 00000000..200042d3 --- /dev/null +++ b/docs/superpowers/plans/2026-06-22-hybrid-auth-test-analysis.md @@ -0,0 +1,315 @@ +# Hybrid Auth Analysis: Integration Test Files + +## Summary Table + +| Test File | Current Auth Mode | Hybrid Possible | Effort | Key Blocker | +|---|---|---|---|---| +| `agent-task-session.test.ts` | **Already hybrid** (SSO default / JWT on CI) | N/A (reference) | — | — | +| `agent-interactive-session.test.ts` | JWT-only (`INCLUDE_JWT_TESTS`) | Partial | High | PTY wizard flows are JWT-only by design; non-interactive asserts could be split | +| `agent-jwt-basic.test.ts` | JWT-only (`INCLUDE_JWT_TESTS`) | Yes | Medium | TC-016/017/021 are generic enough; TC-019 (no-auth negative) is inherently JWT-only | +| `agent-jwt-models.test.ts` | JWT-only (`INCLUDE_JWT_TESTS`) | Yes | Medium | Model selection is auth-agnostic; profile writes need dual paths | +| `cli-commands/doctor.test.ts` | Mixed: SSO-implicit + JWT-gated block | Partial | Low | Base doctor tests already auth-agnostic; TC-003 JWT block can be extended | +| `cli-commands/error-handling.test.ts` | Auth-agnostic | N/A (already hybrid) | None | Never touches auth | +| `cli-commands/models.test.ts` | JWT-only (`INCLUDE_JWT_TESTS`) | Yes | Low | Only needs `writeJwtProfile` → add SSO path with `sso-autotest` profile | +| `cli-commands/profile.test.ts` | Mixed: fake-profile tests + JWT-gated TC-004 | Partial | Low | Profile CRUD tests use fake profiles; only TC-004 needs dual paths | +| `cli-commands/skills.test.ts` | Mixed: SSO auto-detect + JWT-gated TC-012/013 | Partial | Medium | TC-012 requires SSO cookies for catalog API regardless of JWT; TC-013 hybridisable | + +--- + +## The Reference Hybrid Pattern (`agent-task-session.test.ts`) + +This file is the canonical example. Key structural elements to replicate: + +- **Line 78**: `const CI_IS_LOCAL_RUN = getTestEnvFlagOrDefault('CI_IS_LOCAL_RUN', true);` + - Defaults `true` so local developers get SSO without any config change. + - Set to `false` in CI (or `.env.test.local`) for JWT mode. + +- **Lines 89–138 (`beforeAll`)**: Single `if (!CI_IS_LOCAL_RUN)` branch. + - JWT path: `fetchJwtToken()`, `mkdtempSync()` → `jwtHome`, `writeJwtProfile(jwtHome, { jwtToken })`. + - SSO path: reads/writes `~/.codemie/codemie-cli.config.json`, sets `sso-autotest` profile. + +- **Lines 141–156 (`afterAll`)**: Parallel teardown. + - JWT: `rmSync(jwtHome, ...)`. + - SSO: restores `originalActiveProfile`. + +- **Lines 176–210 (test body)**: Separate `spawnSync` calls per mode. + - SSO: `cleanEnv()` (strips all `CODEMIE_*`), no `--jwt-token` arg, `cwd: tempTestDir`. + - JWT: `jwtCleanEnv()` (allowlist), `--jwt-token jwtToken` arg, `cwd: jwtHome`, `CODEMIE_HOME: jwtHome`. + +- **Lines 277–283, 300–303**: `if (CI_IS_LOCAL_RUN)` guards for SSO-only assertions (`sync.conversations.conversationId`, `syncedAt`). + +### The two `cleanEnv()` functions are NOT the same + +| Function | Location | Strategy | When to use | +|---|---|---|---| +| `cleanEnv()` (SSO) | `agent-task-session.ts:62` | Denylist — strips `CODEMIE_*` from full `process.env` | SSO mode; relies on real `~/.codemie` | +| `jwtCleanEnv()` | `helpers/jwt-auth.ts:54` | Allowlist — only PATH + OS vars | JWT mode; prevents any credential leak | + +--- + +## Per-File Analysis + +### 1. `agent-interactive-session.test.ts` + +**Current auth mode:** JWT-only. Outer `describe` at line 47 uses `describe.runIf(INCLUDE_JWT_TESTS)`. All test homes are created with `writeJwtProfile(testHome, { jwtToken })`. + +**Can it be made hybrid?** Partial. + +**TC-level breakdown:** + +| TC | Description | Hybridisable | Notes | +|---|---|---|---| +| TC-014 | setup assistants PTY wizard | Yes (with effort) | Wizard UI is auth-agnostic; SSO env needs `CI_CODEMIE_ASSISTANT_NAME` | +| TC-015 | invalid assistant ID (negative) | Yes | Only `--jwt-token` arg differs | +| TC-024 | in-session /model switch PTY | Yes | `--profile jwt-autotest` → `--profile sso-autotest` | +| TC-025 | skill slash command PTY | Yes (with effort) | Needs `CI_CODEMIE_SKILL_NAME` for SSO env | +| TC-026 | assistants chat non-interactive | Yes | Drop `--jwt-token` and `CODEMIE_JWT_TOKEN` env | + +**What would need to change:** + +- TC-015, TC-026: Remove `--jwt-token jwtToken` arg and `CODEMIE_JWT_TOKEN: jwtToken` from env. Replace `CODEMIE_HOME: testHome` with no override (SSO uses `~/.codemie`). +- TC-024: Replace `--profile jwt-autotest` with `--profile sso-autotest`. +- TC-014, TC-025: As above, plus ensure `CI_CODEMIE_ASSISTANT_NAME` / `CI_CODEMIE_SKILL_NAME` point at a dev-env resource. + +**Problems and risks:** + +1. The file defines its own `cleanEnv()` (lines 33–45) that is actually identical to `jwtCleanEnv()` — not the SSO variant. In a hybrid file, two named functions are needed, or imports from the helpers index. +2. `waitFor(/\d+ assistants total/, 60_000)` (line 89) and `waitFor(/\d+ skills total/, 60_000)` (line 347) make live API calls. Network failures produce different failure modes in SSO vs JWT. +3. `CI_CODEMIE_ASSISTANT_NAME` and `CI_CODEMIE_ASSISTANT_ID` (line 421) are currently JWT CI env vars. SSO mode on a developer machine would need these pointing at a dev-environment assistant. + +**Estimated effort:** High. Five TC scenarios, PTY timing sensitivity, separate `CI_CODEMIE_ASSISTANT_NAME` / `CI_CODEMIE_SKILL_NAME` env vars needed per mode. + +--- + +### 2. `agent-jwt-basic.test.ts` + +**Current auth mode:** JWT-only. Guard at line 30: `describe.runIf(INCLUDE_JWT_TESTS)`. + +**Can it be made hybrid?** Yes, for TC-016, TC-017. TC-018 and TC-019 must stay JWT-only. + +**TC-level breakdown:** + +| TC | Description | Hybridisable | Notes | +|---|---|---|---| +| TC-016 | agent runs with token | Yes | Core assertion (exit 0, session file) is auth-agnostic | +| TC-017 | agent with profile + token override | Yes | SSO provider assertion differs: `/ai-run-sso/i` vs `/bearer-auth/i` | +| TC-018 | invalid JWT token negative | No | JWT-specific negative path — no SSO equivalent | +| TC-019 | no profile / no JWT negative | No | Safe only with empty JWT home; unsafe against real `~/.codemie` | + +**What would need to change:** + +- TC-016 `beforeAll`: add SSO profile write branch; SSO spawn uses `cleanEnv()`, no `--jwt-token`. +- TC-017: SSO path uses `--profile sso-autotest` without `--jwt-token`; provider assertion branches on `CI_IS_LOCAL_RUN`. + +**Problems and risks:** + +1. TC-016 writes sessions to `join(testHome, 'sessions')`. SSO mode sessions go to `~/.codemie/sessions`. The sessions-dir path must be conditional. +2. TC-019 would be dangerous to hybridise — the "no config" negative test assumes a clean slate, which is not true on a developer machine with `~/.codemie` populated. + +**Estimated effort:** Medium. + +--- + +### 3. `agent-jwt-models.test.ts` + +**Current auth mode:** JWT-only. Guard at line 48: `describe.runIf(INCLUDE_JWT_TESTS)`. Uses local `writeModelProfile()` (lines 24–45) to create profiles with specific models. + +**Can it be made hybrid?** Yes. + +**What would need to change:** + +- `writeModelProfile()` (local helper) creates `bearer-auth` profiles. SSO path would write to `~/.codemie` with `sso-autotest` profile including the `model` field — same pattern as `agent-task-session.ts` lines 107–136. +- TC-020: Runs two `spawnSync` calls (sonnet and haiku). JWT path uses `jwtHome` and `haikuHome` (line 74) — two separate temp dirs. SSO path cannot have two simultaneous `~/.codemie` homes; must run sonnet, edit active profile model, then run haiku sequentially. +- TC-021: Single run, straightforward dual-path like TC-016. + +**Problems and risks:** + +1. TC-020 SSO path: sequential profile edits to `~/.codemie` between sonnet and haiku runs. If the test crashes mid-run, the profile is left in the haiku-model state. Add a `try/finally` restore. +2. Model names (`claude-haiku-4-5-20251001`) must exist in the SSO environment's model catalog. If the SSO env has a different catalog this could fail. + +**Estimated effort:** Medium. + +--- + +### 5. `cli-commands/doctor.test.ts` + +**Current auth mode:** Mixed. + +- Base `describe` blocks (lines 21–86): Use `setupTestIsolation()` + `createCLIRunner()`. No auth required — tests only check static output patterns. Already auth-agnostic. +- TC-003 (lines 90–121): `describe.runIf(INCLUDE_JWT_TESTS)`. Checks that a JWT profile name appears in `doctor` output. + +**Can it be made hybrid?** Partial (TC-003 only; base tests unchanged). + +**What would need to change for TC-003:** + +- Add `CI_IS_LOCAL_RUN` flag. +- SSO path: write `sso-autotest` profile to a temp home, run `codemie doctor`, assert `/sso-autotest/i` in output. +- The `spawnSync` env at line 101 uses `{ ...process.env, CODEMIE_HOME: testHome, CI: '1' }` — should use `cleanEnv()` to avoid outer SSO credential bleed. + +**Problems and risks:** + +1. `setupTestIsolation()` has a potential `undefined` assignment bug (see Helpers section below). Not a hybrid concern but a reliability risk. +2. TC-003 SSO path needs the SSO session to be active. If not active, `doctor` will show auth errors rather than the profile name — would need to distinguish between "profile listed" and "profile valid" assertions. + +**Estimated effort:** Low. + +--- + +### 6. `cli-commands/error-handling.test.ts` + +**Current auth mode:** Auth-agnostic. Runs `codemie invalid-command-xyz` which fails before any auth check. + +**Can it be made hybrid?** N/A — already trivially hybrid. + +No changes needed. Adding `CI_IS_LOCAL_RUN` branching would be pure noise. + +--- + +### 7. `cli-commands/models.test.ts` + +**Current auth mode:** JWT-only. Guard at line 12: `describe.runIf(INCLUDE_JWT_TESTS)`. Calls `codemie models list` with `CODEMIE_JWT_TOKEN` in env. + +**Can it be made hybrid?** Yes. + +**What would need to change:** + +- Import `getTestEnvFlagOrDefault` from helpers, set `CI_IS_LOCAL_RUN`. +- SSO `beforeAll`: write `sso-autotest` profile to `~/.codemie` (pattern from `agent-task-session.ts` lines 107–136). +- SSO `spawnSync` env: `{ ...cleanEnv(), CI: '1' }` (no `CODEMIE_HOME` override, no `CODEMIE_JWT_TOKEN`). +- Replace `describe.runIf(INCLUDE_JWT_TESTS)` with unconditional (since `CI_IS_LOCAL_RUN` defaults to `true`). + +**Problems and risks:** + +1. `codemie models list` makes a live API call. SSO env must return a model list that satisfies `process.env.CI_CODEMIE_MODEL ?? 'claude'` assertion regex. +2. Current JWT `spawnSync` env (`{ ...process.env, CODEMIE_HOME, CODEMIE_JWT_TOKEN, CI }`) does not strip outer `CODEMIE_*` vars — inconsistent with the reference pattern. Should use `jwtCleanEnv()` for the JWT path. + +**Estimated effort:** Low. + +--- + +### 8. `cli-commands/profile.test.ts` + +**Current auth mode:** Mixed. + +- TC-005–010, TC-032–033: Use `runCLI()` with fake `fakeProfile()` data (`codeMieUrl: 'https://test.example.com'`). No real auth. Auth-agnostic. +- TC-004 (lines 264–286): `describe.runIf(INCLUDE_JWT_TESTS)`. Uses `fetchJwtToken()` + `writeJwtProfile()` + `profile status`. + +**Can it be made hybrid?** Partial (TC-004 only; TC-005–010, TC-032–033 unchanged). + +**What would need to change for TC-004:** + +- SSO path: skip `fetchJwtToken()`, write `sso-autotest` profile, run `profile status`, assert `/sso-autotest/i` and `/ai-run-sso|sso/i`. +- Drop `CODEMIE_JWT_TOKEN: jwtToken` from `extraEnv` at line 282. + +**Problems and risks:** + +1. `runCLI()` (line 25) passes `{ ...process.env, CODEMIE_HOME, ... }` without stripping `CODEMIE_*` — outer session vars could bleed in. This is consistent with all profile tests but worth noting. +2. TC-004 SSO path: `profile status` may make a network call to validate the profile. If SSO session is not active, the test would get a network error instead of expected status output. + +**Estimated effort:** Low. + +--- + +### 9. `cli-commands/skills.test.ts` + +**Current auth mode:** Three-tier mixed. + +1. Lines 107–156: Auth-agnostic (`--help`, unauthenticated negative). Always runs. +2. Lines 162–262: `describe.runIf(HAS_LOCAL_SSO)` — SSO auto-detected at import time. Already SSO-only by design. +3. Lines 269–395: Two JWT-gated blocks. + - TC-012: `INCLUDE_JWT_TESTS && HAS_LOCAL_SSO` — requires **both** simultaneously. + - TC-013: `INCLUDE_JWT_TESTS` only. + +**Can it be made hybrid?** Partial. + +**TC-level breakdown:** + +| TC / Block | Hybridisable | Notes | +|---|---|---| +| `--help` / unauthenticated negative | N/A (already unconditional) | No change needed | +| HAS_LOCAL_SSO authenticated block | N/A (already SSO) | No change needed | +| TC-012 JWT + SSO lifecycle | No | Skills marketplace catalog API requires SSO cookies regardless of JWT auth | +| TC-013 invalid source (negative) | Yes | Just drop `CODEMIE_JWT_TOKEN` for SSO path | + +**Problems and risks:** + +1. TC-012's `INCLUDE_JWT_TESTS && HAS_LOCAL_SSO` guard is the most complex in the entire suite. TC-012 fundamentally depends on SSO cookies for the `skills find --json` catalog call (line 285 uses `{ ...process.env, CI: '1' }`, not an isolated JWT home). Making it "hybrid" in the `CI_IS_LOCAL_RUN` sense is a misnomer — it already needs SSO. +2. `HAS_LOCAL_SSO` detection (lines 162–178) runs at module load time by probing the CLI. This is correct design and should remain as-is. +3. The `skills find --json` call at line 285 uses full `process.env` — if `CODEMIE_HOME` is set in the test runner's env (e.g. from a parent isolation scope), SSO credentials won't be found. Latent risk if ever composed inside a `setupTestIsolation()` context. + +**Estimated effort:** Medium (TC-013 is Low; TC-012 is a No due to structural dependency on SSO catalog API). + +--- + +## Shared Helpers — What Would Need Updating + +### `tests/helpers/jwt-auth.ts` + +No structural changes needed for existing exports. Additions that would benefit every hybrid file: + +1. **`writeSsoProfile(codemieHome, overrides?)`** — mirrors `writeJwtProfile()` but writes `ai-run-sso` / `sso` auth method. Currently the SSO profile write is inlined in `agent-task-session.test.ts` lines 107–136. Extracting it prevents per-file duplication. + +2. **Export `ssoCleanEnv()` (or rename the inline `cleanEnv()`)** — the CODEMIE_*-stripping variant currently defined inline in `agent-task-session.ts:62`. All hybrid SSO spawns need this. Currently: + - `agent-task-session.ts:62`: strips `CODEMIE_*` from full env (correct for SSO). + - `agent-interactive-session.ts:33`: local `cleanEnv()` that is actually identical to `jwtCleanEnv()` — a copy/paste error that would cause JWT credentials to bleed into SSO spawns if used for the SSO path. + +3. **`saveAndSwitchSsoProfile()` / `restoreOriginalProfile()` helpers** — the read-modify-write of `~/.codemie/codemie-cli.config.json` in `beforeAll`/`afterAll` (lines 95–155) is boilerplate that every SSO-path hybrid file would need. Extracting it reduces copy-paste risk. + +### `tests/helpers/test-env.ts` + +No changes required. `getTestEnvFlagOrDefault('CI_IS_LOCAL_RUN', true)` already serves all hybrid files. + +### `tests/helpers/session-poll.ts` + +No changes required. `pollForSession(sessionsDir, testUuid)` is path-agnostic. + +### `tests/helpers/test-isolation.ts` (potential bug) + +`setupTestIsolation()` uses: +```typescript +testHome = mkdirSync(join(tmpdir(), prefix), { recursive: true }) || + join(tmpdir(), prefix + Math.random().toString(36).slice(2, 9)); +``` +`mkdirSync` with `{ recursive: true }` returns `string | undefined` (returns `undefined` if directory already exists). This could assign `undefined` to `testHome` on some Node versions, causing `process.env.CODEMIE_HOME = "undefined"` (literal string). Not directly a hybrid concern but a reliability risk in `doctor.test.ts` and `profile.test.ts`. + +### `tests/helpers/cli-runner.ts` + +`CLIRunner.runSilent()` inherits `process.env` via `execSync` without any filtering. Tests using `createCLIRunner()` rely on `setupTestIsolation()` having already set `process.env.CODEMIE_HOME`. For hybrid tests using `CLIRunner`, either: +- Add an optional `env` parameter to `runSilent()`. +- Use direct `spawnSync` with explicit env instead of `CLIRunner`. + +--- + +## Recommended Priority and Order + +### Priority 1 — Quick wins (no or minimal effort) + +1. **`cli-commands/error-handling.test.ts`**: No work needed. Already hybrid. +2. **`cli-commands/models.test.ts`**: Single describe, clear dual-path. Add `CI_IS_LOCAL_RUN`, SSO `beforeAll` branch. (~1 hour) + +### Priority 2 — Helper extraction first (multiplier for everything else) + +4. **Extract shared helpers**: Add `writeSsoProfile()`, export `ssoCleanEnv()` (the CODEMIE_*-stripping variant), add `saveAndSwitchSsoProfile()` to `tests/helpers/jwt-auth.ts` (or a new `sso-auth.ts`). (~2 hours) + +### Priority 3 — Medium-effort hybridisations (after Priority 2) + +5. **`agent-jwt-basic.test.ts` TC-016, TC-017**: Dual-path `beforeAll` + spawn. TC-018 and TC-019 stay JWT-only. (~3 hours) +6. **`agent-jwt-models.test.ts`**: Dual-path. TC-020 needs careful sequential profile editing for SSO haiku run. (~3–4 hours) +7. **`cli-commands/doctor.test.ts` TC-003**: Add SSO path to JWT-gated describe. (~1 hour) +8. **`cli-commands/profile.test.ts` TC-004**: Add SSO path. (~1 hour) +9. **`cli-commands/skills.test.ts` TC-013**: Add SSO path to invalid-source test. (~2 hours) + +### Priority 4 — High effort + +10. **`agent-interactive-session.test.ts`**: Staged approach recommended: + - TC-015 and TC-026 (non-interactive) first. (~2 hours) + - TC-024 (/model switch PTY). (~3 hours) + - TC-014 and TC-025 (PTY wizard flows) last — most complex due to live catalog API calls and PTY timing sensitivity. (~4–6 hours each) + +### Do Not Hybridise + +| File / TC | Reason | +|---|---| +| `agent-jwt-basic.test.ts` TC-018 | Tests JWT-specific invalid-token negative path | +| `agent-jwt-basic.test.ts` TC-019 | "No config" negative is unsafe against real `~/.codemie` on developer machines | +| `cli-commands/skills.test.ts` TC-012 | Catalog API requires SSO cookies even when JWT auth is active — structurally impossible to run without SSO | diff --git a/package-lock.json b/package-lock.json index 21e36ff9..97170181 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,7 +24,7 @@ "@modelcontextprotocol/server": "2.0.0-alpha.2", "chalk": "^5.3.0", "cli-table3": "^0.6.5", - "codemie-sdk": "^0.1.428", + "codemie-sdk": "^0.1.462", "commander": "^11.1.0", "cors": "^2.8.5", "dedent": "^1.7.1", @@ -74,6 +74,7 @@ "eslint": "^9.38.0", "husky": "^9.1.7", "lint-staged": "^16.2.7", + "node-pty": "^1.1.0", "tsc-alias": "^1.8.16", "typescript": "^5.3.3", "vitest": "^4.1.5" @@ -4757,9 +4758,9 @@ } }, "node_modules/codemie-sdk": { - "version": "0.1.428", - "resolved": "https://registry.npmjs.org/codemie-sdk/-/codemie-sdk-0.1.428.tgz", - "integrity": "sha512-GpwFkRs+ii7XJJA4LVWf8MNT3M4sT2TcH/8tQodYlOIeDDVO1I1fpG2Jmy4CsB3cX8QlHJavwq0ypqLCCnDsGA==", + "version": "0.1.462", + "resolved": "https://registry.npmjs.org/codemie-sdk/-/codemie-sdk-0.1.462.tgz", + "integrity": "sha512-pXTIp0c6mZcEq0kSGCzjDKlQ5nTzHGV8IgjMptahzO0rIPd+r0Zm+FSZQmvP5+NxsilrmIyVN2XdcCfIgMvlYA==", "license": "Apache-2.0", "dependencies": { "axios": "1.15.0", @@ -7725,6 +7726,24 @@ "integrity": "sha512-73sE9+3UaLYYFmDsFZnqCInzPyh3MqIwZO9cw58yIqAZhONrrabrYyYe3TuIqtIiOuTXVhsGau8hcrhhwSsDIQ==", "license": "MIT" }, + "node_modules/node-pty": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/node-pty/-/node-pty-1.1.0.tgz", + "integrity": "sha512-20JqtutY6JPXTUnL0ij1uad7Qe1baT46lyolh2sSENDd4sTzKZ4nmAFkeAARDKwmlLjPx6XKRlwRUxwjOy+lUg==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "node-addon-api": "^7.1.0" + } + }, + "node_modules/node-pty/node_modules/node-addon-api": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-7.1.1.tgz", + "integrity": "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==", + "dev": true, + "license": "MIT" + }, "node_modules/normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", diff --git a/package.json b/package.json index 6cca35ff..31ea8d74 100644 --- a/package.json +++ b/package.json @@ -34,17 +34,20 @@ "prepare:install-artifacts": "node scripts/prepare-install-artifacts.mjs", "dev": "tsc --watch", "test": "vitest", - "test:unit": "vitest run src", - "test:integration": "vitest run tests/integration", - "test:coverage": "vitest run --coverage", + "test:unit": "vitest run --project unit", + "test:integration": "vitest run --project cli", + "test:integration:cli": "vitest run --project cli", + "test:integration:agent": "vitest run --project agent", + "test:run": "vitest run --project unit --project cli", + "test:all": "vitest run --project unit && vitest run --project cli && vitest run --project agent", + "test:coverage": "vitest run --project unit --coverage", "test:watch": "vitest --watch", "test:ui": "vitest --ui", - "test:run": "vitest run", "typecheck": "tsc --noEmit", "format": "npm run lint:fix", "check:pre-commit": "npm run typecheck && npm run lint", - "lint": "eslint '{src,tests}/**/*.ts' --max-warnings=0", - "lint:fix": "eslint '{src,tests}/**/*.ts' --fix", + "lint": "eslint {src,tests}/**/*.ts --max-warnings=0", + "lint:fix": "eslint {src,tests}/**/*.ts --fix", "commitlint": "commitlint --edit", "commitlint:last": "commitlint --from HEAD~1 --to HEAD --verbose", "validate:secrets": "node scripts/validate-secrets.js", @@ -58,7 +61,7 @@ "lint-staged": { "*.ts": [ "eslint --max-warnings=0 --no-warn-ignored", - "vitest related --run" + "vitest related --run --exclude tests/integration/agent-*.test.ts --exclude tests/integration/cli-commands/models.test.ts" ], "package.json": [ "npm run license-check" @@ -127,7 +130,7 @@ "@modelcontextprotocol/server": "2.0.0-alpha.2", "chalk": "^5.3.0", "cli-table3": "^0.6.5", - "codemie-sdk": "^0.1.428", + "codemie-sdk": "^0.1.462", "commander": "^11.1.0", "cors": "^2.8.5", "dedent": "^1.7.1", @@ -163,6 +166,7 @@ "eslint": "^9.38.0", "husky": "^9.1.7", "lint-staged": "^16.2.7", + "node-pty": "^1.1.0", "tsc-alias": "^1.8.16", "typescript": "^5.3.3", "vitest": "^4.1.5" diff --git a/scripts/validate-secrets.js b/scripts/validate-secrets.js index decbdb72..db986318 100755 --- a/scripts/validate-secrets.js +++ b/scripts/validate-secrets.js @@ -64,6 +64,7 @@ if (!engine) { } const engineBin = resolveCommand(engine); +const quotedEngineBin = engineBin.includes(' ') ? `"${engineBin}"` : engineBin; // Produce the staged diff on the host so gitleaks doesn't need git access // inside the container — required for Apple Containers which cannot run git @@ -95,7 +96,7 @@ if (hasConfig) { console.log('Running Gitleaks secrets detection...'); -const gitleaks = spawn(engineBin, args, { +const gitleaks = spawn(quotedEngineBin, args, { stdio: ['pipe', 'inherit', 'inherit'], shell: isWindows, }); diff --git a/src/agents/core/AgentCLI.ts b/src/agents/core/AgentCLI.ts index 218daacc..a91d56b9 100644 --- a/src/agents/core/AgentCLI.ts +++ b/src/agents/core/AgentCLI.ts @@ -5,6 +5,7 @@ import chalk from 'chalk'; import { AgentAdapter } from './types.js'; import { ConfigLoader, CodeMieConfigOptions } from '../../utils/config.js'; import { ensureApiBase, DEFAULT_CODEMIE_BASE_URL } from '../../providers/core/codemie-auth-helpers.js'; +import { AuthMethod, ProviderName } from '../../providers/core/types.js'; import { JWTTemplate } from '../../providers/plugins/jwt/jwt.template.js'; import { logger } from '../../utils/logger.js'; import { getDirname } from '../../utils/paths.js'; @@ -189,14 +190,14 @@ export class AgentCLI { // JWT token from CLI overrides everything if (options.jwtToken) { process.env.CODEMIE_JWT_TOKEN = options.jwtToken as string; - process.env.CODEMIE_AUTH_METHOD = 'jwt'; + process.env.CODEMIE_AUTH_METHOD = AuthMethod.JWT; const hasNoConfig = !options.provider && !(await ConfigLoader.hasGlobalConfig()) && !(await ConfigLoader.hasLocalConfig(process.cwd())); if (hasNoConfig) { - config.provider = 'bearer-auth'; + config.provider = ProviderName.BEARER_AUTH; if (!config.model) { config.model = JWTTemplate.recommendedModels?.[0]; } @@ -206,7 +207,7 @@ export class AgentCLI { ? ensureApiBase(config.codeMieUrl) : ensureApiBase(DEFAULT_CODEMIE_BASE_URL); } - config.authMethod = 'jwt'; + config.authMethod = AuthMethod.JWT; } // Validate --reasoning-effort (catches both CLI flag and profile defaults) @@ -240,7 +241,7 @@ export class AgentCLI { // Skip apiKey validation for SSO and JWT authentication methods const authMethod = config.authMethod; - const usesAlternativeAuth = authMethod === 'sso' || authMethod === 'jwt'; + const usesAlternativeAuth = authMethod === AuthMethod.SSO || authMethod === AuthMethod.JWT; if (requiresAuth && !config.apiKey && !usesAlternativeAuth) { missingFields.push('apiKey'); @@ -289,7 +290,7 @@ export class AgentCLI { // which gets spread after process.env in BaseAgentAdapter.run(), erasing the // 'jwt' value we set in process.env above and causing the proxy to use the SSO path. if (options.jwtToken) { - providerEnv.CODEMIE_AUTH_METHOD = 'jwt'; + providerEnv.CODEMIE_AUTH_METHOD = AuthMethod.JWT; providerEnv.CODEMIE_JWT_TOKEN = options.jwtToken as string; } diff --git a/src/cli/commands/assistants/__tests__/chat.test.ts b/src/cli/commands/assistants/__tests__/chat.test.ts index a88f5128..55b5090e 100644 --- a/src/cli/commands/assistants/__tests__/chat.test.ts +++ b/src/cli/commands/assistants/__tests__/chat.test.ts @@ -74,9 +74,9 @@ describe('Assistants Chat Command', () => { }); describe('Command Options', () => { - it('should have verbose, conversation-id, load-history, and file options', () => { + it('should have verbose, conversation-id, load-history, file, and jwt-token options', () => { const command = createAssistantsChatCommand(); - expect(command.options).toHaveLength(4); + expect(command.options).toHaveLength(5); }); it('should accept --verbose flag', () => { diff --git a/src/cli/commands/assistants/chat/index.ts b/src/cli/commands/assistants/chat/index.ts index d8781eb1..6371c96c 100644 --- a/src/cli/commands/assistants/chat/index.ts +++ b/src/cli/commands/assistants/chat/index.ts @@ -8,20 +8,20 @@ import { Command } from 'commander'; import chalk from 'chalk'; import ora from 'ora'; import inquirer from 'inquirer'; +import { type CodeMieClient, type FileToUpload } from 'codemie-sdk'; import { logger } from '@/utils/logger.js'; import { ConfigLoader } from '@/utils/config.js'; import { StorageScope } from '@/env/types.js'; import { createErrorContext, formatErrorForUser } from '@/utils/errors.js'; import { getAuthenticatedClient, promptReauthentication } from '@/utils/auth.js'; +import { AuthMethod } from '@/providers/core/types.js'; import type { CodemieAssistant, ProviderProfile } from '@/env/types.js'; -import type { CodeMieClient } from 'codemie-sdk'; import { ROLES, MESSAGES, type HistoryMessage } from '../constants.js'; import { loadConversationHistory } from './historyLoader.js'; import { appendConversationTurn } from './historyPersister.js'; import { isExitCommand, enableVerboseMode } from './utils.js'; import type { ChatCommandOptions, SingleMessageOptions } from './types.js'; import { detectFileUploadsFromSession, readFilesFromPaths, type DetectedFile } from './claudeUploadsDetector.js'; -import type { FileToUpload } from 'codemie-sdk'; /** Assistant label color */ const ASSISTANT_LABEL_COLOR = [177, 185, 249] as const; @@ -42,6 +42,7 @@ export function createAssistantsChatCommand(): Command { .option('-f, --file ', 'File path to upload (can be used multiple times)', (value: string, previous: string[]) => { return previous ? [...previous, value] : [value]; }, [] as string[]) + .option('--jwt-token ', 'JWT bearer token for authentication (bypasses SSO)') .action(async ( assistantId: string | undefined, message: string | undefined, @@ -79,7 +80,13 @@ async function chatWithAssistant( ConfigLoader.loadAssistantsByScope(StorageScope.LOCAL, workingDir).catch(() => [] as CodemieAssistant[]), ]); const registeredAssistants = [...globalAssistants, ...localAssistants]; - const client = await getAuthenticatedClient(config); + + const jwtToken = options.jwtToken ?? process.env.CODEMIE_JWT_TOKEN; + if (jwtToken) { + config.authMethod = AuthMethod.JWT; + config.jwtConfig = { ...config.jwtConfig, token: jwtToken }; + } + const client: CodeMieClient = await getAuthenticatedClient(config); const conversationId = options.conversationId || process.env.CODEMIE_SESSION_ID; const isExplicitConversationId = !!options.conversationId; diff --git a/src/cli/commands/assistants/chat/types.ts b/src/cli/commands/assistants/chat/types.ts index 8336babd..95d0b92e 100644 --- a/src/cli/commands/assistants/chat/types.ts +++ b/src/cli/commands/assistants/chat/types.ts @@ -12,6 +12,7 @@ export interface ChatCommandOptions { conversationId?: string; loadHistory?: boolean; file?: string[]; + jwtToken?: string; } /** diff --git a/src/cli/commands/doctor/checks/JWTAuthCheck.ts b/src/cli/commands/doctor/checks/JWTAuthCheck.ts index d0c43930..d2050a5d 100644 --- a/src/cli/commands/doctor/checks/JWTAuthCheck.ts +++ b/src/cli/commands/doctor/checks/JWTAuthCheck.ts @@ -3,6 +3,8 @@ */ import { CredentialStore } from '../../../../utils/security.js'; +import { resolveJwtTokenEnvVar } from '../../../../providers/plugins/jwt/jwt.utils.js'; +import { AuthMethod } from '../../../../providers/core/types.js'; import { ConfigLoader } from '../../../../utils/config.js'; import { HealthCheck, HealthCheckResult, HealthCheckDetail, ProgressCallback } from '../types.js'; @@ -19,7 +21,7 @@ export class JWTAuthCheck implements HealthCheck { const config = await ConfigLoader.load(); // Only check if profile uses JWT auth - if (config.authMethod !== 'jwt') { + if (config.authMethod !== AuthMethod.JWT) { details.push({ status: 'info', message: 'Not using JWT authentication (skipped)' @@ -29,7 +31,7 @@ export class JWTAuthCheck implements HealthCheck { // Check 1: JWT token available (env var or credential store) onProgress?.('Checking JWT token presence'); - const tokenEnvVar = config.jwtConfig?.tokenEnvVar || 'CODEMIE_JWT_TOKEN'; + const tokenEnvVar = resolveJwtTokenEnvVar(config); const envToken = process.env[tokenEnvVar]; if (!envToken) { diff --git a/src/cli/commands/models.ts b/src/cli/commands/models.ts index b1a7f826..f1f5a1f0 100644 --- a/src/cli/commands/models.ts +++ b/src/cli/commands/models.ts @@ -5,7 +5,7 @@ import { ProviderRegistry } from '../../providers/core/registry.js'; import { logger } from '../../utils/logger.js'; import type { ModelInfo } from '../../providers/core/types.js'; -const UNSUPPORTED_PROVIDERS = new Set(['openai', 'bearer-auth', 'openai-compatible']); +const UNSUPPORTED_PROVIDERS = new Set(['openai', 'openai-compatible']); function formatTable(models: ModelInfo[]): void { const ID_WIDTH = 40; diff --git a/src/cli/commands/skills/add.ts b/src/cli/commands/skills/add.ts index c639c3b6..3a13bbf2 100644 --- a/src/cli/commands/skills/add.ts +++ b/src/cli/commands/skills/add.ts @@ -83,6 +83,7 @@ export function createAddCommand(): Command { const result = await runSkillsCli(args, { cwd, timeoutMs: ADD_GIT_TIMEOUT_MS, + interactive, env: { GIT_TERMINAL_PROMPT: '0', GCM_INTERACTIVE: 'never', @@ -107,6 +108,10 @@ export function createAddCommand(): Command { return; } + if (!interactive && result.stderr) { + process.stderr.write(result.stderr); + } + const errorCode = classifySkillError({ result }); if (shouldShowGitAccessHelp(source, errorCode)) { process.stderr.write(formatGitAccessHelp(sanitizedSource, errorCode)); diff --git a/src/env/types.ts b/src/env/types.ts index 3a9036fc..0896e257 100644 --- a/src/env/types.ts +++ b/src/env/types.ts @@ -84,6 +84,9 @@ export interface ProviderProfile { tokenEnvVar?: string; expiresAt?: number; }; + // Keycloak / SSO auth fields (required by SDK for SSO; not used with jwt_token) + authServerUrl?: string; + authRealm?: string; // AWS Bedrock-specific fields awsProfile?: string; diff --git a/src/providers/core/default-agent-hooks.ts b/src/providers/core/default-agent-hooks.ts new file mode 100644 index 00000000..e4916142 --- /dev/null +++ b/src/providers/core/default-agent-hooks.ts @@ -0,0 +1,55 @@ +/** + * Default agent lifecycle hooks shared by all CodeMie providers. + * + * Installs the agent extension before each run and injects --plugin-dir + * for Claude Code. Any provider template can spread these hooks rather + * than duplicating the logic. + */ + +import type { AgentConfig } from '@/agents/core/types.js'; +import type { ProviderTemplate } from '@/providers/core/types.js'; + +export const defaultAgentHooks: ProviderTemplate['agentHooks'] = { + '*': { + async beforeRun(env: NodeJS.ProcessEnv, config: AgentConfig): Promise { + const agentName = config.agent; + if (!agentName) return env; + + // Dynamic import avoids circular dependency — AgentRegistry loads all plugins + // including provider templates, so top-level import would cause a cycle. + const { AgentRegistry } = await import('@/agents/registry.js'); + const agent = AgentRegistry.getAgent(agentName); + if (!agent) return env; + + const installer = (agent as any).getExtensionInstaller?.(); + if (!installer) return env; + + try { + const result = await installer.install(); + env[`CODEMIE_${agentName.toUpperCase()}_EXTENSION_DIR`] = result.targetPath; + + if (!result.success) { + const { logger } = await import('@/utils/logger.js'); + logger.warn(`[${agentName}] Extension installation returned failure: ${result.error || 'unknown error'}`); + logger.warn(`[${agentName}] Continuing without extension - hooks may not be available`); + } + } catch (error) { + const { logger } = await import('@/utils/logger.js'); + const errorMsg = error instanceof Error ? error.message : String(error); + logger.error(`[${agentName}] Extension installation threw exception: ${errorMsg}`); + logger.warn(`[${agentName}] Continuing without extension - hooks may not be available`); + } + + return env; + } + }, + + 'claude': { + enrichArgs(args: string[], _config: AgentConfig): string[] { + const pluginDir = process.env.CODEMIE_CLAUDE_EXTENSION_DIR; + if (!pluginDir) return args; + if (args.some(arg => arg === '--plugin-dir')) return args; + return ['--plugin-dir', pluginDir, ...args]; + } + } +}; diff --git a/src/providers/core/types.ts b/src/providers/core/types.ts index 05d5b382..66540f09 100644 --- a/src/providers/core/types.ts +++ b/src/providers/core/types.ts @@ -40,6 +40,28 @@ export interface ModelMetadata { */ export type AuthenticationType = 'api-key' | 'sso' | 'oauth' | 'jwt' | 'none'; +/** + * Known provider names — use instead of hardcoded strings + */ +export const ProviderName = { + BEARER_AUTH: 'bearer-auth', + AI_RUN_SSO: 'ai-run-sso', + LITELLM: 'litellm', + BEDROCK: 'bedrock', + OLLAMA: 'ollama', + ANTHROPIC_SUBSCRIPTION: 'anthropic-subscription', +} as const; + +/** + * Auth method values — use instead of hardcoded strings + */ +export const AuthMethod = { + JWT: 'jwt', + SSO: 'sso', + MANUAL: 'manual', + API_KEY: 'api-key', +} as const; + /** * Provider template - declarative metadata * diff --git a/src/providers/plugins/jwt/index.ts b/src/providers/plugins/jwt/index.ts index ee497cd3..590fe56b 100644 --- a/src/providers/plugins/jwt/index.ts +++ b/src/providers/plugins/jwt/index.ts @@ -5,11 +5,14 @@ * Auto-registers when imported. */ -import { ProviderRegistry } from '../../core/registry.js'; +import { ProviderRegistry } from '@/providers/core/registry.js'; +import { ProviderName } from '@/providers/core/types.js'; import { JWTBearerSetupSteps } from './jwt.setup-steps.js'; export { JWTTemplate } from './jwt.template.js'; export { JWTBearerSetupSteps } from './jwt.setup-steps.js'; +export { JWTModelProxy } from './jwt.models.js'; +export { resolveJwtToken, resolveJwtTokenEnvVar, JWT_TOKEN_DEFAULT_ENV_VAR } from './jwt.utils.js'; -// Register setup steps -ProviderRegistry.registerSetupSteps('bearer-auth', JWTBearerSetupSteps); +// Register setup steps (model proxy auto-registers in jwt.models.ts) +ProviderRegistry.registerSetupSteps(ProviderName.BEARER_AUTH, JWTBearerSetupSteps); diff --git a/src/providers/plugins/jwt/jwt.models.ts b/src/providers/plugins/jwt/jwt.models.ts new file mode 100644 index 00000000..db3304e1 --- /dev/null +++ b/src/providers/plugins/jwt/jwt.models.ts @@ -0,0 +1,38 @@ +/** + * JWT Model Proxy + * + * Fetches available models from the CodeMie API using a JWT Bearer token. + */ + +import type { CodeMieConfigOptions } from '@/env/types.js'; +import type { ModelInfo, ProviderModelFetcher } from '@/providers/core/types.js'; +import { ProviderName } from '@/providers/core/types.js'; +import { resolveJwtToken, resolveJwtTokenEnvVar } from '@/providers/plugins/jwt/jwt.utils.js'; +import { fetchCodeMieModels } from '@/providers/plugins/sso/sso.http-client.js'; +import { ProviderRegistry } from '@/providers/core/registry.js'; + +export class JWTModelProxy implements ProviderModelFetcher { + supports(provider: string): boolean { + return provider === ProviderName.BEARER_AUTH; + } + + async fetchModels(config: CodeMieConfigOptions): Promise { + const token = resolveJwtToken(config); + + if (!token) { + throw new Error( + `JWT token not found. Set ${resolveJwtTokenEnvVar(config)} or pass --jwt-token .` + ); + } + + const apiUrl = config.baseUrl; + if (!apiUrl) { + throw new Error('No baseUrl configured for bearer-auth provider.'); + } + + const modelIds = await fetchCodeMieModels(apiUrl, token); + return modelIds.map((id) => ({ id, name: id })); + } +} + +ProviderRegistry.registerModelProxy(ProviderName.BEARER_AUTH, new JWTModelProxy()); diff --git a/src/providers/plugins/jwt/jwt.setup-steps.ts b/src/providers/plugins/jwt/jwt.setup-steps.ts index 2a6da1ee..66e7419c 100644 --- a/src/providers/plugins/jwt/jwt.setup-steps.ts +++ b/src/providers/plugins/jwt/jwt.setup-steps.ts @@ -11,12 +11,14 @@ import type { ProviderSetupSteps, ProviderCredentials, AuthValidationResult -} from '../../core/types.js'; -import type { CodeMieConfigOptions } from '../../../env/types.js'; +} from '@/providers/core/types.js'; +import { ProviderName, AuthMethod } from '@/providers/core/types.js'; +import { JWT_TOKEN_DEFAULT_ENV_VAR, resolveJwtToken, resolveJwtTokenEnvVar } from '@/providers/plugins/jwt/jwt.utils.js'; +import type { CodeMieConfigOptions } from '@/env/types.js'; import { ensureApiBase } from '../../core/codemie-auth-helpers.js'; export const JWTBearerSetupSteps: ProviderSetupSteps = { - name: 'bearer-auth', + name: ProviderName.BEARER_AUTH, async getCredentials(_isUpdate?: boolean): Promise { console.log(chalk.cyan('\n🔐 JWT Bearer Authorization Setup\n')); @@ -63,7 +65,7 @@ export const JWTBearerSetupSteps: ProviderSetupSteps = { } ]); - let tokenEnvVar = 'CODEMIE_JWT_TOKEN'; + let tokenEnvVar = JWT_TOKEN_DEFAULT_ENV_VAR; if (tokenConfigAnswers.customEnvVar) { const envVarAnswers = await inquirer.prompt([ @@ -71,7 +73,7 @@ export const JWTBearerSetupSteps: ProviderSetupSteps = { type: 'input', name: 'envVar', message: 'Environment variable name:', - default: 'CODEMIE_JWT_TOKEN', + default: JWT_TOKEN_DEFAULT_ENV_VAR, validate: (input: string) => { if (!input.trim()) return 'Variable name is required'; if (!/^[A-Z_][A-Z0-9_]*$/.test(input)) { @@ -99,7 +101,7 @@ export const JWTBearerSetupSteps: ProviderSetupSteps = { baseUrl, // Full API URL with suffix additionalConfig: { codeMieUrl, // User's input (base URL) - authMethod: 'jwt', + authMethod: AuthMethod.JWT, jwtConfig: { tokenEnvVar // Note: No apiUrl needed - baseUrl is used for credential storage @@ -130,24 +132,23 @@ export const JWTBearerSetupSteps: ProviderSetupSteps = { | undefined; return { - provider: 'bearer-auth', + provider: ProviderName.BEARER_AUTH, codeMieUrl: credentials.additionalConfig?.codeMieUrl as string | undefined, // Base URL (user input) baseUrl: credentials.baseUrl, // Full API URL with suffix model: selectedModel, - authMethod: 'jwt', + authMethod: AuthMethod.JWT, jwtConfig }; }, async validateAuth(config: CodeMieConfigOptions): Promise { // Check if JWT token is available at runtime - const tokenEnvVar = config.jwtConfig?.tokenEnvVar || 'CODEMIE_JWT_TOKEN'; - const token = process.env[tokenEnvVar] || config.jwtConfig?.token; + const token = resolveJwtToken(config); if (!token) { return { valid: false, - error: `JWT token not found in ${tokenEnvVar} environment variable` + error: `JWT token not found in ${resolveJwtTokenEnvVar(config)} environment variable` }; } diff --git a/src/providers/plugins/jwt/jwt.template.ts b/src/providers/plugins/jwt/jwt.template.ts index a26a25b8..59439e95 100644 --- a/src/providers/plugins/jwt/jwt.template.ts +++ b/src/providers/plugins/jwt/jwt.template.ts @@ -8,16 +8,19 @@ * Auto-registers on import via registerProvider(). */ -import type { ProviderTemplate } from '../../core/types.js'; -import { registerProvider } from '../../core/index.js'; +import type { ProviderTemplate } from '@/providers/core/types.js'; +import { ProviderName, AuthMethod } from '@/providers/core/types.js'; +import { defaultAgentHooks } from '@/providers/core/default-agent-hooks.js'; +import { resolveJwtToken } from '@/providers/plugins/jwt/jwt.utils.js'; +import { registerProvider } from '@/providers/core/index.js'; export const JWTTemplate = registerProvider({ - name: 'bearer-auth', + name: ProviderName.BEARER_AUTH, displayName: 'Bearer Authorization', description: 'JWT token authentication - Provide token via CLI or environment variable', defaultBaseUrl: 'https://codemie.lab.epam.com', requiresAuth: true, - authType: 'jwt', + authType: AuthMethod.JWT, priority: 1, // Show after CodeMie SSO hidden: true, // Not shown in interactive setup - used only for script/auto-configuration defaultProfileName: 'jwt-bearer', @@ -34,6 +37,8 @@ export const JWTTemplate = registerProvider({ tokenSource: 'runtime' // Token provided at runtime, not during setup }, + agentHooks: defaultAgentHooks, + // Environment Variable Export exportEnvVars: (config) => { const env: Record = {}; @@ -44,11 +49,10 @@ export const JWTTemplate = registerProvider({ } // Set auth method to JWT - env.CODEMIE_AUTH_METHOD = 'jwt'; + env.CODEMIE_AUTH_METHOD = AuthMethod.JWT; // Export JWT token if available (from env var or config) - const tokenEnvVar = config.jwtConfig?.tokenEnvVar || 'CODEMIE_JWT_TOKEN'; - const token = process.env[tokenEnvVar] || config.jwtConfig?.token; + const token = resolveJwtToken(config); if (token) { env.CODEMIE_JWT_TOKEN = token; } diff --git a/src/providers/plugins/jwt/jwt.utils.ts b/src/providers/plugins/jwt/jwt.utils.ts new file mode 100644 index 00000000..35d18efc --- /dev/null +++ b/src/providers/plugins/jwt/jwt.utils.ts @@ -0,0 +1,23 @@ +/** + * JWT token resolution utilities + */ + +import type { CodeMieConfigOptions } from '@/env/types.js'; + +export const JWT_TOKEN_DEFAULT_ENV_VAR = 'CODEMIE_JWT_TOKEN'; + +/** + * Returns the env-var name to read the JWT token from. + * Falls back to CODEMIE_JWT_TOKEN when not customised in the profile. + */ +export function resolveJwtTokenEnvVar(config: CodeMieConfigOptions): string { + return config.jwtConfig?.tokenEnvVar ?? JWT_TOKEN_DEFAULT_ENV_VAR; +} + +/** + * Resolves the JWT token for the given profile. + * Priority: env var named by profile → inline token stored in profile. + */ +export function resolveJwtToken(config: CodeMieConfigOptions): string | undefined { + return process.env[resolveJwtTokenEnvVar(config)] ?? config.jwtConfig?.token; +} diff --git a/src/providers/plugins/sso/proxy/sso.proxy.ts b/src/providers/plugins/sso/proxy/sso.proxy.ts index 32d8772c..d78eb7c8 100644 --- a/src/providers/plugins/sso/proxy/sso.proxy.ts +++ b/src/providers/plugins/sso/proxy/sso.proxy.ts @@ -26,6 +26,7 @@ import { createServer, Server, IncomingMessage, ServerResponse } from 'http'; import { randomUUID } from 'crypto'; import { URL } from 'url'; import { ProviderRegistry } from '../../../core/registry.js'; +import { AuthMethod } from '../../../core/types.js'; import type { JWTCredentials, SSOCredentials } from '../../../core/types.js'; import { logger } from '../../../../utils/logger.js'; import { ProxyHTTPClient } from './proxy-http-client.js'; @@ -59,13 +60,13 @@ export class CodeMieProxy { */ async start(): Promise<{ port: number; url: string }> { // 1. Detect auth method from config - const authMethod = this.config.authMethod || 'sso'; // Default: SSO for backward compat + const authMethod = this.config.authMethod || AuthMethod.SSO; // Default: SSO for backward compat // 2. Load credentials based on auth method let credentials: SSOCredentials | JWTCredentials | null = null; let syncCredentials: SSOCredentials | JWTCredentials | null = null; - if (authMethod === 'jwt') { + if (authMethod === AuthMethod.JWT) { // JWT path: token from CLI arg, env var, or credential store const token = this.config.jwtToken || process.env.CODEMIE_JWT_TOKEN diff --git a/src/providers/plugins/sso/sso.template.ts b/src/providers/plugins/sso/sso.template.ts index def6aa81..137674ed 100644 --- a/src/providers/plugins/sso/sso.template.ts +++ b/src/providers/plugins/sso/sso.template.ts @@ -8,9 +8,11 @@ */ import type { ProviderTemplate } from '../../core/types.js'; -import type { AgentConfig } from '../../../agents/core/types.js'; +import { AuthMethod } from '../../core/types.js'; import { registerProvider } from '../../core/index.js'; +import { defaultAgentHooks } from '../../core/default-agent-hooks.js'; import { DEFAULT_CODEMIE_BASE_URL } from '../../core/codemie-auth-helpers.js'; +import { resolveJwtToken } from '../jwt/jwt.utils.js'; export const SSOTemplate = registerProvider({ name: 'ai-run-sso', @@ -42,9 +44,8 @@ export const SSOTemplate = registerProvider({ if (config.authMethod) env.CODEMIE_AUTH_METHOD = config.authMethod; // Export JWT token when auth method is JWT - if (config.authMethod === 'jwt') { - const tokenEnvVar = config.jwtConfig?.tokenEnvVar || 'CODEMIE_JWT_TOKEN'; - const token = process.env[tokenEnvVar] || config.jwtConfig?.token; + if (config.authMethod === AuthMethod.JWT) { + const token = resolveJwtToken(config); if (token) env.CODEMIE_JWT_TOKEN = token; } @@ -56,94 +57,5 @@ export const SSOTemplate = registerProvider({ return env; }, - // Agent lifecycle hooks for session metrics - agentHooks: { - /** - * Wildcard hook for ALL agents - generic extension installation - * Checks if agent has getExtensionInstaller() method - * Installer handles all logging internally - * - * Correct signature: (env, config) - matches lifecycle-helpers.ts - * Agent name is available in config.agent (not as third parameter) - */ - '*': { - async beforeRun(env: NodeJS.ProcessEnv, config: AgentConfig): Promise { - // Get agent name from config (not from third parameter) - const agentName = config.agent; - if (!agentName) { - return env; // No agent name, skip silently - } - - // Dynamic import to avoid circular dependency - // AgentRegistry imports all plugins, which would cause circular dependency - // if imported at module level (SSO template is loaded as side effect) - const { AgentRegistry } = await import('../../../agents/registry.js'); - - // Get agent from registry - const agent = AgentRegistry.getAgent(agentName); - if (!agent) { - return env; // Agent not found, skip silently - } - - // Check if agent has extension installer - const installer = (agent as any).getExtensionInstaller?.(); - if (!installer) { - return env; // No installer, skip silently - } - - // Run installer with error handling (logging happens INSIDE installer) - try { - const result = await installer.install(); - - // Store target path in env (for enrichArgs if needed) - env[`CODEMIE_${agentName.toUpperCase()}_EXTENSION_DIR`] = result.targetPath; - - if (!result.success) { - // Installation failed but returned a result - const { logger } = await import('../../../utils/logger.js'); - logger.warn(`[${agentName}] Extension installation returned failure: ${result.error || 'unknown error'}`); - logger.warn(`[${agentName}] Continuing without extension - hooks may not be available`); - } - } catch (error) { - // Installation threw an exception - const { logger } = await import('../../../utils/logger.js'); - const errorMsg = error instanceof Error ? error.message : String(error); - logger.error(`[${agentName}] Extension installation threw exception: ${errorMsg}`); - logger.warn(`[${agentName}] Continuing without extension - hooks may not be available`); - // Don't throw - continue agent startup even if extension fails - } - - return env; - } - }, - - // Claude-specific: inject --plugin-dir flag - 'claude': { - /** - * Inject --plugin-dir flag for Claude Code - * Only applies when using ai-run-sso provider - * - * Note: enrichArgs is synchronous, so we read the plugin path - * from process.env that was set by beforeRun hook - */ - enrichArgs(args: string[], _config: AgentConfig): string[] { - // Get plugin directory from env (set by beforeRun) - const pluginDir = process.env.CODEMIE_CLAUDE_EXTENSION_DIR; - - if (!pluginDir) { - return args; - } - - // Check if --plugin-dir already specified - const hasPluginDir = args.some(arg => arg === '--plugin-dir'); - - if (hasPluginDir) { - return args; - } - - // Prepend --plugin-dir to arguments - return ['--plugin-dir', pluginDir, ...args]; - } - } - } + agentHooks: defaultAgentHooks }); diff --git a/src/utils/auth.ts b/src/utils/auth.ts index 213b1cf6..21a48837 100644 --- a/src/utils/auth.ts +++ b/src/utils/auth.ts @@ -3,9 +3,11 @@ */ import chalk from 'chalk'; -import type { CodeMieClient } from 'codemie-sdk'; +import { CodeMieClient } from 'codemie-sdk'; import { getCodemieClient } from '@/utils/sdk-client.js'; import { ConfigurationError } from '@/utils/errors.js'; +import { resolveJwtToken, resolveJwtTokenEnvVar } from '@/providers/plugins/jwt/jwt.utils.js'; +import { AuthMethod } from '@/providers/core/types.js'; import type { ProviderProfile } from '@/env/types.js'; import { ProviderRegistry } from '@/providers/core/registry.js'; import { handleAuthValidationFailure } from '@/providers/core/auth-validation.js'; @@ -18,13 +20,32 @@ import { handleAuthValidationFailure } from '@/providers/core/auth-validation.js * @throws ConfigurationError if authentication fails and user declines re-auth */ export async function getAuthenticatedClient(config: ProviderProfile): Promise { + if (config.authMethod === AuthMethod.JWT) { + const token = resolveJwtToken(config); + if (!token) { + throw new ConfigurationError( + `JWT token not found in ${resolveJwtTokenEnvVar(config)} environment variable. ` + + 'Provide it via the environment variable or set it in your profile configuration.' + ); + } + if (!config.baseUrl) { + throw new ConfigurationError( + 'baseUrl is required for JWT authentication. Set it in your profile configuration.' + ); + } + return new CodeMieClient({ + codemie_api_domain: config.baseUrl, + jwt_token: token, + verify_ssl: process.env.CODEMIE_INSECURE !== '1', + }); + } + try { return await getCodemieClient(); } catch (error) { if (error instanceof ConfigurationError && error.message.includes('SSO authentication required')) { const reauthed = await promptReauthentication(config); if (reauthed) { - // Retry getting client after successful re-authentication return await getCodemieClient(); } } diff --git a/src/utils/config.ts b/src/utils/config.ts index f086b8bc..fed611f3 100644 --- a/src/utils/config.ts +++ b/src/utils/config.ts @@ -932,7 +932,7 @@ export class ConfigLoader { ): Promise { const config = await this.loadConfigByScope(scope, workingDir); const assistants = config.codemieAssistants ?? []; - const baseDir = scope === StorageScope.GLOBAL ? os.homedir() : workingDir; + const baseDir = scope === StorageScope.GLOBAL ? (process.env.CODEMIE_HOME ?? os.homedir()) : workingDir; // One fs.access per assistant — acceptable for small lists (<20) but may add // measurable latency on agent startup if the list grows large. diff --git a/src/utils/errors.ts b/src/utils/errors.ts index 44e5e55d..0bc09b60 100644 --- a/src/utils/errors.ts +++ b/src/utils/errors.ts @@ -365,7 +365,8 @@ export function formatErrorForUser( // Error message (just the message, not the name) - wrapped at 100 chars const wrappedError = wrapText(context.error.message, 97, ' '); // 97 to account for "❌ " prefix - lines.push(`❌ ${wrappedError[0].trim()}`); + const firstLine = wrappedError.length > 0 ? wrappedError[0].trim() : context.error.message || '(unknown error)'; + lines.push(`❌ ${firstLine}`); for (let i = 1; i < wrappedError.length; i++) { lines.push(wrappedError[i]); } diff --git a/tests/helpers/index.ts b/tests/helpers/index.ts index 4e488c8f..32509493 100644 --- a/tests/helpers/index.ts +++ b/tests/helpers/index.ts @@ -3,4 +3,11 @@ */ export { CLIRunner, createCLIRunner, createAgentRunner, CommandResult } from './cli-runner.js'; -export { TempWorkspace, createTempWorkspace } from './temp-workspace.js'; +export { TempWorkspace, createTempWorkspace, getTempDir, resolveLongPath } from './temp-workspace.js'; +export { fetchJwtToken, writeJwtProfile, jwtCleanEnv, type JwtProfileOverrides } from './jwt-auth.js'; +export { writeSsoProfile, ssoCleanEnv, copySsoCredentials, setupSsoAutotestProfile, teardownSsoAutotestProfile } from './sso-auth.js'; +export { waitForOutput, cleanKill } from './interactive-helpers.js'; +export { spawnPty, type PtySession } from './pty-session.js'; +export { getLatestMetricsRecord } from './metrics.js'; +export { getTestEnvFlag, getTestEnvFlagOrDefault } from './test-env.js'; +export { pollForSession, type SessionPollOptions, type SessionPollResult } from './session-poll.js'; diff --git a/tests/helpers/interactive-helpers.ts b/tests/helpers/interactive-helpers.ts new file mode 100644 index 00000000..900068b1 --- /dev/null +++ b/tests/helpers/interactive-helpers.ts @@ -0,0 +1,63 @@ +import { createInterface } from 'node:readline'; +import type { ChildProcess } from 'node:child_process'; + +/** + * Resolves with the matching line when stdout (and optionally stderr) matches pattern. + * Rejects on timeout or process exit before match. + */ +export function waitForOutput( + proc: ChildProcess, + pattern: RegExp, + timeoutMs: number, + { includeStderr = false }: { includeStderr?: boolean } = {} +): Promise { + if (!proc.stdout) throw new Error('waitForOutput: process stdout is not piped'); + return new Promise((resolve, reject) => { + const lines: string[] = []; + const interfaces: ReturnType[] = []; + + const handleLine = (line: string): void => { + lines.push(line); + if (pattern.test(line)) { + clearTimeout(timer); + interfaces.forEach(rl => { try { rl.close(); } catch { /* ignore */ } }); + resolve(line); + } + }; + + const stdoutRl = createInterface({ input: proc.stdout! }); + stdoutRl.on('line', handleLine); + interfaces.push(stdoutRl); + + if (includeStderr && proc.stderr) { + const stderrRl = createInterface({ input: proc.stderr }); + stderrRl.on('line', handleLine); + interfaces.push(stderrRl); + } + + const closeAll = (): void => interfaces.forEach(rl => { try { rl.close(); } catch { /* ignore */ } }); + + const timer = setTimeout(() => { + closeAll(); + reject(new Error(`Timeout (${timeoutMs}ms) waiting for ${pattern}.\nGot:\n${lines.join('\n')}`)); + }, timeoutMs); + + proc.on('close', (code) => { + clearTimeout(timer); + closeAll(); + reject(new Error(`Process exited (code ${code ?? 'null'}) before matching ${pattern}.\nGot:\n${lines.join('\n')}`)); + }); + }); +} + +/** + * Send SIGTERM and wait for the process to exit. + * Falls back to SIGKILL after 5 seconds. + */ +export function cleanKill(proc: ChildProcess): Promise { + return new Promise((resolve) => { + const fallback = setTimeout(() => { try { proc.kill('SIGKILL'); } catch { /* ignore */ } }, 5000); + proc.on('close', () => { clearTimeout(fallback); resolve(); }); + try { proc.kill('SIGTERM'); } catch { /* process already exited */ } + }); +} diff --git a/tests/helpers/jwt-auth.ts b/tests/helpers/jwt-auth.ts new file mode 100644 index 00000000..287ba87e --- /dev/null +++ b/tests/helpers/jwt-auth.ts @@ -0,0 +1,123 @@ +import { mkdirSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; + +// Slug used for the test assistant agent file +const CI_ASSISTANT_SLUG = 'ci-assistant'; + +/** + * Fetch a fresh JWT token via Keycloak password grant. + * + * Shortcut: if CI_CODEMIE_JWT_TOKEN is already set, returns it directly + * (useful for manual runs or when the token is pre-fetched in CI). + * + * Otherwise, performs a Keycloak password grant using CI_CODEMIE_USERNAME + * and CI_CODEMIE_PASSWORD. Credentials are trimmed to strip whitespace/CRLF + * that PowerShell env files sometimes introduce. + */ +export async function fetchJwtToken(): Promise { + const preFetched = process.env.CI_CODEMIE_JWT_TOKEN?.trim(); + if (preFetched) return preFetched; + + const username = process.env.CI_CODEMIE_USERNAME?.trim(); + const password = process.env.CI_CODEMIE_PASSWORD?.trim(); + if (!username || !password) + throw new Error('CI_CODEMIE_USERNAME and CI_CODEMIE_PASSWORD should be set in .env.test.local or env variables'); + + const authUrlRaw = process.env.CI_CODEMIE_AUTH_URL?.trim(); + if (!authUrlRaw) throw new Error('CI_CODEMIE_AUTH_URL must be set in .env.test.local or env variables'); + const authUrl = `${authUrlRaw.replace(/\/$/, '')}/realms/codemie-prod/protocol/openid-connect/token`; + + const resp = await fetch(authUrl, { + method: 'POST', + headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, + body: new URLSearchParams({ + grant_type: 'password', + client_id: process.env.CI_CODEMIE_AUTH_CLIENT_ID?.trim() ?? 'codemie-sdk', + username, + password, + }), + }); + if (!resp.ok) { + const body = await resp.text().catch(() => ''); + throw new Error(`JWT token fetch failed: HTTP ${resp.status} ${resp.statusText}\n${body}`); + } + const data = (await resp.json()) as Record; + if (!data.access_token) throw new Error(`JWT token fetch failed: no access_token in response: ${JSON.stringify(data)}`); + return data.access_token as string; +} + +/** + * Minimal allowlist environment for JWT agent spawns. + * Strips everything except essential PATH and platform OS variables so no + * credentials or CODEMIE_* session state leak into the subprocess. + */ +export function jwtCleanEnv(): NodeJS.ProcessEnv { + const pick = (...keys: string[]): NodeJS.ProcessEnv => + Object.fromEntries(keys.flatMap((k) => (process.env[k] !== undefined ? [[k, process.env[k]]] : []))); + return { + PATH: process.env.PATH ?? '', + NODE_PATH: process.env.NODE_PATH ?? '', + ...pick('SystemRoot', 'SYSTEMROOT', 'PATHEXT', 'TEMP', 'TMP', 'WINDIR', 'COMSPEC', + 'USERPROFILE', 'HOMEDRIVE', 'HOMEPATH', 'APPDATA', 'LOCALAPPDATA'), + ...pick('HOME', 'USER', 'LANG', 'LC_ALL', 'SHELL'), + }; +} + +export interface JwtProfileOverrides { + profileName?: string; + model?: string; + codeMieUrl?: string; + baseUrl?: string; + jwtToken?: string; + codeMieProject?: string; + authServerUrl?: string; + authRealm?: string; + /** When set, writes the assistant to LOCAL config + creates its agent file. */ + assistantId?: string; +} + +/** + * Write a bearer-auth profile to ${codemieHome}/codemie-cli.config.json. + * The config location matches getCodemiePath() which uses CODEMIE_HOME as the + * base directory (not ~/.codemie/.codemie). + */ +export function writeJwtProfile(codemieHome: string, overrides: JwtProfileOverrides = {}): void { + const profileName = overrides.profileName ?? 'jwt-autotest'; + const authUrlRaw = process.env.CI_CODEMIE_AUTH_URL?.trim(); + if (!authUrlRaw) throw new Error('CI_CODEMIE_AUTH_URL must be set in .env.test.local or env variables'); + const authBase = authUrlRaw.replace(/\/$/, ''); + const profile: Record = { + name: profileName, + provider: 'bearer-auth', + authMethod: 'jwt', + codeMieUrl: overrides.codeMieUrl ?? process.env.CI_CODEMIE_URL ?? '', + baseUrl: overrides.baseUrl ?? `${(process.env.CI_CODEMIE_URL ?? '').replace(/\/$/, '')}/code-assistant-api`, + model: overrides.model ?? process.env.CI_CODEMIE_MODEL ?? 'claude-sonnet-4-6', + authServerUrl: overrides.authServerUrl ?? authBase, + authRealm: overrides.authRealm ?? 'codemie-prod', + }; + if (overrides.jwtToken) profile.jwtToken = overrides.jwtToken; + if (overrides.codeMieProject) profile.codeMieProject = overrides.codeMieProject; + + const config = { version: 2, activeProfile: profileName, profiles: { [profileName]: profile } }; + mkdirSync(codemieHome, { recursive: true }); + writeFileSync(join(codemieHome, 'codemie-cli.config.json'), JSON.stringify(config, null, 2), 'utf-8'); + + if (overrides.assistantId) { + // Register assistant in the GLOBAL config (codemieHome/codemie-cli.config.json). + // loadAssistantsByScope(GLOBAL) uses process.env.CODEMIE_HOME as baseDir for the + // agent file-existence check, so the stub lives at /.claude/agents/.md. + const assistant = { + id: overrides.assistantId, + name: 'CI Assistant', + slug: CI_ASSISTANT_SLUG, + registeredAt: new Date().toISOString(), + }; + config.codemieAssistants = [assistant]; + writeFileSync(join(codemieHome, 'codemie-cli.config.json'), JSON.stringify(config, null, 2), 'utf-8'); + + const agentDir = join(codemieHome, '.claude', 'agents'); + mkdirSync(agentDir, { recursive: true }); + writeFileSync(join(agentDir, `${CI_ASSISTANT_SLUG}.md`), `# CI Assistant\n`, 'utf-8'); + } +} diff --git a/tests/helpers/metrics.ts b/tests/helpers/metrics.ts new file mode 100644 index 00000000..46b3c6df --- /dev/null +++ b/tests/helpers/metrics.ts @@ -0,0 +1,23 @@ +import { readdirSync, readFileSync, statSync } from 'node:fs'; +import { join } from 'node:path'; + +/** + * Finds the most recently modified *_metrics.jsonl file in sessionsDir and + * returns the last parsed JSON record from it. + */ +export function getLatestMetricsRecord(sessionsDir: string): Record { + const files = readdirSync(sessionsDir) + .filter((f) => f.endsWith('_metrics.jsonl')) + .map((f) => join(sessionsDir, f)) + .sort((a, b) => { + try { + return statSync(b).mtimeMs - statSync(a).mtimeMs; + } catch { + return 0; + } + }); + if (!files.length) throw new Error('No metrics files found in ' + sessionsDir); + const lines = readFileSync(files[0], 'utf-8').trim().split('\n').filter(Boolean); + if (!lines.length) throw new Error('Metrics file is empty: ' + files[0]); + return JSON.parse(lines[lines.length - 1]) as Record; +} diff --git a/tests/helpers/pty-session.ts b/tests/helpers/pty-session.ts new file mode 100644 index 00000000..64b26a98 --- /dev/null +++ b/tests/helpers/pty-session.ts @@ -0,0 +1,138 @@ +import * as pty from 'node-pty'; + +// Strips common ANSI/VT100 escape sequences so pattern matching works on plain text. +const ANSI_RE = + /\x1b\[[0-9;?]*[A-Za-z]|\x1b[()][012AB]|\x1b[=>]|\x07|\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)/g; // eslint-disable-line no-control-regex + +function stripAnsi(s: string): string { + return s.replace(ANSI_RE, ''); +} + +export interface PtySession { + writeLine(text: string): void; + /** Send raw bytes to the PTY without appending \r (use for control characters like \x03). */ + write(raw: string): void; + waitFor(pattern: RegExp, timeoutMs: number, startFromLine?: number): Promise; + /** Wait for the process to exit naturally, force-kill after timeoutMs. */ + exit(timeoutMs?: number): Promise; + /** Return a snapshot of all lines received from the PTY so far. */ + lines(): string[]; +} + +interface Waiter { + pattern: RegExp; + resolve: (line: string) => void; + reject: (err: Error) => void; + timer: ReturnType; +} + +export function spawnPty( + file: string, + args: string[], + options: { cwd: string; env: NodeJS.ProcessEnv }, +): PtySession { + const proc = pty.spawn(file, args, { + name: 'xterm-256color', + cols: 120, + rows: 30, + cwd: options.cwd, + env: options.env, + }); + + const allLines: string[] = []; + const waiters: Waiter[] = []; + let tail = ''; + + proc.onData((raw) => { + const chunk = stripAnsi(raw); + tail += chunk; + const parts = tail.split(/\r?\n/); + tail = parts.pop() ?? ''; + const newLines = parts.map((l) => l.replace(/\r/g, '').trim()).filter((l) => l.length > 0); + allLines.push(...newLines); + for (const line of newLines) { + for (let i = waiters.length - 1; i >= 0; i--) { + if (waiters[i].pattern.test(line)) { + const w = waiters.splice(i, 1)[0]; + clearTimeout(w.timer); + w.resolve(line); + } + } + } + // Also check the incomplete current line (input prompts never emit a trailing \n + // while waiting for user input, so they never appear in allLines). + const trimmedTail = tail.replace(/\r/g, '').trim(); + if (trimmedTail.length > 0) { + for (let i = waiters.length - 1; i >= 0; i--) { + if (waiters[i].pattern.test(trimmedTail)) { + const w = waiters.splice(i, 1)[0]; + clearTimeout(w.timer); + w.resolve(trimmedTail); + } + } + } + }); + + return { + writeLine(text: string): void { + proc.write(text + '\r\n'); + }, + + write(raw: string): void { + proc.write(raw); + }, + + waitFor(pattern: RegExp, timeoutMs: number, startFromLine = 0): Promise { + for (let i = startFromLine; i < allLines.length; i++) { + if (pattern.test(allLines[i])) return Promise.resolve(allLines[i]); + } + // Check the incomplete current line — input prompts sit here waiting for input. + const trimmedTail = tail.replace(/\r/g, '').trim(); + if (trimmedTail.length > 0 && pattern.test(trimmedTail)) { + return Promise.resolve(trimmedTail); + } + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + const idx = waiters.findIndex((w) => w.timer === timer); + if (idx >= 0) waiters.splice(idx, 1); + reject( + new Error( + `Timeout (${timeoutMs}ms) waiting for ${pattern}\nLast lines:\n${allLines.slice(-20).join('\n')}`, + ), + ); + }, timeoutMs); + waiters.push({ pattern, resolve, reject, timer }); + }); + }, + + exit(timeoutMs = 15_000): Promise { + return new Promise((resolve) => { + let resolved = false; + const fallback = setTimeout(() => { + if (!resolved) { + resolved = true; + try { + proc.kill(); + } catch { + /* ignore */ + } + resolve(); + } + }, timeoutMs); + proc.onExit(() => { + if (!resolved) { + resolved = true; + clearTimeout(fallback); + resolve(); + } + }); + // Caller is responsible for initiating exit (e.g. writeLine('/exit')). + // This method only waits and force-kills if the process hasn't exited by timeoutMs. + }); + }, + + lines(): string[] { + return [...allLines]; + }, + }; +} diff --git a/tests/helpers/session-poll.ts b/tests/helpers/session-poll.ts new file mode 100644 index 00000000..3e32b3c8 --- /dev/null +++ b/tests/helpers/session-poll.ts @@ -0,0 +1,76 @@ +/** + * Session polling helper. + * + * Waits for a session conversation file containing a given marker string to + * appear in a sessions directory. Needed because onSessionEnd may still be + * writing/renaming files when the agent process returns — files appear as + * either `{id}_conversation.jsonl` (mid-write) or + * `completed_{id}_conversation.jsonl` (after rename). + */ + +import { existsSync, readdirSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +export interface SessionPollOptions { + /** Maximum time to wait in milliseconds. Default: 30 000 */ + timeoutMs?: number; + /** Polling interval in milliseconds. Default: 1 000 */ + intervalMs?: number; +} + +export interface SessionPollResult { + /** Session ID (with completed_ prefix if renamed), or null if timed out */ + sessionId: string | null; + /** Human-readable description of sessions dir contents for error messages */ + dirContents: string; +} + +/** + * Poll a sessions directory until a `*_conversation.jsonl` file containing + * `marker` appears, then return its session ID. + * + * Returns `{ sessionId: null, dirContents }` if the timeout is reached without + * finding a match — callers should assert `sessionId !== null` with + * `dirContents` in the failure message. + */ +export async function pollForSession( + sessionsDir: string, + marker: string, + options: SessionPollOptions = {}, +): Promise { + const timeoutMs = options.timeoutMs ?? 30_000; + const intervalMs = options.intervalMs ?? 1_000; + + let sessionId: string | null = null; + const pollStart = Date.now(); + + while (sessionId === null && Date.now() - pollStart < timeoutMs) { + if (existsSync(sessionsDir)) { + for (const fileName of readdirSync(sessionsDir).filter(f => f.endsWith('_conversation.jsonl'))) { + try { + if (readFileSync(join(sessionsDir, fileName), 'utf-8').includes(marker)) { + sessionId = fileName.replace('_conversation.jsonl', ''); + break; + } + } catch { + continue; + } + } + } + + if (sessionId === null) { + await new Promise(r => setTimeout(r, intervalMs)); + } + } + + let dirContents = '(dir missing)'; + if (existsSync(sessionsDir)) { + try { + dirContents = readdirSync(sessionsDir).join(', ') || '(empty)'; + } catch { + dirContents = '(read error)'; + } + } + + return { sessionId, dirContents }; +} diff --git a/tests/helpers/sso-auth.ts b/tests/helpers/sso-auth.ts new file mode 100644 index 00000000..d05ebb80 --- /dev/null +++ b/tests/helpers/sso-auth.ts @@ -0,0 +1,115 @@ +import { copyFileSync, existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { homedir } from 'node:os'; +import { join } from 'node:path'; + +const SSO_PROFILE_NAME = 'sso-autotest'; + +function getCodemieConfigDir(): string { + return join(homedir(), '.codemie'); +} + +function buildSsoProfile(): Record { + const ciCodemieUrl = (process.env.CI_CODEMIE_URL ?? '').replace(/\/$/, ''); + return { + name: SSO_PROFILE_NAME, + provider: 'ai-run-sso', + authMethod: 'sso', + codeMieUrl: process.env.CI_CODEMIE_URL ?? '', + baseUrl: `${ciCodemieUrl}/code-assistant-api`, + apiKey: 'sso-authenticated', + model: process.env.CODEMIE_MODEL ?? 'claude-sonnet-4-6', + timeout: 300, + debug: false, + }; +} + +/** + * Write a fresh SSO profile config to the given codemieHome directory. + * Mirrors writeJwtProfile but for SSO auth — writes an sso-autotest profile + * so subprocesses can authenticate via the OS keychain using codeMieUrl. + */ +export function writeSsoProfile(codemieHome: string): void { + const config = { + version: 2, + activeProfile: SSO_PROFILE_NAME, + profiles: { [SSO_PROFILE_NAME]: buildSsoProfile() }, + }; + mkdirSync(codemieHome, { recursive: true }); + writeFileSync(join(codemieHome, 'codemie-cli.config.json'), JSON.stringify(config, null, 2), 'utf-8'); +} + +/** + * Strip CODEMIE_* vars from the process environment for SSO subprocess spawns. + * Uses a denylist (vs jwtCleanEnv's allowlist) to preserve HOME, proxy settings, + * and other vars that the OS keychain and network calls depend on. + */ +export function ssoCleanEnv(): NodeJS.ProcessEnv { + return Object.fromEntries( + Object.entries(process.env).filter(([key]) => !key.startsWith('CODEMIE_')), + ) as NodeJS.ProcessEnv; +} + +/** + * Upsert the sso-autotest profile into ~/.codemie/codemie-cli.config.json and + * set it as the active profile. All existing profiles are preserved. + * + * Returns the previous activeProfile value for later restoration via + * teardownSsoAutotestProfile(). + */ +export function setupSsoAutotestProfile(): string | undefined { + const configDir = getCodemieConfigDir(); + const configFilePath = join(configDir, 'codemie-cli.config.json'); + + let originalActiveProfile: string | undefined; + if (existsSync(configFilePath)) { + try { + const existing = JSON.parse(readFileSync(configFilePath, 'utf-8')) as Record; + originalActiveProfile = existing.activeProfile as string | undefined; + } catch { /* ignore parse errors */ } + } + + mkdirSync(configDir, { recursive: true }); + + let config: Record = { version: 2, activeProfile: SSO_PROFILE_NAME, profiles: {} }; + if (existsSync(configFilePath)) { + try { config = JSON.parse(readFileSync(configFilePath, 'utf-8')) as Record; } catch { /* use defaults */ } + } + (config.profiles as Record)[SSO_PROFILE_NAME] = buildSsoProfile(); + config.activeProfile = SSO_PROFILE_NAME; + writeFileSync(configFilePath, JSON.stringify(config, null, 2)); + + return originalActiveProfile; +} + +/** + * Restore the ~/.codemie active profile to the value saved by + * setupSsoAutotestProfile(). No-op if originalActiveProfile is undefined. + */ +export function teardownSsoAutotestProfile(originalActiveProfile: string | undefined): void { + const configFilePath = join(getCodemieConfigDir(), 'codemie-cli.config.json'); + if (originalActiveProfile !== undefined && existsSync(configFilePath)) { + try { + const config = JSON.parse(readFileSync(configFilePath, 'utf-8')) as Record; + config.activeProfile = originalActiveProfile; + writeFileSync(configFilePath, JSON.stringify(config, null, 2)); + } catch { /* ignore restore errors */ } + } +} + +/** + * Copy SSO credential files from ~/.codemie/credentials/ into testHome/credentials/. + * + * The encryption key is machine-specific (hostname+platform+arch), not path-specific, + * so the copied files are decryptable by any subprocess running on the same machine. + * Call this after writeSsoProfile() so the subprocess finds credentials at the + * CODEMIE_HOME-relative path it derives from CREDENTIALS_DIR at startup. + */ +export function copySsoCredentials(testHome: string): void { + const realCredsDir = join(getCodemieConfigDir(), 'credentials'); + if (!existsSync(realCredsDir)) return; + const testCredsDir = join(testHome, 'credentials'); + mkdirSync(testCredsDir, { recursive: true }); + for (const file of readdirSync(realCredsDir)) { + copyFileSync(join(realCredsDir, file), join(testCredsDir, file)); + } +} diff --git a/tests/helpers/temp-workspace.ts b/tests/helpers/temp-workspace.ts index a519c0ba..5b436bab 100644 --- a/tests/helpers/temp-workspace.ts +++ b/tests/helpers/temp-workspace.ts @@ -4,7 +4,7 @@ * Creates temporary directories with helper methods for file operations */ -import { mkdtempSync, rmSync, writeFileSync, mkdirSync, readFileSync } from 'fs'; +import { mkdtempSync, rmSync, writeFileSync, mkdirSync, readFileSync, realpathSync } from 'fs'; import { join } from 'path'; import { tmpdir } from 'os'; @@ -115,3 +115,27 @@ export class TempWorkspace { export function createTempWorkspace(prefix?: string): TempWorkspace { return new TempWorkspace(prefix); } + +/** + * Resolve Windows 8.3 short path names to full long paths. + * Equivalent to ctypes.windll.kernel32.GetLongPathNameW in Python. + * Needed when passing temp dirs as cwd to subprocesses — short paths cause + * path comparison mismatches inside the agent on Windows. + */ +export function resolveLongPath(p: string): string { + if (process.platform !== 'win32') return p; + try { return realpathSync.native(p); } catch { return p; } +} + +/** + * Returns a temp directory base path free of Windows 8.3 short names. + * os.tmpdir() returns e.g. C:\Users\MAKSYM~1\AppData\Local\Temp on Windows, + * which confuses path comparisons inside the agent. LOCALAPPDATA always + * contains the full username, so we derive Temp from it instead. + */ +export function getTempDir(): string { + if (process.platform === 'win32' && process.env.LOCALAPPDATA) { + return join(process.env.LOCALAPPDATA, 'Temp'); + } + return tmpdir(); +} diff --git a/tests/helpers/test-env.ts b/tests/helpers/test-env.ts new file mode 100644 index 00000000..72694e8e --- /dev/null +++ b/tests/helpers/test-env.ts @@ -0,0 +1,64 @@ +/** + * Test environment flag helpers. + * + * Reads boolean mode flags with file-first priority: when .env.test.local + * exists the file value always wins, preventing stale shell exports from + * overriding local test configuration. When the file is absent (CI pipeline), + * falls back to process.env where the CI sets flags as real environment variables. + */ + +import { existsSync, readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; + +function parseDotEnvFile(filePath: string): Record { + try { + return Object.fromEntries( + readFileSync(filePath, 'utf-8').split('\n') + .map(l => l.trim()) + .filter(l => l && !l.startsWith('#')) + .map(l => l.replace(/^export\s+/, '').match(/^([^=]+)=(.*)$/)) + .filter((m): m is RegExpMatchArray => m !== null) + .map(m => [m[1].trim(), m[2].trim().replace(/^(["'])(.*)\1$/, '$2')]), + ); + } catch { return {}; } +} + +const DOT_ENV_PATH = resolve(process.cwd(), '.env.test.local'); +const _dotEnvExists = existsSync(DOT_ENV_PATH); +const _fileEnv = _dotEnvExists ? parseDotEnvFile(DOT_ENV_PATH) : {}; + +/** + * Read a boolean test flag with file-first priority. + * + * When .env.test.local exists: returns whether the flag is set to 'true' in + * the file, regardless of shell environment. Commenting out the line or setting + * it to 'false' in the file is always sufficient to disable the flag locally. + * + * When .env.test.local is absent (CI): reads from process.env, where the CI + * pipeline sets flags as real environment variables. + */ +export function getTestEnvFlag(name: string): boolean { + return _dotEnvExists + ? _fileEnv[name] === 'true' + : process.env[name] === 'true'; +} + +/** + * Read a boolean test flag with file-first priority and an explicit default. + * + * Useful for flags that should be ON unless explicitly disabled — e.g. + * CI_IS_LOCAL_RUN defaults to true so SSO mode runs locally with no config, + * and only setting CI_IS_LOCAL_RUN=false in .env.test.local (or as a CI env var) + * switches to JWT mode. + * + * Priority: file value (if key present) > env var (if no file) > defaultValue. + */ +export function getTestEnvFlagOrDefault(name: string, defaultValue: boolean): boolean { + if (_dotEnvExists) { + if (name in _fileEnv) return _fileEnv[name] === 'true'; + return defaultValue; + } + const envVal = process.env[name]; + if (envVal !== undefined) return envVal === 'true'; + return defaultValue; +} diff --git a/tests/integration/agent-assistant.test.ts b/tests/integration/agent-assistant.test.ts new file mode 100644 index 00000000..50c667b1 --- /dev/null +++ b/tests/integration/agent-assistant.test.ts @@ -0,0 +1,343 @@ +/** + * Assistant management tests — TC-014, TC-015, TC-026 + * + * Run with: npm run test:integration:agent + * + * Auth mode (CI_IS_LOCAL_RUN in .env.test.local): + * true (default) — SSO mode; uses developer's sso-autotest profile in ~/.codemie + * false — JWT mode; isolates to a temp CODEMIE_HOME with bearer-auth profile + * + * A fresh assistant (AutoAssistantRandomGenerator) is created in the outer beforeAll + * via the SDK and deleted in afterAll, removing the static CI_CODEMIE_ASSISTANT_ID + * dependency. + * + * TC-014: Setup assistants wizard via PTY — registers the created assistant as a skill. + * TC-015: Assistants chat with invalid ID — negative test, exits non-zero. + * TC-026: Non-interactive assistant chat random-number test. + */ + +import '../setup/load-test-env.js'; +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { spawnSync } from 'node:child_process'; +import { randomBytes } from 'node:crypto'; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { homedir } from 'node:os'; +import { join, dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { CodeMieClient } from 'codemie-sdk'; +import { getCodemieClient } from '@/utils/sdk-client.js'; +import { createAssistant, deleteAssistant, listAssistants } from '@/cli/commands/sdk/services/assistants.js'; +import { + fetchJwtToken, + writeJwtProfile, + writeSsoProfile, + copySsoCredentials, + getTempDir, + spawnPty, + jwtCleanEnv, + ssoCleanEnv, + setupSsoAutotestProfile, + teardownSsoAutotestProfile, + getTestEnvFlagOrDefault, +} from '../helpers/index.js'; + +const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../..'); +const CLI_BIN = join(REPO_ROOT, 'bin', 'codemie.js'); +const CLAUDE_BIN = join(REPO_ROOT, 'bin', 'codemie-claude.js'); + +const CI_IS_LOCAL_RUN = getTestEnvFlagOrDefault('CI_IS_LOCAL_RUN', true); + +// Unique per-run suffix so concurrent runs and other users' leftover assistants don't collide. +const RUN_SUFFIX = randomBytes(3).toString('hex'); +const ASSISTANT_NAME = `AutoAssistantRandomGenerator-${RUN_SUFFIX}`; +const ASSISTANT_SYSTEM_PROMPT = [ + 'You are random generator', + 'You should answer on any user message with single number from 1 to 10', + 'Nothing else shouldn\'t be provided except random number', +].join('\n'); +const ASSISTANT_SLUG = ASSISTANT_NAME.toLowerCase().replace(/[^a-z0-9]/g, ''); + +function registerAssistantInConfig(codemieHome: string, id: string, name: string, slug: string): void { + const configPath = join(codemieHome, 'codemie-cli.config.json'); + const config = JSON.parse(readFileSync(configPath, 'utf-8')) as Record; + config.codemieAssistants = [{ id, name, slug, registeredAt: new Date().toISOString() }]; + writeFileSync(configPath, JSON.stringify(config, null, 2), 'utf-8'); + const agentDir = join(codemieHome, '.claude', 'agents'); + mkdirSync(agentDir, { recursive: true }); + writeFileSync(join(agentDir, `${slug}.md`), `# ${name}\n`, 'utf-8'); +} + +describe.runIf(process.env.SSO_AVAILABLE !== 'false')('Assistant management tests', () => { + let jwtToken: string; + let jwtHome: string; + let sdkClient: CodeMieClient; + let createdAssistantId: string; + let originalActiveProfile: string | undefined; + + beforeAll(async () => { + const ciCodemieUrl = (process.env.CI_CODEMIE_URL ?? '').replace(/\/$/, ''); + + if (!CI_IS_LOCAL_RUN) { + // JWT mode: isolated temp home + jwtToken = await fetchJwtToken(); + jwtHome = mkdtempSync(join(getTempDir(), 'codemie-asst-')); + writeJwtProfile(jwtHome, { jwtToken }); + sdkClient = new CodeMieClient({ + codemie_api_domain: `${ciCodemieUrl}/code-assistant-api`, + jwt_token: jwtToken, + verify_ssl: process.env.CODEMIE_INSECURE !== '1', + }); + } else { + // SSO mode: upsert sso-autotest profile into ~/.codemie + originalActiveProfile = setupSsoAutotestProfile(); + sdkClient = await getCodemieClient(true); + } + + const aboutUser = await sdkClient.users.aboutMe(); + const project = aboutUser.applications[0]; + if (!project) throw new Error('No accessible project found for this user'); + + // Remove any same-named assistants left over from interrupted previous runs. + const stale = await listAssistants(sdkClient); + for (const a of stale.filter((a) => a.name === ASSISTANT_NAME)) { + try { await deleteAssistant(sdkClient, a.id!); } catch { /* best-effort */ } + } + + await createAssistant(sdkClient, { + name: ASSISTANT_NAME, + description: 'Integration test assistant — auto-created and deleted by the test suite', + system_prompt: ASSISTANT_SYSTEM_PROMPT, + slug: ASSISTANT_SLUG, + project, + }); + + const assistants = await listAssistants(sdkClient); + const created = assistants.find((a) => a.name === ASSISTANT_NAME); + if (!created?.id) throw new Error(`Failed to find created assistant "${ASSISTANT_NAME}" after creation`); + createdAssistantId = created.id; + }, 60_000); + + afterAll(async () => { + if (createdAssistantId && sdkClient) { + try { await deleteAssistant(sdkClient, createdAssistantId); } catch { /* best-effort */ } + } + if (!CI_IS_LOCAL_RUN) { + if (jwtHome) rmSync(jwtHome, { recursive: true, force: true }); + } else { + teardownSsoAutotestProfile(originalActiveProfile); + } + }); + + // ── TC-014: Setup assistants wizard via PTY ──────────────────────────────── + // Drives the `codemie setup assistants` interactive wizard via PTY for the + // dynamically created AutoAssistantRandomGenerator assistant: + // 1. Searches for the assistant in the picker and selects it. + // 2. Chooses "Agent Skills" registration mode (gets a /slug command). + // 3. Keeps Global storage scope. + // 4. Confirms Target Agents screen. + // Verifies the config is updated, then checks the /slug command works in + // a live codemie-claude session. + describe('TC-014 — setup assistants wizard registers assistant as skill', () => { + let testHome: string; + + beforeAll(async () => { + testHome = mkdtempSync(join(getTempDir(), 'codemie-setup-asst-')); + if (!CI_IS_LOCAL_RUN) { + writeJwtProfile(testHome, { jwtToken }); + } else { + writeSsoProfile(testHome); + copySsoCredentials(testHome); + } + mkdirSync(join(testHome, '.claude'), { recursive: true }); + + const wizardEnv = CI_IS_LOCAL_RUN + ? { ...ssoCleanEnv(), CODEMIE_HOME: testHome, TERM: 'xterm-256color' } + : { ...process.env, CODEMIE_HOME: testHome, CODEMIE_JWT_TOKEN: jwtToken, TERM: 'xterm-256color' }; + + const setupProc = spawnPty( + process.execPath, + [CLI_BIN, 'setup', 'assistants'], + { cwd: testHome, env: wizardEnv }, + ); + + try { + // Step 1: Assistants picker — search by name, select, then Continue. + await setupProc.waitFor(/\d+ assistants total/, 60_000); + await new Promise((r) => setTimeout(r, 1_500)); + setupProc.write('\x1B[A'); // Arrow Up → focus search box + await new Promise((r) => setTimeout(r, 300)); + for (const char of ASSISTANT_NAME) { + setupProc.write(char); + await new Promise((r) => setTimeout(r, 150)); + } + await new Promise((r) => setTimeout(r, 4_000)); // Debounce + search API response + setupProc.write('\x1B[B'); // Arrow Down → focus first result + await new Promise((r) => setTimeout(r, 300)); + setupProc.write(' '); // Space to select + await new Promise((r) => setTimeout(r, 200)); + setupProc.write('\x1B[B'); // Arrow Down → focus Continue + await new Promise((r) => setTimeout(r, 200)); + setupProc.write('\r'); // Enter to confirm Continue + + // Step 2: Mode selection — arrow down once to "Agent Skills", then Enter. + await setupProc.waitFor(/Configure Registration|How would you like to register/, 45_000); + await new Promise((r) => setTimeout(r, 300)); + setupProc.write('\x1B[B'); // Arrow Down → Agent Skills + await new Promise((r) => setTimeout(r, 200)); + setupProc.write('\r'); // Enter to confirm + + // Step 3: Storage scope — keep Global default. + await setupProc.waitFor(/Where would you like to save/, 30_000); + await new Promise((r) => setTimeout(r, 200)); + setupProc.write('\r'); // Enter to accept Global + + // Step 4: Target Agents — arrow down twice to reach Continue, then Enter. + await setupProc.waitFor(/Target Agents/, 30_000); + await new Promise((r) => setTimeout(r, 300)); + setupProc.write('\x1B[B'); // Arrow Down #1 + await new Promise((r) => setTimeout(r, 200)); + setupProc.write('\x1B[B'); // Arrow Down #2 → Continue button + await new Promise((r) => setTimeout(r, 200)); + setupProc.write('\r'); // Enter to confirm + + // Step 5: Wait for success confirmation. + await setupProc.waitFor(/Updated \d+ assistant/, 30_000); + } finally { + await setupProc.exit(15_000); + } + }, 180_000); + + afterAll(async () => { + await new Promise((r) => setTimeout(r, 500)); + rmSync(testHome, { recursive: true, force: true }); + rmSync(join(homedir(), '.claude', 'skills', ASSISTANT_SLUG), { recursive: true, force: true }); + }); + + it('codemie-cli.config.json contains the registered assistant slug', () => { + const configPath = join(testHome, 'codemie-cli.config.json'); + const raw = readFileSync(configPath, 'utf-8'); + expect( + raw.includes(ASSISTANT_SLUG), + `Expected config to contain slug "${ASSISTANT_SLUG}".\nConfig: ${raw}`, + ).toBe(true); + }); + + it('agent responds to / and returns a number 1-10', async () => { + const sessionArgs = CI_IS_LOCAL_RUN + ? [CLAUDE_BIN] + : [CLAUDE_BIN, '--profile', 'jwt-autotest', '--jwt-token', jwtToken]; + const sessionEnv = CI_IS_LOCAL_RUN + ? { ...ssoCleanEnv(), CODEMIE_HOME: testHome, TERM: 'xterm-256color' } + : { ...jwtCleanEnv(), CODEMIE_HOME: testHome, TERM: 'xterm-256color' }; + + const proc = spawnPty(process.execPath, sessionArgs, { cwd: testHome, env: sessionEnv }); + + try { + await proc.waitFor(/Model\s*[│|]/i, 60_000); + // On macOS, Claude Code shows a workspace-trust prompt for new temp directories + // before rendering the startup box. Start a background handler that accepts the + // prompt ('1' = Yes, I trust this folder) if it appears, so the startup box can + // render. The handler is a no-op on platforms where the prompt does not appear. + void proc.waitFor(/trust.*folder|trustthisfolder/i, 15_000) + .then(() => proc.writeLine('1')) + .catch(() => { /* no trust prompt on this platform */ }); + await proc.waitFor(/╰─/, 60_000); + await new Promise((r) => setTimeout(r, 1_000)); + proc.writeLine(`/${ASSISTANT_SLUG} hi`); + await proc.waitFor(/\b([1-9]|10)\b/, 90_000); + } finally { + proc.writeLine('/exit'); + await proc.exit(90_000); + } + + const lines = proc.lines(); + const matchedLine = lines.find((l) => /\b([1-9]|10)\b/.test(l)); + expect( + matchedLine, + `Expected a line with a number 1-10 from /${ASSISTANT_SLUG}.\nLast PTY lines:\n${lines.slice(-20).join('\n')}`, + ).toBeTruthy(); + const num = parseInt(matchedLine!.match(/\b([1-9]|10)\b/)![1], 10); + expect(num).toBeGreaterThanOrEqual(1); + expect(num).toBeLessThanOrEqual(10); + }, 240_000); + }); + + // ── TC-015: Assistants chat with invalid ID (negative) ───────────────────── + // Verifies that `codemie assistants chat` with an unknown assistant ID exits + // non-zero and shows an appropriate error message. + describe('TC-015 — assistants chat with invalid ID (negative)', () => { + let testHome: string; + let chatResult: ReturnType; + + beforeAll(() => { + testHome = mkdtempSync(join(getTempDir(), 'codemie-asst-invalid-')); + if (!CI_IS_LOCAL_RUN) { + writeJwtProfile(testHome, { jwtToken }); + } else { + writeSsoProfile(testHome); + copySsoCredentials(testHome); + } + const chatArgs = CI_IS_LOCAL_RUN + ? [CLI_BIN, 'assistants', 'chat', 'nonexistent-assistant-id-000', 'Say hello'] + : [CLI_BIN, 'assistants', 'chat', '--jwt-token', jwtToken, 'nonexistent-assistant-id-000', 'Say hello']; + const chatEnv = CI_IS_LOCAL_RUN + ? { ...ssoCleanEnv(), CODEMIE_HOME: testHome, CI: '1' } + : { ...jwtCleanEnv(), CODEMIE_HOME: testHome, CODEMIE_JWT_TOKEN: jwtToken, CI: '1' }; + chatResult = spawnSync( + process.execPath, + chatArgs, + { cwd: testHome, env: chatEnv, encoding: 'utf-8', timeout: 30_000 }, + ); + }, 60_000); + + afterAll(() => rmSync(testHome, { recursive: true, force: true })); + + it('exits non-zero with an invalid assistant ID', () => { + expect(chatResult.status).not.toBe(0); + }); + + it('shows an error indicating the assistant was not found or is not registered', () => { + const out = (chatResult.stdout ?? '') + (chatResult.stderr ?? ''); + expect(out).toMatch(/not found|not registered|register|error|failed|unknown/i); + }); + }); + + // ── TC-026: Assistant chat non-interactive ────────────────────────────────── + // Uses the dynamically created AutoAssistantRandomGenerator which always + // responds with a random number 1-10. + describe('TC-026 — assistants chat non-interactive (random number test)', () => { + let testHome: string; + let chatResult: ReturnType; + + beforeAll(() => { + testHome = mkdtempSync(join(getTempDir(), 'codemie-asst-chat-')); + if (!CI_IS_LOCAL_RUN) { + writeJwtProfile(testHome, { jwtToken }); + } else { + writeSsoProfile(testHome); + copySsoCredentials(testHome); + } + registerAssistantInConfig(testHome, createdAssistantId, ASSISTANT_NAME, ASSISTANT_SLUG); + + const chatArgs = CI_IS_LOCAL_RUN + ? [CLI_BIN, 'assistants', 'chat', createdAssistantId, 'hi'] + : [CLI_BIN, 'assistants', 'chat', '--jwt-token', jwtToken, createdAssistantId, 'hi']; + const chatEnv = CI_IS_LOCAL_RUN + ? { ...ssoCleanEnv(), CODEMIE_HOME: testHome, CI: '1' } + : { ...jwtCleanEnv(), CODEMIE_HOME: testHome, CODEMIE_JWT_TOKEN: jwtToken, CI: '1' }; + chatResult = spawnSync( + process.execPath, + chatArgs, + { cwd: testHome, env: chatEnv, encoding: 'utf-8', timeout: 60_000 }, + ); + }, 90_000); + + afterAll(() => rmSync(testHome, { recursive: true, force: true })); + + it('exits 0 and returns a number 1-10', () => { + const out = (chatResult.stdout ?? '') + (chatResult.stderr ?? ''); + expect(chatResult.status, `stdout: ${chatResult.stdout ?? ''}\nstderr: ${chatResult.stderr ?? ''}`).toBe(0); + expect(out).toMatch(/\b([1-9]|10)\b/); + }); + }); +}); diff --git a/tests/integration/agent-jwt-token.test.ts b/tests/integration/agent-jwt-token.test.ts new file mode 100644 index 00000000..5365ed6a --- /dev/null +++ b/tests/integration/agent-jwt-token.test.ts @@ -0,0 +1,176 @@ +/** + * JWT token tests — TC-017, TC-027 + * + * Run with: npm run test:integration:agent + * Requires: CI_IS_LOCAL_RUN=false (JWT mode) + CI_CODEMIE_* env vars + * + * JWT-ONLY: these tests exercise CLI flag paths that are specific to the + * --jwt-token mechanism. Skipped when CI_IS_LOCAL_RUN=true (SSO mode). + * + * TC-017: Config has two profiles — an SSO profile set as active and a JWT + * profile. Running with --profile --jwt-token + * verifies that --profile overrides the active profile and --jwt-token + * overrides the auth, even when the active profile is SSO-configured. + * + * TC-027: --jwt-token passed with no pre-written profile and an empty + * CODEMIE_HOME. Verifies the agent authenticates and completes a + * --task using only the token supplied on the command line. + */ + +import '../setup/load-test-env.js'; +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { spawnSync } from 'node:child_process'; +import { mkdirSync, mkdtempSync, rmSync, readdirSync, statSync, readFileSync, writeFileSync } from 'node:fs'; +import { join, dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + fetchJwtToken, + getTempDir, + jwtCleanEnv, + getTestEnvFlagOrDefault, +} from '../helpers/index.js'; + +const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../..'); +const CLAUDE_BIN = join(REPO_ROOT, 'bin', 'codemie-claude.js'); + +const CI_IS_LOCAL_RUN = getTestEnvFlagOrDefault('CI_IS_LOCAL_RUN', true); + +/** + * Write a config with two profiles: + * profile-sso-active — SSO, set as the activeProfile + * profile-jwt-override — JWT bearer-auth, not active + * + * Used by TC-017 to verify that --profile + --jwt-token override both the + * active profile selection and the auth method at runtime. + */ +function writeTwoProfileConfig(testHome: string): void { + const ciCodemieUrl = (process.env.CI_CODEMIE_URL ?? '').replace(/\/$/, ''); + const authBase = (process.env.CI_CODEMIE_AUTH_URL ?? '').replace(/\/$/, ''); + const config = { + version: 2, + activeProfile: 'profile-sso-active', + profiles: { + 'profile-sso-active': { + name: 'profile-sso-active', + provider: 'ai-run-sso', + authMethod: 'sso', + codeMieUrl: process.env.CI_CODEMIE_URL ?? '', + baseUrl: `${ciCodemieUrl}/code-assistant-api`, + apiKey: 'sso-authenticated', + model: process.env.CI_CODEMIE_MODEL ?? 'claude-sonnet-4-6', + timeout: 300, + debug: false, + }, + 'profile-jwt-override': { + name: 'profile-jwt-override', + provider: 'bearer-auth', + authMethod: 'jwt', + codeMieUrl: process.env.CI_CODEMIE_URL ?? '', + baseUrl: `${ciCodemieUrl}/code-assistant-api`, + model: process.env.CI_CODEMIE_MODEL ?? 'claude-sonnet-4-6', + authServerUrl: authBase, + authRealm: 'codemie-prod', + }, + }, + }; + mkdirSync(testHome, { recursive: true }); + writeFileSync(join(testHome, 'codemie-cli.config.json'), JSON.stringify(config, null, 2), 'utf-8'); +} + +function getLatestSessionFile(sessionsDir: string): Record { + const files = readdirSync(sessionsDir) + .filter((f) => f.endsWith('.json')) + .map((f) => join(sessionsDir, f)) + .sort((a, b) => statSync(b).mtimeMs - statSync(a).mtimeMs); + if (!files.length) throw new Error('No session files found in ' + sessionsDir); + return JSON.parse(readFileSync(files[0], 'utf-8')); +} + +describe.runIf(!CI_IS_LOCAL_RUN)( + 'JWT token tests [JWT-only, skipped when CI_IS_LOCAL_RUN=true]', + () => { + let jwtToken: string; + + beforeAll(async () => { + jwtToken = await fetchJwtToken(); + }, 30_000); + + // ── TC-017: --profile + --jwt-token override active SSO profile ──────────── + // Two profiles are written to the config with an SSO profile set as active. + // Running with --profile profile-jwt-override --jwt-token must use + // the JWT profile, not the active SSO one, and authenticate via the token. + describe('TC-017 — --profile and --jwt-token override active SSO profile', () => { + let testHome: string; + let result: ReturnType; + + beforeAll(() => { + testHome = mkdtempSync(join(getTempDir(), 'codemie-jwt-override-')); + writeTwoProfileConfig(testHome); + result = spawnSync( + process.execPath, + [ + CLAUDE_BIN, + '--profile', 'profile-jwt-override', + '--jwt-token', jwtToken, + '--task', 'Say READY', + ], + { + cwd: testHome, + env: { ...jwtCleanEnv(), CODEMIE_HOME: testHome }, + encoding: 'utf-8', + timeout: 120_000, + }, + ); + }, 180_000); + + afterAll(() => rmSync(testHome, { recursive: true, force: true })); + + it('exits 0', () => { + expect( + result.status, + `stdout: ${result.stdout ?? ''}\nstderr: ${result.stderr ?? ''}`, + ).toBe(0); + }); + + it('session file records bearer-auth provider (not SSO)', () => { + const session = getLatestSessionFile(join(testHome, 'sessions')); + expect(String(session.provider ?? session.providerName ?? '')).toMatch(/bearer-auth/i); + }); + }); + + // ── TC-027: --jwt-token with no profile ──────────────────────────────────── + // Empty CODEMIE_HOME, no profile written, token supplied only via CLI flag. + // Every other JWT test pre-writes a bearer-auth profile first; this test + // exercises the token-only code path that skips profile resolution entirely. + describe('TC-027 — --jwt-token without profile exits 0 and prints agent response', () => { + let testHome: string; + let result: ReturnType; + + beforeAll(() => { + testHome = mkdtempSync(join(getTempDir(), 'codemie-jwt-token-')); + result = spawnSync( + process.execPath, + [CLAUDE_BIN, '--task', 'Say the word READY and nothing else', '--jwt-token', jwtToken], + { + env: { ...jwtCleanEnv(), CODEMIE_HOME: testHome }, + encoding: 'utf-8', + timeout: 120_000, + }, + ); + }, 180_000); + + afterAll(() => rmSync(testHome, { recursive: true, force: true })); + + it('exits 0', () => { + expect( + result.status, + `stdout: ${result.stdout ?? ''}\nstderr: ${result.stderr ?? ''}`, + ).toBe(0); + }); + + it('agent response appears in stdout', () => { + expect(result.stdout).toMatch(/READY/i); + }); + }); + }, +); diff --git a/tests/integration/agent-model.test.ts b/tests/integration/agent-model.test.ts new file mode 100644 index 00000000..a16905ee --- /dev/null +++ b/tests/integration/agent-model.test.ts @@ -0,0 +1,316 @@ +/** + * Model tests — TC-020, TC-021, TC-022, TC-024 + * + * Run with: npm run test:integration:agent + * + * Auth mode (CI_IS_LOCAL_RUN in .env.test.local): + * true (default) — SSO mode; uses developer's sso-autotest profile in ~/.codemie + * false — JWT mode; isolates to a temp CODEMIE_HOME with bearer-auth profile + * + * TC-020: Session uses the model configured in the profile (sonnet and haiku variants). + * TC-021: Metrics records the configured model in the models array. + * TC-022: codemie models list returns the configured model in its output. + * TC-024: In-session model switch via /model slash command records new model in metrics. + */ + +import '../setup/load-test-env.js'; +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { spawnSync } from 'node:child_process'; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { join, dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + fetchJwtToken, + writeJwtProfile, + writeSsoProfile, + copySsoCredentials, + getTempDir, + spawnPty, + jwtCleanEnv, + ssoCleanEnv, + getLatestMetricsRecord, + getTestEnvFlagOrDefault, +} from '../helpers/index.js'; + +const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../..'); +const CLAUDE_BIN = join(REPO_ROOT, 'bin', 'codemie-claude.js'); +const CLI_BIN = join(REPO_ROOT, 'bin', 'codemie.js'); + +const CI_IS_LOCAL_RUN = getTestEnvFlagOrDefault('CI_IS_LOCAL_RUN', true); + +/** + * Write a profile that selects a specific model, in SSO or JWT format + * depending on CI_IS_LOCAL_RUN. Used by TC-020 and TC-021 to verify that + * the model field in codemie-cli.config.json is honoured at runtime. + */ +function writeProfileWithModel(codemieHome: string, profileName: string, model: string): void { + const ciCodemieUrl = (process.env.CI_CODEMIE_URL ?? '').replace(/\/$/, ''); + const profile = CI_IS_LOCAL_RUN + ? { + name: profileName, + provider: 'ai-run-sso', + authMethod: 'sso', + codeMieUrl: process.env.CI_CODEMIE_URL ?? '', + baseUrl: `${ciCodemieUrl}/code-assistant-api`, + apiKey: 'sso-authenticated', + model, + timeout: 300, + debug: false, + } + : { + name: profileName, + provider: 'bearer-auth', + authMethod: 'jwt', + codeMieUrl: process.env.CI_CODEMIE_URL ?? '', + baseUrl: `${ciCodemieUrl}/code-assistant-api`, + model, + }; + mkdirSync(codemieHome, { recursive: true }); + writeFileSync( + join(codemieHome, 'codemie-cli.config.json'), + JSON.stringify({ version: 2, activeProfile: profileName, profiles: { [profileName]: profile } }, null, 2), + 'utf-8', + ); +} + +describe.runIf(process.env.SSO_AVAILABLE !== 'false')('Model tests', () => { + let jwtToken: string; + + // SSO mode: the sso-autotest profile is configured once by the global setup + // (agent-build-setup.ts) before any test files start. No per-file setup is + // needed here — calling setupSsoAutotestProfile() again would write to + // ~/.codemie/codemie-cli.config.json concurrently with other test workers + // that use ~/.codemie directly, creating a race condition. + beforeAll(async () => { + if (!CI_IS_LOCAL_RUN) { + jwtToken = await fetchJwtToken(); + } + }, 30_000); + + // ── TC-020: Profile model selection ─────────────────────────────────────────── + // Runs two --task sessions back-to-back, each with a different model profile, + // then checks that the model recorded in _metrics.jsonl matches the profile. + // Separate temp homes for each model guarantee unambiguous mtime ordering when + // reading the latest record. + describe('TC-020 — session uses model from profile', () => { + let sonnetHome: string; + let haikuHome: string; + let sonnetMetrics: Record; + let haikuMetrics: Record; + + beforeAll(() => { + sonnetHome = mkdtempSync(join(getTempDir(), 'codemie-model-sonnet-')); + writeProfileWithModel(sonnetHome, 'profile-sonnet', 'claude-sonnet-4-6'); + if (CI_IS_LOCAL_RUN) copySsoCredentials(sonnetHome); + spawnSync( + process.execPath, + CI_IS_LOCAL_RUN + ? [CLAUDE_BIN, '--profile', 'profile-sonnet', '--task', 'Say READY'] + : [CLAUDE_BIN, '--profile', 'profile-sonnet', '--jwt-token', jwtToken, '--task', 'Say READY'], + { cwd: sonnetHome, env: { ...(CI_IS_LOCAL_RUN ? ssoCleanEnv() : jwtCleanEnv()), CODEMIE_HOME: sonnetHome }, encoding: 'utf-8', timeout: 180_000 }, + ); + sonnetMetrics = getLatestMetricsRecord(join(sonnetHome, 'sessions')); + + haikuHome = mkdtempSync(join(getTempDir(), 'codemie-model-haiku-')); + writeProfileWithModel(haikuHome, 'profile-haiku', 'claude-haiku-4-5-20251001'); + if (CI_IS_LOCAL_RUN) copySsoCredentials(haikuHome); + spawnSync( + process.execPath, + CI_IS_LOCAL_RUN + ? [CLAUDE_BIN, '--profile', 'profile-haiku', '--task', 'Say READY'] + : [CLAUDE_BIN, '--profile', 'profile-haiku', '--jwt-token', jwtToken, '--task', 'Say READY'], + { cwd: haikuHome, env: { ...(CI_IS_LOCAL_RUN ? ssoCleanEnv() : jwtCleanEnv()), CODEMIE_HOME: haikuHome }, encoding: 'utf-8', timeout: 180_000 }, + ); + haikuMetrics = getLatestMetricsRecord(join(haikuHome, 'sessions')); + }, 300_000); + + afterAll(() => { + if (sonnetHome) rmSync(sonnetHome, { recursive: true, force: true }); + if (haikuHome) rmSync(haikuHome, { recursive: true, force: true }); + }); + + it('metrics models array contains sonnet for claude-sonnet-4-6 profile', () => { + const models = (sonnetMetrics.models as string[]) ?? []; + expect( + models.some((m) => /sonnet/i.test(m)), + `Expected models to contain sonnet, got: ${JSON.stringify(models)}`, + ).toBe(true); + }); + + it('metrics models array contains haiku for claude-haiku-4-5-20251001 profile', () => { + const models = (haikuMetrics.models as string[]) ?? []; + expect( + models.some((m) => /haiku/i.test(m)), + `Expected models to contain haiku, got: ${JSON.stringify(models)}`, + ).toBe(true); + }); + }); + + // ── TC-021: Metrics models array populated ───────────────────────────────── + // Sanity check: after a minimal --task run, the models array in _metrics.jsonl + // is non-empty and reflects the model that was configured in the profile. + describe('TC-021 — metrics records the configured model', () => { + let testHome: string; + let metrics: Record; + + beforeAll(() => { + testHome = mkdtempSync(join(getTempDir(), 'codemie-model-tiers-')); + writeProfileWithModel(testHome, 'profile-tiers', 'claude-sonnet-4-6'); + if (CI_IS_LOCAL_RUN) copySsoCredentials(testHome); + spawnSync( + process.execPath, + CI_IS_LOCAL_RUN + ? [CLAUDE_BIN, '--profile', 'profile-tiers', '--task', 'Say READY'] + : [CLAUDE_BIN, '--profile', 'profile-tiers', '--jwt-token', jwtToken, '--task', 'Say READY'], + { cwd: testHome, env: { ...(CI_IS_LOCAL_RUN ? ssoCleanEnv() : jwtCleanEnv()), CODEMIE_HOME: testHome }, encoding: 'utf-8', timeout: 120_000 }, + ); + metrics = getLatestMetricsRecord(join(testHome, 'sessions')); + }, 180_000); + + afterAll(() => rmSync(testHome, { recursive: true, force: true })); + + it('metrics models array is non-empty and contains the configured model', () => { + const models = (metrics.models as string[]) ?? []; + expect(models.length, 'models array must not be empty').toBeGreaterThan(0); + expect( + models.some((m) => /sonnet/i.test(m)), + `Expected models to contain the configured sonnet model, got: ${JSON.stringify(models)}`, + ).toBe(true); + }); + }); + + // ── TC-022: codemie models list returns available models ────────────────── + // Calls the codemie CLI (not codemie-claude) with real auth. Verifies that + // the models list command contacts the provider and returns at least the + // configured model name. + describe('TC-022 — codemie models list returns available models', () => { + let testHome: string; + let listResult: ReturnType; + + beforeAll(() => { + testHome = mkdtempSync(join(getTempDir(), 'codemie-models-')); + if (!CI_IS_LOCAL_RUN) { + writeJwtProfile(testHome, { jwtToken }); + } else { + writeSsoProfile(testHome); + copySsoCredentials(testHome); + } + listResult = spawnSync( + process.execPath, + [CLI_BIN, 'models', 'list'], + { + cwd: testHome, + env: { + ...(CI_IS_LOCAL_RUN ? ssoCleanEnv() : jwtCleanEnv()), + CODEMIE_HOME: testHome, + ...(CI_IS_LOCAL_RUN ? {} : { CODEMIE_JWT_TOKEN: jwtToken }), + CI: '1', + }, + encoding: 'utf-8', + timeout: 30_000, + }, + ); + }, 60_000); + + afterAll(() => rmSync(testHome, { recursive: true, force: true })); + + it('exits 0', () => { + expect( + listResult.status, + `stdout: ${listResult.stdout ?? ''}\nstderr: ${listResult.stderr ?? ''}`, + ).toBe(0); + }); + + it('output contains the expected model name', () => { + const out = (listResult.stdout ?? '') + (listResult.stderr ?? ''); + expect(out).toMatch(new RegExp(process.env.CI_CODEMIE_MODEL ?? 'claude', 'i')); + }); + }); + + // ── TC-024: In-session /model switch via PTY ──────────────────────────────── + // Uses node-pty to give the process a real TTY (isTTY=true), which is required + // for the /model slash command to be available inside a running agent session. + // Verifies that the switched model appears in the session metrics file. + describe('TC-024 — in-session /model switch records new model in metrics', () => { + let testHome: string; + + beforeAll(() => { + testHome = mkdtempSync(join(getTempDir(), 'codemie-model-switch-')); + if (!CI_IS_LOCAL_RUN) { + writeJwtProfile(testHome, { jwtToken }); + } else { + writeSsoProfile(testHome); + copySsoCredentials(testHome); + } + }); + + afterAll(async () => { + await new Promise((r) => setTimeout(r, 500)); + rmSync(testHome, { recursive: true, force: true }); + }); + + it('agent processes /model switch and records new model in metrics', async () => { + const sessionArgs = CI_IS_LOCAL_RUN + ? [CLAUDE_BIN] + : [CLAUDE_BIN, '--profile', 'jwt-autotest', '--jwt-token', jwtToken]; + const sessionEnv = CI_IS_LOCAL_RUN + ? { ...ssoCleanEnv(), CODEMIE_HOME: testHome, TERM: 'xterm-256color' } + : { ...jwtCleanEnv(), CODEMIE_HOME: testHome, TERM: 'xterm-256color' }; + + const proc = spawnPty(process.execPath, sessionArgs, { cwd: testHome, env: sessionEnv }); + + try { + // Wait for the profile info table rendered before Claude enters interactive mode. + await proc.waitFor(/Model\s*[│|]/i, 60_000); + // On macOS, Claude Code shows a workspace-trust prompt for new temp directories + // before rendering the startup box. Start a background handler that accepts the + // prompt ('1' = Yes, I trust this folder) if it appears, so the startup box can + // render. The handler is a no-op on platforms where the prompt does not appear. + void proc.waitFor(/trust.*folder|trustthisfolder/i, 15_000) + .then(() => proc.writeLine('1')) + .catch(() => { /* no trust prompt on this platform */ }); + // Wait for Claude Code's startup box to fully render (╰─ is its bottom-left + // corner). Sending commands before this point causes them to pile up in the + // ConPTY input buffer and be drained by readline as ONE combined input when it + // finally starts — that is the root cause of the "model=...SayPONG" 400 error. + // Once the startup box is visible, the TUI is rendered and readline is actively + // waiting for keystrokes, so commands sent now are processed individually. + await proc.waitFor(/╰─/, 60_000); + // 1 s buffer for the prompt area to settle after the startup box closes. + await new Promise((r) => setTimeout(r, 1_000)); + // Switch model in-session via slash command — readline IS ready at this point. + proc.writeLine('/model claude-haiku-4-5-20251001'); + // Wait for /model to be processed. Do NOT use waitFor(/haiku/) here because + // the PTY echoes the input line back (writeLine sends \r\n = proper line) and + // that echo would match /haiku/ before any Claude Code processing happens. + await new Promise((r) => setTimeout(r, 8_000)); + // Send a message so haiku is actually used and recorded in metrics. + const pongCursor = proc.lines().length; + proc.writeLine('Say PONG and nothing else'); + // Only match PONG in lines received AFTER the message was sent (pongCursor). + // waitFor scans allLines from startFromLine, so historical output cannot cause + // a false-positive match. The lookbehind still excludes the echoed input line + // "Say PONG and nothing else" (PONG preceded by "Say "). + await proc.waitFor(/(? the original 3 s. + await new Promise((r) => setTimeout(r, 5_000)); + } finally { + // /exit is a local slash command in the Claude Code REPL that exits + // gracefully, firing SessionEnd → codemie hook → renameFiles. + proc.writeLine('/exit'); + // Wait up to 90 s for Claude Code to exit and all hooks to complete. + await proc.exit(90_000); + } + + const ptyLines = proc.lines(); + const metrics = getLatestMetricsRecord(join(testHome, 'sessions')); + const models = (metrics.models as string[]) ?? []; + expect( + models.some((m) => /haiku/i.test(m)), + `Expected metrics.models to contain haiku after /model switch.\nGot: ${JSON.stringify(models)}\nLast PTY lines:\n${ptyLines.slice(-30).join('\n')}`, + ).toBe(true); + }, 240_000); + }); +}); diff --git a/tests/integration/agent-negative.test.ts b/tests/integration/agent-negative.test.ts new file mode 100644 index 00000000..75d9f839 --- /dev/null +++ b/tests/integration/agent-negative.test.ts @@ -0,0 +1,117 @@ +/** + * Agent negative cases — TC-018, TC-019 + * + * Run with: npm run test:integration:agent + * + * Auth mode (CI_IS_LOCAL_RUN in .env.test.local): + * true (default) — SSO mode + * false — JWT mode + * + * TC-018: Invalid JWT token — exits non-zero with an auth error. + * JWT-only: tests the --jwt-token code path directly; skipped in SSO mode. + * TC-019: No profile and no token — exits non-zero with a setup/config error. + * Dual-mode: an empty CODEMIE_HOME fails the same way in both auth modes. + */ + +import '../setup/load-test-env.js'; +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs'; +import { join, dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + writeJwtProfile, + getTempDir, + jwtCleanEnv, + ssoCleanEnv, + getTestEnvFlagOrDefault, +} from '../helpers/index.js'; + +const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../..'); +const CLAUDE_BIN = join(REPO_ROOT, 'bin', 'codemie-claude.js'); + +const CI_IS_LOCAL_RUN = getTestEnvFlagOrDefault('CI_IS_LOCAL_RUN', true); + +describe.runIf(process.env.SSO_AVAILABLE !== 'false')('Agent negative cases', () => { + // ── TC-018: Invalid JWT token ─────────────────────────────────────────────── + // Hardcoded invalid token — no fetchJwtToken() needed. JWT-only because the + // --jwt-token flag and bearer-auth profile are JWT-specific concepts. + describe.runIf(!CI_IS_LOCAL_RUN)( + 'TC-018 — invalid JWT token [JWT-only, skipped when CI_IS_LOCAL_RUN=true]', + () => { + let testHome: string; + let result: ReturnType; + + beforeAll(() => { + testHome = mkdtempSync(join(getTempDir(), 'codemie-jwt-invalid-')); + writeJwtProfile(testHome, { jwtToken: 'INVALID_TOKEN_VALUE' }); + result = spawnSync( + process.execPath, + [CLAUDE_BIN, '--task', 'Say hello', '--jwt-token', 'INVALID_TOKEN_VALUE'], + { + cwd: testHome, + env: { ...jwtCleanEnv(), CODEMIE_HOME: testHome }, + encoding: 'utf-8', + timeout: 60_000, + }, + ); + }, 90_000); + + afterAll(() => rmSync(testHome, { recursive: true, force: true })); + + it('exits non-zero with an invalid JWT token', () => { + expect(result.status).not.toBe(0); + }); + + it('shows an auth error message', () => { + expect((result.stdout ?? '') + (result.stderr ?? '')).toMatch( + /auth|unauthorized|401|invalid|token|malformed|empty.*response|API Error/i, + ); + }); + }, + ); + + // ── TC-019: No profile and no token ──────────────────────────────────────── + // Empty CODEMIE_HOME, no profile written, no token flag. Fails in both SSO + // and JWT mode because the agent cannot find any auth configuration. + describe('TC-019 — no profile and no token (negative)', () => { + let testHome: string; + let result: ReturnType; + + beforeAll(() => { + testHome = mkdtempSync(join(getTempDir(), 'codemie-no-config-')); + // Write a config with no profiles so ConfigLoader finds a real file in + // testHome and does not fall back to ~/.codemie (which has the + // sso-autotest profile from globalSetup). Without this, the CLI would + // attempt SSO re-auth via inquirer and crash without a TTY. + mkdirSync(testHome, { recursive: true }); + writeFileSync( + join(testHome, 'codemie-cli.config.json'), + JSON.stringify({ version: 2, profiles: {} }), + 'utf-8', + ); + result = spawnSync( + process.execPath, + [CLAUDE_BIN, '--task', 'Say hello'], + { + cwd: testHome, + env: { ...(CI_IS_LOCAL_RUN ? ssoCleanEnv() : jwtCleanEnv()), CODEMIE_HOME: testHome }, + encoding: 'utf-8', + timeout: 30_000, + }, + ); + }, 60_000); + + afterAll(() => rmSync(testHome, { recursive: true, force: true })); + + it('exits non-zero with empty CODEMIE_HOME and no auth', () => { + expect(result.status).not.toBe(0); + }); + + it('shows a setup/configuration error message', () => { + expect((result.stdout ?? '') + (result.stderr ?? '')).toMatch( + /no profile|not configured|setup|profile/i, + ); + }); + }); +}); diff --git a/tests/integration/agent-setup.test.ts b/tests/integration/agent-setup.test.ts new file mode 100644 index 00000000..eef5400f --- /dev/null +++ b/tests/integration/agent-setup.test.ts @@ -0,0 +1,155 @@ +/** + * TC-029: codemie setup wizard — SSO profile creation (PTY) + * + * Walks through the interactive `codemie setup` wizard, creates a fresh SSO + * profile, then verifies the written config file. + * + * SSO-only: step 4–5 opens a browser for authentication. + * Run with: npm run test:integration:agent + * + * Isolation: the wizard runs with CODEMIE_HOME pointing to a temp dir so it + * never touches ~/.codemie — safe to run in parallel with other agent tests. + */ + +import '../setup/load-test-env.js'; +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { existsSync, readFileSync, mkdtempSync, rmSync } from 'node:fs'; +import { homedir } from 'node:os'; +import { join, dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { spawnPty } from '../helpers/pty-session.js'; +import { ssoCleanEnv, getTempDir } from '../helpers/index.js'; + +const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../..'); +const CODEMIE_BIN = join(REPO_ROOT, 'bin', 'codemie.js'); + +/** Deterministic test profile name typed in step 8. */ +const TEST_PROFILE_NAME = 'setup-test-sso'; + +describe.runIf(process.env.SSO_AVAILABLE !== 'false')('TC-029 — codemie setup wizard (SSO)', () => { + let testHome: string; + + beforeAll(() => { + // Fresh isolated config home — wizard writes here, never touches ~/.codemie. + testHome = mkdtempSync(join(getTempDir(), 'codemie-setup-')); + }); + + afterAll(() => { + rmSync(testHome, { recursive: true, force: true }); + }); + + it( + 'walks the wizard, creates an SSO profile, and verifies the written config', + async () => { + const proc = spawnPty(process.execPath, [CODEMIE_BIN, 'setup'], { + cwd: homedir(), + // CODEMIE_HOME isolation: wizard reads/writes testHome, not ~/.codemie. + env: { ...ssoCleanEnv(), CODEMIE_HOME: testHome }, + }); + + // ── Step 1 (conditional): "What would you like to do?" ───────────────────── + // Appears only when existing profiles are present. Fresh testHome has none, + // so this is skipped. The conditional guard handles both cases. + const firstLine = await proc.waitFor( + /what would you like to do|where would you like to store/i, + 30_000, + ); + if (/what would you like to do/i.test(firstLine)) { + await new Promise(r => setTimeout(r, 200)); + proc.write('\r'); // accept default: add a new profile + await proc.waitFor(/where would you like to store/i, 15_000); + } + + // ── Step 2: "Where would you like to store?" → global (default) ──────────── + await new Promise(r => setTimeout(r, 200)); + proc.write('\r'); + + // ── Step 3: "Choose your LLM provider" → SSO (first, priority 0) ────────── + await proc.waitFor(/choose your llm provider/i, 15_000); + await new Promise(r => setTimeout(r, 200)); + proc.write('\r'); + + // ── Step 4: Organization URL → accept default (codemie prod) ─────────────── + // The input prompt ("? CodeMie organization URL:") never emits a trailing \n + // while waiting for input. waitFor now checks the incomplete tail line so + // the pattern will match once the prompt is rendered. + // The saved URL is cross-verified via the config-file assertion below. + await proc.waitFor(/organization url|codemie.*url|enter.*url/i, 15_000); + await new Promise(r => setTimeout(r, 200)); + proc.write('\r'); + + // ── Step 5: Browser SSO flow ───────────────────────────────────────────────── + // The wizard opens the browser; wait up to 2 minutes for the user to log in. + await proc.waitFor(/Authentication successful/i, 120_000); + + // ── Step 6: "Select your project:" → first option ────────────────────────── + await proc.waitFor(/Select your project/i, 30_000); + await new Promise(r => setTimeout(r, 500)); + proc.write('\r'); + // Capture "✓ Selected project: " for cross-verification with config. + const projectLine = await proc.waitFor(/Selected project:/i, 15_000); + const selectedProject = projectLine.match(/Selected project:\s*(\S+)/i)?.[1]; + + // ── Step 7: Model selection → first option ────────────────────────────────── + await proc.waitFor(/\(Use arrow keys\)/i, 15_000); + await new Promise(r => setTimeout(r, 500)); + proc.write('\r'); + + // ── Step 8: Profile name → clear default, type test name ──────────────────── + await proc.waitFor(/Enter a name for this profile/i, 15_000); + await new Promise(r => setTimeout(r, 200)); + proc.write('\x15'); // Ctrl+U — clears the entire readline input + await new Promise(r => setTimeout(r, 100)); + proc.writeLine(TEST_PROFILE_NAME); + + // Wait for save confirmation: '✔ Profile "..." saved to global config' + await proc.waitFor(/Profile .+ saved to (global|local) config/i, 15_000); + + // ── Step 9: "Switch to profile as active?" → confirm ──────────────────────── + // Safe: wizard writes to testHome only; ~/.codemie is not touched. + await proc.waitFor(/Switch to profile/i, 10_000); + await new Promise(r => setTimeout(r, 200)); + proc.writeLine('y'); + + await proc.exit(30_000); + + // ── Verify config ──────────────────────────────────────────────────────────── + const configPath = join(testHome, 'codemie-cli.config.json'); + expect(existsSync(configPath), 'config file must exist in testHome after setup').toBe(true); + + const cfg = JSON.parse(readFileSync(configPath, 'utf-8')) as { + activeProfile?: string; + profiles?: Record>; + }; + const profile = cfg.profiles?.[TEST_PROFILE_NAME]; + + expect(profile, `profile "${TEST_PROFILE_NAME}" must exist in config`).toBeDefined(); + expect(profile!.name, 'name must match the typed profile key').toBe(TEST_PROFILE_NAME); + expect(profile!.provider, 'provider must be ai-run-sso').toBe('ai-run-sso'); + expect(String(profile!.apiKey ?? ''), 'apiKey must be sso-provided').toBe('sso-provided'); + expect(String(profile!.codeMieUrl ?? ''), 'codeMieUrl must be the prod URL').toMatch( + /codemie\.lab\.epam\.com/, + ); + expect(String(profile!.baseUrl ?? ''), 'baseUrl must include code-assistant-api').toMatch( + /code-assistant-api/, + ); + + // Step 9 verification: profile was activated + expect(cfg.activeProfile, 'activeProfile must be set to the new profile').toBe( + TEST_PROFILE_NAME, + ); + + // Verify the selected project was persisted (captured from PTY + checked in config) + expect(String(profile!.codeMieProject ?? ''), 'codeMieProject must not be empty').not.toBe(''); + if (selectedProject) { + expect(profile!.codeMieProject, `codeMieProject must match selected "${selectedProject}"`).toBe( + selectedProject, + ); + } + + // Verify the selected model was persisted (read from config — more reliable than PTY capture) + expect(String(profile!.model ?? ''), 'model must not be empty').not.toBe(''); + }, + 180_000, // 3 min: allows 2 min for browser auth + PTY interactions + ); +}); diff --git a/tests/integration/agent-shortcuts.test.ts b/tests/integration/agent-shortcuts.test.ts index a0f9bcb6..f2aad0f9 100644 --- a/tests/integration/agent-shortcuts.test.ts +++ b/tests/integration/agent-shortcuts.test.ts @@ -11,7 +11,7 @@ import { statSync, readFileSync } from 'fs'; import { resolve } from 'path'; import { setupTestIsolation } from '../helpers/test-isolation.js'; -describe('Agent Shortcuts - Integration', () => { +describe.runIf(process.env.SSO_AVAILABLE !== 'false')('Agent Shortcuts - Integration', () => { // Setup isolated CODEMIE_HOME for this test suite setupTestIsolation(); diff --git a/tests/integration/agent-skills.test.ts b/tests/integration/agent-skills.test.ts new file mode 100644 index 00000000..99859027 --- /dev/null +++ b/tests/integration/agent-skills.test.ts @@ -0,0 +1,223 @@ +/** + * Skill tests — TC-025 + * + * Run with: npm run test:integration:agent + * + * Auth mode (CI_IS_LOCAL_RUN in .env.test.local): + * true (default) — SSO mode; uses developer's sso-autotest profile in ~/.codemie + * false — JWT mode; isolates to a temp CODEMIE_HOME with bearer-auth profile + * + * A fresh skill (auto-skill-random-gen) is created in the outer beforeAll via the SDK + * and deleted in afterAll, removing any static skill dependency. + * + * TC-025: Skill slash command invocation inside a running agent session. + * Installs the dynamically created skill via the interactive setup wizard, + * then verifies the slash command is available and returns a number 1-10. + */ + +import '../setup/load-test-env.js'; +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { randomBytes } from 'node:crypto'; +import { mkdirSync, mkdtempSync, rmSync } from 'node:fs'; +import { join, dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { CodeMieClient } from 'codemie-sdk'; +import { getCodemieClient } from '@/utils/sdk-client.js'; +import { createSkill, deleteSkill } from '@/cli/commands/sdk/services/skills.js'; +import { + fetchJwtToken, + writeJwtProfile, + writeSsoProfile, + copySsoCredentials, + getTempDir, + spawnPty, + jwtCleanEnv, + ssoCleanEnv, + setupSsoAutotestProfile, + teardownSsoAutotestProfile, + getTestEnvFlagOrDefault, +} from '../helpers/index.js'; + +const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../..'); +const CLAUDE_BIN = join(REPO_ROOT, 'bin', 'codemie-claude.js'); +const CLI_BIN = join(REPO_ROOT, 'bin', 'codemie.js'); + +const CI_IS_LOCAL_RUN = getTestEnvFlagOrDefault('CI_IS_LOCAL_RUN', true); + +// Unique per-run suffix so concurrent runs and other users' leftover skills don't collide. +const RUN_SUFFIX = randomBytes(3).toString('hex'); +const SKILL_NAME = `auto-skill-random-gen-${RUN_SUFFIX}`; +const SKILL_DESCRIPTION = 'Integration test skill — auto-created and deleted by the test suite. Returns a random number from 1 to 10.'; +const SKILL_CONTENT = [ + '# Random Number Generator', + '', + 'When invoked, respond with a single random number between 1 and 10.', + 'Your entire response must be exactly the number — no words, punctuation, or explanation.', +].join('\n'); + +describe.runIf(process.env.SSO_AVAILABLE !== 'false')('Skill tests', () => { + let jwtToken: string; + let jwtHome: string; + let sdkClient: CodeMieClient; + let createdSkillId: string; + let originalActiveProfile: string | undefined; + + beforeAll(async () => { + const ciCodemieUrl = (process.env.CI_CODEMIE_URL ?? '').replace(/\/$/, ''); + + if (!CI_IS_LOCAL_RUN) { + jwtToken = await fetchJwtToken(); + jwtHome = mkdtempSync(join(getTempDir(), 'codemie-skill-jwt-')); + writeJwtProfile(jwtHome, { jwtToken }); + sdkClient = new CodeMieClient({ + codemie_api_domain: `${ciCodemieUrl}/code-assistant-api`, + jwt_token: jwtToken, + verify_ssl: process.env.CODEMIE_INSECURE !== '1', + }); + } else { + originalActiveProfile = setupSsoAutotestProfile(); + sdkClient = await getCodemieClient(true); + } + + const aboutUser = await sdkClient.users.aboutMe(); + const project = aboutUser.applications[0]; + if (!project) throw new Error('No accessible project found for this user'); + + const created = await createSkill(sdkClient, { + name: SKILL_NAME, + description: SKILL_DESCRIPTION, + content: SKILL_CONTENT, + project, + }); + createdSkillId = created.id; + }, 60_000); + + afterAll(async () => { + if (createdSkillId && sdkClient) { + try { await deleteSkill(sdkClient, createdSkillId); } catch { /* best-effort */ } + } + if (!CI_IS_LOCAL_RUN) { + if (jwtHome) rmSync(jwtHome, { recursive: true, force: true }); + } else { + teardownSsoAutotestProfile(originalActiveProfile); + } + }); + + // ── TC-025: Skill invocation inside running session ───────────────────────── + // Installs the dynamically created platform skill via the interactive codemie + // setup skills wizard (driven by PTY), then verifies that the skill's slash + // command is available in a Claude Code session and returns a number 1-10. + describe('TC-025 — skill slash command in running session', () => { + let testHome: string; + + beforeAll(async () => { + testHome = mkdtempSync(join(getTempDir(), 'codemie-skill-')); + if (!CI_IS_LOCAL_RUN) { + writeJwtProfile(testHome, { jwtToken }); + } else { + writeSsoProfile(testHome); + copySsoCredentials(testHome); + } + // .claude/ marker causes auto-detection to include Claude Code as a target agent. + mkdirSync(join(testHome, '.claude'), { recursive: true }); + + const setupArgs = CI_IS_LOCAL_RUN + ? [CLI_BIN, 'setup', 'skills'] + : [CLI_BIN, 'setup', 'skills', '--profile', 'jwt-autotest']; + const setupEnv = CI_IS_LOCAL_RUN + ? { ...ssoCleanEnv(), CODEMIE_HOME: testHome, TERM: 'xterm-256color' } + // Full process.env for proxy/TLS/server-URL vars; CODEMIE_JWT_TOKEN set explicitly + // because the token is fetched into jwtToken but never exported to process.env. + : { ...process.env, CODEMIE_HOME: testHome, CODEMIE_JWT_TOKEN: jwtToken, TERM: 'xterm-256color' }; + + const setupProc = spawnPty(process.execPath, setupArgs, { cwd: testHome, env: setupEnv }); + + try { + // Step 1: Disclaimer screen. + await setupProc.waitFor(/Press Enter to continue/, 30_000); + setupProc.write('\r'); + + // Step 2: Storage scope — keep Global default, just Enter. + // Using Global + CODEMIE_HOME ensures skills write to testHome's config. + await setupProc.waitFor(/Where would you like to save/, 30_000); + setupProc.write('\r'); + + // Step 3: Target Agents — pre-selected; Enter confirms. + await setupProc.waitFor(/Target Agents/, 30_000); + setupProc.write('\r'); + + // Step 4: Skills picker — wait for the count line unique to this screen. + // Default focus is on list item 0 (not the search box). Arrow Up moves + // focus to search. The search field requires individual keypresses — bulk + // write does not trigger its keystroke handler. With the list filtered to + // one result, one Arrow Down after Space reaches the Continue button. + await setupProc.waitFor(/\d+ skills total/, 60_000); + await new Promise((r) => setTimeout(r, 500)); // Let the picker fully render + setupProc.write('\x1B[A'); // Arrow Up → focus search box + await new Promise((r) => setTimeout(r, 200)); + // Type letter-by-letter — the search field processes one keypress at a time. + for (const char of SKILL_NAME) { + setupProc.write(char); + await new Promise((r) => setTimeout(r, 50)); + } + await new Promise((r) => setTimeout(r, 1_500)); // Debounce (500ms) + API fetch + setupProc.write('\x1B[B'); // Arrow Down → unfocus search, cursor=0 + await new Promise((r) => setTimeout(r, 300)); + setupProc.write(' '); // Space to select (1 filtered result) + await new Promise((r) => setTimeout(r, 200)); + setupProc.write('\x1B[B'); // Arrow Down → focus Continue button + await new Promise((r) => setTimeout(r, 200)); + setupProc.write('\r'); // Enter to confirm (Continue button) + + await setupProc.waitFor(/Registered \d+ skill/, 30_000); + } finally { + await setupProc.exit(15_000); + } + }, 120_000); + + afterAll(async () => { + // Small delay for Windows to release file handles from PTY processes. + await new Promise((r) => setTimeout(r, 500)); + rmSync(testHome, { recursive: true, force: true }); + }); + + it(`agent responds to /${SKILL_NAME} and returns a number 1-10`, async () => { + const sessionArgs = CI_IS_LOCAL_RUN + ? [CLAUDE_BIN] + : [CLAUDE_BIN, '--profile', 'jwt-autotest', '--jwt-token', jwtToken]; + const sessionEnv = CI_IS_LOCAL_RUN + ? { ...ssoCleanEnv(), CODEMIE_HOME: testHome, TERM: 'xterm-256color' } + : { ...jwtCleanEnv(), CODEMIE_HOME: testHome, TERM: 'xterm-256color' }; + + const proc = spawnPty(process.execPath, sessionArgs, { cwd: testHome, env: sessionEnv }); + + try { + await proc.waitFor(/Model\s*[│|]/i, 60_000); + // On macOS, Claude Code shows a workspace-trust prompt for new temp directories + // before rendering the startup box. Start a background handler that accepts the + // prompt ('1' = Yes, I trust this folder) if it appears, so the startup box can + // render. The handler is a no-op on platforms where the prompt does not appear. + void proc.waitFor(/trust.*folder|trustthisfolder/i, 15_000) + .then(() => proc.writeLine('1')) + .catch(() => { /* no trust prompt on this platform */ }); + await proc.waitFor(/╰─/, 60_000); + await new Promise((r) => setTimeout(r, 1_000)); + proc.writeLine(`/${SKILL_NAME} hi`); + await proc.waitFor(/\b([1-9]|10)\b/, 90_000); + } finally { + proc.writeLine('/exit'); + await proc.exit(90_000); + } + + const lines = proc.lines(); + const matchedLine = lines.find((l) => /\b([1-9]|10)\b/.test(l)); + expect( + matchedLine, + `Expected a line containing a number 1-10.\nLast PTY lines:\n${lines.slice(-20).join('\n')}`, + ).toBeTruthy(); + const num = parseInt(matchedLine!.match(/\b([1-9]|10)\b/)![1], 10); + expect(num).toBeGreaterThanOrEqual(1); + expect(num).toBeLessThanOrEqual(10); + }, 240_000); + }); +}); diff --git a/tests/integration/agent-task-session.test.ts b/tests/integration/agent-task-session.test.ts new file mode 100644 index 00000000..fa7f4c22 --- /dev/null +++ b/tests/integration/agent-task-session.test.ts @@ -0,0 +1,254 @@ +/** + * Agent task execution and session artifact validation. + * + * Migrated from: codemie-sdk/test-harness/.../test_codemie_cli_claude.py + * + * Run with: npm run test:integration:agent + * + * Auth mode (CI_IS_LOCAL_RUN in .env.test.local): + * true (default) — SSO mode; uses developer's sso-autotest profile in ~/.codemie + * false — JWT mode; isolates to a temp CODEMIE_HOME with bearer-auth profile + * + * Environment variables: + * - CI_IS_LOCAL_RUN: "true" (default) for SSO, "false" for JWT + * - DEFAULT_TIMEOUT: Command timeout in seconds (default: 60) + * - CI_CODEMIE_URL: CodeMie frontend URL (both modes) + * - CI_CODEMIE_URL: CodeMie URL — API domain is derived as CI_CODEMIE_URL/code-assistant-api + * - CI_CODEMIE_USERNAME / CI_CODEMIE_PASSWORD / CI_CODEMIE_AUTH_URL: JWT mode only + */ + +import '../setup/load-test-env.js'; +import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach } from 'vitest'; +import { spawnSync } from 'child_process'; +import { + mkdtempSync, + rmSync, + existsSync, + readFileSync, + readdirSync, +} from 'fs'; +import { homedir, tmpdir } from 'os'; +import { join, dirname, resolve } from 'path'; +import { fileURLToPath } from 'url'; +import { randomUUID } from 'crypto'; +import { + SessionDataSchema, + MetricsRecordSchema, + ConversationRecordSchema, + UserMessageSchema, + AssistantMessageSchema, +} from './models/index.js'; +import { fetchJwtToken, writeJwtProfile, getTempDir, jwtCleanEnv, resolveLongPath, getTestEnvFlagOrDefault, pollForSession, ssoCleanEnv, setupSsoAutotestProfile, teardownSsoAutotestProfile } from '../helpers/index.js'; +import { validateSchema } from './models/index.js'; + +// Timeout from environment (seconds → milliseconds) +const CLI_TIMEOUT_MS = parseInt(process.env.DEFAULT_TIMEOUT ?? '60', 10) * 1000; + +// Setup hooks (installs) can take much longer than individual commands +const SETUP_TIMEOUT_MS = CLI_TIMEOUT_MS * 5; + +// Repo root is 2 levels up from tests/integration/ +const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), '../..'); + +// Path to the local codemie-claude entry point (uses dist/ from this repo) +const CLAUDE_BIN = join(repoRoot, 'bin', 'codemie-claude.js'); + +// true (default) = SSO mode (local dev); false = JWT mode (CI pipeline) +const CI_IS_LOCAL_RUN = getTestEnvFlagOrDefault('CI_IS_LOCAL_RUN', true); + +describe.runIf(process.env.SSO_AVAILABLE !== 'false')('agent task execution and session artifact validation', () => { + const getConfigDir = (): string => join(homedir(), '.codemie'); + + let originalActiveProfile: string | undefined; + let jwtToken: string; + let jwtHome: string; + + // build + npm link are handled once by agent-build-setup.ts globalSetup. + beforeAll(async () => { + if (!CI_IS_LOCAL_RUN) { + jwtToken = await fetchJwtToken(); + jwtHome = mkdtempSync(join(getTempDir(), 'codemie-task-jwt-')); + writeJwtProfile(jwtHome, { jwtToken }); + } else { + originalActiveProfile = setupSsoAutotestProfile(); + } + }, SETUP_TIMEOUT_MS); + + afterAll(() => { + if (!CI_IS_LOCAL_RUN) { + if (jwtHome) rmSync(jwtHome, { recursive: true, force: true }); + } else { + teardownSsoAutotestProfile(originalActiveProfile); + } + }); + + // temp_test_dir fixture equivalent + let tempTestDir: string; + + beforeEach(() => { + tempTestDir = mkdtempSync(join(tmpdir(), 'codemie_test_')); + // Expand Windows 8.3 short path names to full long paths + tempTestDir = resolveLongPath(tempTestDir); + }); + afterEach(() => { + if (existsSync(tempTestDir)) { + rmSync(tempTestDir, { recursive: true, force: true }); + } + }); + + it('should create java file with task mode and validate session metrics', async () => { + // Generate unique UUID to track this test session + const testUuid = randomUUID(); + + const taskDir = CI_IS_LOCAL_RUN ? tempTestDir : jwtHome; + const sessionsDir = CI_IS_LOCAL_RUN + ? join(getConfigDir(), 'sessions') + : join(jwtHome, 'sessions'); + + // Run the local codemie-claude entry point (bin/codemie-claude.js → dist/) + // so the test always uses the current branch build, not a globally installed binary. + // Use a clean environment (strip outer CODEMIE_* session vars) so the process + // reads config from ~/.codemie, not the inherited session of the test runner. + const result = CI_IS_LOCAL_RUN + ? spawnSync( + process.execPath, + [ + CLAUDE_BIN, + '--task', + `Create java file with helloworld app that prints: ${testUuid}`, + '--permission-mode', 'acceptEdits', + ], + { env: ssoCleanEnv(), cwd: tempTestDir, input: 'Y\n', + encoding: 'utf-8', timeout: CLI_TIMEOUT_MS }, + ) + : spawnSync( + process.execPath, + [ + CLAUDE_BIN, + '--task', + `Create java file with helloworld app that prints: ${testUuid}`, + '--permission-mode', 'acceptEdits', + '--jwt-token', jwtToken, + ], + // --permission-mode acceptEdits required: this task creates files + // (existing JWT tests omit it because they use text-only tasks) + { cwd: jwtHome, env: { ...jwtCleanEnv(), CODEMIE_HOME: jwtHome }, + encoding: 'utf-8', timeout: CLI_TIMEOUT_MS }, + ); + + // Assert command completed successfully + expect( + result.status, + `Command failed with stderr: ${result.stderr}\nstdout: ${result.stdout}`, + ).toBe(0); + + // Find Java files created in the temporary directory + const javaFiles = readdirSync(taskDir).filter(f => f.endsWith('.java')); + + // Assert at least one Java file was created + expect( + javaFiles.length, + `No Java files were created in ${taskDir}. Directory contents: ${readdirSync(taskDir).join(', ')}`, + ).toBeGreaterThan(0); + + // Read and validate the first Java file + const javaFilePath = join(taskDir, javaFiles[0]); + const javaContent = readFileSync(javaFilePath, 'utf-8'); + + // Assert file is not empty + expect(javaContent).not.toBe(''); + + // Assert file contains HelloWorld-related Java patterns + expect( + javaContent.toLowerCase().includes('class') || javaContent.toLowerCase().includes('public'), + `Java file doesn't contain class definition: ${javaContent}`, + ).toBe(true); + + // ── Session file verification ──────────────────────────────────────────────── + // Use the same ceiling as the CLI command itself so the poll always + // outlasts the session-hook rename that happens after process exit. + const SESSION_POLL_TIMEOUT_MS = CLI_TIMEOUT_MS; + + const { sessionId, dirContents } = await pollForSession(sessionsDir, testUuid, { + timeoutMs: SESSION_POLL_TIMEOUT_MS, + }); + + expect( + sessionId, + `Could not find session containing UUID ${testUuid} in ${sessionsDir} ` + + `after ${SESSION_POLL_TIMEOUT_MS / 1000}s. ` + + `Sessions dir contents: ${dirContents}`, + ).not.toBeNull(); + + // Strip 'completed_' prefix to get the bare session ID + const bareSessionId = sessionId!.replace(/^completed_/, ''); + + // Build paths for all 3 session files + const sessionFile = join(sessionsDir, `${sessionId}.json`); + const conversationFile = join(sessionsDir, `${sessionId}_conversation.jsonl`); + const metricsFile = join(sessionsDir, `${sessionId}_metrics.jsonl`); + + // Assert all 3 files exist + expect(existsSync(sessionFile), `Session file not found: ${sessionFile}`).toBe(true); + expect(existsSync(conversationFile), `Conversation file not found: ${conversationFile}`).toBe(true); + expect(existsSync(metricsFile), `Metrics file not found: ${metricsFile}`).toBe(true); + + // ── completed_*.json ────────────────────────────────────────────────────── + const sessionRaw = JSON.parse(readFileSync(sessionFile, 'utf-8')); + const session = validateSchema(SessionDataSchema, sessionRaw, `session file ${sessionId}.json`); + + expect(session.sessionId, 'sessionId does not match filename').toBe(bareSessionId); + expect(session.agentName, 'agentName must not be empty').toBeTruthy(); + expect(session.provider, 'provider must not be empty').toBeTruthy(); + expect(session.workingDirectory, 'workingDirectory must not be empty').toBeTruthy(); + + // SSO-only: conversation sync must have run and produced a conversationId + if (CI_IS_LOCAL_RUN) { + const syncConv = session.sync?.conversations as Record | undefined; + expect(syncConv?.totalSyncAttempts, 'SSO sync must have attempted at least once').toBeGreaterThan(0); + expect(syncConv?.conversationId, 'SSO sync must have produced a conversationId').toBeTruthy(); + expect(syncConv?.lastSyncAt, 'SSO sync must have recorded a lastSyncAt timestamp').toBeGreaterThan(0); + } + + // ── completed_*_metrics.jsonl ───────────────────────────────────────────── + const metricsLines = readFileSync(metricsFile, 'utf-8').split('\n').filter(Boolean); + const metricsRaw = metricsLines + .map(line => { try { return JSON.parse(line); } catch { return null; } }) + .find((r): r is Record => r !== null && JSON.stringify(r).includes(testUuid)); + + expect( + metricsRaw, + `No metrics record containing UUID ${testUuid} in ${metricsFile}`, + ).not.toBeNull(); + + const metrics = validateSchema(MetricsRecordSchema, metricsRaw, `metrics file ${sessionId}_metrics.jsonl`); + + expect(metrics.sessionId, 'metrics.sessionId does not match filename').toBe(bareSessionId); + expect(metrics.userPrompts[0].text, 'userPrompts[0].text must contain the test UUID').toContain(testUuid); + + // SSO-only: sync must have run and stamped a syncedAt timestamp + if (CI_IS_LOCAL_RUN) { + expect(metrics.syncedAt, 'SSO sync must have recorded a syncedAt timestamp').toBeGreaterThan(0); + } + + // ── completed_*_conversation.jsonl ──────────────────────────────────────── + const convLines = readFileSync(conversationFile, 'utf-8').split('\n').filter(Boolean); + const convRaw = convLines + .map(line => { try { return JSON.parse(line); } catch { return null; } }) + .find((r): r is Record => r !== null && JSON.stringify(r).includes(testUuid)); + + expect( + convRaw, + `No conversation record containing UUID ${testUuid} in ${conversationFile}`, + ).not.toBeNull(); + + const conv = validateSchema(ConversationRecordSchema, convRaw, `conversation file ${sessionId}_conversation.jsonl`); + + const userMsg = validateSchema(UserMessageSchema, conv.payload.history[0], 'conversation history[0] (user message)'); + const assistantMsg = validateSchema(AssistantMessageSchema, conv.payload.history[1], 'conversation history[1] (assistant message)'); + + expect(userMsg.message, 'history[0].message must contain the test UUID').toContain(testUuid); + expect(assistantMsg.message, 'history[1].message must not be empty').toBeTruthy(); + + }, CLI_TIMEOUT_MS + 60_000); +}); diff --git a/tests/integration/agent-task.test.ts b/tests/integration/agent-task.test.ts new file mode 100644 index 00000000..f50804d2 --- /dev/null +++ b/tests/integration/agent-task.test.ts @@ -0,0 +1,100 @@ +/** + * Task output tests — TC-016 + * + * Run with: npm run test:integration:agent + * + * Auth mode (CI_IS_LOCAL_RUN in .env.test.local): + * true (default) — SSO mode; uses developer's sso-autotest profile in ~/.codemie + * false — JWT mode; isolates to a temp CODEMIE_HOME with bearer-auth profile + * + * TC-016: --task run exits 0 and the agent response appears in stdout. + * Verifies that non-interactive task output is correctly routed to the + * caller's stdout (not swallowed or written only to session files). + */ + +import '../setup/load-test-env.js'; +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { join, dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + fetchJwtToken, + writeJwtProfile, + writeSsoProfile, + copySsoCredentials, + getTempDir, + jwtCleanEnv, + ssoCleanEnv, + setupSsoAutotestProfile, + teardownSsoAutotestProfile, + getTestEnvFlagOrDefault, +} from '../helpers/index.js'; + +const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../..'); +const CLAUDE_BIN = join(REPO_ROOT, 'bin', 'codemie-claude.js'); + +const CI_IS_LOCAL_RUN = getTestEnvFlagOrDefault('CI_IS_LOCAL_RUN', true); + +describe.runIf(process.env.SSO_AVAILABLE !== 'false')('Task output tests', () => { + let jwtToken: string; + let originalActiveProfile: string | undefined; + + beforeAll(async () => { + if (!CI_IS_LOCAL_RUN) { + jwtToken = await fetchJwtToken(); + } else { + originalActiveProfile = setupSsoAutotestProfile(); + } + }, 30_000); + + afterAll(() => { + if (CI_IS_LOCAL_RUN) { + teardownSsoAutotestProfile(originalActiveProfile); + } + }); + + // ── TC-016: --task exits 0 and response appears in stdout ───────────────── + // Checks the non-interactive output path specifically. PTY-based tests + // (TC-024, TC-025) verify interactive session output; this test verifies + // that --task mode routes the agent response to the caller's stdout. + describe('TC-016 — --task run exits 0 and prints agent response to stdout', () => { + let testHome: string; + let result: ReturnType; + + beforeAll(() => { + testHome = mkdtempSync(join(getTempDir(), 'codemie-task-')); + if (!CI_IS_LOCAL_RUN) { + writeJwtProfile(testHome, { jwtToken }); + } else { + writeSsoProfile(testHome); + copySsoCredentials(testHome); + } + result = spawnSync( + process.execPath, + CI_IS_LOCAL_RUN + ? [CLAUDE_BIN, '--task', 'Say the word READY and nothing else'] + : [CLAUDE_BIN, '--task', 'Say the word READY and nothing else', '--jwt-token', jwtToken], + { + cwd: testHome, + env: { ...(CI_IS_LOCAL_RUN ? ssoCleanEnv() : jwtCleanEnv()), CODEMIE_HOME: testHome }, + encoding: 'utf-8', + timeout: 120_000, + }, + ); + }, 180_000); + + afterAll(() => rmSync(testHome, { recursive: true, force: true })); + + it('exits 0', () => { + expect( + result.status, + `stdout: ${result.stdout ?? ''}\nstderr: ${result.stderr ?? ''}`, + ).toBe(0); + }); + + it('agent response appears in stdout', () => { + expect(result.stdout).toMatch(/READY/i); + }); + }); +}); diff --git a/tests/integration/cli-commands/self-update.test.ts b/tests/integration/cli-commands/self-update.test.ts new file mode 100644 index 00000000..a6583124 --- /dev/null +++ b/tests/integration/cli-commands/self-update.test.ts @@ -0,0 +1,64 @@ +/** + * TC-030: codemie self-update --check + * + * Verifies the self-update command exits 0 and reports the current version. + * Uses --check to query the npm registry without triggering an actual install. + * No auth required: the command only queries the public npm registry. + */ + +import { describe, it, expect } from 'vitest'; +import { spawnSync } from 'node:child_process'; +import { readFileSync } from 'node:fs'; +import { join, dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../../..'); +const CODEMIE_BIN = join(REPO_ROOT, 'bin', 'codemie.js'); + +function getLocalVersion(): string | undefined { + try { + const pkg = JSON.parse( + readFileSync(join(REPO_ROOT, 'package.json'), 'utf-8'), + ) as { version?: string }; + return pkg.version; + } catch { + return undefined; + } +} + +describe('TC-030 — self-update command', () => { + it('exits 0 and reports the current version', () => { + const version = getLocalVersion(); + + // Use --check to avoid triggering an actual npm install. + // FORCE_COLOR=0 strips ANSI codes so plain-text matching works. + const result = spawnSync( + process.execPath, + [CODEMIE_BIN, 'self-update', '--check'], + { + encoding: 'utf-8', + timeout: 30_000, + env: { ...process.env, FORCE_COLOR: '0' }, + }, + ); + + // Ora spinner writes to stderr; console.log output goes to stdout. + const combined = (result.stdout ?? '') + (result.stderr ?? ''); + + expect( + result.status, + `self-update --check failed\nstdout: ${result.stdout}\nstderr: ${result.stderr}`, + ).toBe(0); + + // Command must produce one of the two expected messages. + expect(combined, 'expected self-update status output').toMatch( + /is up to date|Update available/i, + ); + + // When already up to date, the version from package.json must appear. + // Skipped automatically when npm has a newer release (Update available path). + if (/is up to date/i.test(combined) && version) { + expect(combined, `expected version ${version} in "up to date" message`).toContain(version); + } + }); +}); diff --git a/tests/integration/models/conversation.ts b/tests/integration/models/conversation.ts new file mode 100644 index 00000000..42dbfeaa --- /dev/null +++ b/tests/integration/models/conversation.ts @@ -0,0 +1,38 @@ +import { z } from 'zod'; + +export const UserMessageSchema = z.object({ + date: z.string(), + file_names: z.array(z.string()), + history_index: z.number().int(), + message: z.string(), + message_raw: z.string(), + role: z.string(), +}).passthrough(); + +export const AssistantMessageSchema = z.object({ + assistant_id: z.string(), + date: z.string(), + history_index: z.number().int(), + message: z.string(), + message_raw: z.string(), + response_time: z.number(), + role: z.string(), +}).passthrough(); + +const ConversationPayloadSchema = z.object({ + conversationId: z.string(), + history: z.array(z.record(z.string(), z.unknown())), +}).passthrough(); + +export const ConversationRecordSchema = z.object({ + historyIndices: z.array(z.number().int()), + isTurnContinuation: z.boolean(), + messageCount: z.number().int(), + payload: ConversationPayloadSchema, + status: z.string(), + timestamp: z.number().int(), +}).passthrough(); + +export type UserMessage = z.infer; +export type AssistantMessage = z.infer; +export type ConversationRecord = z.infer; diff --git a/tests/integration/models/index.ts b/tests/integration/models/index.ts new file mode 100644 index 00000000..97c3005a --- /dev/null +++ b/tests/integration/models/index.ts @@ -0,0 +1,21 @@ +export { SessionDataSchema, type SessionData } from './session.js'; +export { MetricsRecordSchema } from './metrics.js'; +export { ConversationRecordSchema, UserMessageSchema, AssistantMessageSchema } from './conversation.js'; + +import { z } from 'zod'; + +/** + * Parse and validate data against a Zod schema. + * Throws a descriptive error listing all validation failures if the data + * does not conform, making test assertion failures easy to diagnose. + */ +export function validateSchema(schema: z.ZodType, data: unknown, label: string): T { + const result = schema.safeParse(data); + if (!result.success) { + const errors = result.error.issues + .map(e => ` [${e.path.join('.')}] ${e.message}`) + .join('\n'); + throw new Error(`${label} failed schema validation:\n${errors}`); + } + return result.data; +} diff --git a/tests/integration/models/metrics.ts b/tests/integration/models/metrics.ts new file mode 100644 index 00000000..239f1f63 --- /dev/null +++ b/tests/integration/models/metrics.ts @@ -0,0 +1,32 @@ +import { z } from 'zod'; + +const FileOperationSchema = z.object({ + format: z.string(), + language: z.string(), + path: z.string(), + type: z.string(), + linesAdded: z.number().int(), +}).passthrough(); + +const UserPromptSchema = z.object({ + count: z.number().int(), + text: z.string(), +}).passthrough(); + +export const MetricsRecordSchema = z.object({ + agentSessionId: z.string(), + fileOperations: z.array(FileOperationSchema), + gitBranch: z.string(), + models: z.array(z.string()), + recordId: z.string(), + sessionId: z.string(), + syncAttempts: z.number().int(), + syncStatus: z.string(), + syncedAt: z.number().int().optional(), // absent when SSO sync has not run (e.g. JWT mode) + timestamp: z.string(), + toolStatus: z.record(z.string(), z.unknown()), + tools: z.record(z.string(), z.unknown()), + userPrompts: z.array(UserPromptSchema), +}).passthrough(); + +export type MetricsRecord = z.infer; diff --git a/tests/integration/models/session.ts b/tests/integration/models/session.ts new file mode 100644 index 00000000..70deddfc --- /dev/null +++ b/tests/integration/models/session.ts @@ -0,0 +1,44 @@ +import { z } from 'zod'; + +const CorrelationSchema = z.object({ + status: z.string(), + agentSessionId: z.string(), + agentSessionFile: z.string(), + retryCount: z.number().int(), +}).passthrough(); + +const SyncMetricsSchema = z.object({ + lastProcessedTimestamp: z.number().int(), + processedRecordIds: z.array(z.string()), + totalDeltas: z.number().int(), + totalSynced: z.number().int(), + totalFailed: z.number().int(), +}).passthrough(); + +const SyncConversationsSchema = z.object({ + lastSyncedMessageUuid: z.string(), + lastSyncedHistoryIndex: z.number().int(), + totalMessagesSynced: z.number().int(), + totalSyncAttempts: z.number().int(), + conversationId: z.string().optional(), // absent when SSO sync has not run (e.g. JWT mode) + lastSyncAt: z.number().int().optional(), // absent when SSO sync has not run (e.g. JWT mode) +}).passthrough(); + +const SyncSchema = z.object({ + metrics: SyncMetricsSchema, + conversations: SyncConversationsSchema, +}).passthrough(); + +export const SessionDataSchema = z.object({ + sessionId: z.string(), + agentName: z.string(), + provider: z.string(), + startTime: z.number().int(), + workingDirectory: z.string(), + status: z.string(), + activeDurationMs: z.number().int(), + correlation: CorrelationSchema, + sync: SyncSchema, +}).passthrough(); + +export type SessionData = z.infer; diff --git a/tests/setup/agent-build-setup.ts b/tests/setup/agent-build-setup.ts new file mode 100644 index 00000000..a07f883d --- /dev/null +++ b/tests/setup/agent-build-setup.ts @@ -0,0 +1,142 @@ +import { execSync } from 'node:child_process'; +import { existsSync, readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { dirname, join, resolve } from 'node:path'; +import { homedir } from 'node:os'; +import { config as loadEnv } from 'dotenv'; +import { setupSsoAutotestProfile, teardownSsoAutotestProfile } from '../helpers/sso-auth.js'; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const root = resolve(__dirname, '../..'); + +function getActiveProfileProvider(): string | undefined { + const configPath = join(homedir(), '.codemie', 'codemie-cli.config.json'); + if (!existsSync(configPath)) return undefined; + try { + const config = JSON.parse(readFileSync(configPath, 'utf-8')) as Record; + const active = config.activeProfile as string | undefined; + if (!active) return undefined; + const profiles = config.profiles as Record> | undefined; + return profiles?.[active]?.provider as string | undefined; + } catch { + return undefined; + } +} + +let originalSsoProfile: string | undefined; + +/** + * Vitest globalSetup — runs once per test session before any test file. + * Equivalent to pytest scope="session" fixture. + * Ensures dist/ exists and the claude CLI is installed before agent tests run. + */ +export async function setup(): Promise { + loadEnv({ path: resolve(root, '.env.test.local'), override: true }); + + // Default to the public prod instance when no .env.test.local is present. + process.env.CI_CODEMIE_URL ??= 'https://codemie.lab.epam.com'; + + console.log('\n[agent-integration] Building dist/ (runs once per session)...'); + execSync('npm run build', { cwd: root, stdio: 'inherit' }); + console.log('[agent-integration] Build complete.'); + + // The native Claude installer places the binary at ~/.local/bin/claude(.exe). + // On Windows CI runners this directory is not in PATH by default, so we add it + // to process.env.PATH before checking and after installing. + const localBin = join(homedir(), '.local', 'bin'); + const pathSep = process.platform === 'win32' ? ';' : ':'; + if (!(process.env.PATH ?? '').includes(localBin)) { + process.env.PATH = `${localBin}${pathSep}${process.env.PATH ?? ''}`; + } + + try { + execSync('claude --version', { stdio: 'pipe' }); + console.log('[agent-integration] claude CLI found.\n'); + } catch { + console.log('[agent-integration] claude CLI not found — installing via codemie...'); + try { + // Installer may exit non-zero on Windows when it warns that ~/.local/bin + // is not yet in the system PATH — installation itself succeeds. + execSync(`node ${resolve(root, 'bin/codemie.js')} install claude`, { cwd: root, stdio: 'inherit' }); + } catch { + // Ignore exit code — verify the binary is actually present below. + } + // Re-add localBin in case the installer modified PATH during its run. + if (!(process.env.PATH ?? '').includes(localBin)) { + process.env.PATH = `${localBin}${pathSep}${process.env.PATH ?? ''}`; + } + execSync('claude --version', { stdio: 'pipe' }); // throws if install genuinely failed + console.log('[agent-integration] claude CLI installed.\n'); + } + + // Link the local build to global PATH so `codemie hook` resolves when + // Claude fires it via hooks.json during a test session. + console.log('[agent-integration] Linking local build to global PATH...'); + execSync('npm link', { cwd: root, stdio: 'pipe' }); + console.log('[agent-integration] Linked.'); + + // For SSO (local dev) runs: validate credentials before any test subprocess + // tries to use them. If credentials are missing or expired, launch the + // browser SSO flow immediately (no "Re-authenticate now?" prompt). + // JWT (CI) runs skip this — each test fetches a fresh JWT token itself. + const isLocalRun = (process.env.CI_IS_LOCAL_RUN ?? 'true') !== 'false'; + if (isLocalRun) { + const activeProvider = getActiveProfileProvider(); + if (activeProvider !== 'ai-run-sso') { + console.log( + `[agent-integration] Active profile provider is "${activeProvider ?? 'none'}" — not CodeMie SSO.`, + ); + console.log('[agent-integration] Agent SSO tests will be skipped.'); + console.log('[agent-integration] Use npm run test:run for unit + CLI tests without credentials.\n'); + process.env.SSO_AVAILABLE = 'false'; + return; + } + console.log('[agent-integration] SSO mode — validating credentials...'); + originalSsoProfile = setupSsoAutotestProfile(); + try { + const { getCodemieClient } = await import( + resolve(root, 'dist/utils/sdk-client.js') + ) as { getCodemieClient: (quiet: boolean) => Promise }; + await getCodemieClient(true); + console.log('[agent-integration] SSO credentials valid.\n'); + } catch { + // Credentials missing or expired — launch browser SSO login directly. + // Using stdio: 'inherit' so the user sees and can complete the flow. + console.log('[agent-integration] SSO credentials missing or expired — launching login...\n'); + const codemieUrl = process.env.CI_CODEMIE_URL ?? ''; + execSync( + `node ${resolve(root, 'bin/codemie.js')} profile login --url ${codemieUrl}`, + { cwd: root, stdio: 'inherit' }, + ); + } + } + + // Pre-install the Claude CodeMie extension once before parallel tests start. + // Without this, each parallel test triggers installer.install() simultaneously. + // When the source version differs from the installed version, every installer + // does rm -rf ~/.codemie/claude-plugin then cp — racing each other. A test's + // Claude Code process that starts mid-race gets a missing/partial plugin dir, + // the hooks never fire, and sessions/ is never created (ENOENT). + // Pre-installing here ensures all concurrent callers see action=already_exists + // and skip the destructive rm/cp entirely. + console.log('[agent-integration] Pre-installing Claude CodeMie extension...'); + try { + const { ClaudePluginInstaller } = await import( + resolve(root, 'dist/agents/plugins/claude/claude.plugin-installer.js') + ) as { ClaudePluginInstaller: new (m: { name: string }) => { install(): Promise<{ action: string; targetPath: string }> } }; + const installer = new ClaudePluginInstaller({ name: 'claude' }); + const result = await installer.install(); + console.log(`[agent-integration] Claude extension ${result.action} at ${result.targetPath}.\n`); + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + console.warn(`[agent-integration] Claude extension pre-install warning (non-fatal): ${msg}\n`); + } +} + +/** + * Vitest globalTeardown — runs once after all test files complete. + * Restores the user's original active SSO profile if it was changed during setup(). + */ +export async function teardown(): Promise { + teardownSsoAutotestProfile(originalSsoProfile); +} diff --git a/tests/setup/load-test-env.ts b/tests/setup/load-test-env.ts new file mode 100644 index 00000000..5f314a8e --- /dev/null +++ b/tests/setup/load-test-env.ts @@ -0,0 +1,6 @@ +import { config } from 'dotenv'; +import { resolve } from 'node:path'; + +// override: true — file always wins over stale shell exports when running locally. +// On CI there is no .env.test.local, so CI env vars are never touched. +config({ path: resolve(process.cwd(), '.env.test.local'), override: true }); diff --git a/tests/unit/cli/commands/assistants/chat/index.test.ts b/tests/unit/cli/commands/assistants/chat/index.test.ts index 05326827..eb7c0d96 100644 --- a/tests/unit/cli/commands/assistants/chat/index.test.ts +++ b/tests/unit/cli/commands/assistants/chat/index.test.ts @@ -67,7 +67,7 @@ describe('Chat Command Structure', () => { }); it('should have all expected options', () => { - expect(command.options).toHaveLength(4); + expect(command.options).toHaveLength(5); }); }); diff --git a/vitest.config.ts b/vitest.config.ts index b38fb6cc..783ee2af 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -1,47 +1,84 @@ -import { defineConfig } from 'vitest/config'; +import { defineConfig, defineProject } from 'vitest/config'; export default defineConfig({ test: { - globals: true, - environment: 'node', - include: ['src/**/*.test.ts', 'src/**/*.spec.ts', 'tests/**/*.test.ts'], - exclude: ['node_modules', 'dist'], - // Force color output for consistent test behavior (chalk output length varies with/without colors) - env: { - FORCE_COLOR: '1', - NODE_ENV: 'test', // Skip auto-update checks during testing - }, + projects: [ + // ── Unit tests (src/) ──────────────────────────────────────────────── + defineProject({ + test: { + name: 'unit', + include: ['src/**/*.test.ts', 'src/**/*.spec.ts'], + globals: true, + environment: 'node', + testTimeout: 30_000, + hookTimeout: 10_000, + isolate: true, + env: { + FORCE_COLOR: '1', + NODE_ENV: 'test', + }, + coverage: { + provider: 'v8', + reporter: ['text', 'json', 'html'], + exclude: [ + 'node_modules/', + 'dist/', + '**/*.test.ts', + '**/*.spec.ts', + '**/types.ts', + 'bin/', + 'tests/', + ], + }, + }, + resolve: { + alias: { '@': '/src' }, + }, + }), - // Enable parallel execution with isolated environments - pool: 'threads', - poolOptions: { - threads: { - maxThreads: 8, - minThreads: 2, - }, - }, - // Isolate each test file for safety - isolate: true, + // ── CLI integration tests (no network auth) ────────────────────────── + defineProject({ + test: { + name: 'cli', + include: ['tests/integration/**/*.test.ts'], + exclude: ['tests/integration/agent-*.test.ts'], + globals: true, + environment: 'node', + testTimeout: 30_000, + hookTimeout: 10_000, + isolate: true, + sequence: { groupOrder: 1 }, + env: { + FORCE_COLOR: '1', + NODE_ENV: 'test', + }, + }, + resolve: { + alias: { '@': '/src' }, + }, + }), - coverage: { - provider: 'v8', - reporter: ['text', 'json', 'html'], - exclude: [ - 'node_modules/', - 'dist/', - '**/*.test.ts', - '**/*.spec.ts', - '**/types.ts', - 'bin/', - 'tests/', - ], - }, - testTimeout: 30000, - hookTimeout: 10000, - }, - resolve: { - alias: { - '@': '/src', - }, + // ── Agent integration tests (real network, SSO/JWT auth) ───────────── + defineProject({ + test: { + name: 'agent', + include: ['tests/integration/agent-*.test.ts'], + globalSetup: ['tests/setup/agent-build-setup.ts'], + testTimeout: 180_000, + hookTimeout: 300_000, + maxWorkers: parseInt(process.env.CI_AGENT_MAX_WORKERS ?? '2', 10), + isolate: true, + sequence: { groupOrder: 2 }, + reporters: ['verbose'], + env: { + FORCE_COLOR: '1', + NODE_ENV: 'test', + }, + }, + resolve: { + alias: { '@': '/src' }, + }, + }), + ], }, });