From 76d23746fd2e47a588cdce0b3ece1e55d5997e65 Mon Sep 17 00:00:00 2001 From: Sviatoslav Likhtarchyk Date: Tue, 7 Jul 2026 16:51:00 +0300 Subject: [PATCH 1/3] fix(analytics): exclude non-CodeMie-owned native sessions from output Native sessions tagged native-external by hasOwnershipMarker were only labeled, not filtered, so untracked claude/codex usage still counted toward totals. SessionsSource now drops them by default; --include-external restores the prior unfiltered behavior for comparison. EPMCDME-13367 --- src/cli/commands/analytics/index.ts | 7 +- .../sources/__tests__/sessions-source.test.ts | 106 ++++++++++++++++++ .../analytics/sources/sessions-source.ts | 6 +- src/cli/commands/analytics/sources/types.ts | 2 + src/cli/commands/analytics/types.ts | 2 + 5 files changed, 119 insertions(+), 4 deletions(-) create mode 100644 src/cli/commands/analytics/sources/__tests__/sessions-source.test.ts diff --git a/src/cli/commands/analytics/index.ts b/src/cli/commands/analytics/index.ts index 16a44ba7..c9907c50 100644 --- a/src/cli/commands/analytics/index.ts +++ b/src/cli/commands/analytics/index.ts @@ -20,6 +20,7 @@ export function createAnalyticsCommand(): Command { // Default source: local CodeMie-tracked sessions + native agent logs. applyCommonOptions(command) .option('--no-scan-native', 'Skip native agent-log discovery (use only CodeMie-tracked sessions)') + .option('--include-external', 'Include non-CodeMie-owned native sessions in output (opt-in; matches pre-fix behavior)') .action((options: AnalyticsOptions) => runAnalytics(options, new SessionsSource())); // `codemie analytics otel --file ` — OTEL file source. @@ -62,7 +63,11 @@ function applyCommonOptions(command: Command): Command { async function runAnalytics(options: AnalyticsOptions, source: AnalyticsSource): Promise { try { const filter = parseFilterOptions(options); - const { rawSessions, cost } = await source.load({ filter, scanNative: options.scanNative }); + const { rawSessions, cost } = await source.load({ + filter, + scanNative: options.scanNative, + includeExternal: options.includeExternal + }); if (rawSessions.length === 0) { console.log(chalk.yellow('\nNo sessions found matching the specified criteria.')); diff --git a/src/cli/commands/analytics/sources/__tests__/sessions-source.test.ts b/src/cli/commands/analytics/sources/__tests__/sessions-source.test.ts new file mode 100644 index 00000000..a1c39ade --- /dev/null +++ b/src/cli/commands/analytics/sources/__tests__/sessions-source.test.ts @@ -0,0 +1,106 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import type { RawSessionData } from '../../data-loader.js'; + +const mockLoadSessions = vi.fn(); +const mockSessionMatchesFilter = vi.fn(); + +vi.mock('../../data-loader.js', () => ({ + MetricsDataLoader: vi.fn().mockImplementation(function MockMetricsDataLoader() { + this.loadSessions = mockLoadSessions; + this.sessionMatchesFilter = mockSessionMatchesFilter; + }) +})); + +const mockLoadNativeSessions = vi.fn(); + +vi.mock('../../native-loader.js', () => ({ + loadNativeSessions: mockLoadNativeSessions +})); + +function makeSession(sessionId: string, provider: string): RawSessionData { + return { + sessionId, + startEvent: { + recordId: sessionId, + type: 'session_start', + timestamp: 0, + codeMieSessionId: sessionId, + agentName: 'claude', + syncStatus: 'synced', + data: { provider, workingDirectory: '/tmp', startTime: 0 } + }, + deltas: [] + }; +} + +describe('SessionsSource', () => { + beforeEach(() => { + mockLoadSessions.mockReturnValue([]); + mockSessionMatchesFilter.mockReturnValue(true); + mockLoadNativeSessions.mockResolvedValue([]); + }); + + afterEach(() => { + vi.clearAllMocks(); + }); + + it('excludes native-external sessions by default', async () => { + const owned = makeSession('owned-1', 'native'); + const external = makeSession('external-1', 'native-external'); + mockLoadNativeSessions.mockResolvedValue([owned, external]); + + const { SessionsSource } = await import('../sessions-source.js'); + const source = new SessionsSource(); + const result = await source.load({ filter: {} }); + + expect(result.rawSessions.map((s) => s.sessionId)).toEqual(['owned-1']); + }); + + it('includes native-external sessions when includeExternal is true', async () => { + const owned = makeSession('owned-1', 'native'); + const external = makeSession('external-1', 'native-external'); + mockLoadNativeSessions.mockResolvedValue([owned, external]); + + const { SessionsSource } = await import('../sessions-source.js'); + const source = new SessionsSource(); + const result = await source.load({ filter: {}, includeExternal: true }); + + expect(result.rawSessions.map((s) => s.sessionId).sort()).toEqual(['external-1', 'owned-1']); + }); + + it('always includes owned native sessions regardless of includeExternal', async () => { + const owned = makeSession('owned-1', 'native'); + mockLoadNativeSessions.mockResolvedValue([owned]); + + const { SessionsSource } = await import('../sessions-source.js'); + const source = new SessionsSource(); + const result = await source.load({ filter: {} }); + + expect(result.rawSessions.map((s) => s.sessionId)).toEqual(['owned-1']); + }); + + it('applies sessionMatchesFilter before the external-session filter', async () => { + const external = makeSession('external-1', 'native-external'); + mockLoadNativeSessions.mockResolvedValue([external]); + mockSessionMatchesFilter.mockReturnValue(false); + + const { SessionsSource } = await import('../sessions-source.js'); + const source = new SessionsSource(); + const result = await source.load({ filter: {}, includeExternal: true }); + + expect(result.rawSessions).toEqual([]); + expect(mockSessionMatchesFilter).toHaveBeenCalledWith(external, {}); + }); + + it('includes tracked sessions from the loader unconditionally', async () => { + const tracked = makeSession('tracked-1', 'native'); + mockLoadSessions.mockReturnValue([tracked]); + mockLoadNativeSessions.mockResolvedValue([]); + + const { SessionsSource } = await import('../sessions-source.js'); + const source = new SessionsSource(); + const result = await source.load({ filter: {} }); + + expect(result.rawSessions.map((s) => s.sessionId)).toEqual(['tracked-1']); + }); +}); diff --git a/src/cli/commands/analytics/sources/sessions-source.ts b/src/cli/commands/analytics/sources/sessions-source.ts index 96e56b5a..bcb896de 100644 --- a/src/cli/commands/analytics/sources/sessions-source.ts +++ b/src/cli/commands/analytics/sources/sessions-source.ts @@ -17,9 +17,9 @@ export class SessionsSource implements AnalyticsSource { if (opts.scanNative !== false) { try { const { loadNativeSessions } = await import('../native-loader.js'); - const natives = (await loadNativeSessions(opts.filter)).filter((s) => - loader.sessionMatchesFilter(s, opts.filter) - ); + const natives = (await loadNativeSessions(opts.filter)) + .filter((s) => loader.sessionMatchesFilter(s, opts.filter)) + .filter((s) => opts.includeExternal || s.startEvent?.data.provider !== 'native-external'); rawSessions.push(...natives); } catch (error) { logger.debug('Native session discovery failed (continuing with tracked sessions):', error); diff --git a/src/cli/commands/analytics/sources/types.ts b/src/cli/commands/analytics/sources/types.ts index 47e4c5cc..a822227c 100644 --- a/src/cli/commands/analytics/sources/types.ts +++ b/src/cli/commands/analytics/sources/types.ts @@ -20,6 +20,8 @@ export interface SourceLoadOptions { filter: AnalyticsFilter; /** Sessions source only: skip native agent-log discovery. Ignored by other sources. */ scanNative?: boolean; + /** Sessions source only: include non-CodeMie-owned native sessions (tagged 'native-external') in output. Ignored by other sources. */ + includeExternal?: boolean; } export interface SourceResult { diff --git a/src/cli/commands/analytics/types.ts b/src/cli/commands/analytics/types.ts index ad81c704..233036b6 100644 --- a/src/cli/commands/analytics/types.ts +++ b/src/cli/commands/analytics/types.ts @@ -248,6 +248,8 @@ export interface AnalyticsOptions { reportFormat?: 'html' | 'json' | 'both'; /** When false (via --no-scan-native), skip native-log discovery and use tracked sessions only. */ scanNative?: boolean; + /** When true (via --include-external), include non-CodeMie-owned native sessions in output (matches pre-fix behavior). */ + includeExternal?: boolean; } /** Options for the `analytics otel` subcommand: the shared base plus OTEL-specific flags. */ From 257e70372379c72b69235f3f72ffe14812de5698 Mon Sep 17 00:00:00 2001 From: Sviatoslav Likhtarchyk Date: Tue, 7 Jul 2026 16:51:33 +0300 Subject: [PATCH 2/3] chore: add EPMCDME-13367 superpowers artifacts Spec, plan, technical analysis, and task run artifacts from the brainstorming/planning workflow behind the analytics exclusion fix. EPMCDME-13367 --- ...367-analytics-exclude-external-sessions.md | 307 +++++++++++ ...lytics-exclude-external-sessions-design.md | 120 +++++ .../complexity-assessment.json | 25 + .../decisions.jsonl | 2 + .../events.jsonl | 2 + .../plan.md | 307 +++++++++++ .../spec.md | 120 +++++ .../technical-analysis.md | 494 ++++++++++++++++++ 8 files changed, 1377 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-07-EPMCDME-13367-analytics-exclude-external-sessions.md create mode 100644 docs/superpowers/specs/2026-07-07-analytics-exclude-external-sessions-design.md create mode 100644 docs/superpowers/tasks/2026-07-07-analytics-exclude-external-sessions/complexity-assessment.json create mode 100644 docs/superpowers/tasks/2026-07-07-analytics-exclude-external-sessions/decisions.jsonl create mode 100644 docs/superpowers/tasks/2026-07-07-analytics-exclude-external-sessions/events.jsonl create mode 100644 docs/superpowers/tasks/2026-07-07-analytics-exclude-external-sessions/plan.md create mode 100644 docs/superpowers/tasks/2026-07-07-analytics-exclude-external-sessions/spec.md create mode 100644 docs/superpowers/tasks/2026-07-07-analytics-exclude-external-sessions/technical-analysis.md diff --git a/docs/superpowers/plans/2026-07-07-EPMCDME-13367-analytics-exclude-external-sessions.md b/docs/superpowers/plans/2026-07-07-EPMCDME-13367-analytics-exclude-external-sessions.md new file mode 100644 index 00000000..55df51f0 --- /dev/null +++ b/docs/superpowers/plans/2026-07-07-EPMCDME-13367-analytics-exclude-external-sessions.md @@ -0,0 +1,307 @@ +# Exclude non-CodeMie-owned sessions from analytics (EPMCDME-13367) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** By default, `codemie analytics` excludes non-CodeMie-owned native sessions (tagged `provider: 'native-external'` by `native-loader.ts`) from output and from all aggregated totals, with an opt-in `--include-external` flag that restores today's exact behavior. + +**Architecture:** Add a single filter step in `SessionsSource.load()` — the one seam where natively-discovered sessions merge into the pipeline shared by console output, JSON/CSV export, and the HTML report — that drops any session whose `startEvent.data.provider === 'native-external'` unless a new `includeExternal` option is set. Thread that option from a new `--include-external` CLI flag through `AnalyticsOptions` → `SourceLoadOptions` → the filter. No changes to `native-loader.ts`, `aggregator.ts`, or `formatter.ts`. + +**Tech Stack:** TypeScript (ES modules), Commander.js (CLI), Vitest (unit tests). + +## Global Constraints + +- ES modules with `.js` extensions on all relative imports (per `AGENTS.md`). +- `native-loader.ts` is NOT modified — preserves CR-002 (bounded transcript scan), CR-003 (index-first ownership check), CR-R-002 (`_metrics.json` suffix filtering). +- `aggregator.ts` and `formatter.ts` are NOT modified. +- `--include-external` means "restore full pre-fix behavior" (included, tagged, contributing to totals) — not a diagnostic-only/display-only mode (architecturally infeasible without touching 15+ `reduce()` call sites in `aggregator.ts`; out of scope per spec non-goals). +- No changes to `AnalyticsFilter` — `includeExternal` is a source-level behavior modifier, not a filter criterion. +- Unit test coverage only; no integration-level (`tests/integration/`) CLI test (per spec non-goals). + +--- + +### Task 1: Exclude `native-external` sessions in `SessionsSource.load()` + +**Files:** +- Modify: `src/cli/commands/analytics/sources/types.ts` (add `includeExternal?: boolean` to `SourceLoadOptions`) +- Modify: `src/cli/commands/analytics/types.ts:233-251` (add `includeExternal?: boolean` to `AnalyticsOptions`) +- Modify: `src/cli/commands/analytics/sources/sessions-source.ts` +- Test: `src/cli/commands/analytics/sources/__tests__/sessions-source.test.ts` (new file) + +**Interfaces:** +- Consumes: `RawSessionData` (from `../data-loader.js`) — `{ sessionId: string, startEvent?: SessionStartEvent, endEvent?: SessionEndEvent, deltas: MetricDelta[], agentSessionFile?: string }`. `SessionStartEvent.data.provider` is a plain `string` (`'native'` or `'native-external'` for native sessions). +- Consumes: `loadNativeSessions(filter?: AnalyticsFilter, deps?: NativeLoaderDeps): Promise` (from `../native-loader.js`, unchanged). +- Consumes: `MetricsDataLoader` (from `../data-loader.js`, unchanged) — `loadSessions(filter?: AnalyticsFilter): RawSessionData[]`, `sessionMatchesFilter(s: RawSessionData, filter?: AnalyticsFilter): boolean`. +- Produces: `SourceLoadOptions.includeExternal?: boolean` — consumed by Task 2's CLI wiring in `index.ts`. + +- [ ] **Step 1: Write the failing tests** + +Create `src/cli/commands/analytics/sources/__tests__/sessions-source.test.ts`: + +```typescript +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import type { RawSessionData } from '../../data-loader.js'; + +const mockLoadSessions = vi.fn(); +const mockSessionMatchesFilter = vi.fn(); + +vi.mock('../../data-loader.js', () => ({ + MetricsDataLoader: vi.fn().mockImplementation(() => ({ + loadSessions: mockLoadSessions, + sessionMatchesFilter: mockSessionMatchesFilter + })) +})); + +const mockLoadNativeSessions = vi.fn(); + +vi.mock('../../native-loader.js', () => ({ + loadNativeSessions: mockLoadNativeSessions +})); + +function makeSession(sessionId: string, provider: string): RawSessionData { + return { + sessionId, + startEvent: { + recordId: sessionId, + type: 'session_start', + timestamp: 0, + codeMieSessionId: sessionId, + agentName: 'claude', + syncStatus: 'synced', + data: { provider, workingDirectory: '/tmp', startTime: 0 } + }, + deltas: [] + }; +} + +describe('SessionsSource', () => { + beforeEach(() => { + mockLoadSessions.mockReturnValue([]); + mockSessionMatchesFilter.mockReturnValue(true); + mockLoadNativeSessions.mockResolvedValue([]); + }); + + afterEach(() => { + vi.clearAllMocks(); + }); + + it('excludes native-external sessions by default', async () => { + const owned = makeSession('owned-1', 'native'); + const external = makeSession('external-1', 'native-external'); + mockLoadNativeSessions.mockResolvedValue([owned, external]); + + const { SessionsSource } = await import('../sessions-source.js'); + const source = new SessionsSource(); + const result = await source.load({ filter: {} }); + + expect(result.rawSessions.map((s) => s.sessionId)).toEqual(['owned-1']); + }); + + it('includes native-external sessions when includeExternal is true', async () => { + const owned = makeSession('owned-1', 'native'); + const external = makeSession('external-1', 'native-external'); + mockLoadNativeSessions.mockResolvedValue([owned, external]); + + const { SessionsSource } = await import('../sessions-source.js'); + const source = new SessionsSource(); + const result = await source.load({ filter: {}, includeExternal: true }); + + expect(result.rawSessions.map((s) => s.sessionId).sort()).toEqual(['external-1', 'owned-1']); + }); + + it('always includes owned native sessions regardless of includeExternal', async () => { + const owned = makeSession('owned-1', 'native'); + mockLoadNativeSessions.mockResolvedValue([owned]); + + const { SessionsSource } = await import('../sessions-source.js'); + const source = new SessionsSource(); + const result = await source.load({ filter: {} }); + + expect(result.rawSessions.map((s) => s.sessionId)).toEqual(['owned-1']); + }); + + it('applies sessionMatchesFilter before the external-session filter', async () => { + const external = makeSession('external-1', 'native-external'); + mockLoadNativeSessions.mockResolvedValue([external]); + mockSessionMatchesFilter.mockReturnValue(false); + + const { SessionsSource } = await import('../sessions-source.js'); + const source = new SessionsSource(); + const result = await source.load({ filter: {}, includeExternal: true }); + + expect(result.rawSessions).toEqual([]); + expect(mockSessionMatchesFilter).toHaveBeenCalledWith(external, {}); + }); + + it('includes tracked sessions from the loader unconditionally', async () => { + const tracked = makeSession('tracked-1', 'native'); + mockLoadSessions.mockReturnValue([tracked]); + mockLoadNativeSessions.mockResolvedValue([]); + + const { SessionsSource } = await import('../sessions-source.js'); + const source = new SessionsSource(); + const result = await source.load({ filter: {} }); + + expect(result.rawSessions.map((s) => s.sessionId)).toEqual(['tracked-1']); + }); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `npx vitest run src/cli/commands/analytics/sources/__tests__/sessions-source.test.ts` +Expected: FAIL — the `includeExternal` case fails because `SourceLoadOptions` has no such property yet (TS compile error under vitest's esbuild transform is treated as a runtime failure), and the default-exclusion test fails because `sessions-source.ts` currently pushes every native session unfiltered. + +- [ ] **Step 3: Add `includeExternal` to the option types** + +In `src/cli/commands/analytics/sources/types.ts`, change: + +```typescript +export interface SourceLoadOptions { + filter: AnalyticsFilter; + /** Sessions source only: skip native agent-log discovery. Ignored by other sources. */ + scanNative?: boolean; +} +``` + +to: + +```typescript +export interface SourceLoadOptions { + filter: AnalyticsFilter; + /** Sessions source only: skip native agent-log discovery. Ignored by other sources. */ + scanNative?: boolean; + /** Sessions source only: include non-CodeMie-owned native sessions (tagged 'native-external') in output. Ignored by other sources. */ + includeExternal?: boolean; +} +``` + +In `src/cli/commands/analytics/types.ts`, change the `AnalyticsOptions` interface (lines 233-251) so the block right after `scanNative` reads: + +```typescript + /** When false (via --no-scan-native), skip native-log discovery and use tracked sessions only. */ + scanNative?: boolean; + /** When true (via --include-external), include non-CodeMie-owned native sessions in output (matches pre-fix behavior). */ + includeExternal?: boolean; +} +``` + +- [ ] **Step 4: Implement the filter in `SessionsSource.load()`** + +In `src/cli/commands/analytics/sources/sessions-source.ts`, change: + +```typescript + if (opts.scanNative !== false) { + try { + const { loadNativeSessions } = await import('../native-loader.js'); + const natives = (await loadNativeSessions(opts.filter)).filter((s) => + loader.sessionMatchesFilter(s, opts.filter) + ); + rawSessions.push(...natives); + } catch (error) { + logger.debug('Native session discovery failed (continuing with tracked sessions):', error); + } + } +``` + +to: + +```typescript + if (opts.scanNative !== false) { + try { + const { loadNativeSessions } = await import('../native-loader.js'); + const natives = (await loadNativeSessions(opts.filter)) + .filter((s) => loader.sessionMatchesFilter(s, opts.filter)) + .filter((s) => opts.includeExternal || s.startEvent?.data.provider !== 'native-external'); + rawSessions.push(...natives); + } catch (error) { + logger.debug('Native session discovery failed (continuing with tracked sessions):', error); + } + } +``` + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `npx vitest run src/cli/commands/analytics/sources/__tests__/sessions-source.test.ts` +Expected: PASS — all 5 tests green. + +- [ ] **Step 6: Typecheck** + +Run: `npm run typecheck` +Expected: no errors. + +- [ ] **Step 7: Commit** + +```bash +git add src/cli/commands/analytics/sources/types.ts src/cli/commands/analytics/types.ts src/cli/commands/analytics/sources/sessions-source.ts src/cli/commands/analytics/sources/__tests__/sessions-source.test.ts +git commit -m "fix(analytics): exclude non-CodeMie-owned native sessions by default (EPMCDME-13367)" +``` + +--- + +### Task 2: Wire `--include-external` CLI flag + +**Files:** +- Modify: `src/cli/commands/analytics/index.ts:21-23` (register the flag on the default source command) +- Modify: `src/cli/commands/analytics/index.ts:65` (thread the option into `source.load()`) + +**Interfaces:** +- Consumes: `SourceLoadOptions.includeExternal` (produced by Task 1) and `AnalyticsOptions.includeExternal` (produced by Task 1). +- Produces: the `--include-external` CLI flag, parsed by Commander into `options.includeExternal: boolean | undefined`. + +**Test-first: no** — this task only wires an existing, already-tested option through Commander's option parser into a call site; there is no new branching logic to unit-test, and an end-to-end CLI test is explicitly out of scope per the spec's non-goals (unit coverage at the seam in Task 1 is sufficient). Verified manually in Step 3 below instead. + +- [ ] **Step 1: Register the flag** + +In `src/cli/commands/analytics/index.ts`, change: + +```typescript + // Default source: local CodeMie-tracked sessions + native agent logs. + applyCommonOptions(command) + .option('--no-scan-native', 'Skip native agent-log discovery (use only CodeMie-tracked sessions)') + .action((options: AnalyticsOptions) => runAnalytics(options, new SessionsSource())); +``` + +to: + +```typescript + // Default source: local CodeMie-tracked sessions + native agent logs. + applyCommonOptions(command) + .option('--no-scan-native', 'Skip native agent-log discovery (use only CodeMie-tracked sessions)') + .option('--include-external', 'Include non-CodeMie-owned native sessions in output (opt-in; matches pre-fix behavior)') + .action((options: AnalyticsOptions) => runAnalytics(options, new SessionsSource())); +``` + +- [ ] **Step 2: Thread the option into `source.load()`** + +In `src/cli/commands/analytics/index.ts`, change: + +```typescript + const { rawSessions, cost } = await source.load({ filter, scanNative: options.scanNative }); +``` + +to: + +```typescript + const { rawSessions, cost } = await source.load({ + filter, + scanNative: options.scanNative, + includeExternal: options.includeExternal + }); +``` + +- [ ] **Step 3: Manually verify the flag is registered** + +Run: `node bin/codemie.js analytics --help` +Expected: output includes a line for `--include-external Include non-CodeMie-owned native sessions in output (opt-in; matches pre-fix behavior)`. + +- [ ] **Step 4: Typecheck and run the full analytics test suite** + +Run: `npm run typecheck && npx vitest run src/cli/commands/analytics` +Expected: no errors; all analytics tests (including Task 1's new file, `aggregator.test.ts`, and `native-loader.test.ts`, which are unchanged and still pass) are green. + +- [ ] **Step 5: Commit** + +```bash +git add src/cli/commands/analytics/index.ts +git commit -m "feat(analytics): add --include-external escape hatch for the exclusion fix (EPMCDME-13367)" +``` diff --git a/docs/superpowers/specs/2026-07-07-analytics-exclude-external-sessions-design.md b/docs/superpowers/specs/2026-07-07-analytics-exclude-external-sessions-design.md new file mode 100644 index 00000000..67c3280e --- /dev/null +++ b/docs/superpowers/specs/2026-07-07-analytics-exclude-external-sessions-design.md @@ -0,0 +1,120 @@ +# Design: Exclude non-CodeMie-owned sessions from analytics (EPMCDME-13367) + +## Problem + +The parent issue (EPMCDME-12992, merged via PR #403) added ownership validation that gates the +RESUME path — the analytics command was left out of scope. `codemie analytics` still scans and +reports every native session it discovers, whether or not CodeMie owns it. Today, +`src/cli/commands/analytics/native-loader.ts` already computes ownership via +`hasOwnershipMarker(descriptor.filePath)` and tags unowned sessions +`startEvent.data.provider = 'native-external'` — but the session is still pushed unfiltered into +the returned array (`out.push(raw)`) and flows through aggregation and output uncounted-but-present. +`formatter.ts` only shows a cosmetic yellow warning; it never excludes the session from totals. + +This intentionally reverses one line of the original PR #403 design decision ("No sessions are +hidden — all are surfaced, external ones are clearly marked") — but only for the analytics +pipeline. The RESUME-path UX (informing/blocking on external session resume) is unaffected. + +## Goals + +- By default, `codemie analytics` (console, JSON/CSV export, HTML report) excludes non-CodeMie-owned + native sessions from output and from all aggregated totals. +- Provide an opt-in escape hatch (`--include-external`) for diagnostics that restores today's + exact behavior (included, tagged, contributing to totals, shown with the existing warning label). +- Add regression coverage proving the default path no longer "blindly scrapes" external sessions. +- Do not touch the RESUME-path ownership primitives (`session-ownership.ts`, + `session-origin-audit.ts`) or the plugin layer (`claude.plugin.ts`, `codex.plugin.ts`) — they are + already correct and out of scope. +- Preserve prior code-review fixes in `native-loader.ts` (CR-002 bounded scan, CR-003 index-first + ordering, CR-R-002 `_metrics.json` suffix check) by not touching that file at all. + +## Non-goals + +- Reconciling `native-loader.ts`'s parallel ownership-index implementation with + `session-ownership.ts`'s `scanSessionsForClaudeId` — left as a separate future concern; both + already work correctly for their respective use cases (bulk scan vs. single-id lookup). +- A "diagnostic-only, excluded from totals" mode for `--include-external`. `aggregator.ts` computes + every total (duration, turns, file ops, tool calls, lines, model calls, etc.) by reducing directly + over the same session list that's displayed — there is no separate display-only channel. Building + one would mean threading an exclusion flag through 15+ reduce call sites across an 868-line file, + well beyond this ticket's scope. `--include-external` therefore means "go back to today's exact + behavior" — an explicit, opt-in escape hatch, not a hidden data leak. +- An integration-level (`tests/integration/`) end-to-end CLI test. Unit coverage at the exact seam + where the fix lives is sufficient and matches existing test conventions. + +## Design + +### Fix point: `src/cli/commands/analytics/sources/sessions-source.ts` + +This is the single place where natively-discovered sessions merge into the pipeline shared by +console output, JSON/CSV export, and the HTML report (`SessionsSource.load()` is called once by +`analytics/index.ts`, upstream of the aggregator and all output formats). Today: + +```ts +if (opts.scanNative !== false) { + const { loadNativeSessions } = await import('../native-loader.js'); + const natives = (await loadNativeSessions(opts.filter)).filter((s) => + loader.sessionMatchesFilter(s, opts.filter) + ); + rawSessions.push(...natives); +} +``` + +Change: after the existing `sessionMatchesFilter` filter, drop any session tagged +`native-external` unless `opts.includeExternal` is set: + +```ts +const natives = (await loadNativeSessions(opts.filter)) + .filter((s) => loader.sessionMatchesFilter(s, opts.filter)) + .filter((s) => opts.includeExternal || s.startEvent?.data.provider !== 'native-external'); +``` + +`native-loader.ts` itself is **not modified** — it keeps computing `hasOwnershipMarker` and tagging +`provider = 'native-external'` exactly as it does today. `aggregator.ts` and `formatter.ts` are also +**not modified**: by the time a `RawSessionData` reaches them, it has already been excluded (default) +or is meant to be fully counted (`--include-external`), so the existing warning-label branch in +`formatter.ts` becomes the diagnostic display for the flagged case rather than dead code. + +### Flag wiring + +| Layer | Change | +|---|---| +| `src/cli/commands/analytics/types.ts` | Add `includeExternal?: boolean` to `AnalyticsOptions`, doc-commented like the existing `scanNative` field. | +| `src/cli/commands/analytics/sources/types.ts` | Add `includeExternal?: boolean` to `SourceLoadOptions`, mirroring `scanNative`. | +| `src/cli/commands/analytics/index.ts` | Add `.option('--include-external', 'Include non-CodeMie-owned native sessions in output (opt-in; matches pre-fix behavior)')`; thread `options.includeExternal` into `source.load({ filter, scanNative: options.scanNative, includeExternal: options.includeExternal })`. | +| `src/cli/commands/analytics/sources/sessions-source.ts` | Consume `opts.includeExternal` in the filter above. | + +No changes needed to `AnalyticsFilter` (date/session/project matching stays untouched — +`includeExternal` is a source-level behavior modifier, not a filter criterion, so it lives on +`SourceLoadOptions`/`AnalyticsOptions` alongside `scanNative`, not on `AnalyticsFilter`). + +This applies uniformly across output modes (console table, `--export json|csv`, `--report`) +because they all consume the same `rawSessions` returned by `SessionsSource.load()`. + +## Testing + +- New file `src/cli/commands/analytics/sources/__tests__/sessions-source.test.ts` (no prior coverage + exists for this file): asserts (a) default mode excludes a `native-external`-tagged session from + `rawSessions`, (b) `includeExternal: true` restores it, (c) owned native sessions are always + included, (d) existing `sessionMatchesFilter`-based filtering (date range etc.) is unaffected. + Follows the project's dependency-injection/dynamic-import mocking convention + (`.ai-run/guides/testing/testing-patterns.md`) by mocking `../native-loader.js`'s + `loadNativeSessions` export via `vi.mock`/dynamic import, and a `MetricsDataLoader` stub — + no real filesystem access needed. +- `native-loader.test.ts` and `aggregator.test.ts`: **no changes** — neither file's behavior + changes under this design, so existing tests (including the "external session labeling" suite in + `native-loader.test.ts`, which still accurately describes current tag-only behavior in that file) + remain correct as-is. + +This satisfies the ticket's acceptance criterion that "regression coverage confirms analytics does +not scrape all sessions blindly and excludes external sessions" at the exact seam where the +exclusion is implemented. + +## Risks + +- `--include-external` restores full inclusion in totals, not diagnostic-only display, per the + non-goals section above — must be documented in the CLI help text and the ticket/PR description + so it isn't mistaken for a safe always-on diagnostic mode. +- `codex.plugin.ts` has no `resolveResumeOwnership` implementation (pre-existing, RESUME-path-only + gap, unrelated to this fix) — verified not to matter here since `hasOwnershipMarker` in + `native-loader.ts` is already agent-agnostic and unchanged by this design. diff --git a/docs/superpowers/tasks/2026-07-07-analytics-exclude-external-sessions/complexity-assessment.json b/docs/superpowers/tasks/2026-07-07-analytics-exclude-external-sessions/complexity-assessment.json new file mode 100644 index 00000000..cf7b9c96 --- /dev/null +++ b/docs/superpowers/tasks/2026-07-07-analytics-exclude-external-sessions/complexity-assessment.json @@ -0,0 +1,25 @@ +{ + "schema": 1, + "task": "Fix the analytics command's native-loader.ts to exclude (not just label) non-CodeMie-owned native Claude/Codex sessions from output, retire the now-dead formatter.ts display branch, and add regression coverage proving exclusion.", + "generated": "2026-07-07", + "dimensions": { + "component_scope": { "score": 3, "label": "M" }, + "requirements_clarity": { "score": 2, "label": "S" }, + "technical_risk": { "score": 3, "label": "M" }, + "file_change_estimate": { "score": 3, "label": "M" }, + "dependencies": { "score": 1, "label": "XS" }, + "affected_layers": { "score": 2, "label": "S" } + }, + "total": 14, + "size": "S", + "routing": "writing-plans", + "key_reasoning": [ + { "dimension": "component_scope", "reason": "Two components across two layers: native-loader.ts (session discovery/exclusion logic) and formatter.ts (retiring the dead 'native-external' display branch). Pattern is clear — hasOwnershipMarker signal already exists, only the tag-vs-exclude behavior needs to change." }, + { "dimension": "technical_risk", "reason": "Bumped from S to M by the security/data-exposure red flag: this fix reverses the original PR #403 design decision ('no sessions are hidden, external ones are clearly marked') specifically for analytics output, preventing non-CodeMie-owned session data from leaking into aggregated results. Otherwise low novelty — reuses an already-computed, already-tested ownership signal." }, + { "dimension": "file_change_estimate", "reason": "Estimated 4-6 files touched: native-loader.ts and formatter.ts (modified), native-loader.test.ts (rewritten to assert exclusion instead of labeling), likely aggregator.test.ts (new 'native-external' fixture case), and probably a new integration test under tests/integration/ per the ticket's explicit call for regression coverage proving analytics 'does not scrape all sessions blindly'. Spans src/cli/commands/analytics/, its __tests__/ folder, and possibly tests/integration/." } + ], + "red_flags_applied": [ + "Technical Risk bumped from S to M: security/data-exposure concern — the fix must prevent non-CodeMie-owned session data from being exposed in analytics output, intentionally reversing the prior 'never hide, only label' design decision for the analytics pipeline specifically." + ], + "split_recommendation": null +} diff --git a/docs/superpowers/tasks/2026-07-07-analytics-exclude-external-sessions/decisions.jsonl b/docs/superpowers/tasks/2026-07-07-analytics-exclude-external-sessions/decisions.jsonl new file mode 100644 index 00000000..20a025c5 --- /dev/null +++ b/docs/superpowers/tasks/2026-07-07-analytics-exclude-external-sessions/decisions.jsonl @@ -0,0 +1,2 @@ +{"ts":"2026-07-07T13:15:01Z","gate_id":"spec.approved","mode":"hitl","verdict":{"decision":"approve","rationale":"(provided by user)","follow_ups":[],"confidence":"high","source":"hitl"},"escalated":false,"prior_context":{"question":"Approve spec.md for EPMCDME-13367 analytics external-session exclusion fix?","options":["Approve","Request changes","Abort"],"phase":3,"risk_flags":[],"artifact_refs":["docs/superpowers/tasks/2026-07-07-analytics-exclude-external-sessions/spec.md (sha256:aae3bee87bbb78a0d2a195affa82fe9bf11d79c62d5705877e94cebbaf06f8ac)"]}} +{"ts":"2026-07-07T13:20:38Z","gate_id":"plan.approved","mode":"hitl","verdict":{"decision":"approve","rationale":"(provided by user; initial 'Abort' selection was a misclick, corrected to approve in the same turn)","follow_ups":[],"confidence":"high","source":"hitl"},"escalated":false,"prior_context":{"question":"Approve plan.md for EPMCDME-13367 analytics external-session exclusion fix?","options":["Approve","Request changes","Abort"],"phase":4,"risk_flags":[],"artifact_refs":["docs/superpowers/tasks/2026-07-07-analytics-exclude-external-sessions/plan.md (sha256:5f5f250cb0e5701a0f9fddcf9aa6202d43f6146207af5a9648001ecd17660a36)"]}} diff --git a/docs/superpowers/tasks/2026-07-07-analytics-exclude-external-sessions/events.jsonl b/docs/superpowers/tasks/2026-07-07-analytics-exclude-external-sessions/events.jsonl new file mode 100644 index 00000000..bd635181 --- /dev/null +++ b/docs/superpowers/tasks/2026-07-07-analytics-exclude-external-sessions/events.jsonl @@ -0,0 +1,2 @@ +{"schema":1,"ts":"2026-07-07T13:15:01Z","event":"decision.recorded","run_id":"analytics-exclude-external-sessions","phase":3,"actor":"decision-router","summary":"Decision recorded for spec.approved: approve","artifacts":["decisions.jsonl"],"data":{"gate_id":"spec.approved","mode":"hitl","decision":"approve","source":"hitl","escalated":false,"prior_context":{"question":"Approve spec.md for EPMCDME-13367 analytics external-session exclusion fix?","phase":3,"risk_flags":[],"artifact_refs":["docs/superpowers/tasks/2026-07-07-analytics-exclude-external-sessions/spec.md"]}}} +{"schema":1,"ts":"2026-07-07T13:20:38Z","event":"decision.recorded","run_id":"analytics-exclude-external-sessions","phase":4,"actor":"decision-router","summary":"Decision recorded for plan.approved: approve","artifacts":["decisions.jsonl"],"data":{"gate_id":"plan.approved","mode":"hitl","decision":"approve","source":"hitl","escalated":false,"prior_context":{"question":"Approve plan.md for EPMCDME-13367 analytics external-session exclusion fix?","phase":4,"risk_flags":[],"artifact_refs":["docs/superpowers/tasks/2026-07-07-analytics-exclude-external-sessions/plan.md"]}}} diff --git a/docs/superpowers/tasks/2026-07-07-analytics-exclude-external-sessions/plan.md b/docs/superpowers/tasks/2026-07-07-analytics-exclude-external-sessions/plan.md new file mode 100644 index 00000000..55df51f0 --- /dev/null +++ b/docs/superpowers/tasks/2026-07-07-analytics-exclude-external-sessions/plan.md @@ -0,0 +1,307 @@ +# Exclude non-CodeMie-owned sessions from analytics (EPMCDME-13367) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** By default, `codemie analytics` excludes non-CodeMie-owned native sessions (tagged `provider: 'native-external'` by `native-loader.ts`) from output and from all aggregated totals, with an opt-in `--include-external` flag that restores today's exact behavior. + +**Architecture:** Add a single filter step in `SessionsSource.load()` — the one seam where natively-discovered sessions merge into the pipeline shared by console output, JSON/CSV export, and the HTML report — that drops any session whose `startEvent.data.provider === 'native-external'` unless a new `includeExternal` option is set. Thread that option from a new `--include-external` CLI flag through `AnalyticsOptions` → `SourceLoadOptions` → the filter. No changes to `native-loader.ts`, `aggregator.ts`, or `formatter.ts`. + +**Tech Stack:** TypeScript (ES modules), Commander.js (CLI), Vitest (unit tests). + +## Global Constraints + +- ES modules with `.js` extensions on all relative imports (per `AGENTS.md`). +- `native-loader.ts` is NOT modified — preserves CR-002 (bounded transcript scan), CR-003 (index-first ownership check), CR-R-002 (`_metrics.json` suffix filtering). +- `aggregator.ts` and `formatter.ts` are NOT modified. +- `--include-external` means "restore full pre-fix behavior" (included, tagged, contributing to totals) — not a diagnostic-only/display-only mode (architecturally infeasible without touching 15+ `reduce()` call sites in `aggregator.ts`; out of scope per spec non-goals). +- No changes to `AnalyticsFilter` — `includeExternal` is a source-level behavior modifier, not a filter criterion. +- Unit test coverage only; no integration-level (`tests/integration/`) CLI test (per spec non-goals). + +--- + +### Task 1: Exclude `native-external` sessions in `SessionsSource.load()` + +**Files:** +- Modify: `src/cli/commands/analytics/sources/types.ts` (add `includeExternal?: boolean` to `SourceLoadOptions`) +- Modify: `src/cli/commands/analytics/types.ts:233-251` (add `includeExternal?: boolean` to `AnalyticsOptions`) +- Modify: `src/cli/commands/analytics/sources/sessions-source.ts` +- Test: `src/cli/commands/analytics/sources/__tests__/sessions-source.test.ts` (new file) + +**Interfaces:** +- Consumes: `RawSessionData` (from `../data-loader.js`) — `{ sessionId: string, startEvent?: SessionStartEvent, endEvent?: SessionEndEvent, deltas: MetricDelta[], agentSessionFile?: string }`. `SessionStartEvent.data.provider` is a plain `string` (`'native'` or `'native-external'` for native sessions). +- Consumes: `loadNativeSessions(filter?: AnalyticsFilter, deps?: NativeLoaderDeps): Promise` (from `../native-loader.js`, unchanged). +- Consumes: `MetricsDataLoader` (from `../data-loader.js`, unchanged) — `loadSessions(filter?: AnalyticsFilter): RawSessionData[]`, `sessionMatchesFilter(s: RawSessionData, filter?: AnalyticsFilter): boolean`. +- Produces: `SourceLoadOptions.includeExternal?: boolean` — consumed by Task 2's CLI wiring in `index.ts`. + +- [ ] **Step 1: Write the failing tests** + +Create `src/cli/commands/analytics/sources/__tests__/sessions-source.test.ts`: + +```typescript +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import type { RawSessionData } from '../../data-loader.js'; + +const mockLoadSessions = vi.fn(); +const mockSessionMatchesFilter = vi.fn(); + +vi.mock('../../data-loader.js', () => ({ + MetricsDataLoader: vi.fn().mockImplementation(() => ({ + loadSessions: mockLoadSessions, + sessionMatchesFilter: mockSessionMatchesFilter + })) +})); + +const mockLoadNativeSessions = vi.fn(); + +vi.mock('../../native-loader.js', () => ({ + loadNativeSessions: mockLoadNativeSessions +})); + +function makeSession(sessionId: string, provider: string): RawSessionData { + return { + sessionId, + startEvent: { + recordId: sessionId, + type: 'session_start', + timestamp: 0, + codeMieSessionId: sessionId, + agentName: 'claude', + syncStatus: 'synced', + data: { provider, workingDirectory: '/tmp', startTime: 0 } + }, + deltas: [] + }; +} + +describe('SessionsSource', () => { + beforeEach(() => { + mockLoadSessions.mockReturnValue([]); + mockSessionMatchesFilter.mockReturnValue(true); + mockLoadNativeSessions.mockResolvedValue([]); + }); + + afterEach(() => { + vi.clearAllMocks(); + }); + + it('excludes native-external sessions by default', async () => { + const owned = makeSession('owned-1', 'native'); + const external = makeSession('external-1', 'native-external'); + mockLoadNativeSessions.mockResolvedValue([owned, external]); + + const { SessionsSource } = await import('../sessions-source.js'); + const source = new SessionsSource(); + const result = await source.load({ filter: {} }); + + expect(result.rawSessions.map((s) => s.sessionId)).toEqual(['owned-1']); + }); + + it('includes native-external sessions when includeExternal is true', async () => { + const owned = makeSession('owned-1', 'native'); + const external = makeSession('external-1', 'native-external'); + mockLoadNativeSessions.mockResolvedValue([owned, external]); + + const { SessionsSource } = await import('../sessions-source.js'); + const source = new SessionsSource(); + const result = await source.load({ filter: {}, includeExternal: true }); + + expect(result.rawSessions.map((s) => s.sessionId).sort()).toEqual(['external-1', 'owned-1']); + }); + + it('always includes owned native sessions regardless of includeExternal', async () => { + const owned = makeSession('owned-1', 'native'); + mockLoadNativeSessions.mockResolvedValue([owned]); + + const { SessionsSource } = await import('../sessions-source.js'); + const source = new SessionsSource(); + const result = await source.load({ filter: {} }); + + expect(result.rawSessions.map((s) => s.sessionId)).toEqual(['owned-1']); + }); + + it('applies sessionMatchesFilter before the external-session filter', async () => { + const external = makeSession('external-1', 'native-external'); + mockLoadNativeSessions.mockResolvedValue([external]); + mockSessionMatchesFilter.mockReturnValue(false); + + const { SessionsSource } = await import('../sessions-source.js'); + const source = new SessionsSource(); + const result = await source.load({ filter: {}, includeExternal: true }); + + expect(result.rawSessions).toEqual([]); + expect(mockSessionMatchesFilter).toHaveBeenCalledWith(external, {}); + }); + + it('includes tracked sessions from the loader unconditionally', async () => { + const tracked = makeSession('tracked-1', 'native'); + mockLoadSessions.mockReturnValue([tracked]); + mockLoadNativeSessions.mockResolvedValue([]); + + const { SessionsSource } = await import('../sessions-source.js'); + const source = new SessionsSource(); + const result = await source.load({ filter: {} }); + + expect(result.rawSessions.map((s) => s.sessionId)).toEqual(['tracked-1']); + }); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `npx vitest run src/cli/commands/analytics/sources/__tests__/sessions-source.test.ts` +Expected: FAIL — the `includeExternal` case fails because `SourceLoadOptions` has no such property yet (TS compile error under vitest's esbuild transform is treated as a runtime failure), and the default-exclusion test fails because `sessions-source.ts` currently pushes every native session unfiltered. + +- [ ] **Step 3: Add `includeExternal` to the option types** + +In `src/cli/commands/analytics/sources/types.ts`, change: + +```typescript +export interface SourceLoadOptions { + filter: AnalyticsFilter; + /** Sessions source only: skip native agent-log discovery. Ignored by other sources. */ + scanNative?: boolean; +} +``` + +to: + +```typescript +export interface SourceLoadOptions { + filter: AnalyticsFilter; + /** Sessions source only: skip native agent-log discovery. Ignored by other sources. */ + scanNative?: boolean; + /** Sessions source only: include non-CodeMie-owned native sessions (tagged 'native-external') in output. Ignored by other sources. */ + includeExternal?: boolean; +} +``` + +In `src/cli/commands/analytics/types.ts`, change the `AnalyticsOptions` interface (lines 233-251) so the block right after `scanNative` reads: + +```typescript + /** When false (via --no-scan-native), skip native-log discovery and use tracked sessions only. */ + scanNative?: boolean; + /** When true (via --include-external), include non-CodeMie-owned native sessions in output (matches pre-fix behavior). */ + includeExternal?: boolean; +} +``` + +- [ ] **Step 4: Implement the filter in `SessionsSource.load()`** + +In `src/cli/commands/analytics/sources/sessions-source.ts`, change: + +```typescript + if (opts.scanNative !== false) { + try { + const { loadNativeSessions } = await import('../native-loader.js'); + const natives = (await loadNativeSessions(opts.filter)).filter((s) => + loader.sessionMatchesFilter(s, opts.filter) + ); + rawSessions.push(...natives); + } catch (error) { + logger.debug('Native session discovery failed (continuing with tracked sessions):', error); + } + } +``` + +to: + +```typescript + if (opts.scanNative !== false) { + try { + const { loadNativeSessions } = await import('../native-loader.js'); + const natives = (await loadNativeSessions(opts.filter)) + .filter((s) => loader.sessionMatchesFilter(s, opts.filter)) + .filter((s) => opts.includeExternal || s.startEvent?.data.provider !== 'native-external'); + rawSessions.push(...natives); + } catch (error) { + logger.debug('Native session discovery failed (continuing with tracked sessions):', error); + } + } +``` + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `npx vitest run src/cli/commands/analytics/sources/__tests__/sessions-source.test.ts` +Expected: PASS — all 5 tests green. + +- [ ] **Step 6: Typecheck** + +Run: `npm run typecheck` +Expected: no errors. + +- [ ] **Step 7: Commit** + +```bash +git add src/cli/commands/analytics/sources/types.ts src/cli/commands/analytics/types.ts src/cli/commands/analytics/sources/sessions-source.ts src/cli/commands/analytics/sources/__tests__/sessions-source.test.ts +git commit -m "fix(analytics): exclude non-CodeMie-owned native sessions by default (EPMCDME-13367)" +``` + +--- + +### Task 2: Wire `--include-external` CLI flag + +**Files:** +- Modify: `src/cli/commands/analytics/index.ts:21-23` (register the flag on the default source command) +- Modify: `src/cli/commands/analytics/index.ts:65` (thread the option into `source.load()`) + +**Interfaces:** +- Consumes: `SourceLoadOptions.includeExternal` (produced by Task 1) and `AnalyticsOptions.includeExternal` (produced by Task 1). +- Produces: the `--include-external` CLI flag, parsed by Commander into `options.includeExternal: boolean | undefined`. + +**Test-first: no** — this task only wires an existing, already-tested option through Commander's option parser into a call site; there is no new branching logic to unit-test, and an end-to-end CLI test is explicitly out of scope per the spec's non-goals (unit coverage at the seam in Task 1 is sufficient). Verified manually in Step 3 below instead. + +- [ ] **Step 1: Register the flag** + +In `src/cli/commands/analytics/index.ts`, change: + +```typescript + // Default source: local CodeMie-tracked sessions + native agent logs. + applyCommonOptions(command) + .option('--no-scan-native', 'Skip native agent-log discovery (use only CodeMie-tracked sessions)') + .action((options: AnalyticsOptions) => runAnalytics(options, new SessionsSource())); +``` + +to: + +```typescript + // Default source: local CodeMie-tracked sessions + native agent logs. + applyCommonOptions(command) + .option('--no-scan-native', 'Skip native agent-log discovery (use only CodeMie-tracked sessions)') + .option('--include-external', 'Include non-CodeMie-owned native sessions in output (opt-in; matches pre-fix behavior)') + .action((options: AnalyticsOptions) => runAnalytics(options, new SessionsSource())); +``` + +- [ ] **Step 2: Thread the option into `source.load()`** + +In `src/cli/commands/analytics/index.ts`, change: + +```typescript + const { rawSessions, cost } = await source.load({ filter, scanNative: options.scanNative }); +``` + +to: + +```typescript + const { rawSessions, cost } = await source.load({ + filter, + scanNative: options.scanNative, + includeExternal: options.includeExternal + }); +``` + +- [ ] **Step 3: Manually verify the flag is registered** + +Run: `node bin/codemie.js analytics --help` +Expected: output includes a line for `--include-external Include non-CodeMie-owned native sessions in output (opt-in; matches pre-fix behavior)`. + +- [ ] **Step 4: Typecheck and run the full analytics test suite** + +Run: `npm run typecheck && npx vitest run src/cli/commands/analytics` +Expected: no errors; all analytics tests (including Task 1's new file, `aggregator.test.ts`, and `native-loader.test.ts`, which are unchanged and still pass) are green. + +- [ ] **Step 5: Commit** + +```bash +git add src/cli/commands/analytics/index.ts +git commit -m "feat(analytics): add --include-external escape hatch for the exclusion fix (EPMCDME-13367)" +``` diff --git a/docs/superpowers/tasks/2026-07-07-analytics-exclude-external-sessions/spec.md b/docs/superpowers/tasks/2026-07-07-analytics-exclude-external-sessions/spec.md new file mode 100644 index 00000000..67c3280e --- /dev/null +++ b/docs/superpowers/tasks/2026-07-07-analytics-exclude-external-sessions/spec.md @@ -0,0 +1,120 @@ +# Design: Exclude non-CodeMie-owned sessions from analytics (EPMCDME-13367) + +## Problem + +The parent issue (EPMCDME-12992, merged via PR #403) added ownership validation that gates the +RESUME path — the analytics command was left out of scope. `codemie analytics` still scans and +reports every native session it discovers, whether or not CodeMie owns it. Today, +`src/cli/commands/analytics/native-loader.ts` already computes ownership via +`hasOwnershipMarker(descriptor.filePath)` and tags unowned sessions +`startEvent.data.provider = 'native-external'` — but the session is still pushed unfiltered into +the returned array (`out.push(raw)`) and flows through aggregation and output uncounted-but-present. +`formatter.ts` only shows a cosmetic yellow warning; it never excludes the session from totals. + +This intentionally reverses one line of the original PR #403 design decision ("No sessions are +hidden — all are surfaced, external ones are clearly marked") — but only for the analytics +pipeline. The RESUME-path UX (informing/blocking on external session resume) is unaffected. + +## Goals + +- By default, `codemie analytics` (console, JSON/CSV export, HTML report) excludes non-CodeMie-owned + native sessions from output and from all aggregated totals. +- Provide an opt-in escape hatch (`--include-external`) for diagnostics that restores today's + exact behavior (included, tagged, contributing to totals, shown with the existing warning label). +- Add regression coverage proving the default path no longer "blindly scrapes" external sessions. +- Do not touch the RESUME-path ownership primitives (`session-ownership.ts`, + `session-origin-audit.ts`) or the plugin layer (`claude.plugin.ts`, `codex.plugin.ts`) — they are + already correct and out of scope. +- Preserve prior code-review fixes in `native-loader.ts` (CR-002 bounded scan, CR-003 index-first + ordering, CR-R-002 `_metrics.json` suffix check) by not touching that file at all. + +## Non-goals + +- Reconciling `native-loader.ts`'s parallel ownership-index implementation with + `session-ownership.ts`'s `scanSessionsForClaudeId` — left as a separate future concern; both + already work correctly for their respective use cases (bulk scan vs. single-id lookup). +- A "diagnostic-only, excluded from totals" mode for `--include-external`. `aggregator.ts` computes + every total (duration, turns, file ops, tool calls, lines, model calls, etc.) by reducing directly + over the same session list that's displayed — there is no separate display-only channel. Building + one would mean threading an exclusion flag through 15+ reduce call sites across an 868-line file, + well beyond this ticket's scope. `--include-external` therefore means "go back to today's exact + behavior" — an explicit, opt-in escape hatch, not a hidden data leak. +- An integration-level (`tests/integration/`) end-to-end CLI test. Unit coverage at the exact seam + where the fix lives is sufficient and matches existing test conventions. + +## Design + +### Fix point: `src/cli/commands/analytics/sources/sessions-source.ts` + +This is the single place where natively-discovered sessions merge into the pipeline shared by +console output, JSON/CSV export, and the HTML report (`SessionsSource.load()` is called once by +`analytics/index.ts`, upstream of the aggregator and all output formats). Today: + +```ts +if (opts.scanNative !== false) { + const { loadNativeSessions } = await import('../native-loader.js'); + const natives = (await loadNativeSessions(opts.filter)).filter((s) => + loader.sessionMatchesFilter(s, opts.filter) + ); + rawSessions.push(...natives); +} +``` + +Change: after the existing `sessionMatchesFilter` filter, drop any session tagged +`native-external` unless `opts.includeExternal` is set: + +```ts +const natives = (await loadNativeSessions(opts.filter)) + .filter((s) => loader.sessionMatchesFilter(s, opts.filter)) + .filter((s) => opts.includeExternal || s.startEvent?.data.provider !== 'native-external'); +``` + +`native-loader.ts` itself is **not modified** — it keeps computing `hasOwnershipMarker` and tagging +`provider = 'native-external'` exactly as it does today. `aggregator.ts` and `formatter.ts` are also +**not modified**: by the time a `RawSessionData` reaches them, it has already been excluded (default) +or is meant to be fully counted (`--include-external`), so the existing warning-label branch in +`formatter.ts` becomes the diagnostic display for the flagged case rather than dead code. + +### Flag wiring + +| Layer | Change | +|---|---| +| `src/cli/commands/analytics/types.ts` | Add `includeExternal?: boolean` to `AnalyticsOptions`, doc-commented like the existing `scanNative` field. | +| `src/cli/commands/analytics/sources/types.ts` | Add `includeExternal?: boolean` to `SourceLoadOptions`, mirroring `scanNative`. | +| `src/cli/commands/analytics/index.ts` | Add `.option('--include-external', 'Include non-CodeMie-owned native sessions in output (opt-in; matches pre-fix behavior)')`; thread `options.includeExternal` into `source.load({ filter, scanNative: options.scanNative, includeExternal: options.includeExternal })`. | +| `src/cli/commands/analytics/sources/sessions-source.ts` | Consume `opts.includeExternal` in the filter above. | + +No changes needed to `AnalyticsFilter` (date/session/project matching stays untouched — +`includeExternal` is a source-level behavior modifier, not a filter criterion, so it lives on +`SourceLoadOptions`/`AnalyticsOptions` alongside `scanNative`, not on `AnalyticsFilter`). + +This applies uniformly across output modes (console table, `--export json|csv`, `--report`) +because they all consume the same `rawSessions` returned by `SessionsSource.load()`. + +## Testing + +- New file `src/cli/commands/analytics/sources/__tests__/sessions-source.test.ts` (no prior coverage + exists for this file): asserts (a) default mode excludes a `native-external`-tagged session from + `rawSessions`, (b) `includeExternal: true` restores it, (c) owned native sessions are always + included, (d) existing `sessionMatchesFilter`-based filtering (date range etc.) is unaffected. + Follows the project's dependency-injection/dynamic-import mocking convention + (`.ai-run/guides/testing/testing-patterns.md`) by mocking `../native-loader.js`'s + `loadNativeSessions` export via `vi.mock`/dynamic import, and a `MetricsDataLoader` stub — + no real filesystem access needed. +- `native-loader.test.ts` and `aggregator.test.ts`: **no changes** — neither file's behavior + changes under this design, so existing tests (including the "external session labeling" suite in + `native-loader.test.ts`, which still accurately describes current tag-only behavior in that file) + remain correct as-is. + +This satisfies the ticket's acceptance criterion that "regression coverage confirms analytics does +not scrape all sessions blindly and excludes external sessions" at the exact seam where the +exclusion is implemented. + +## Risks + +- `--include-external` restores full inclusion in totals, not diagnostic-only display, per the + non-goals section above — must be documented in the CLI help text and the ticket/PR description + so it isn't mistaken for a safe always-on diagnostic mode. +- `codex.plugin.ts` has no `resolveResumeOwnership` implementation (pre-existing, RESUME-path-only + gap, unrelated to this fix) — verified not to matter here since `hasOwnershipMarker` in + `native-loader.ts` is already agent-agnostic and unchanged by this design. diff --git a/docs/superpowers/tasks/2026-07-07-analytics-exclude-external-sessions/technical-analysis.md b/docs/superpowers/tasks/2026-07-07-analytics-exclude-external-sessions/technical-analysis.md new file mode 100644 index 00000000..a89efc26 --- /dev/null +++ b/docs/superpowers/tasks/2026-07-07-analytics-exclude-external-sessions/technical-analysis.md @@ -0,0 +1,494 @@ +# Technical Research + +**Task**: analytics session-ownership session-origin claude codex native-loader +**Generated**: 2026-07-07 +**Research path**: filesystem + +--- + +## 1. Original Context + +Jira ticket EPMCDME-13367 — "Fix analytics command to exclude non-CodeMie-owned external sessions" + +Description: The main fix for resuming non-CodeMie-owned sessions was handled in the parent +issue EPMCDME-12992 (already merged to main via PR #403 "feat(agents): restrict session +ingestion to CodeMie-owned sessions", commit 3015cd2). That PR introduced +src/agents/core/session/session-ownership.ts (scanSessionsForClaudeId) and +src/agents/core/session/session-origin-audit.ts, and modified src/agents/core/AgentCLI.ts, +src/agents/plugins/claude/claude.plugin.ts, src/agents/plugins/codex/codex.plugin.ts, and their +conversations-processors to gate RESUME on session ownership. + +However, the `codemie analytics` command still scans and reports ALL detected local sessions +(Claude, Codex, etc.) without filtering out sessions that were not created by CodeMie. This +sub-task covers the remaining analytics-specific gap: CodeMie Analytics must include only +CodeMie-owned sessions and must not expose unrelated external session data. + +Acceptance criteria from the ticket: +- The analytics command does not include sessions created outside CodeMie. +- Session filtering is based on CodeMie ownership/origin validation before analytics + aggregation/output. +- Non-CodeMie-owned sessions under local Claude session directories are ignored by analytics + processing. +- Analytics output remains correct for valid CodeMie-owned sessions. +- Regression coverage confirms analytics does not scrape all sessions blindly and excludes + external sessions. + +Research goal: understand exactly how the analytics pipeline currently discovers/loads sessions +(native-loader.ts, sessions-source.ts, data-loader.ts, aggregator.ts, per-plugin +conversations-processor files under src/cli/commands/analytics/), how ownership validation +works today (session-ownership.ts, session-origin-audit.ts, AgentCLI.ts resume-path usage, +types.ts additions from PR #403), and identify the precise integration point(s) where an +ownership/origin check should be applied to analytics session discovery for each agent plugin +(Claude, Codex, and any others with local session files e.g. Gemini/OpenCode/Kimi). Also note +existing test patterns for analytics loaders and for session-ownership so a regression test can +follow established conventions. + +--- + +## 2. Codebase Findings + +### Existing Implementations + +- `src/cli/commands/analytics/native-loader.ts` — discovers untracked native Claude/Codex logs + and synthesizes `RawSessionData`. **This is the defect location.** `NATIVE_AGENTS = ['claude', + 'codex']` — hardcoded, and share one discovery loop. `hasOwnershipMarker(filePath)` (from + `NativeLoaderDeps`) is already computed at **line ~518** but only used to *tag* the session: + `if (!deps.hasOwnershipMarker(descriptor.filePath) && raw.startEvent) { raw.startEvent.data.provider + = 'native-external'; }` — the session is still unconditionally pushed to the output array at + line 521 (`out.push(raw)`) and flows downstream unfiltered. + Key exports: `loadNativeSessions(filter?: AnalyticsFilter, deps: NativeLoaderDeps = + realNativeDeps): Promise`, `synthesizeRawSession(agentName, descriptor, + parsed)`, `synthesizeCodexRawSession(...)`, interface `NativeLoaderDeps` (`hasOwnershipMarker`, + `trackedLogPaths`, `discover`, `parse`, `realPath`), `realNativeDeps` (fs-backed impl). + `realNativeDeps.hasOwnershipMarker` (lines ~135-199) implements its own + `buildOwnershipIndex()`: a cached `Set` (`ownershipIndexCache`, process-lifetime) + built from correlation `agentSessionFile` real-paths (from tracked-session metadata files) + + `transcriptPath` values from sidecar `-codemie-marker.json` files (fast path), falling back to + a **bounded 4KB / 10-line** scan of the transcript for a `codemie_session_start` JSON line + (slow path, per CR-002 fix — never a full-file read). +- `src/cli/commands/analytics/sources/sessions-source.ts` — `SessionsSource.load()` builds + tracked sessions via `MetricsDataLoader`, then (unless `opts.scanNative === false`) dynamically + imports `loadNativeSessions` and merges filtered natives in. This is the top of the discovery + pipeline that `analytics/index.ts` calls, and an alternative integration point for a + post-hoc `.filter(...)` if exclusion is applied outside `native-loader.ts` itself. +- `src/cli/commands/analytics/sources/types.ts` — `AnalyticsSource.load(opts: + SourceLoadOptions): Promise`; `SourceLoadOptions { filter, scanNative? }`; + `SourceResult { rawSessions, cost? }`. +- `src/cli/commands/analytics/data-loader.ts` — `MetricsDataLoader.loadSessions(filter?)` reads + `~/.codemie/sessions/*.json` (tracked, CodeMie-owned by construction — no ownership check + needed here). `sessionMatchesFilter` is exposed `public` specifically so native/untracked + sessions can be run through the same filter logic. +- `src/cli/commands/analytics/aggregator.ts` (~line 405-410) — `AnalyticsAggregator` builds + `SessionAnalytics` from `RawSessionData`, copying `provider: startEvent.data.provider` through + with **no ownership-based exclusion**. Only existing filter is an empty-deltas exclusion + (`aggregate()` lines 43-45, `keepSessionIds`). +- `src/cli/commands/analytics/formatter.ts` (lines 156-166) — `displaySession()` only + cosmetically flags `provider === 'native-external'` with a yellow "external ⚠ not + CodeMie-managed" label; does not exclude the session from any totals/output. This branch + becomes dead code (or should be repurposed behind a debug/opt-in flag) once exclusion happens + upstream. +- `src/cli/commands/analytics/types.ts` — `SessionAnalytics.provider: string` (values include + `'native' | 'native-external' | 'claude' | 'codex' | ...`); `AnalyticsFilter`; + `AnalyticsOptions.scanNative?: boolean` (backs the existing `--no-scan-native` CLI flag — a + coarse on/off switch, distinct from the fine-grained per-session ownership filter this ticket + needs). +- `src/cli/commands/analytics/index.ts` — `createAnalyticsCommand()` wires `--no-scan-native` → + `SourceLoadOptions.scanNative`; `runAnalytics()` calls `source.load(...)`. +- `src/agents/core/session/session-ownership.ts` — `scanSessionsForClaudeId(claudeSessionId: + string, sessionsDir?: string): boolean` scans `~/.codemie/sessions/*.json` (excluding + `_metrics.json` via `.endsWith('_metrics.json')`, per CR-R-002 fix — not `.includes`) for a + record whose `correlation.agentSessionId === claudeSessionId`. Despite the "Claude" naming it + is generically usable (keys off an arbitrary agent session id). **Not currently reused by + native-loader.ts**, which reimplements a parallel ownership-index scan inline — a drift risk + between the RESUME-path ownership check and the analytics-path ownership check. +- `src/agents/core/session/session-origin-audit.ts` — `appendAuditEvent(event: AuditEventName, + data, logsDir?)` writes JSONL to `~/.codemie/logs/session-origin-audit.jsonl`; + `AuditEventName = 'transcript_marker_written' | 'resume_blocked' | 'resume_external_confirmed'` + (would need a new value, e.g. `'analytics_external_excluded'`, if analytics adopts audit + logging for exclusions). `appendTranscriptMarker(transcriptPath, codemieSessionId, + codemieAgent)` writes the sidecar marker `~/.codemie/sessions/{id}-codemie-marker.json` (moved + off the live transcript per CR-006/CR-007 to avoid `appendFileSync` races) plus a best-effort + `codemie_session_start` transcript line for legacy compat. Called from `src/cli/commands/ + hook.ts` (~lines 706, 747-751) at session-start — this is what makes `hasOwnershipMarker` + able to detect ownership later. +- `src/agents/core/AgentCLI.ts` (~lines 308-378) — reference RESUME-gating integration pattern: + builds `resumeId`, calls `adapter.resolveResumeOwnership({resumeId, cwd, env})`, computes + `isExternal = ownership?.supported === true && ownership.owned === false`, prompts + (`promptExternalResume`), and audits via `appendAuditEvent('resume_blocked' | + 'resume_external_confirmed', ...)` (dynamically imported). Fails open when the adapter does + not implement the optional capability. +- `src/agents/plugins/claude/claude.plugin.ts` (lines 325-340) — implements + `resolveResumeOwnership()` via dynamic import of `scanSessionsForClaudeId`. +- `src/agents/plugins/codex/codex.plugin.ts` — implements `extractNativeResumeId(args)` but + does **not** implement `resolveResumeOwnership` — a gap on the RESUME path. This does not + block the analytics fix, however, because `native-loader.ts`'s `hasOwnershipMarker` is + agent-agnostic (keyed on file path/content, not plugin-specific format) and already covers + both Claude and Codex through the single shared loop. +- `src/agents/core/types.ts` (lines ~658-693) — `ResumeOwnershipInput { resumeId, cwd, env }`; + `ResumeOwnershipResult { supported, owned?, fallbackResumeCommand?, auditData? }`; + `AgentAdapter.resolveResumeOwnership?(...)` and `AgentAdapter.extractNativeResumeId?(...)` are + both optional capabilities. Analytics has **no equivalent typed contract** yet — it only has + the untyped boolean `hasOwnershipMarker`. +- `src/agents/core/session/discovery-types.ts` — `SessionDescriptor` has no ownership field; + ownership is resolved out-of-band via marker files, not encoded in the descriptor itself. +- `src/cli/commands/analytics/cost/codex-agent.ts` — `isCodexFamilyAgent`, + `agentMatchesAnalyticsFilter` — reusable per-plugin dispatch pattern already used by both + `native-loader.ts` and `data-loader.ts`. +- **Scope confirmation**: Gemini (`src/agents/plugins/gemini/...`, `dataPaths.home = '.gemini'`), + OpenCode (`src/agents/plugins/opencode/opencode.paths.ts`), and Kimi + (`src/agents/plugins/kimi/...`, `dataPaths.home = '.kimi-code'`) each have their own + `discoverSessions()` session-adapter implementations, but **none of them are included in + `NATIVE_AGENTS` today** — analytics does not scan their native session files at all + (neither includes nor needs to exclude them). The ticket's "e.g. Gemini/OpenCode/Kimi" phrasing + in feature_area is therefore not an active concern for this fix; only Claude and Codex have + native sessions flowing through `native-loader.ts` today. + +### Architecture and Layers Affected + +- CLI command layer: `src/cli/commands/analytics/index.ts` (flag parsing, dispatch) +- Analytics source layer: `src/cli/commands/analytics/sources/sessions-source.ts`, + `otel-source.ts` (implements `AnalyticsSource`) +- Session discovery layer: `native-loader.ts` (native/untracked discovery + synthesis, **defect + location**) and `data-loader.ts` (tracked `~/.codemie` session loading, unaffected) +- Session ownership/origin layer (agents core): `src/agents/core/session/session-ownership.ts`, + `session-origin-audit.ts` — reusable primitives, not yet consumed by analytics +- Agent adapter layer: `src/agents/plugins/claude/claude.plugin.ts`, + `src/agents/plugins/codex/codex.plugin.ts` +- Orchestration/CLI-run layer: `src/agents/core/AgentCLI.ts` (reference RESUME-gating consumer, + not itself in scope for this ticket) +- Aggregation/presentation layer: `aggregator.ts`, `formatter.ts`, `exporter.ts`, + `report/payload-builder.ts` + +### Integration Points + +- `src/cli/commands/analytics/native-loader.ts:518-521` — primary fix point. Change the + tag-only branch to skip/`continue` (exclude from `out`) when `!deps.hasOwnershipMarker(...)`, + for both Claude and Codex since they share the loop. Optionally gate behind a new + default-true option with an opt-in escape hatch (e.g. `--include-external`) if diagnostic + visibility into excluded sessions is still desired. +- `src/cli/commands/analytics/native-loader.ts:135-199` (`realNativeDeps.hasOwnershipMarker`) — + candidate to reconcile with `session-ownership.ts`'s `scanSessionsForClaudeId` logic for + consistency between the RESUME-path and analytics-path ownership checks, rather than + maintaining two parallel implementations. +- `src/cli/commands/analytics/sources/sessions-source.ts:19-23` — alternative/secondary + integration point: filter the merged `natives` array here if exclusion is applied post-hoc + rather than inside `native-loader.ts`. +- `src/cli/commands/analytics/types.ts:233-251` (`AnalyticsOptions`) — would need a new option + (e.g. `includeExternal?: boolean`) mirrored in `index.ts:createAnalyticsCommand()` if an + explicit opt-in escape hatch is desired. +- `src/cli/commands/analytics/formatter.ts:162-166` — current display-only `'native-external'` + branch should be retired or repurposed for a debug-only mode once exclusion happens upstream. +- No plugin-specific integration point is needed for Claude/Codex today — `NATIVE_AGENTS` + covers both through the single generic check. Gemini/OpenCode/Kimi are confirmed out of scope + since they are not in `NATIVE_AGENTS` and analytics does not discover their native sessions at + all currently. + +### Patterns and Conventions + +- **RESUME-gating pattern (reference)**: adapter exposes an optional capability + (`resolveResumeOwnership`) → orchestrator calls it, interprets `{supported, owned}`, and only + acts when the adapter opts in; plugins without the capability fail open (no-op). The + analytics fix should mirror the "opt-in exclusion" spirit but does not need a new adapter + capability since `hasOwnershipMarker` is already agent-agnostic. +- **Dependency-injection boundary for testability**: `NativeLoaderDeps` interface isolates + fs/registry calls; tests inject fakes (`hasOwnershipMarker: () => boolean`) without touching + disk. Any new filtering logic should extend this same seam. +- **Index-then-fallback ownership check**: fast path via cached realpath `Set`, slow path via + bounded transcript scan for legacy sessions — agent-agnostic, reused for both Claude and + Codex. +- **Tag-not-filter anti-pattern (current defect)**: labeling instead of excluding is exactly + what this ticket must fix. +- **Non-fatal, best-effort audit/marker writes**: all fs operations in `session-origin-audit.ts` + swallow errors and never break the primary flow; any new audit event should follow this + convention. +- **Dynamic `import()` for optional/heavy modules**: `AgentCLI.ts`, `claude.plugin.ts`, and + `sessions-source.ts` all lazy-import session-ownership/native-loader modules only when the + code path is actually hit — new ownership-check wiring in analytics should follow the same + convention. +- **Filter composition pattern**: `MetricsDataLoader.sessionMatchesFilter` is exposed `public` + so untracked (native) sessions can be run through the same filter logic as tracked sessions — + a similarly exported/reusable ownership-check function is the idiomatic shape for this fix. + +--- + +## 3. Documentation Findings + +### Guides and Architecture Docs + +- `.ai-run/guides/architecture/architecture.md` — plugin-based 5-layer architecture (CLI → + Registry → Plugin → Core → Utils); documents `src/agents/core/session/` as the session + adapter layer, meaning new session-domain logic belongs there, agent-specific specifics stay + in the plugin layer. +- `.ai-run/guides/integration/external-integrations.md` — documents the Session Analytics Flow + (`onSessionEnd` hook → JSONL deltas → `SessionSyncer` → `v1/metrics`); background for what + "excluding" a session from analytics affects downstream. +- `.ai-run/guides/testing/testing-patterns.md` — mandates dynamic-import mocking for Vitest + (static imports bypass `vi.spyOn`), `vi.restoreAllMocks()` in `afterEach` — directly + applicable to any new test for this fix. +- `.ai-run/guides/standards/code-quality.md` — naming/file-size/no-`console.log` conventions + applicable to the native-loader.ts change. +- No guide specifically names "session-ownership" or "analytics origin exclusion" as a topic; + conventions below are derived from code and prior review artifacts. + +### Architectural Decisions + +- `docs/superpowers/specs/2026-07-01-EPMCDME-12992-session-origin-validation-design.md` — + **the original design explicitly decided**: "No sessions are hidden — all are surfaced, + external ones are clearly marked." This is precisely the decision EPMCDME-13367 must reverse + for the analytics command specifically (the RESUME-path UX of informing users about external + sessions can remain unchanged; only analytics aggregation/output needs to exclude them). +- `docs/superpowers/reviews/2026-07-01-EPMCDME-12992/code-review-final.json` (CR-002) — bounded + 4KB/10-line transcript scan fix (avoid OOM on huge transcripts) — must be preserved in any + refactor of `hasOwnershipMarker`. +- Same review (CR-003) — index-first ownership check must precede the transcript-marker scan, + to avoid false-positive "external" labeling of re-entered CodeMie sessions — must be + preserved if `buildOwnershipIndex()` logic is touched. +- Same review (CR-006/CR-007) — ownership marker moved to a sidecar file instead of appending + to the live transcript, to avoid `appendFileSync` races. +- `docs/superpowers/specs/2026-07-06-generic-resume-ownership-validation-design.md` (commit + dffcddc) — made resume ownership validation a generic, optional adapter capability; fail-open + when unsupported; preserves slug-based resume IDs. Relevant context, not directly touched by + this ticket. +- `docs/superpowers/reviews/2026-07-01-EPMCDME-12992-2/code-review-final.json` (CR-R-002) — + `_metrics.json` filtering must use `.endsWith('_metrics.json')`, not `.includes('_metrics')`, + for consistency between `native-loader.ts` and `session-ownership.ts`. +- `docs/superpowers/tasks/2026-07-01-epmcdme-12992-session-origin-validation/qa-report.md` — QA + note: lint required `\p{Cc}/gu` control-char stripping instead of an ESLint + `no-control-regex`-triggering `[\x00-\x1F...]` pattern — relevant if any new sanitization is + added. + +### Derived Conventions + +- Ownership resolution is index-first (correlation JSON + sidecar marker), falling back to a + bounded transcript scan for legacy sessions — never a full-file read. +- Non-fatal try/catch around every filesystem read; failures fall back conservatively to + "treat as external/not owned" (fail-closed for ownership determination, as opposed to the + fail-open adapter-capability pattern used on the RESUME path — these are different concerns + and should not be conflated). +- Ownership index results are process-lifetime cached, acceptable because analytics CLI runs + are short-lived. +- New session-domain logic belongs in `src/agents/core/session/*.ts`, agent-agnostic; agent + specifics stay in the plugin layer, exposed via optional adapter capability methods only when + genuinely agent-specific (not needed for this fix, since `hasOwnershipMarker` is already + agent-agnostic). + +--- + +## 4. Testing Landscape + +### Existing Coverage + +- `src/agents/core/__tests__/session-ownership.test.ts` — unit tests for + `scanSessionsForClaudeId`; real temp dir via `node:fs`/`node:os` `tmpdir()`, hand-written JSON + fixtures via a local `writeSession()` helper. Covers match/no-match/empty-dir/missing-dir/ + malformed-JSON/`_metrics.json`-skip cases. +- `src/agents/core/__tests__/session-origin-audit.test.ts` — unit tests for `appendAuditEvent`/ + `appendTranscriptMarker`; same tmpdir pattern; asserts JSONL line shape and non-fatal + behavior on missing paths. +- `src/agents/core/__tests__/AgentCLI-resume.test.ts` — reference for CLI-level ownership + gating: tests resume-ownership flow end to end using `vi.spyOn` on `ConfigLoader`, + `ProviderRegistry`, `logger`, and the `session-origin-audit` module; injects adapter overrides + (DI, no real fs). +- `src/cli/commands/analytics/__tests__/native-loader.test.ts` — **already has** a + `describe('loadNativeSessions — external session labeling')` block asserting + `startEvent.data.provider` becomes `'native-external'` when `hasOwnershipMarker()` returns + false, and stays `'native'` when true, via a `makeDeps(hasMarker)` fake. **This is the closest + existing ownership coverage for analytics, but it only tests the label, not exclusion** — it + must be updated to assert `results` excludes/omits unowned sessions (e.g. + `expect(results).toHaveLength(0)` when marker absent) once the fix lands. +- `src/cli/commands/analytics/__tests__/aggregator.test.ts` — tests `aggregate()`; fixtures use + `provider: 'native'` but never `'native-external'`; asserts only the empty-deltas exclusion. + No ownership-based filtering coverage. +- `src/cli/commands/analytics/__tests__/otel-loader.test.ts`, `model-normalizer.test.ts`, + `otel-report.integration.test.ts`, `report/__tests__/*`, `cost/__tests__/*` — unrelated to + ownership filtering. +- `src/agents/plugins/claude/__tests__/claude.conversations-processor.test.ts`, + `src/agents/plugins/codex/__tests__/codex.conversations-processor.test.ts` — transcript + parsing only, no ownership filtering. +- **No test files exist** for `src/cli/commands/analytics/data-loader.ts` or + `src/cli/commands/analytics/sources/sessions-source.ts` (confirmed via glob — files exist, no + `__tests__` counterpart). + +### Testing Framework and Patterns + +- Vitest `^4.1.5`, TypeScript `^5.3.3`, ESM throughout (`.js` extensions on relative imports). +- `vitest.config.ts`: `globals: true`, `environment: 'node'`, `include: ['src/**/*.test.ts', + 'src/**/*.spec.ts', 'tests/**/*.test.ts']`, thread pool, `isolate: true`. +- **Colocated `__tests__/` folders** next to the source file under test (e.g. + `src/agents/core/__tests__/session-ownership.test.ts`) is the convention for unit tests. The + top-level `tests/` tree (`tests/unit`, `tests/integration`) is reserved for CLI-command/e2e + integration tests, distinct from the `src/**/__tests__` unit convention. +- Real temp directories via `node:fs`/`node:os` `tmpdir()` keyed by `process.pid`, cleaned in + `beforeEach`/`afterEach` — used for session-ownership/session-origin-audit tests (no + `mock-fs`/`memfs`). +- Dependency-injection boundary pattern (`NativeLoaderDeps`) — tests build fake `deps` objects + inline; this is the pattern a new analytics regression test should reuse. +- `vi.spyOn` + `vi.restoreAllMocks()` in `afterEach` for module-level mocking; adapter/dep + override factory pattern (`createAdapter(overrides)`) for AgentCLI tests. +- Session JSON fixtures are hand-built inline plain objects, not a shared fixture library. + +### Coverage Gaps + +- **Confirmed: analytics has no test today asserting it filters/excludes sessions by + ownership.** `native-loader.test.ts` verifies only the cosmetic `provider` label — it does not + assert that `loadNativeSessions`, `sessions-source.ts`, or `aggregator.ts` actually drop + `'native-external'` sessions from counts or output. This is precisely the gap EPMCDME-13367 + must close, and no regression test currently guards against "blindly scraping all sessions." +- No tests exist for `data-loader.ts` (`MetricsDataLoader`) or `sources/sessions-source.ts` + (`SessionsSource.load`) at all — the latter is the natural seam for an end-to-end assertion + that native sessions are filtered by ownership before being returned/aggregated. +- `aggregator.test.ts` has no `'native-external'` fixture case, so no coverage that aggregation + totals correctly exclude such sessions. +- No integration-level test (`tests/integration/`) exercises `codemie analytics` end-to-end with + a mix of CodeMie-owned and non-CodeMie session files on disk — the acceptance criteria's + language ("does not scrape all sessions blindly") suggests this level of test may be warranted + in addition to the unit-level fix. + +--- + +## 5. Configuration and Environment + +### Environment Variables + +- `CODEMIE_HOME` — overrides `~/.codemie` (tracked sessions, correlation index, audit logs). + Read in `src/utils/paths.ts` (`getCodemieHome()`/`getCodemiePath()`). +- `CODEX_HOME` — overrides `~/.codex` for Codex rollout discovery. Read in + `src/agents/plugins/codex/codex.paths.ts` (`getCodexHomePath()`/ + `getCodexDiscoverySessionRoots()`). +- `OPENCODE_STORAGE_PATH` — explicit override of OpenCode's session storage root. Read in + `src/agents/plugins/opencode/opencode.paths.ts`. Not relevant today since OpenCode is not in + `NATIVE_AGENTS`. +- `XDG_DATA_HOME` / `XDG_CONFIG_HOME` — standard XDG fallbacks used by OpenCode's storage/config + path resolution. +- **No analytics-specific or ownership-filtering env var exists** (e.g. no + `CODEMIE_ANALYTICS_*`) — nothing controls ownership-filtering behavior via environment + variable today; any new behavior toggle would need a CLI flag (see Feature Flags below) rather + than an env var, to match existing analytics conventions. + +### Configuration Files + +- `src/utils/paths.ts` — `getCodemieHome()`/`getCodemiePath()`: resolves `~/.codemie` (or + `CODEMIE_HOME` override); used for tracked sessions dir, correlation metadata, sidecar + markers, and the audit log. +- `src/agents/plugins/codex/codex.paths.ts` — `getCodexHomePath()`/ + `getCodexDiscoverySessionRoots()`: resolves `~/.codex` (or `CODEX_HOME` override) and + enumerates both native (`sessions/`) and CodeMie-managed (`codemie/home/sessions/`) Codex + roots. +- `src/agents/plugins/opencode/opencode.paths.ts` — resolves OpenCode's XDG-style storage dir + (not currently consumed by analytics). +- No dedicated analytics config file beyond the CLI flags defined in `index.ts`. + +### Feature Flags and Deployment Concerns + +- `--no-scan-native` (existing, `AnalyticsOptions.scanNative`) — the only existing toggle; + disables native-log discovery entirely (tracked-only). It does **not** selectively exclude + external/non-CodeMie native sessions while still including CodeMie-tracked native ones — this + is the closest existing lever, but not a substitute for the fine-grained per-session ownership + filter this ticket requires. +- No existing flag distinguishes "include native-external sessions" vs "exclude them" — this is + the exact gap the ticket needs to close. A new opt-in flag (e.g. `--include-external`) may be + worth introducing during implementation if diagnostic visibility into excluded sessions is + desired. +- **Session directories confirmed per plugin** (relevant to "local Claude session directories" + language in the acceptance criteria): + - Claude: `~/.claude/projects/` (`getProjectsDir()` in `claude.session.ts`, + `join(homedir(), '.claude', 'projects')`). + - Codex: `~/.codex/sessions/` (native) and `~/.codex/codemie/home/sessions/` + (CodeMie-managed), or `${CODEX_HOME}/sessions` if set (`getCodexDiscoverySessionRoots()`). + - Gemini: `~/.gemini/` — has a session adapter but is not in `NATIVE_AGENTS`, so not scanned + by analytics today. + - OpenCode: XDG-based storage path — has a session adapter but is not in `NATIVE_AGENTS`, so + not scanned by analytics today. + - Kimi: `~/.kimi-code/` — has a session adapter but is not in `NATIVE_AGENTS`, so not scanned + by analytics today. + - CodeMie's own tracked-session store (all plugins funnel correlation metadata here): + `~/.codemie/sessions/` via `getCodemiePath('sessions')`. +- No deployment/CI concerns identified; this is a pure application-logic fix within the CLI. + +--- + +## 6. Risk Indicators + +- No existing test asserts analytics excludes ownership-unowned sessions — `native-loader.test.ts` + covers only cosmetic labeling, exactly the gap this ticket must close with new regression + coverage. +- Zero test coverage today for `data-loader.ts` and `sources/sessions-source.ts` — any new + filter logic touching these files starts from an untested baseline. +- `native-loader.ts`'s `hasOwnershipMarker`/`buildOwnershipIndex` reimplements ownership-index + logic in parallel with `session-ownership.ts`'s `scanSessionsForClaudeId` rather than reusing + it — risk of drift/inconsistency between the RESUME-path ownership check and the + analytics-path ownership check if the two are not reconciled or explicitly kept separate by + design. +- `codex.plugin.ts` has no `resolveResumeOwnership` implementation on the RESUME path (a + pre-existing gap unrelated to this ticket) — must confirm the analytics fix does not + implicitly assume a Claude-only ownership mechanism; it should not, since `hasOwnershipMarker` + is already agent-agnostic, but this should be explicitly verified during implementation. +- The original PR #403 design **explicitly decided** not to hide external sessions ("No + sessions are hidden — all are surfaced, external ones are clearly marked"). This ticket + reverses that decision for analytics specifically; care is needed to avoid inadvertently + changing the RESUME-path UX (which should keep informing users about external sessions when + resuming) — the exclusion must be scoped to the analytics pipeline only. +- Prior code-review fixes must be preserved when touching `hasOwnershipMarker`/ + `buildOwnershipIndex`: CR-002 (bounded 4KB/10-line scan, avoid full-file reads), CR-003 + (index-first check ordering), CR-R-002 (`_metrics.json` via `.endsWith`, not `.includes`). + Regressing any of these would reintroduce previously-fixed defects. +- No integration-level test exercises `codemie analytics` end-to-end with a mix of owned/ + external session files on disk — the acceptance criteria's explicit call for regression + coverage that analytics "does not scrape all sessions blindly" suggests this level of test may + be expected, not just a unit-level assertion. +- `formatter.ts`'s `'native-external'` display branch becomes dead/unreachable code once + exclusion happens upstream at the loader/source level — needs an explicit decision (remove, + or repurpose behind a new `--include-external` debug flag). +- `NATIVE_AGENTS` is hardcoded to `['claude', 'codex']`; Gemini/OpenCode/Kimi have their own + `discoverSessions()` implementations but are not included in `NATIVE_AGENTS`, so analytics + does not scan them at all today. This is confirmed out of scope for this fix (no code changes + needed for those plugins), but should be called out explicitly in the plan/spec so the fix is + not scoped too broadly based on the ticket's "e.g. Gemini/OpenCode/Kimi" phrasing — only + Claude and Codex native sessions are actually affected by the current bug. +- No analytics-specific environment variable exists to toggle ownership filtering — any new + configurability should be a CLI flag (matching the `--no-scan-native` convention), not an env + var, to stay consistent with existing analytics configuration conventions. + +--- + +## 7. Summary for Complexity Assessment + +This is a narrowly-scoped, well-localized fix. The defect lives almost entirely in one file — +`src/cli/commands/analytics/native-loader.ts`, lines ~518-521 — where an existing ownership +signal (`hasOwnershipMarker`) is already computed but only used to *label* a session as +`'native-external'` rather than exclude it from the returned array. The primary layers touched +are the analytics session-discovery layer (`native-loader.ts`, and possibly +`sources/sessions-source.ts` as an alternate/secondary filter point) and the presentation layer +(`formatter.ts`, where the now-obsolete display-only branch needs to be removed or repurposed). +No changes are required to the agents/core ownership primitives themselves +(`session-ownership.ts`, `session-origin-audit.ts`) or to the plugin layer +(`claude.plugin.ts`, `codex.plugin.ts`) — both Claude and Codex already flow through the single +shared `NATIVE_AGENTS` loop and the agent-agnostic `hasOwnershipMarker` check, and Gemini/ +OpenCode/Kimi are confirmed out of scope since they are not in `NATIVE_AGENTS` at all today. +Estimated file change surface is small: 1-2 source files for the core fix (`native-loader.ts`, +possibly `sessions-source.ts` and/or `types.ts`/`index.ts` if an opt-in `--include-external` +flag is added), plus `formatter.ts` cleanup, plus test files. + +Technical novelty is low — this follows an established pattern (the RESUME-path ownership +gating from PR #403) closely, reusing the same `hasOwnershipMarker` boolean signal that already +exists and is already tested for its labeling behavior. No new environment variables or config +files are needed; any new toggle should be a CLI flag consistent with the existing +`--no-scan-native` convention. The main judgment call is whether to reuse/reconcile the existing +`session-ownership.ts` primitive versus continuing with `native-loader.ts`'s parallel index +implementation, and whether to preserve an escape hatch (debug flag) for seeing excluded +sessions. Neither introduces new architectural concepts. + +Test coverage posture is mixed-to-thin and is itself part of the deliverable: the closest +existing test (`native-loader.test.ts`'s external-session-labeling suite) asserts the current +buggy tag-only behavior and must be rewritten to assert actual exclusion; `data-loader.ts` and +`sessions-source.ts` have zero existing tests; and no integration test exists for the CLI +end-to-end. Key risk factors for complexity scoring: (1) the behavior-reversal risk relative to +the explicit prior design decision ("no sessions are hidden") — this is an intentional, +ticket-mandated policy change scoped to analytics only, not a regression, but must be +communicated as such in the implementation; (2) the need to preserve three prior code-review +fixes (bounded scan, index-first ordering, `_metrics.json` suffix check) while modifying +adjacent logic; and (3) the acceptance criteria's explicit demand for regression coverage that +proves analytics no longer "blindly scrapes" — this likely requires both an updated unit test in +`native-loader.test.ts` and a new integration-level test under `tests/integration/`, pushing this +from a "trivial one-line fix" toward a "small, well-defined, low-risk" change with a +correspondingly modest test-writing effort attached. From 61b1f848d4086def421c44c5dd8a51cfc858e5f0 Mon Sep 17 00:00:00 2001 From: Sviatoslav Likhtarchyk Date: Tue, 7 Jul 2026 18:11:26 +0300 Subject: [PATCH 3/3] fix(analytics): prevent native sessions from being tagged native-external (EPMCDME-13367) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two bugs caused CodeMie-managed sessions to lose their ownership marker and appear as native-external in analytics: 1. Sidecar overwrite on resume: appendTranscriptMarker keyed the sidecar by the CodeMie session ID, so resuming with a new Claude session overwrote the prior transcript's entry in the ownership index. Fixed by keying the sidecar by the Claude session ID (transcript basename without .jsonl) — each Claude session now gets its own non-overwriting marker. Write is idempotent; skips if already exists. 2. Empty transcript_path at SessionStart: the transcript file does not exist when the SessionStart hook fires, so the marker was never written for the initial session. Fixed by calling appendTranscriptMarker on the first UserPromptSubmit/Stop event that carries transcript_path (deferred write, idempotent). Also adds a fallback for sessions started via the native claude CLI (no CODEMIE_SESSION_ID env var): writes a minimal ownership sidecar from the SessionStart event and exits cleanly instead of throwing. Generated with AI Co-Authored-By: codemie-ai --- .../core/session/session-origin-audit.ts | 33 ++++++++++++------- src/cli/commands/hook.ts | 13 ++++++++ 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/src/agents/core/session/session-origin-audit.ts b/src/agents/core/session/session-origin-audit.ts index dab022ce..c2efdaf5 100644 --- a/src/agents/core/session/session-origin-audit.ts +++ b/src/agents/core/session/session-origin-audit.ts @@ -1,5 +1,5 @@ import { appendFileSync, mkdirSync, writeFileSync, existsSync } from 'node:fs'; -import { join } from 'node:path'; +import { basename, join } from 'node:path'; import { randomUUID } from 'node:crypto'; import { getCodemiePath } from '../../../utils/paths.js'; import { logger } from '../../../utils/logger.js'; @@ -35,20 +35,29 @@ export function appendTranscriptMarker( // CR-006: write a race-condition-free sidecar marker in ~/.codemie/sessions/ instead of // appending to the live Claude transcript (avoids byte-level interleaving on concurrent writes). + // Key by the Claude session ID (transcript basename) so each distinct Claude session gets its + // own non-overwriting marker — prevents a session resume from clobbering the prior transcript's + // entry in the ownership index. The write is idempotent: skip if the file already exists. try { const sessionsDir = getCodemiePath('sessions'); mkdirSync(sessionsDir, { recursive: true }); - writeFileSync( - join(sessionsDir, `${codemieSessionId}-codemie-marker.json`), - JSON.stringify({ - transcriptPath, - codemieSessionId, - codemieAgent, - timestamp: new Date().toISOString(), - }), - 'utf-8', - ); - logger.debug(`[session-origin] Sidecar marker written for session ${codemieSessionId}`); + const claudeSessionId = basename(transcriptPath, '.jsonl'); + const sidecarPath = join(sessionsDir, `${claudeSessionId}-codemie-marker.json`); + if (existsSync(sidecarPath)) { + logger.debug(`[session-origin] Sidecar marker already exists for ${claudeSessionId}, skipping`); + } else { + writeFileSync( + sidecarPath, + JSON.stringify({ + transcriptPath, + codemieSessionId, + codemieAgent, + timestamp: new Date().toISOString(), + }), + 'utf-8', + ); + logger.debug(`[session-origin] Sidecar marker written for ${claudeSessionId} (codemie: ${codemieSessionId})`); + } } catch (err) { logger.debug(`[session-origin] Failed to write sidecar marker (non-fatal): ${err}`); } diff --git a/src/cli/commands/hook.ts b/src/cli/commands/hook.ts index e5881e28..baadcffd 100644 --- a/src/cli/commands/hook.ts +++ b/src/cli/commands/hook.ts @@ -619,6 +619,19 @@ async function routeHookEvent(event: BaseHookEvent, rawInput: string, sessionId: return; } + // Deferred ownership marker: covers sessions where transcript_path was empty at + // SessionStart (transcript file not yet created). appendTranscriptMarker is idempotent — + // it skips if the per-Claude-session sidecar already exists. + if ( + event.transcript_path && + (normalizedEventName === 'UserPromptSubmit' || normalizedEventName === 'Stop') + ) { + const { appendTranscriptMarker } = await import( + '../../agents/core/session/session-origin-audit.js' + ); + appendTranscriptMarker(event.transcript_path, sessionId, agentName); + } + const duration = Date.now() - startTime; logger.info(`[hook:router] Event handled successfully: ${normalizedEventName} (${duration}ms)`);