feat: implement high-performance atom cache and maintenance tools (Issue #29)

README.md

@@ -107,6 +107,7 @@ lore search --text "session" --confidence high
 | Command | Description |
 |---------|-------------|
 | `lore init` | Initialize `.lore/` config in repository |
+| `lore cache --clean` | Clear all local cache data |
 | `lore commit` | Create a Lore-enriched commit |
 | `lore context <target>` | Full lore summary for a code region |
 | `lore constraints <target>` | Active constraints for a code region |
@@ -131,6 +132,7 @@ lore search --text "session" --confidence high
 | `--format <type>` | Output format: `text` (default) or `json` |
 | `--no-color` | Disable colored output |
 | `--no-update-notifier` | Disable update notification |
+| `--no-cache` | Bypass caching in the CLI |
 | `--limit <n>` | Limit number of results |
 | `--since <ref>` | Only consider commits since ref/date |
 
@@ -155,7 +157,7 @@ Every Lore-enriched commit carries a `Lore-id` and any combination of these trai
 
 ## Configuration
 
-`lore init` creates `.lore/config.toml`:
+`lore init` creates `.lore/config.toml` and ensures `.lore/cache` is added to your `.gitignore` to prevent local cache files from being tracked.
 
 ```toml
 [protocol]
@@ -188,6 +190,7 @@ update_check = true    # Set to false to disable update notifications
 
 | Variable | Description |
 |----------|-------------|
+| `LORE_NO_CACHE` | Set to `1` or `true` to bypass the atom cache |
 | `LORE_NO_UPDATE_CHECK` | Set to `1` or `true` to disable update notifications |
 | `NO_UPDATE_NOTIFIER` | Standard variable to disable update notifications (set to `1` or `true`) |
 | `CI` | Set to `true` or `1` to disable notifications (automatic in most CI) |

documents/PROJECT_ARCHITECTURE.md

@@ -1,7 +1,7 @@
 # Lore CLI -- Project Architecture
 
 > Authoritative reference for contributors (human or AI) to the lore-cli codebase.
-> Last updated 2026-03-22. Reflects the codebase after PR #1-15 merge cycle.
+> Last updated 2026-05-11. Reflects the codebase after the Atom Cache implementation.
 
 ---
 
@@ -148,6 +148,12 @@ No command or service instantiates its own dependencies. All wiring is centraliz
 - **Dependencies**: None.
 - **Dependents**: `TerminalPrompt` (implements), `commit.ts`, `main.ts`.
 
+#### `src/interfaces/atom-cache.ts`
+- **Contains**: `IAtomCache` interface.
+- **Single Responsibility**: Contract for sharded immutable file list caching.
+- **Dependencies**: None.
+- **Dependents**: `AtomCache` (implements), `NullAtomCache` (implements), `AtomRepository`.
+
 #### `src/interfaces/commit-input-reader.ts`
 - **Contains**: `ICommitInputReader` interface.
 - **Single Responsibility**: Strategy contract for reading commit input from any source.
@@ -168,6 +174,12 @@ No command or service instantiates its own dependencies. All wiring is centraliz
 - **Dependencies**: `output.ts` (`ValidationIssue`).
 - **Dependents**: `GitClient`, `commit.ts`, `squash.ts`, `trace.ts`, `why.ts`, `main.ts`.
 
+#### `src/util/cache-check.ts`
+- **Contains**: `shouldBypassCache()` function.
+- **Single Responsibility**: Determines whether the atom cache should be bypassed based on CLI flags, environment variables, and config. Extracted to avoid side effects during testing.
+- **Dependencies**: None.
+- **Dependents**: `main.ts`.
+
 #### `src/util/constants.ts`
 - **Contains**: All protocol constants: `LORE_TRAILER_KEYS`, `ARRAY_TRAILER_KEYS`, `ENUM_TRAILER_KEYS`, valid enum value arrays, `LORE_ID_PATTERN` regex, `REFERENCE_TRAILER_KEYS`, default limits/thresholds, config file names, exit codes.
 - **Single Responsibility**: Central registry of all protocol-level constants. Changing a trailer name or adding a new one starts here.
@@ -197,10 +209,17 @@ No command or service instantiates its own dependencies. All wiring is centraliz
 - **Dependents**: `CommitBuilder`, `SquashMerger`, `main.ts`.
 - **Key methods**: `generate()`.
 
+#### `src/services/atom-cache.ts`
+- **Contains**: `AtomCache` and `NullAtomCache` classes.
+- **Single Responsibility**: Implements sharded immutable file list caching on the filesystem. Uses a sharded structure (e.g., `.lore/cache/atoms/ab/cd/ef...`) to avoid directory entry limits.
+- **Dependencies**: `IAtomCache`, Node.js `fs`, `path`.
+- **Dependents**: `main.ts`, `AtomRepository`.
+- **Key methods**: `getFiles()`, `setFiles()`.
+
 #### `src/services/atom-repository.ts`
 - **Contains**: `AtomRepository` class.
-- **Single Responsibility**: Central query engine. Retrieves `LoreAtom` objects from git history by target path, Lore-id, revision range, scope, or globally. Handles follow-link BFS traversal.
-- **Dependencies**: `IGitClient`, `TrailerParser`, `domain.ts`, `query.ts`, `constants.ts`.
+- **Single Responsibility**: Central query engine. Retrieves `LoreAtom` objects from git history by target path, Lore-id, revision range, scope, or globally. Handles follow-link BFS traversal and file-list caching.
+- **Dependencies**: `IGitClient`, `TrailerParser`, `IAtomCache`, `domain.ts`, `query.ts`, `constants.ts`.
 - **Dependents**: Commands (`context`, `constraints`, `rejected`, `directives`, `tested`, `search`, `log`, `stale`, `trace`, `squash`, `doctor`), `Validator`, `main.ts`.
 - **Key methods**: `findByTarget()`, `findByLoreId()`, `findByRange()`, `findAll()`, `findByScope()`, `resolveFollowLinks()`.
 
@@ -329,7 +348,12 @@ No command or service instantiates its own dependencies. All wiring is centraliz
 
 #### `src/commands/init.ts`
 - **Contains**: `registerInitCommand()` function.
-- **Single Responsibility**: Creates `.lore/config.toml` with default content. Shows existing config if already present.
+- **Single Responsibility**: Creates `.lore/config.toml` with default content and ensures `.lore/cache` is ignored via `.gitignore`. Shows existing config if already present.
+- **Dependencies**: `IOutputFormatter`, `constants.ts`, Node.js `fs`.
+
+#### `src/commands/cache.ts`
+- **Contains**: `registerCacheCommand()` function.
+- **Single Responsibility**: Provides local cache management, including clearing the `.lore/cache` directory.
 - **Dependencies**: `IOutputFormatter`, `constants.ts`, Node.js `fs`.
 
 #### `src/commands/context.ts`
@@ -597,6 +621,7 @@ graph LR
         LIG[LoreIdGenerator]
         CL[ConfigLoader]
         TMP[TerminalPrompt]
+        AC[AtomCache]
     end
 
     subgraph Composed Services
@@ -614,6 +639,7 @@ graph LR
     MAIN --> LIG
     MAIN --> CL
     MAIN --> TMP
+    MAIN --> AC
 
     MAIN --> AR
     MAIN --> SR
@@ -624,6 +650,7 @@ graph LR
 
     AR -.->|IGitClient| GC
     AR -.-> TP
+    AR -.->|IAtomCache| AC
     SD -.->|IGitClient| GC
     CB -.-> TP
     CB -.-> LIG
@@ -697,6 +724,7 @@ graph LR
 AtomRepository(
   gitClient: IGitClient,
   trailerParser: TrailerParser,
+  atomCache: IAtomCache,
   customTrailerKeys: readonly string[]
 )
 

src/commands/cache.ts

@@ -0,0 +1,39 @@
+import type { Command } from 'commander';
+import type { IOutputFormatter } from '../interfaces/output-formatter.js';
+import { rm } from 'node:fs/promises';
+
+/**
+ * Register the `lore cache` command.
+ * Provides management utilities for the local sharded cache.
+ */
+export function registerCacheCommand(
+  program: Command,
+  deps: {
+    getFormatter: () => IOutputFormatter;
+    cacheDir: string;
+  },
+): void {
+  program
+    .command('cache')
+    .description('Manage the local Lore cache')
+    .option('--clean', 'Clear all cached atom and query data')
+    .action(async (options) => {
+      const formatter = deps.getFormatter();
+
+      if (options.clean) {
+        try {
+          await rm(deps.cacheDir, { recursive: true, force: true });
+          console.log(formatter.formatSuccess('Successfully cleared local cache.'));
+        } catch (error: unknown) {
+          const message = error instanceof Error ? error.message : String(error);
+          console.error(formatter.formatError(1, [{ severity: 'error', message: `Failed to clear cache: ${message}` }]));
+          process.exitCode = 1;
+          return;
+        }
+        return;
+      }
+
+      // If no options are provided, show help
+      program.commands.find(c => c.name() === 'cache')?.help();
+    });
+}

src/commands/init.ts

@@ -27,13 +27,15 @@ default_format = "text"
 max_depth = 3
 
 [cli]
+cache = true
 update_check = true
 `;
 
 /**
  * Register the `lore init` command.
  * Creates .lore/config.toml with default content.
  * If the config file already exists, prints its content and exits.
+ * Also ensures .lore/cache is added to .gitignore.
  */
 export function registerInitCommand(
   program: Command,
@@ -64,15 +66,52 @@ export function registerInitCommand(
           `Config already exists at ${CONFIG_DIR}/${CONFIG_FILENAME}:`,
         ));
         console.log(content);
-        return;
-      }
+      } else {
+        // Create directory and write config
+        await mkdir(configDir, { recursive: true });
+        await writeFile(configPath, DEFAULT_CONFIG_CONTENT, 'utf-8');
 
-      // Create directory and write config
-      await mkdir(configDir, { recursive: true });
-      await writeFile(configPath, DEFAULT_CONFIG_CONTENT, 'utf-8');
+        console.log(formatter.formatSuccess(
+          `Created ${CONFIG_DIR}/${CONFIG_FILENAME} with protocol version 1.0`,
+        ));
+      }
 
-      console.log(formatter.formatSuccess(
-        `Created ${CONFIG_DIR}/${CONFIG_FILENAME} with protocol version 1.0`,
-      ));
+      // Ensure cache is ignored
+      await ensureCacheIgnored(formatter);
     });
 }
+
+/**
+ * Ensures that .lore/cache is added to the .gitignore in the current directory.
+ * Uses process.cwd() to match where `lore init` creates .lore/config.toml.
+ * Idempotent: does nothing if the pattern is already present.
+ */
+async function ensureCacheIgnored(formatter: IOutputFormatter): Promise<void> {
+  const gitignorePath = join(process.cwd(), '.gitignore');
+  const ignorePattern = '.lore/cache';
+  let content = '';
+  let exists = false;
+
+  try {
+    content = await readFile(gitignorePath, 'utf-8');
+    exists = true;
+  } catch {
+    // File does not exist
+  }
+
+  const lines = content.split('\n').map((l) => l.trim());
+  if (lines.includes(ignorePattern)) {
+    return;
+  }
+
+  const suffix = content === '' || content.endsWith('\n') ? '' : '\n';
+  const newContent = `${content}${suffix}${ignorePattern}\n`;
+
+  await writeFile(gitignorePath, newContent, 'utf-8');
+
+  if (exists) {
+    console.log(formatter.formatSuccess(`Updated .gitignore to ignore ${ignorePattern}`));
+  } else {
+    console.log(formatter.formatSuccess(`Created .gitignore to ignore ${ignorePattern}`));
+  }
+}

src/interfaces/atom-cache.ts

@@ -0,0 +1,12 @@
+export interface IAtomCache {
+  /**
+   * Get cached list of files changed for a commit hash.
+   * Returns null if not in cache.
+   */
+  getFiles(hash: string): Promise<readonly string[] | null>;
+
+  /**
+   * Cache the list of files changed for a commit hash.
+   */
+  setFiles(hash: string, files: readonly string[]): Promise<void>;
+}

src/main.ts

@@ -16,6 +16,7 @@ import { PathResolver } from './services/path-resolver.js';
 import { LoreIdGenerator } from './services/lore-id-generator.js';
 import { ConfigLoader } from './services/config-loader.js';
 import { AtomRepository } from './services/atom-repository.js';
+import { AtomCache, NullAtomCache } from './services/atom-cache.js';
 import { SupersessionResolver } from './services/supersession-resolver.js';
 import { StalenessDetector } from './services/staleness-detector.js';
 import { CommitBuilder } from './services/commit-builder.js';
@@ -26,10 +27,12 @@ import { CommitInputResolver } from './services/commit-input-resolver.js';
 import { HeadLoreIdReader } from './services/head-lore-id-reader.js';
 import { SearchFilter } from './services/search-filter.js';
 
+import { join } from 'node:path';
 import { TextFormatter } from './formatters/text-formatter.js';
 import { JsonFormatter } from './formatters/json-formatter.js';
 
 import { registerInitCommand } from './commands/init.js';
+import { registerCacheCommand } from './commands/cache.js';
 import { registerContextCommand } from './commands/context.js';
 import { registerConstraintsCommand } from './commands/constraints.js';
 import { registerRejectedCommand } from './commands/rejected.js';
@@ -47,6 +50,8 @@ import { registerDoctorCommand } from './commands/doctor.js';
 
 import { LoreError, ValidationError } from './util/errors.js';
 import { shouldCheckForUpdate } from './util/update-check.js';
+import { shouldBypassCache } from './util/cache-check.js';
+import { resolveLoreRoot } from './services/root-resolver.js';
 
 /**
  * Composition root: constructs all dependencies and wires them together.
@@ -68,7 +73,9 @@ async function main(): Promise<void> {
     .option('--json', 'Shorthand for --format json')
     .option('--format <type>', 'Output format: text or json', 'text')
     .option('--no-color', 'Disable colored output')
-    .option('--no-update-notifier', 'Disable update notification');
+    .option('--no-update-notifier', 'Disable update notification')
+    .option('--no-cache', 'Bypass caching in the CLI');
+
 
   // 1. Create concrete implementations
   const gitClient: IGitClient = new GitClient();
@@ -87,13 +94,31 @@ async function main(): Promise<void> {
     config = DEFAULT_CONFIG;
   }
 
-  // 3. Update notification (fire-and-forget, respects env vars and config)
+  // 3. Resolve project root for caching and config
+  const loreRoot = await resolveLoreRoot(process.cwd(), configLoader, gitClient);
+
+  const cacheDir = join(loreRoot, '.lore', 'cache');
+  const atomCacheDir = join(cacheDir, 'atoms');
+
+  // 4. Update notification (fire-and-forget, respects env vars and config)
   if (shouldCheckForUpdate(config.cli.updateCheck)) {
     simpleUpdateNotifier({ pkg }).catch(() => {});
   }
 
-  // 4. Create services that depend on others
-  const atomRepository = new AtomRepository(gitClient, trailerParser, config.trailers.custom);
+  // 5. Determine if caching is bypassed via command line, env, or config
+  const bypassCache = shouldBypassCache(config.cli.cache);
+
+  const atomCache = bypassCache
+    ? new NullAtomCache()
+    : new AtomCache(atomCacheDir);
+
+  // 6. Create services that depend on others
+  const atomRepository = new AtomRepository(
+    gitClient,
+    trailerParser,
+    atomCache,
+    config.trailers.custom,
+  );
   const supersessionResolver = new SupersessionResolver();
   const stalenessDetector = new StalenessDetector(gitClient, config);
   const commitBuilder = new CommitBuilder(trailerParser, loreIdGenerator, config);
@@ -104,7 +129,7 @@ async function main(): Promise<void> {
   const commitInputResolver = new CommitInputResolver(prompt);
   const headLoreIdReader = new HeadLoreIdReader(gitClient, trailerParser);
 
-  // 5. Formatter factory (reads --format/--json from program options at call time)
+  // 7. Formatter factory (reads --format/--json from program options at call time)
   // Memoized: the formatter is created once on first call and reused thereafter.
   let cachedFormatter: IOutputFormatter | null = null;
   const getFormatter = (): IOutputFormatter => {
@@ -120,9 +145,10 @@ async function main(): Promise<void> {
     return cachedFormatter;
   };
 
-  // 6. Register all commands with their dependencies
+  // 8. Register all commands with their dependencies
 
   registerInitCommand(program, { getFormatter });
+  registerCacheCommand(program, { getFormatter, cacheDir });
 
   const pathQueryDeps = {
     atomRepository,
@@ -197,7 +223,7 @@ async function main(): Promise<void> {
     getFormatter,
   });
 
-  // 7. Parse and run
+  // 9. Parse and run
   await program.parseAsync(process.argv);
 }