From c6460a6d2d0a3153c98cbeee3dcb048bdc06eae5 Mon Sep 17 00:00:00 2001 From: Geoff Douglas Date: Wed, 4 Mar 2026 11:31:22 -0700 Subject: [PATCH] Make ingest pipeline configurable via .nexo/config.json Add four optional fields to the ingest config section: - samTemplate: explicit SAM template path (skips candidate search) - appEntry: explicit React app entry path (skips candidate search) - handlerSourceRoots: dirs to search for handler source files - skipDirs: additional dirs to skip during source file walk Auto-discovery remains the default when fields are not configured. Candidate lists and built-in skip dirs moved to DEFAULTS for single source of truth. Error messages hint at config fields to set. Co-Authored-By: Claude Opus 4.6 --- src/cli/commands/ingest.ts | 4 ++ src/config/schema.ts | 24 ++++++++++ src/ingest/parsers/sourceFiles.ts | 19 +++++--- src/ingest/sourceMap.ts | 21 +++++---- src/ingest/sync.ts | 73 ++++++++++++++++++++++++++++--- 5 files changed, 117 insertions(+), 24 deletions(-) diff --git a/src/cli/commands/ingest.ts b/src/cli/commands/ingest.ts index ebdc252..92d8c31 100644 --- a/src/cli/commands/ingest.ts +++ b/src/cli/commands/ingest.ts @@ -33,6 +33,10 @@ export const ingestCommand = new Command("ingest") frontendPath: frontend, backendPath: backend, apply: Boolean(opts.apply), + samTemplate: cfg.ingest?.samTemplate, + appEntry: cfg.ingest?.appEntry, + handlerSourceRoots: cfg.ingest?.handlerSourceRoots, + skipDirs: cfg.ingest?.skipDirs, }); printSyncResults(results, Boolean(opts.apply)); diff --git a/src/config/schema.ts b/src/config/schema.ts index 5c56929..7c3f7f7 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -9,12 +9,17 @@ export const NexoConfigSchema = z.object({ database: z.string().optional(), username: z.string().optional(), password: z.string().optional(), + caFile: z.string().optional(), }) .optional(), ingest: z .object({ frontend: z.string().optional(), backend: z.string().optional(), + samTemplate: z.string().optional(), + appEntry: z.string().optional(), + handlerSourceRoots: z.array(z.string()).optional(), + skipDirs: z.array(z.string()).optional(), }) .optional(), web: z @@ -39,4 +44,23 @@ export const DEFAULTS = { port: 3000, host: "127.0.0.1", }, + ingest: { + samTemplateCandidates: [ + "unified-stack/template.yaml", + "template.yaml", + "infra/web.yaml", + ], + appEntryCandidates: [ + "src/App.tsx", + "src/App.jsx", + ], + handlerSourceRoots: [ + "unified-stack/src", + "src", + ], + builtinSkipDirs: [ + "node_modules", "dist", ".git", ".cache", "build", "coverage", + "__tests__", "__mocks__", "test", "tests", ".aws-sam", + ], + }, } as const; diff --git a/src/ingest/parsers/sourceFiles.ts b/src/ingest/parsers/sourceFiles.ts index f3d2185..a69d184 100644 --- a/src/ingest/parsers/sourceFiles.ts +++ b/src/ingest/parsers/sourceFiles.ts @@ -8,10 +8,9 @@ export interface ParsedSourceFile { layer: string; } -const SKIP_DIRS = new Set([ - "node_modules", "dist", ".git", ".cache", "build", "coverage", - "__tests__", "__mocks__", "test", "tests", ".aws-sam", -]); +import { DEFAULTS } from "../../config/schema.js"; + +const BUILTIN_SKIP_DIRS = new Set(DEFAULTS.ingest.builtinSkipDirs); const LANG_MAP: Record = { ".jsx": "jsx", ".tsx": "tsx", ".js": "js", ".ts": "ts", @@ -25,11 +24,16 @@ const LANG_MAP: Record = { export function parseSourceFiles( rootPath: string, repo: string, + extraSkipDirs?: string[], ): ParsedSourceFile[] { if (!existsSync(rootPath)) return []; + const skipDirs = extraSkipDirs?.length + ? new Set([...BUILTIN_SKIP_DIRS, ...extraSkipDirs]) + : BUILTIN_SKIP_DIRS; + const files: ParsedSourceFile[] = []; - walkDir(rootPath, rootPath, repo, files); + walkDir(rootPath, rootPath, repo, files, skipDirs); return files; } @@ -38,6 +42,7 @@ function walkDir( rootPath: string, repo: string, files: ParsedSourceFile[], + skipDirs: Set, ): void { let entries; try { @@ -48,12 +53,12 @@ function walkDir( for (const entry of entries) { if (entry.name.startsWith(".")) continue; - if (SKIP_DIRS.has(entry.name)) continue; + if (skipDirs.has(entry.name)) continue; const fullPath = join(dir, entry.name); if (entry.isDirectory()) { - walkDir(fullPath, rootPath, repo, files); + walkDir(fullPath, rootPath, repo, files, skipDirs); } else if (entry.isFile()) { const ext = extname(entry.name); const language = LANG_MAP[ext]; diff --git a/src/ingest/sourceMap.ts b/src/ingest/sourceMap.ts index 1e80095..920d2fe 100644 --- a/src/ingest/sourceMap.ts +++ b/src/ingest/sourceMap.ts @@ -2,6 +2,7 @@ import { existsSync, readdirSync } from "fs"; import { join, resolve } from "path"; import type { Surreal } from "surrealdb"; import { listEdges } from "../db/edges.js"; +import { DEFAULTS } from "../config/schema.js"; import type { Node } from "../schema/types.js"; import type { ParsedEndpoint } from "./parsers/sam.js"; import type { ParsedScreen } from "./parsers/routes.js"; @@ -9,6 +10,7 @@ import type { ParsedScreen } from "./parsers/routes.js"; export interface SourceMapConfig { frontendRoot?: string; backendRoot?: string; + handlerSourceRoots?: string[]; } /** @@ -23,19 +25,16 @@ export function endpointSourceFile( // Handler format: "api/trips.handler" → src/api/trips.js (or .ts, .mjs, etc.) const handlerPath = ep.handler.replace(/\.[^.]+$/, ""); // strip .handler - const srcRoot = join(resolve(config.backendRoot), "unified-stack", "src"); + const roots = config.handlerSourceRoots ?? DEFAULTS.ingest.handlerSourceRoots; + const backendRoot = resolve(config.backendRoot); const extensions = [".js", ".ts", ".mjs", ".cjs"]; - for (const ext of extensions) { - const candidate = join(srcRoot, handlerPath + ext); - if (existsSync(candidate)) return candidate; - } - - // Try without unified-stack subdirectory - const altRoot = resolve(config.backendRoot); - for (const ext of extensions) { - const candidate = join(altRoot, "src", handlerPath + ext); - if (existsSync(candidate)) return candidate; + for (const root of roots) { + const srcRoot = join(backendRoot, root); + for (const ext of extensions) { + const candidate = join(srcRoot, handlerPath + ext); + if (existsSync(candidate)) return candidate; + } } return null; diff --git a/src/ingest/sync.ts b/src/ingest/sync.ts index ba508ac..9fca417 100644 --- a/src/ingest/sync.ts +++ b/src/ingest/sync.ts @@ -6,7 +6,9 @@ import { generateNodeId } from "../schema/ids.js"; import type { Node } from "../schema/types.js"; import type { ParsedEndpoint } from "./parsers/sam.js"; import type { ParsedScreen } from "./parsers/routes.js"; -import { basename, resolve } from "node:path"; +import { basename, resolve, join } from "node:path"; +import { existsSync } from "node:fs"; +import { DEFAULTS } from "../config/schema.js"; import type { ParsedSourceFile } from "./parsers/sourceFiles.js"; import { endpointName, parseSamTemplate } from "./parsers/sam.js"; import { screenName, parseRoutes } from "./parsers/routes.js"; @@ -18,6 +20,10 @@ export interface SyncOptions { frontendPath?: string; backendPath?: string; apply?: boolean; + samTemplate?: string; + appEntry?: string; + handlerSourceRoots?: string[]; + skipDirs?: string[]; } export interface SyncResults { @@ -37,6 +43,60 @@ interface PropsUpdate { [key: string]: { old: unknown; new: unknown }; } +/** + * Find the SAM template file within a backend path. + * If `configured` is set, resolves it directly; otherwise searches candidates. + */ +function findSamTemplate(backendPath: string, configured?: string): string { + if (configured) { + const resolved = join(backendPath, configured); + if (existsSync(resolved)) return resolved; + throw new Error( + `Configured SAM template not found: ${resolved}\n` + + ` (set via ingest.samTemplate in .nexo/config.json)` + ); + } + + const candidates = DEFAULTS.ingest.samTemplateCandidates.map( + (c) => join(backendPath, c) + ); + for (const candidate of candidates) { + if (existsSync(candidate)) return candidate; + } + throw new Error( + `No SAM template found in ${backendPath}. Checked:\n` + + candidates.map((c) => ` - ${c}`).join("\n") + + `\n Hint: set ingest.samTemplate in .nexo/config.json` + ); +} + +/** + * Find the React app entry file (App.tsx or App.jsx). + * If `configured` is set, resolves it directly; otherwise searches candidates. + */ +function findAppEntry(frontendPath: string, configured?: string): string { + if (configured) { + const resolved = join(frontendPath, configured); + if (existsSync(resolved)) return resolved; + throw new Error( + `Configured app entry not found: ${resolved}\n` + + ` (set via ingest.appEntry in .nexo/config.json)` + ); + } + + const candidates = DEFAULTS.ingest.appEntryCandidates.map( + (c) => join(frontendPath, c) + ); + for (const candidate of candidates) { + if (existsSync(candidate)) return candidate; + } + throw new Error( + `No App entry file found in ${frontendPath}. Checked:\n` + + candidates.map((c) => ` - ${c}`).join("\n") + + `\n Hint: set ingest.appEntry in .nexo/config.json` + ); +} + /** * Run the full ingest sync. Dry-run by default unless opts.apply is true. */ @@ -53,14 +113,14 @@ export async function runSync(db: Surreal, opts: SyncOptions): Promise 0) { await syncSourceFiles(db, allSourceFiles, opts, results.sourceFiles); @@ -237,6 +297,7 @@ async function syncImplementedInEdges( const config = { frontendRoot: opts.frontendPath, backendRoot: opts.backendPath, + handlerSourceRoots: opts.handlerSourceRoots, }; // Match API endpoints to handler files