diff --git a/README.md b/README.md index b53bcaa..63c80b3 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,8 @@ For more information on configuring the CLI, see [this documentation section](ht ## Usage +### Pull + ```bash npx @dittowords/cli pull ``` @@ -70,9 +72,19 @@ See our demo projects for examples of how to integrate the Ditto CLI in differen - [iOS mobile app](https://github.com/dittowords/ditto-react-demo) - [Android mobile app](https://github.com/dittowords/ditto-react-demo) +### Scan + +```bash +npx @dittowords/cli scan +``` + +Scans your codebase at `` to identify user-facing text, and then prompts you to navigate to the Ditto web app where you can create a full content system off of the identified text. + +This feature is currently in beta, and only accessible to select partners, so message us at [support@dittowords.com](mailto:support@dittowords.com) if you would like access! + ## Legacy Setup -Beginning with `v5.0.0`, the Ditto CLI points at the new Ditto experience by default. To run the CLI compatible with legacy Ditto, append the `--legacy` flag to any legacy command, and the CLI will work as it did in the `4.x` version. All existing legacy commands remain fully functional at this time. +Beginning with `v5.0.0`, the Ditto CLI points at the new Ditto experience by default. To run the CLI compatible with legacy Ditto, append the `--legacy` flag to any legacy command, and the CLI will work as it did in the `4.x` version. All existing legacy commands remain fully functional at this time. Only the `pull` command is supported with the `--legacy` flag. ## Feedback diff --git a/esbuild.mjs b/esbuild.mjs index 383e680..2852c3e 100644 --- a/esbuild.mjs +++ b/esbuild.mjs @@ -1,4 +1,5 @@ import * as esbuild from "esbuild"; +import { execSync } from "child_process"; let define = {}; const KEYS_TO_DEFINE = [ @@ -35,6 +36,10 @@ const config = { async function main() { const result = await esbuild.build(config); + execSync( + "npx dts-bundle-generator --no-check --out-file bin/ditto.d.ts lib/ditto.ts", + { stdio: "inherit" } + ); // Output build metafile so we can analyze the bundle // size over time and check if anything unexpected is being bundled in. if (process.env.ENV === "production") { diff --git a/jest.config.ts b/jest.config.ts index 9607fea..0c16749 100644 --- a/jest.config.ts +++ b/jest.config.ts @@ -2,6 +2,10 @@ import type { Config } from "jest"; const config: Config = { transformIgnorePatterns: [], + moduleNameMapper: { + "^unicorn-magic/node$": + "/node_modules/unicorn-magic/node.js", + }, maxWorkers: 1, verbose: true, testPathIgnorePatterns: [ diff --git a/lib/ditto.ts b/lib/ditto.ts index 5f3cc54..bdf04eb 100755 --- a/lib/ditto.ts +++ b/lib/ditto.ts @@ -25,4 +25,6 @@ const main = async () => { } }; +export type * from "./src/scan/types"; + main(); diff --git a/lib/src/commands/scan.ts b/lib/src/commands/scan.ts new file mode 100644 index 0000000..9ebc3ab --- /dev/null +++ b/lib/src/commands/scan.ts @@ -0,0 +1,163 @@ +import "dotenv/config"; +import fs from "fs/promises"; +import path from "path"; + +import { prompt } from "enquirer"; +import open from "open"; + +import logger from "../utils/logger"; +import { DittoScanExtractSummary, runExtract } from "../scan/extract"; +import { DittoScanCandidate } from "../scan/types"; +import { quit } from "../utils/quit"; +import initAPIToken from "../services/apiToken/initAPIToken"; +import appContext from "../utils/appContext"; +import { + initiateClassify, + initiateScan, + uploadCandidatesToS3, +} from "../http/scan"; +import chalk from "chalk"; + +// Yarn sets INIT_CWD to the directory the user invoked yarn from, which +// matters when we proxy via `cd product-text-detection && yarn ptd`. +function resolveUserPath(input: string): string { + if (path.isAbsolute(input)) return input; + return path.resolve(process.env.INIT_CWD ?? process.cwd(), input); +} + +// Builds the standard output file paths for a given directory and optional +// prefix. Files are named "{prefix}-{artifact}.ext" when a prefix is given, +// or just "{artifact}.ext" when omitted. +function buildOutputPaths(outDir: string, prefix?: string) { + const p = prefix ? `${prefix}-` : ""; + return { + candidates: path.join(outDir, `${p}candidates.ndjson`), + }; +} + +// Serialize extracted candidates as newline-delimited JSON. Used by both the +// standalone `extract` command and `run`, which keeps the same artifact on disk. +async function writeCandidatesNdjson( + candidates: DittoScanCandidate[], + outputPath: string +): Promise { + await fs.mkdir(path.dirname(path.resolve(outputPath)), { recursive: true }); + const ndjson = + candidates.map((c) => JSON.stringify(c)).join("\n") + + (candidates.length > 0 ? "\n" : ""); + await fs.writeFile(outputPath, ndjson, "utf8"); +} + +// Telemetry for the extract phase — framework detection, file counts, +// per-detection-kind candidate counts, and the output path. +function logExtractSummary( + summary: DittoScanExtractSummary, + outputPath?: string +): void { + process.stderr.write( + `[ditto-cli scan][extract] framework: ${ + summary.framework.length > 0 + ? summary.framework.join(", ") + : "(none detected)" + }\n` + ); + process.stderr.write( + `[ditto-cli scan][extract] scanned ${summary.filesScanned} files in ${summary.elapsedMs}ms\n` + ); + if (summary.i18nFileDiscovery) { + const d = summary.i18nFileDiscovery; + process.stderr.write( + `[ditto-cli scan][extract] i18n file discovery (${d.task}): ${ + d.heuristicConfirmed + d.autoIncluded + }/${d.totalCandidates} files matched in ${d.elapsedMs}ms (preFiltered=${ + d.preFiltered + }, autoIncluded=${d.autoIncluded}, heuristicConsidered=${ + d.heuristicConsidered + }, heuristicConfirmed=${d.heuristicConfirmed})\n` + ); + } + for (const [kind, count] of Object.entries(summary.filesByKind)) { + process.stderr.write(` files.${kind}: ${count}\n`); + } + if (summary.filesSkippedMinified > 0) { + process.stderr.write( + ` files.skipped_minified: ${summary.filesSkippedMinified}\n` + ); + } + process.stderr.write( + `[ditto-cli scan][extract] emitted ${ + summary.candidatesEmitted + } candidates -> ${outputPath ?? "Ditto"}\n` + ); + for (const [kind, count] of Object.entries(summary.candidatesByKind)) { + if (count > 0) process.stderr.write(` candidates.${kind}: ${count}\n`); + } +} + +interface ISyncOptions { + local: boolean; + outDir?: string; + prefix?: string; +} +export const scan = async ( + path: string, + { local, outDir = "", prefix = "" }: ISyncOptions +) => { + if (local && !outDir) { + return await quit( + logger.errorText( + "Must specify --out-dir if outputting candidates locally" + ), + 2 + ); + } + + const resolvedInput = resolveUserPath(path); + + const { candidates, summary: extractSummary } = await runExtract({ + inputPath: resolvedInput, + }); + + if (candidates.length === 0) { + logger.warnText( + `[ditto scan] no candidates extracted; writing empty classify output\n` + ); + } + + if (local) { + const resolvedOutDir = resolveUserPath(outDir); + await fs.mkdir(resolvedOutDir, { recursive: true }); + + const { candidates: candidatesPath } = buildOutputPaths( + resolvedOutDir, + prefix + ); + + await writeCandidatesNdjson(candidates, candidatesPath); + logExtractSummary(extractSummary, candidatesPath); + } else { + const token = await initAPIToken(); + appContext.setApiToken(token); + const { + candidatesSignedS3Url, + record: { _id: recordId }, + } = await initiateScan(path); + await uploadCandidatesToS3(candidates, candidatesSignedS3Url); + await initiateClassify(recordId); + logExtractSummary(extractSummary); + const url = `https://app.dittowords.com/scan/${recordId}`; + console.log( + `Scan initiated! Visit ${chalk.blueBright.underline( + url + )} to view progress and see results.` + ); + const { openUrl } = await prompt<{ openUrl: boolean }>({ + type: "confirm", + name: "openUrl", + message: "Open in browser?", + initial: true, + }); + if (openUrl) await open(url); + await quit(null, 0); + } +}; diff --git a/lib/src/formatters/mixins/javascriptCodegenMixin.ts b/lib/src/formatters/mixins/javascriptCodegenMixin.ts index f2028ef..af4f6e8 100644 --- a/lib/src/formatters/mixins/javascriptCodegenMixin.ts +++ b/lib/src/formatters/mixins/javascriptCodegenMixin.ts @@ -1,6 +1,6 @@ import { Constructor } from "../shared"; -interface NamedImport { +export interface NamedImport { name: string; alias?: string; } @@ -9,9 +9,9 @@ export default function javascriptCodegenMixin( Base: TBase ) { return class JavascriptCodegenHelpers extends Base { - protected indentSpaces: number = 2; + public indentSpaces: number = 2; - protected sanitizeStringForJSVariableName(str: string) { + public sanitizeStringForJSVariableName(str: string) { return str.replace(/[^a-zA-Z0-9]/g, "_"); } @@ -20,7 +20,7 @@ export default function javascriptCodegenMixin( * @param modules array of { name: string, alias?: string }, each named import * @returns a string of comma-separated module names/aliases, sorted */ - protected formatNamedModules(modules: NamedImport[]) { + public formatNamedModules(modules: NamedImport[]) { return modules .map((m) => { if (m.alias) { @@ -39,7 +39,7 @@ export default function javascriptCodegenMixin( * @param moduleName the name of the file or package to import from * @returns i.e `import { foo, bar as name } from "./file";` */ - protected codegenNamedImport(modules: NamedImport[], moduleName: string) { + public codegenNamedImport(modules: NamedImport[], moduleName: string) { const formattedModules = this.formatNamedModules(modules); return `import { ${formattedModules} } from "${moduleName}";\n`; @@ -52,7 +52,7 @@ export default function javascriptCodegenMixin( * @param moduleName the name of the file or package to import from * @returns i.e `const { foo, bar as name } = require("./file");` */ - protected codegenNamedRequire(modules: NamedImport[], moduleName: string) { + public codegenNamedRequire(modules: NamedImport[], moduleName: string) { const formattedModules = this.formatNamedModules(modules); return `const { ${formattedModules} } = require("${moduleName}");\n`; @@ -64,7 +64,7 @@ export default function javascriptCodegenMixin( * @param moduleName the name of the file or package to import from * @returns i.e codegenDefaultImport("item", "./file") => `import item from "./file";` */ - protected codegenDefaultImport(module: string, moduleName: string) { + public codegenDefaultImport(module: string, moduleName: string) { return `import ${module} from "${moduleName}";\n`; } @@ -74,7 +74,7 @@ export default function javascriptCodegenMixin( * @param moduleName the name of the file or package to import from * @returns i.e codegenDefaultRequire("item", "./file") => `const item = require("./file)";` */ - protected codegenDefaultRequire(module: string, moduleName: string) { + public codegenDefaultRequire(module: string, moduleName: string) { return `const ${module} = require("${moduleName}");\n`; } @@ -83,7 +83,7 @@ export default function javascriptCodegenMixin( * @param module the name of the module to export * @returns i.e codegenDefaultExport("item") => "export default item;" */ - protected codegenDefaultExport(module: string) { + public codegenDefaultExport(module: string) { return `export default ${module};`; } @@ -92,11 +92,11 @@ export default function javascriptCodegenMixin( * @param module the name of the module to export * @returns i.e codegenModuleExports("item") => "module.exports = item;" */ - protected codegenCommonJSModuleExports(module: string) { + public codegenCommonJSModuleExports(module: string) { return `module.exports = ${module};`; } - protected codegenPad(depth: number) { + public codegenPad(depth: number) { return " ".repeat(depth * this.indentSpaces); } }; diff --git a/lib/src/http/scan.ts b/lib/src/http/scan.ts new file mode 100644 index 0000000..3cbc7b6 --- /dev/null +++ b/lib/src/http/scan.ts @@ -0,0 +1,60 @@ +import axios, { AxiosError } from "axios"; +import getHttpClient from "./client"; +import { IInitiateScanResponse, ZInitiateScanResponse } from "./types"; +import { DittoScanCandidate } from "../scan/types"; +import { Blob } from "buffer"; + +export async function initiateScan( + path: string +): Promise { + try { + const httpClient = getHttpClient({}); + const response = await httpClient.post("/v2/scan", { path }); + return ZInitiateScanResponse.parse(response.data); + } catch (e) { + if (!(e instanceof AxiosError)) { + throw new Error( + "Sorry! We're having trouble reaching the Ditto API. Please try again later." + ); + } + throw e; + } +} + +export async function initiateClassify(scanId: string): Promise { + try { + const httpClient = getHttpClient({}); + await httpClient.post(`/v2/scan/${scanId}/classify`, {}); + } catch (e) { + if (!(e instanceof AxiosError)) { + throw new Error( + "Sorry! We're having trouble reaching the Ditto API. Please try again later." + ); + } + throw e; + } +} + +export async function uploadCandidatesToS3( + candidates: DittoScanCandidate[], + signedUrl: string +) { + // Convert TypeScript array to NDJSON format (JSON objects separated by \n) + const ndjsonString = candidates + .map((candidate) => JSON.stringify(candidate)) + .join("\n"); + + // Create a Blob from the NDJSON string + const blob = new Blob([ndjsonString], { type: "application/x-ndjson" }); + + // Upload via PUT request to the pre-signed URL + // Needs to be done with a non-ditto http client so we don't pass our auth token + const httpClient = axios.create({}); + const response = await httpClient.put(signedUrl, blob, { + headers: { + // S3 requires the Content-Length header for PUT requests + "Content-Length": blob.size, + }, + }); + return response; +} diff --git a/lib/src/http/types.ts b/lib/src/http/types.ts index 96f6deb..e56b714 100644 --- a/lib/src/http/types.ts +++ b/lib/src/http/types.ts @@ -177,3 +177,12 @@ export const ZExportSwiftFileRequest = z.object({ }); export type IExportSwiftFileRequest = z.infer; + +export const ZInitiateScanBodySchema = z.object({ + path: z.string(), +}); +export const ZInitiateScanResponse = z.object({ + record: z.object({ _id: z.string() }), + candidatesSignedS3Url: z.string(), +}); +export type IInitiateScanResponse = z.infer; diff --git a/lib/src/index.ts b/lib/src/index.ts index b197503..294e512 100644 --- a/lib/src/index.ts +++ b/lib/src/index.ts @@ -3,132 +3,100 @@ import * as Sentry from "@sentry/node"; import { program } from "commander"; import { pull } from "./commands/pull"; +import { scan } from "./commands/scan"; import { quit } from "./utils/quit"; import { version } from "../../package.json"; import logger from "./utils/logger"; import initAPIToken from "./services/apiToken/initAPIToken"; import { initProjectConfig } from "./services/projectConfig"; import appContext from "./utils/appContext"; -import type commander from "commander"; import { ErrorType, isDittoError, isDittoErrorType } from "./utils/DittoError"; import processCommandMetaFlag from "./utils/processCommandMetaFlag"; -type Command = "pull"; - -interface CommandConfig { - name: T; - description: string; - commands?: CommandConfig<"add" | "remove">[]; - flags?: { - [flag: string]: { description: string; processor?: (value: string) => any }; - }; -} - -const COMMANDS: CommandConfig[] = [ - { - name: "pull", - description: "Sync copy from Ditto", - flags: { - "-c, --config [value]": { - description: - "Relative path to the project config file. Defaults to `./ditto/config.yml`. Alternatively, you can set the DITTO_PROJECT_CONFIG_FILE environment variable.", - }, - }, - }, -]; +const handleCommandError = async (error: unknown) => { + if (process.env.DEBUG === "true") { + console.error(logger.info("Development stack trace:\n"), error); + } -const setupCommands = () => { - program.name("ditto-cli"); + let sentryOptions = undefined; + let exitCode = undefined; + let errorText = + "Something went wrong. Please contact support or try again later."; - COMMANDS.forEach((commandConfig) => { - const cmd = program - .command(commandConfig.name) - .description(commandConfig.description) - .action((options) => { - return executeCommand(commandConfig.name, options); - }); + if (isDittoError(error)) { + exitCode = error.exitCode; - if (commandConfig.flags) { - Object.entries(commandConfig.flags).forEach( - ([flags, { description, processor }]) => { - if (processor) { - cmd.option(flags, description, processor); - } else { - cmd.option(flags, description); - } - } - ); + if (isDittoErrorType(error, ErrorType.ConfigYamlLoadError)) { + errorText = error.message; + } else if (isDittoErrorType(error, ErrorType.ConfigParseError)) { + errorText = `${error.data.messagePrefix}\n\n${error.data.formattedError}`; } - }); -}; -const setupOptions = () => { - program.option("--legacy", "Run in legacy mode"); - program.option( - "-m, --meta ", - "Include arbitrary data in requests to the Ditto API. Ex: -m githubActionRequest:true trigger:manual" - ); - program.version(version, "-v, --version", "Output the current version"); -}; + sentryOptions = { + extra: { message: errorText, ...(error.data || {}) }, + }; + } -const executeCommand = async ( - commandName: Command | "none", - command: commander.Command -): Promise => { - try { - const options = command.opts(); - const token = await initAPIToken(); - appContext.setApiToken(token); + const eventId = Sentry.captureException(error, sentryOptions); + const eventStr = `\n\nError ID: ${logger.info(eventId)}`; - await initProjectConfig(options); + return await quit(logger.errorText(errorText) + eventStr, exitCode); +}; - const { meta } = program.opts(); +const appEntry = async () => { + program.name("ditto-cli"); - switch (commandName) { - case "none": - case "pull": { - return await pull(processCommandMetaFlag(meta)); - } - default: { - await quit(`Invalid command: ${commandName}. Exiting Ditto CLI...`); - return; + // ditto pull + program + .command("pull") + .description("Sync copy from Ditto") + .option( + "-c, --config [value]", + "Relative path to the project config file. Defaults to `./ditto/config.yml`. Alternatively, you can set the DITTO_PROJECT_CONFIG_FILE environment variable." + ) + .option( + "-m, --meta ", + "Include arbitrary data in requests to the Ditto API. Ex: -m githubActionRequest:true trigger:manual" + ) + .option("--legacy", "Run in legacy mode") + .action(async (opts: { config?: string; meta?: string[] }) => { + try { + const token = await initAPIToken(); + appContext.setApiToken(token); + await initProjectConfig(opts); + return await pull(processCommandMetaFlag(opts.meta ?? null)); + } catch (error) { + handleCommandError(error); } - } - } catch (error) { - if (process.env.DEBUG === "true") { - console.error(logger.info("Development stack trace:\n"), error); - } - - let sentryOptions = undefined; - let exitCode = undefined; - let errorText = - "Something went wrong. Please contact support or try again later."; - - if (isDittoError(error)) { - exitCode = error.exitCode; - - if (isDittoErrorType(error, ErrorType.ConfigYamlLoadError)) { - errorText = error.message; - } else if (isDittoErrorType(error, ErrorType.ConfigParseError)) { - errorText = `${error.data.messagePrefix}\n\n${error.data.formattedError}`; + }); + + // ditto scan + program + .command("scan ") + .description( + "Run extract + classify + infer end-to-end; emits schema.json and stagings.ndjson to --out-dir" + ) + .option( + "--local", + "outputs the candidates file locally to the out-dir with the given prefix", + false + ) + .option("--out-dir ", "output directory", "") + .option("--prefix ", "prefix for output files", "") + .action( + async ( + inputPath: string, + opts: { local: boolean; outDir: string; prefix?: string } + ) => { + try { + return await scan(inputPath, opts); + } catch (error) { + handleCommandError(error); + } } + ); - sentryOptions = { - extra: { message: errorText, ...(error.data || {}) }, - }; - } - - const eventId = Sentry.captureException(error, sentryOptions); - const eventStr = `\n\nError ID: ${logger.info(eventId)}`; - - return await quit(logger.errorText(errorText) + eventStr, exitCode); - } -}; - -const appEntry = async () => { - setupCommands(); - setupOptions(); - + program.version(version, "-v, --version", "Output the current version"); program.parse(process.argv); }; diff --git a/lib/src/scan/extract.ts b/lib/src/scan/extract.ts new file mode 100644 index 0000000..fdde4f0 --- /dev/null +++ b/lib/src/scan/extract.ts @@ -0,0 +1,311 @@ +import fs from "fs/promises"; +import { globby } from "globby"; +import path from "path"; + +import { + DittoScanDetectionKindSchema, + type DittoScanCandidate, + type DittoScanDetectionKind, +} from "./types"; +import { createHash } from "crypto"; +import type { FileDiscoveryStats } from "./lang/file-discovery"; +import { shouldEmit } from "./rules"; +import { walkCodebase } from "./walk"; + +export interface DittoScanExtractOptions { + inputPath: string; +} + +export interface DittoScanExtractResult { + candidates: DittoScanCandidate[]; + summary: DittoScanExtractSummary; +} + +export interface DittoScanExtractSummary { + filesScanned: number; + filesByKind: Record; + filesSkippedMinified: number; + candidatesEmitted: number; + candidatesByKind: Record; + framework: string[]; + elapsedMs: number; + i18nFileDiscovery: FileDiscoveryStats | null; +} + +const CONTEXT_LINES = 3; +const MAX_CONTEXT_LINE_CHARS = 200; + +// Maps a dependency name in package.json to the framework token we surface +// to the LLM. Only frameworks that meaningfully shift the user-facing +// likelihood of strings are listed (UI frameworks, server frameworks). +const FRAMEWORK_MARKERS: ReadonlyArray< + readonly [pattern: RegExp, token: string] +> = [ + [/^react-native$/, "react-native"], + [/^react(-dom)?$/, "react"], + [/^next$/, "next"], + [/^@remix-run\//, "remix"], + [/^vue$/, "vue"], + [/^nuxt$/, "nuxt"], + [/^svelte$/, "svelte"], + [/^@sveltejs\/kit$/, "sveltekit"], + [/^@angular\/core$/, "angular"], + [/^solid-js$/, "solid"], + [/^preact$/, "preact"], + [/^astro$/, "astro"], + [/^express$/, "express"], + [/^fastify$/, "fastify"], + [/^koa$/, "koa"], + [/^@nestjs\/core$/, "nestjs"], + [/^@hapi\/hapi$/, "hapi"], +]; + +function zeroKindCounts(): Record { + return Object.fromEntries( + DittoScanDetectionKindSchema.options.map((k) => [k, 0]) + ) as Record; +} + +function buildSourceContext(lines: string[], targetLine: number): string { + const start = Math.max(1, targetLine - CONTEXT_LINES); + const end = Math.min(lines.length, targetLine + CONTEXT_LINES); + const out: string[] = []; + for (let i = start; i <= end; i++) { + const raw = lines[i - 1] ?? ""; + const text = + raw.length > MAX_CONTEXT_LINE_CHARS + ? raw.slice(0, MAX_CONTEXT_LINE_CHARS) + "…(truncated)" + : raw; + out.push(`${i}: ${text}`); + } + return out.join("\n"); +} + +const VCS_MARKERS = [".git", ".hg", ".svn"] as const; + +// Aggregates deps from every package.json: +// - Walking *up* from inputPath to the repo root, since monorepo +// subpackages often have a near-empty package.json with the real +// deps living one or more levels up. +// - Walking *down* from inputPath via gitignore-aware globby, since +// the inverse is also common: pnpm/yarn workspace monorepos where +// the root package.json is empty and react/vue/etc. live in +// `packages/*/package.json`. Without this pass, large frontend +// monorepos (excalidraw, nx-style repos) would surface +// `framework: (none detected)`. +// +// On top of the package.json passes, we sniff for native Android/iOS +// project markers anywhere under inputPath so a Gradle-only or +// Xcode-only project still surfaces an `android` / `ios` token to the +// LLM. We never read the marker contents — presence alone is the signal. +async function detectFramework(inputPath: string): Promise { + const tokens = new Set(); + + const ingestPackageJson = (raw: string) => { + let pkg: { + dependencies?: Record; + devDependencies?: Record; + }; + try { + pkg = JSON.parse(raw); + } catch { + return; + } + for (const name of Object.keys({ + ...(pkg.dependencies ?? {}), + ...(pkg.devDependencies ?? {}), + })) { + for (const [pattern, token] of FRAMEWORK_MARKERS) { + if (pattern.test(name)) tokens.add(token); + } + } + }; + + // Upward walk (inputPath → repo root). + let dir = path.resolve(inputPath); + while (true) { + try { + ingestPackageJson( + await fs.readFile(path.join(dir, "package.json"), "utf8") + ); + } catch { + // no package.json here, keep walking + } + if (await isRepoRoot(dir)) break; + const parent = path.dirname(dir); + if (parent === dir) break; + dir = parent; + } + + // Downward walk (inputPath/**/package.json). Honors .gitignore and + // skips the usual large-directory denylist so we don't spelunk through + // node_modules / build artifacts. + const nested = await globby(["**/package.json"], { + cwd: path.resolve(inputPath), + gitignore: true, + onlyFiles: true, + ignore: [ + "**/node_modules/**", + "**/.git/**", + "**/build/**", + "**/dist/**", + "**/.next/**", + "**/.nuxt/**", + ], + followSymbolicLinks: false, + suppressErrors: true, + absolute: true, + }); + for (const pkgPath of nested) { + try { + ingestPackageJson(await fs.readFile(pkgPath, "utf8")); + } catch { + // unreadable / malformed, ignore + } + } + + for (const platformToken of await detectMobilePlatforms(inputPath)) { + tokens.add(platformToken); + } + + return [...tokens].sort(); +} + +const ANDROID_MARKER_RE = + /(?:^|\/)(?:AndroidManifest\.xml|build\.gradle(?:\.kts)?|settings\.gradle(?:\.kts)?)$/; +const IOS_MARKER_RE = + /(?:^|\/)(?:[^/]+\.xcodeproj\/project\.pbxproj|Package\.swift|Podfile)$/; + +async function detectMobilePlatforms(inputPath: string): Promise { + const tokens = new Set(); + const matches = await globby( + [ + "**/AndroidManifest.xml", + "**/build.gradle", + "**/build.gradle.kts", + "**/settings.gradle", + "**/settings.gradle.kts", + "**/*.xcodeproj/project.pbxproj", + "**/Package.swift", + "**/Podfile", + ], + { + cwd: path.resolve(inputPath), + gitignore: true, + onlyFiles: true, + ignore: ["**/node_modules/**", "**/.git/**", "**/build/**", "**/Pods/**"], + followSymbolicLinks: false, + suppressErrors: true, + } + ); + for (const m of matches) { + if (ANDROID_MARKER_RE.test(m)) tokens.add("android"); + if (IOS_MARKER_RE.test(m)) tokens.add("ios"); + } + return [...tokens]; +} + +async function isRepoRoot(dir: string): Promise { + for (const marker of VCS_MARKERS) { + try { + await fs.access(path.join(dir, marker)); + return true; + } catch { + // marker not present, try the next one + } + } + return false; +} + +// Deterministic id for a candidate so the same string in the same place gets +// the same id across runs. +export function makeCandidateId( + file: string, + line: number, + column: number, + value: string +): string { + return createHash("sha1") + .update(`${file}:${line}:${column}:${value}`) + .digest("hex") + .slice(0, 12); +} + +export async function runExtract( + opts: DittoScanExtractOptions +): Promise { + const t0 = Date.now(); + const framework = await detectFramework(opts.inputPath); + const { files, filesSkippedMinified, i18nFileDiscovery } = await walkCodebase( + opts.inputPath + ); + + const filesByKind: Record = {}; + for (const f of files) + filesByKind[f.language.id] = (filesByKind[f.language.id] ?? 0) + 1; + + const candidatesByKind = zeroKindCounts(); + const candidates: DittoScanCandidate[] = []; + + for (const file of files) { + const lang = file.language; + let hits; + try { + hits = await lang.extractor.extract({ + source: file.source, + kind: lang.id, + }); + } catch (e) { + process.stderr.write( + `[ptd extract] failed on ${file.relPath} (${lang.id}): ${ + (e as Error).message + }\n` + ); + continue; + } + + const lines = file.source.split(/\r?\n/); + + for (const hit of hits) { + if (!shouldEmit(hit.value, hit.context)) continue; + const candidate: DittoScanCandidate = { + id: makeCandidateId( + file.relPath, + hit.location.line, + hit.location.column, + hit.value + ), + value_raw: hit.value, + detection_kind: hit.context.parentRole, + location: { + file: file.relPath, + line: hit.location.line, + column: hit.location.column, + }, + language: file.languageLabel, + locale_key: hit.localeKey ?? file.localeKey, + i18n_key: hit.i18nKey ?? null, + framework, + source_context: buildSourceContext(lines, hit.location.line), + context_identifiers: hit.context.identifiers, + usage_evidence: null, + }; + candidates.push(candidate); + candidatesByKind[hit.context.parentRole]++; + } + } + + return { + candidates, + summary: { + filesScanned: files.length, + filesByKind, + filesSkippedMinified, + candidatesEmitted: candidates.length, + candidatesByKind, + framework, + elapsedMs: Date.now() - t0, + i18nFileDiscovery, + }, + }; +} diff --git a/lib/src/scan/lang/extractors/android-resources.test.ts b/lib/src/scan/lang/extractors/android-resources.test.ts new file mode 100644 index 0000000..e880325 --- /dev/null +++ b/lib/src/scan/lang/extractors/android-resources.test.ts @@ -0,0 +1,67 @@ +import { androidResourceExtractor } from "./android-resources"; + +const extract = (source: string) => androidResourceExtractor.extract({ source, kind: "android_resources" }); + +describe("androidResourceExtractor", () => { + test("emits values keyed by name", async () => { + const source = [ + ``, + ``, + ` Hello, world!`, + ` Goodbye`, + ``, + ``, + ].join("\n"); + const hits = await extract(source); + expect(hits.map((h) => ({ v: h.value, ids: h.context.identifiers }))).toEqual([ + { v: "Hello, world!", ids: ["hello"] }, + { v: "Goodbye", ids: ["goodbye"] }, + ]); + expect(hits[0].context.parentRole).toBe("resource_value"); + }); + + test("emits one hit per item, tagged with the quantity", async () => { + const source = [ + ``, + ` `, + ` %d item`, + ` %d items`, + ` `, + ``, + ``, + ].join("\n"); + const hits = await extract(source); + expect(hits).toHaveLength(2); + expect(hits[0].context.identifiers).toEqual(["items", "one"]); + expect(hits[1].context.identifiers).toEqual(["items", "other"]); + }); + + test("emits one hit per item, indexed numerically", async () => { + const source = [ + ``, + ` `, + ` Mercury`, + ` Venus`, + ` `, + ``, + ``, + ].join("\n"); + const hits = await extract(source); + expect(hits.map((h) => h.value)).toEqual(["Mercury", "Venus"]); + expect(hits[0].context.identifiers).toEqual(["planets", "0"]); + expect(hits[1].context.identifiers).toEqual(["planets", "1"]); + }); + + test("recovers CDATA-wrapped values via the regex sweep", async () => { + const source = [ + ``, + ` Welcome]]>`, + ``, + ``, + ].join("\n"); + const hits = await extract(source); + const welcome = hits.find((h) => h.value === "Welcome"); + expect(welcome).toBeDefined(); + expect(welcome?.context.identifiers).toEqual(["welcome"]); + }); +}); diff --git a/lib/src/scan/lang/extractors/android-resources.ts b/lib/src/scan/lang/extractors/android-resources.ts new file mode 100644 index 0000000..ae874ff --- /dev/null +++ b/lib/src/scan/lang/extractors/android-resources.ts @@ -0,0 +1,112 @@ +import { Lang, parse, type SgNode } from "@ast-grep/napi"; + +import type { ExtractedHit, LanguageExtractor } from "../types"; +import { offsetToLineCol } from "./util"; +import { elementAttribute, emitTextHit, findCdataElements, tagName } from "./xml"; + +/** + * Android `res/values/*.xml` resource files. Three top-level shapes: + * + * Hello, world! + * + * %d item + * %d items + * + * + * Mercury + * Venus + * + * + * Every value is intentional product copy, so all hits flow out as + * `resource_value`. The resource key (and the variant for plurals or the + * index for arrays) is surfaced through `identifiers` so downstream sees + * what's going on without re-parsing. + */ +export const androidResourceExtractor: LanguageExtractor = { + async extract({ source }) { + const root = parse(Lang.Html, source).root(); + const out: ExtractedHit[] = []; + + for (const el of root.findAll({ rule: { kind: "element" } })) { + const tag = tagName(el); + if (tag === "string") { + emitStringElement(el, out, source); + } else if (tag === "plurals" || tag === "string-array") { + const parentName = elementAttribute(el, "name") ?? ""; + let index = 0; + for (const child of el.children()) { + if (child.kind() !== "element" || tagName(child) !== "item") continue; + const variant = tag === "plurals" ? elementAttribute(child, "quantity") ?? "" : String(index); + // The resource name is the lookup key; quantity/index are selectors. + emitTextHit(child, [parentName, variant], out, source, parentName || undefined); + index++; + } + } + } + + emitCdataValues(source, out); + + return out; + }, +}; + +const NAME_ATTR_RE = /\bname\s*=\s*"([^"]*)"/; +const QUANTITY_ATTR_RE = /\bquantity\s*=\s*"([^"]*)"/; + +function emitCdataValues(source: string, out: ExtractedHit[]): void { + for (const m of findCdataElements(source, ["string", "item"])) { + const name = NAME_ATTR_RE.exec(m.attrsRaw)?.[1]; + const quantity = QUANTITY_ATTR_RE.exec(m.attrsRaw)?.[1]; + const parent = m.tag === "item" ? findEnclosingResourceParent(source, m.offset) : null; + if (m.tag === "item" && !parent) continue; + const { line, column } = offsetToLineCol(source, m.valueOffset); + out.push({ + value: m.value, + location: { line, column }, + context: { + parentRole: "resource_value", + identifiers: buildItemIdentifiers({ name, quantity, parent }), + }, + i18nKey: name || parent || undefined, + }); + } +} + +function buildItemIdentifiers({ + name, + quantity, + parent, +}: { + name: string | undefined; + quantity: string | undefined; + parent: string | null; +}): string[] { + // `` CDATA: name carries the identity directly. + if (name) return quantity ? [name, quantity] : [name]; + // `` CDATA inside /: use the enclosing parent's + // name so the identifier matches the non-CDATA path's [parentName, variant]. + // The array-item index isn't reconstructable from a regex sweep, so plain + // items collapse to just [parentName]. + if (parent) return quantity ? [parent, quantity] : [parent]; + return quantity ? [quantity] : []; +} + +// Walks open/close tags up to `before` and returns the `name` of the +// innermost active ``/`` ancestor, or null if there +// is no enclosing one. +function findEnclosingResourceParent(source: string, before: number): string | null { + const re = /<(plurals|string-array)\b([^>]*)>|<\/(plurals|string-array)>/g; + const stack: string[] = []; + let m: RegExpExecArray | null; + while ((m = re.exec(source)) !== null) { + if (m.index >= before) break; + if (m[1]) stack.push(NAME_ATTR_RE.exec(m[2])?.[1] ?? ""); + else stack.pop(); + } + return stack[stack.length - 1] ?? null; +} + +function emitStringElement(el: SgNode, out: ExtractedHit[], source: string): void { + const name = elementAttribute(el, "name"); + emitTextHit(el, name ? [name] : [], out, source, name ?? undefined); +} diff --git a/lib/src/scan/lang/extractors/arb.ts b/lib/src/scan/lang/extractors/arb.ts new file mode 100644 index 0000000..a43dd7e --- /dev/null +++ b/lib/src/scan/lang/extractors/arb.ts @@ -0,0 +1,23 @@ +import type { LanguageExtractor } from "../types"; +import { jsonI18nExtractor } from "./json-i18n"; + +/** + * Flutter Application Resource Bundles (`.arb`). ARB is plain JSON with one + * convention: keys prefixed with `@` are metadata, not product copy. + * + * { + * "@@locale": "en", + * "hello": "Hello", + * "@hello": { "description": "greeting", "placeholders": {...} } + * } + * + * `@@locale` / `@@last_modified` are file-level metadata; `@` holds the + * description and placeholder spec for ``. Reuse the JSON walker and + * drop any hit whose identifier chain crosses an `@`-prefixed key. + */ +export const arbExtractor: LanguageExtractor = { + async extract(opts) { + const hits = await jsonI18nExtractor.extract(opts); + return hits.filter((h) => !h.context.identifiers.some((id) => id.startsWith("@"))); + }, +}; diff --git a/lib/src/scan/lang/extractors/fallback.test.ts b/lib/src/scan/lang/extractors/fallback.test.ts new file mode 100644 index 0000000..b4982be --- /dev/null +++ b/lib/src/scan/lang/extractors/fallback.test.ts @@ -0,0 +1,41 @@ +import { fallbackExtractor } from "./fallback"; + +const extract = (source: string) => fallbackExtractor.extract({ source, kind: "regex_fallback" }); + +describe("fallbackExtractor", () => { + test("emits one hit per quoted literal with 1-based line/column", async () => { + const hits = await extract(`const greeting = "Hello, world";\n`); + expect(hits).toEqual([ + { + value: "Hello, world", + location: { line: 1, column: 18 }, + context: { parentRole: "other", identifiers: [] }, + }, + ]); + }); + + test("handles single, double, and backtick quotes", async () => { + const hits = await extract(`x = "a"\ny = 'b'\nz = \`c\`\n`); + expect(hits.map((h) => h.value)).toEqual(["a", "b", "c"]); + expect(hits.map((h) => h.location.line)).toEqual([1, 2, 3]); + }); + + test("emits multiple literals on the same line", async () => { + const hits = await extract(`pair("a", "b")\n`); + expect(hits).toHaveLength(2); + expect(hits[0].value).toBe("a"); + expect(hits[1].value).toBe("b"); + expect(hits[0].location.column).toBeLessThan(hits[1].location.column); + }); + + test("respects escaped quotes inside a literal", async () => { + const hits = await extract(`msg = "He said \\"hi\\""\n`); + expect(hits).toHaveLength(1); + expect(hits[0].value).toBe('He said \\"hi\\"'); + }); + + test("returns an empty array when no literals are present", async () => { + const hits = await extract(`const x = 42;\n// no strings here\n`); + expect(hits).toEqual([]); + }); +}); diff --git a/lib/src/scan/lang/extractors/fallback.ts b/lib/src/scan/lang/extractors/fallback.ts new file mode 100644 index 0000000..e1973e3 --- /dev/null +++ b/lib/src/scan/lang/extractors/fallback.ts @@ -0,0 +1,43 @@ +import type { ExtractedHit, LanguageExtractor } from "../types"; + +/** + * Regex-based string finder used for any source file that isn't supported by + * one of our extractors. + * + * Scan each line for `"..."`, `'...'`, or `` `...` `` literals + * and emit them with `parentRole: "other"`. The LLM + * phase reads `source_context` to decide whether each hit is user-facing. + * + * Known limitations - anything caught here is acceptable noise that + * downstream filters or the LLM will handle: + * - Multi-line strings (Python triple-quoted, Go backticks, etc.) get + * truncated at the first newline. + * - Block comments are not stripped; string-like content inside `\/* *\/` + * or `"""..."""` leaks through. + * - Line comments are not stripped either — strings inside `# foo`, + * `// foo`, `-- foo` leak through. (Stripping these would also drop + * real strings like `"#abc"` or `"https://..."`, so we don't bother.) + * - Escape sequences inside strings are matched but not interpreted. + */ +const STRING_RE = /(["'`])((?:(?!\1)[^\\\n]|\\.)*)\1/g; + +export const fallbackExtractor: LanguageExtractor = { + async extract({ source }) { + const out: ExtractedHit[] = []; + const lines = source.split(/\r?\n/); + + for (let i = 0; i < lines.length; i++) { + STRING_RE.lastIndex = 0; + let m: RegExpExecArray | null; + while ((m = STRING_RE.exec(lines[i])) !== null) { + out.push({ + value: m[2], + location: { line: i + 1, column: m.index + 1 }, + context: { parentRole: "other", identifiers: [] }, + }); + } + } + + return out; + }, +}; diff --git a/lib/src/scan/lang/extractors/html-markup.test.ts b/lib/src/scan/lang/extractors/html-markup.test.ts new file mode 100644 index 0000000..58de204 --- /dev/null +++ b/lib/src/scan/lang/extractors/html-markup.test.ts @@ -0,0 +1,50 @@ +import { htmlMarkupExtractor } from "./html-markup"; + +const extract = (source: string) => htmlMarkupExtractor.extract({ source, kind: "html" }); + +describe("htmlMarkupExtractor", () => { + test("emits element text as markup_text tagged with its parent tag", async () => { + const hits = await extract(`

Welcome home

\n`); + const text = hits.find((h) => h.value.trim() === "Welcome home"); + expect(text?.context.parentRole).toBe("markup_text"); + expect(text?.context.parentTag).toBe("p"); + }); + + test("emits plain attribute values as markup_attr with attribute name", async () => { + const hits = await extract(`\n`); + const email = hits.find((h) => h.value === "Email"); + expect(email?.context.parentRole).toBe("markup_attr"); + expect(email?.context.identifiers).toEqual(["placeholder"]); + }); + + test("skips contents of \n

Real copy

\n` + ); + const values = hits.map((h) => h.value.trim()); + expect(values).toContain("Real copy"); + expect(values).not.toContain("Hi"); + }); + + test("skips framework directive attributes (`@click`, `:foo`, `v-on`)", async () => { + const hits = await extract(`\n`); + const attrNames = hits.filter((h) => h.context.parentRole === "markup_attr").flatMap((h) => h.context.identifiers); + expect(attrNames).not.toContain("@click"); + expect(attrNames).not.toContain(":class"); + expect(attrNames).not.toContain("v-bind:id"); + expect(hits.some((h) => h.value.trim() === "Go")).toBe(true); + }); + + test("suppresses parentTag for descendants of /
", async () => {
+    const hits = await extract(`
foo bar
\n`);
+    const inner = hits.find((h) => h.value.trim() === "foo bar");
+    expect(inner?.context.parentRole).toBe("markup_text");
+    expect(inner?.context.parentTag).toBeUndefined();
+  });
+
+  test("extracts inner literals from template interpolation expressions", async () => {
+    const hits = await extract(`
{{ isFollowing ? 'Unfollow' : 'Follow' }}
\n`);
+    const values = hits.filter((h) => h.context.parentRole === "markup_text").map((h) => h.value);
+    expect(values).toEqual(expect.arrayContaining(["Unfollow", "Follow"]));
+  });
+});
diff --git a/lib/src/scan/lang/extractors/html-markup.ts b/lib/src/scan/lang/extractors/html-markup.ts
new file mode 100644
index 0000000..d0239b8
--- /dev/null
+++ b/lib/src/scan/lang/extractors/html-markup.ts
@@ -0,0 +1,181 @@
+import { Lang, parse, type SgNode } from "@ast-grep/napi";
+
+import type { ExtractedHit, LanguageExtractor } from "../types";
+
+// Tags whose contents are code-shaped, not human copy. If the candidate
+// sits inside any of these up the ancestor chain we suppress parentTag so
+// the markup_text accept rule falls through to the LLM.
+const CODE_LIKE_HTML_ANCESTORS: ReadonlySet = new Set(["pre", "code", "kbd", "samp", "var", "script", "style"]);
+const INTERPOLATION_LITERAL_RE = /(["'`])((?:\\.|(?!\1).)+?)\1/g;
+
+export const htmlMarkupExtractor: LanguageExtractor = {
+  async extract({ source }) {
+    return extractHtmlMarkup(source);
+  },
+};
+
+export function extractHtmlMarkup(source: string): ExtractedHit[] {
+  const root = parse(Lang.Html, source).root();
+  const out: ExtractedHit[] = [];
+
+  for (const el of root.findAll({ rule: { kind: "element" } })) {
+    if (isInsideScriptOrStyle(el)) continue;
+    emitElementMarkup(el, out);
+  }
+
+  return out;
+}
+
+/**
+ * ``,
+      ``,
+    ].join("\n");
+    const hits = await extract(source);
+
+    const hello = hits.find((h) => h.value.trim() === "Hello world");
+    expect(hello?.context.parentRole).toBe("markup_text");
+    expect(hello?.context.parentTag).toBe("p");
+
+    const fromScript = hits.find((h) => h.value === '"Hi from script"');
+    expect(fromScript?.context.parentRole).toBe("other");
+  });
+
+  test("script positions are reported against the original .vue file", async () => {
+    const source = [``, ``, ``].join("\n");
+    const hits = await extract(source);
+    const fromScript = hits.find((h) => h.value === '"row2"');
+    expect(fromScript?.location.line).toBe(3);
+  });
+
+  test("ignores `, ``].join(
+      "\n"
+    );
+    const hits = await extract(source);
+    const values = hits.map((h) => h.value.trim());
+    expect(values).toContain("Visible");
+    expect(values).not.toContain("hidden");
+  });
+
+  test("returns empty array for an empty SFC", async () => {
+    const hits = await extract(`\n\n`);
+    expect(hits).toEqual([]);
+  });
+});
diff --git a/lib/src/scan/lang/extractors/vue.ts b/lib/src/scan/lang/extractors/vue.ts
new file mode 100644
index 0000000..470690c
--- /dev/null
+++ b/lib/src/scan/lang/extractors/vue.ts
@@ -0,0 +1,89 @@
+import { Lang, parse, type SgNode } from "@ast-grep/napi";
+
+import type { ExtractedHit, LanguageExtractor } from "../types";
+import { extractHtmlMarkup } from "./html-markup";
+import { javascriptExtractor } from "./javascript";
+
+/**
+ * Vue Single-File Component (SFC) extractor. A `.vue` file has three
+ * top-level blocks:
+ *
+ *