[DIT-13008] Implement baseline 'scan' command

README.md

@@ -54,6 +54,8 @@ For more information on configuring the CLI, see [this documentation section](ht
 
 ## Usage
 
+### Pull
+
 ```bash
 npx @dittowords/cli pull
 ```
@@ -70,6 +72,28 @@ See our demo projects for examples of how to integrate the Ditto CLI in differen
 - [iOS mobile app](https://github.com/dittowords/ditto-react-demo)
 - [Android mobile app](https://github.com/dittowords/ditto-react-demo)
 
+### Scan
+
+```bash
+npx @dittowords/cli scan <path>
+```
+
+Scans your codebase at `<path>` to identify user-facing text. Results are written to an output directory as a set of structured artifacts.
+
+**Options:**
+
+| Option              | Description                     | Default  |
+| ------------------- | ------------------------------- | -------- |
+| `--out-dir <dir>`   | Directory to write output files | `./out`  |
+| `--prefix <prefix>` | Prefix for output file names    | _(none)_ |
+
+**Output files** (written to `--out-dir`):
+
+- `candidates.ndjson` — all string candidates extracted from the codebase
+- `results.ndjson` — classified results for each candidate
+- `summary.json` — aggregate statistics from the classify phase
+- `resultsToVerify.ndjson` — results flagged for manual review
+
 ## Legacy Setup
 
 Beginning with `v5.0.0`, the Ditto CLI points at the new Ditto experience by default. To run the CLI compatible with legacy Ditto, append the `--legacy` flag to any legacy command, and the CLI will work as it did in the `4.x` version. All existing legacy commands remain fully functional at this time.

esbuild.mjs

@@ -1,4 +1,5 @@
 import * as esbuild from "esbuild";
+import { execSync } from "child_process";
 
 let define = {};
 const KEYS_TO_DEFINE = [
@@ -35,6 +36,10 @@ const config = {
 
 async function main() {
   const result = await esbuild.build(config);
+  execSync(
+    "npx dts-bundle-generator --no-check --out-file bin/ditto.d.ts lib/ditto.ts",
+    { stdio: "inherit" }
+  );
   // Output build metafile so we can analyze the bundle
   // size over time and check if anything unexpected is being bundled in.
   if (process.env.ENV === "production") {

jest.config.ts

@@ -2,6 +2,10 @@ import type { Config } from "jest";
 
 const config: Config = {
   transformIgnorePatterns: [],
+  moduleNameMapper: {
+    "^unicorn-magic/node$":
+      "<rootDir>/node_modules/unicorn-magic/node.js",
+  },
   maxWorkers: 1,
   verbose: true,
   testPathIgnorePatterns: [

lib/ditto.ts

@@ -25,4 +25,6 @@ const main = async () => {
   }
 };
 
+export type * from "./src/scan/types";
+
 main();

lib/src/commands/scan.ts

@@ -0,0 +1,187 @@
+import "dotenv/config";
+import fs from "fs/promises";
+import path from "path";
+
+import logger from "../utils/logger";
+import { quit } from "../utils/quit";
+import { DittoScanExtractSummary, runExtract } from "../scan/extract";
+import { DittoScanCandidate, DittoScanResult } from "../scan/types";
+import { preClassify } from "../scan/preClassify";
+import { DittoScanClassifyResult, runClassify } from "../scan/classify";
+
+// Yarn sets INIT_CWD to the directory the user invoked yarn from, which
+// matters when we proxy via `cd product-text-detection && yarn ptd`.
+function resolveUserPath(input: string): string {
+  if (path.isAbsolute(input)) return input;
+  return path.resolve(process.env.INIT_CWD ?? process.cwd(), input);
+}
+
+// Builds the standard output file paths for a given directory and optional
+// prefix. Files are named "{prefix}-{artifact}.ext" when a prefix is given,
+// or just "{artifact}.ext" when omitted.
+function buildOutputPaths(outDir: string, prefix?: string) {
+  const p = prefix ? `${prefix}-` : "";
+  return {
+    candidates: path.join(outDir, `${p}candidates.ndjson`),
+    results: path.join(outDir, `${p}results.ndjson`),
+    summary: path.join(outDir, `${p}summary.json`),
+    verify: path.join(outDir, `${p}resultsToVerify.ndjson`),
+    analysis: path.join(outDir, `${p}analysis.json`),
+    schema: path.join(outDir, `${p}schema.json`),
+    stagings: path.join(outDir, `${p}stagings.ndjson`),
+  };
+}
+
+// Serialize extracted candidates as newline-delimited JSON. Used by both the
+// standalone `extract` command and `run`, which keeps the same artifact on disk.
+async function writeCandidatesNdjson(
+  candidates: DittoScanCandidate[],
+  outputPath: string
+): Promise<void> {
+  await fs.mkdir(path.dirname(path.resolve(outputPath)), { recursive: true });
+  const ndjson =
+    candidates.map((c) => JSON.stringify(c)).join("\n") +
+    (candidates.length > 0 ? "\n" : "");
+  await fs.writeFile(outputPath, ndjson, "utf8");
+}
+
+// Telemetry for the extract phase — framework detection, file counts,
+// per-detection-kind candidate counts, and the output path.
+function logExtractSummary(
+  summary: DittoScanExtractSummary,
+  outputPath: string
+): void {
+  process.stderr.write(
+    `[ditto-cli scan][extract] framework: ${
+      summary.framework.length > 0
+        ? summary.framework.join(", ")
+        : "(none detected)"
+    }\n`
+  );
+  process.stderr.write(
+    `[ditto-cli scan][extract] scanned ${summary.filesScanned} files in ${summary.elapsedMs}ms\n`
+  );
+  if (summary.i18nFileDiscovery) {
+    const d = summary.i18nFileDiscovery;
+    process.stderr.write(
+      `[ditto-cli scan][extract] llm file discovery (${d.task}): ${
+        d.llmConfirmed + d.autoIncluded
+      }/${d.totalCandidates} files matched in ${d.elapsedMs}ms (preFiltered=${
+        d.preFiltered
+      }, autoIncluded=${d.autoIncluded}, llmConsidered=${
+        d.llmConsidered
+      }, llmConfirmed=${d.llmConfirmed}, tokensIn=${
+        d.promptTokens
+      }, tokensOut=${d.completionTokens}, calls=${d.llmCalls})\n`
+    );
+  }
+  for (const [kind, count] of Object.entries(summary.filesByKind)) {
+    process.stderr.write(`  files.${kind}: ${count}\n`);
+  }
+  if (summary.filesSkippedMinified > 0) {
+    process.stderr.write(
+      `  files.skipped_minified: ${summary.filesSkippedMinified}\n`
+    );
+  }
+  process.stderr.write(
+    `[ditto-cli scan][extract] emitted ${summary.candidatesEmitted} candidates -> ${outputPath}\n`
+  );
+  for (const [kind, count] of Object.entries(summary.candidatesByKind)) {
+    if (count > 0) process.stderr.write(`  candidates.${kind}: ${count}\n`);
+  }
+  const { accept, reject, llm } = summary.candidatesByVerdict;
+  if (accept > 0 || reject > 0 || llm > 0) {
+    process.stderr.write(
+      `[ditto-cli scan][extract] rule verdicts: accept=${accept}, reject=${reject}, llm=${llm}\n`
+    );
+    for (const [name, count] of Object.entries(summary.ruleHits)) {
+      process.stderr.write(`  rule.${name}: ${count}\n`);
+    }
+  }
+}
+
+// Telemetry for the classify phase — per-status counts, LLM call/token
+// totals, and the paths to the results and summary artifacts.
+function logClassifySummary(
+  result: DittoScanClassifyResult,
+  summaryPath: string
+): void {
+  process.stderr.write(
+    `[ditto-cli scan][classify] classified ${result.summary.candidate_total} candidates\n`
+  );
+  for (const [status, count] of Object.entries(result.summary.by_status)) {
+    process.stderr.write(`  ${status}: ${count}\n`);
+  }
+  process.stderr.write(
+    `[ditto-cli scan][classify] llm_calls: ${result.summary.llm_calls}, tokens_in: ${result.summary.llm_tokens_in}, tokens_out: ${result.summary.llm_tokens_out}\n`
+  );
+  process.stderr.write(
+    `[ditto-cli scan][classify] to_manually_validate: ${result.summary.num_results_to_manually_verify}\n`
+  );
+  process.stderr.write(
+    `[ditto-cli scan][classify] pct_results_needing_manual_verification: ${result.summary.pct_results_needing_manual_verification}\n`
+  );
+  for (const [status, filePath] of Object.entries(
+    result.writtenPaths.resultsByStatus
+  )) {
+    process.stderr.write(`[ptd classify] results.${status} -> ${filePath}\n`);
+  }
+  process.stderr.write(`[ptd classify] summary  -> ${summaryPath}\n`);
+  if (result.writtenPaths.verify) {
+    process.stderr.write(
+      `[ditto-cli scan][classify] verify   -> ${result.writtenPaths.verify}\n`
+    );
+  }
+}
+
+export const scan = async (
+  path: string,
+  outDir: string = "./out",
+  prefix: string = ""
+) => {
+  const apiKey = process.env.GEMINI_API_KEY;
+  if (!apiKey || apiKey.trim().length === 0) {
+    return await quit(
+      logger.errorText("GEMINI_API_KEY is not set. Aborting."),
+      2
+    );
+  }
+
+  const resolvedInput = resolveUserPath(path);
+  const resolvedOutDir = resolveUserPath(outDir);
+  await fs.mkdir(resolvedOutDir, { recursive: true });
+
+  const {
+    candidates: candidatesPath,
+    results: resultsPath,
+    summary: summaryPath,
+    verify: verifyPath,
+    schema: schemaPath,
+    stagings: stagingsPath,
+  } = buildOutputPaths(resolvedOutDir, prefix);
+
+  const {
+    candidates,
+    verdicts,
+    summary: extractSummary,
+  } = await runExtract({ inputPath: resolvedInput });
+  await writeCandidatesNdjson(candidates, candidatesPath);
+  logExtractSummary(extractSummary, candidatesPath);
+
+  if (candidates.length === 0) {
+    logger.warnText(
+      `[ditto scan] no candidates extracted; writing empty classify output\n`
+    );
+  }
+
+  const { llmCandidates, preClassified } = preClassify(candidates, verdicts);
+
+  const classifyResult = await runClassify({
+    candidates: llmCandidates,
+    preClassified,
+    outputPath: resultsPath,
+    summaryPath,
+    verifyPath,
+  });
+  logClassifySummary(classifyResult, summaryPath);
+};

lib/src/formatters/mixins/javascriptCodegenMixin.ts

@@ -1,6 +1,6 @@
 import { Constructor } from "../shared";
 
-interface NamedImport {
+export interface NamedImport {
   name: string;
   alias?: string;
 }
@@ -9,9 +9,9 @@ export default function javascriptCodegenMixin<TBase extends Constructor>(
   Base: TBase
 ) {
   return class JavascriptCodegenHelpers extends Base {
-    protected indentSpaces: number = 2;
+    public indentSpaces: number = 2;
 
-    protected sanitizeStringForJSVariableName(str: string) {
+    public sanitizeStringForJSVariableName(str: string) {
       return str.replace(/[^a-zA-Z0-9]/g, "_");
     }
 
@@ -20,7 +20,7 @@ export default function javascriptCodegenMixin<TBase extends Constructor>(
      * @param modules array of { name: string, alias?: string }, each named import
      * @returns a string of comma-separated module names/aliases, sorted
      */
-    protected formatNamedModules(modules: NamedImport[]) {
+    public formatNamedModules(modules: NamedImport[]) {
       return modules
         .map((m) => {
           if (m.alias) {
@@ -39,7 +39,7 @@ export default function javascriptCodegenMixin<TBase extends Constructor>(
      * @param moduleName the name of the file or package to import from
      * @returns i.e `import { foo, bar as name } from "./file";`
      */
-    protected codegenNamedImport(modules: NamedImport[], moduleName: string) {
+    public codegenNamedImport(modules: NamedImport[], moduleName: string) {
       const formattedModules = this.formatNamedModules(modules);
 
       return `import { ${formattedModules} } from "${moduleName}";\n`;
@@ -52,7 +52,7 @@ export default function javascriptCodegenMixin<TBase extends Constructor>(
      * @param moduleName the name of the file or package to import from
      * @returns i.e `const { foo, bar as name } = require("./file");`
      */
-    protected codegenNamedRequire(modules: NamedImport[], moduleName: string) {
+    public codegenNamedRequire(modules: NamedImport[], moduleName: string) {
       const formattedModules = this.formatNamedModules(modules);
 
       return `const { ${formattedModules} } = require("${moduleName}");\n`;
@@ -64,7 +64,7 @@ export default function javascriptCodegenMixin<TBase extends Constructor>(
      * @param moduleName the name of the file or package to import from
      * @returns i.e codegenDefaultImport("item", "./file") => `import item from "./file";`
      */
-    protected codegenDefaultImport(module: string, moduleName: string) {
+    public codegenDefaultImport(module: string, moduleName: string) {
       return `import ${module} from "${moduleName}";\n`;
     }
 
@@ -74,7 +74,7 @@ export default function javascriptCodegenMixin<TBase extends Constructor>(
      * @param moduleName the name of the file or package to import from
      * @returns i.e codegenDefaultRequire("item", "./file") => `const item = require("./file)";`
      */
-    protected codegenDefaultRequire(module: string, moduleName: string) {
+    public codegenDefaultRequire(module: string, moduleName: string) {
       return `const ${module} = require("${moduleName}");\n`;
     }
 
@@ -83,7 +83,7 @@ export default function javascriptCodegenMixin<TBase extends Constructor>(
      * @param module the name of the module to export
      * @returns i.e codegenDefaultExport("item") => "export default item;"
      */
-    protected codegenDefaultExport(module: string) {
+    public codegenDefaultExport(module: string) {
       return `export default ${module};`;
     }
 
@@ -92,11 +92,11 @@ export default function javascriptCodegenMixin<TBase extends Constructor>(
      * @param module the name of the module to export
      * @returns i.e codegenModuleExports("item") => "module.exports = item;"
      */
-    protected codegenCommonJSModuleExports(module: string) {
+    public codegenCommonJSModuleExports(module: string) {
       return `module.exports = ${module};`;
     }
 
-    protected codegenPad(depth: number) {
+    public codegenPad(depth: number) {
       return " ".repeat(depth * this.indentSpaces);
     }
   };