Ditto scan

README.md

@@ -54,6 +54,8 @@ For more information on configuring the CLI, see [this documentation section](ht
 
 ## Usage
 
+### Pull
+
 ```bash
 npx @dittowords/cli pull
 ```
@@ -70,9 +72,19 @@ See our demo projects for examples of how to integrate the Ditto CLI in differen
 - [iOS mobile app](https://github.com/dittowords/ditto-react-demo)
 - [Android mobile app](https://github.com/dittowords/ditto-react-demo)
 
+### Scan
+
+```bash
+npx @dittowords/cli scan <path>
+```
+
+Scans your codebase at `<path>` to identify user-facing text, and then prompts you to navigate to the Ditto web app where you can create a full content system off of the identified text.
+
+This feature is currently in beta, and only accessible to select partners, so message us at [support@dittowords.com](mailto:support@dittowords.com) if you would like access!
+
 ## Legacy Setup
 
-Beginning with `v5.0.0`, the Ditto CLI points at the new Ditto experience by default. To run the CLI compatible with legacy Ditto, append the `--legacy` flag to any legacy command, and the CLI will work as it did in the `4.x` version. All existing legacy commands remain fully functional at this time.
+Beginning with `v5.0.0`, the Ditto CLI points at the new Ditto experience by default. To run the CLI compatible with legacy Ditto, append the `--legacy` flag to any legacy command, and the CLI will work as it did in the `4.x` version. All existing legacy commands remain fully functional at this time. Only the `pull` command is supported with the `--legacy` flag.
 
 ## Feedback
 

esbuild.mjs

@@ -1,4 +1,5 @@
 import * as esbuild from "esbuild";
+import { execSync } from "child_process";
 
 let define = {};
 const KEYS_TO_DEFINE = [
@@ -35,6 +36,10 @@ const config = {
 
 async function main() {
   const result = await esbuild.build(config);
+  execSync(
+    "npx dts-bundle-generator --no-check --out-file bin/ditto.d.ts lib/ditto.ts",
+    { stdio: "inherit" }
+  );
   // Output build metafile so we can analyze the bundle
   // size over time and check if anything unexpected is being bundled in.
   if (process.env.ENV === "production") {

jest.config.ts

@@ -2,6 +2,10 @@ import type { Config } from "jest";
 
 const config: Config = {
   transformIgnorePatterns: [],
+  moduleNameMapper: {
+    "^unicorn-magic/node$":
+      "<rootDir>/node_modules/unicorn-magic/node.js",
+  },
   maxWorkers: 1,
   verbose: true,
   testPathIgnorePatterns: [

lib/ditto.ts

@@ -25,4 +25,6 @@ const main = async () => {
   }
 };
 
+export type * from "./src/scan/types";
+
 main();

lib/src/commands/scan.ts

@@ -0,0 +1,163 @@
+import "dotenv/config";
+import fs from "fs/promises";
+import path from "path";
+
+import { prompt } from "enquirer";
+import open from "open";
+
+import logger from "../utils/logger";
+import { DittoScanExtractSummary, runExtract } from "../scan/extract";
+import { DittoScanCandidate } from "../scan/types";
+import { quit } from "../utils/quit";
+import initAPIToken from "../services/apiToken/initAPIToken";
+import appContext from "../utils/appContext";
+import {
+  initiateClassify,
+  initiateScan,
+  uploadCandidatesToS3,
+} from "../http/scan";
+import chalk from "chalk";
+
+// Yarn sets INIT_CWD to the directory the user invoked yarn from, which
+// matters when we proxy via `cd product-text-detection && yarn ptd`.
+function resolveUserPath(input: string): string {
+  if (path.isAbsolute(input)) return input;
+  return path.resolve(process.env.INIT_CWD ?? process.cwd(), input);
+}
+
+// Builds the standard output file paths for a given directory and optional
+// prefix. Files are named "{prefix}-{artifact}.ext" when a prefix is given,
+// or just "{artifact}.ext" when omitted.
+function buildOutputPaths(outDir: string, prefix?: string) {
+  const p = prefix ? `${prefix}-` : "";
+  return {
+    candidates: path.join(outDir, `${p}candidates.ndjson`),
+  };
+}
+
+// Serialize extracted candidates as newline-delimited JSON. Used by both the
+// standalone `extract` command and `run`, which keeps the same artifact on disk.
+async function writeCandidatesNdjson(
+  candidates: DittoScanCandidate[],
+  outputPath: string
+): Promise<void> {
+  await fs.mkdir(path.dirname(path.resolve(outputPath)), { recursive: true });
+  const ndjson =
+    candidates.map((c) => JSON.stringify(c)).join("\n") +
+    (candidates.length > 0 ? "\n" : "");
+  await fs.writeFile(outputPath, ndjson, "utf8");
+}
+
+// Telemetry for the extract phase — framework detection, file counts,
+// per-detection-kind candidate counts, and the output path.
+function logExtractSummary(
+  summary: DittoScanExtractSummary,
+  outputPath?: string
+): void {
+  process.stderr.write(
+    `[ditto-cli scan][extract] framework: ${
+      summary.framework.length > 0
+        ? summary.framework.join(", ")
+        : "(none detected)"
+    }\n`
+  );
+  process.stderr.write(
+    `[ditto-cli scan][extract] scanned ${summary.filesScanned} files in ${summary.elapsedMs}ms\n`
+  );
+  if (summary.i18nFileDiscovery) {
+    const d = summary.i18nFileDiscovery;
+    process.stderr.write(
+      `[ditto-cli scan][extract] i18n file discovery (${d.task}): ${
+        d.heuristicConfirmed + d.autoIncluded
+      }/${d.totalCandidates} files matched in ${d.elapsedMs}ms (preFiltered=${
+        d.preFiltered
+      }, autoIncluded=${d.autoIncluded}, heuristicConsidered=${
+        d.heuristicConsidered
+      }, heuristicConfirmed=${d.heuristicConfirmed})\n`
+    );
+  }
+  for (const [kind, count] of Object.entries(summary.filesByKind)) {
+    process.stderr.write(`  files.${kind}: ${count}\n`);
+  }
+  if (summary.filesSkippedMinified > 0) {
+    process.stderr.write(
+      `  files.skipped_minified: ${summary.filesSkippedMinified}\n`
+    );
+  }
+  process.stderr.write(
+    `[ditto-cli scan][extract] emitted ${
+      summary.candidatesEmitted
+    } candidates -> ${outputPath ?? "Ditto"}\n`
+  );
+  for (const [kind, count] of Object.entries(summary.candidatesByKind)) {
+    if (count > 0) process.stderr.write(`  candidates.${kind}: ${count}\n`);
+  }
+}
+
+interface ISyncOptions {
+  local: boolean;
+  outDir?: string;
+  prefix?: string;
+}
+export const scan = async (
+  path: string,
+  { local, outDir = "", prefix = "" }: ISyncOptions
+) => {
+  if (local && !outDir) {
+    return await quit(
+      logger.errorText(
+        "Must specify --out-dir if outputting candidates locally"
+      ),
+      2
+    );
+  }
+
+  const resolvedInput = resolveUserPath(path);
+
+  const { candidates, summary: extractSummary } = await runExtract({
+    inputPath: resolvedInput,
+  });
+
+  if (candidates.length === 0) {
+    logger.warnText(
+      `[ditto scan] no candidates extracted; writing empty classify output\n`
+    );
+  }
+
+  if (local) {
+    const resolvedOutDir = resolveUserPath(outDir);
+    await fs.mkdir(resolvedOutDir, { recursive: true });
+
+    const { candidates: candidatesPath } = buildOutputPaths(
+      resolvedOutDir,
+      prefix
+    );
+
+    await writeCandidatesNdjson(candidates, candidatesPath);
+    logExtractSummary(extractSummary, candidatesPath);
+  } else {
+    const token = await initAPIToken();
+    appContext.setApiToken(token);
+    const {
+      candidatesSignedS3Url,
+      record: { _id: recordId },
+    } = await initiateScan(path);
+    await uploadCandidatesToS3(candidates, candidatesSignedS3Url);
+    await initiateClassify(recordId);
+    logExtractSummary(extractSummary);
+    const url = `https://app.dittowords.com/scan/${recordId}`;
+    console.log(
+      `Scan initiated! Visit ${chalk.blueBright.underline(
+        url
+      )} to view progress and see results.`
+    );
+    const { openUrl } = await prompt<{ openUrl: boolean }>({
+      type: "confirm",
+      name: "openUrl",
+      message: "Open in browser?",
+      initial: true,
+    });
+    if (openUrl) await open(url);
+    await quit(null, 0);
+  }
+};

lib/src/formatters/mixins/javascriptCodegenMixin.ts

@@ -1,6 +1,6 @@
 import { Constructor } from "../shared";
 
-interface NamedImport {
+export interface NamedImport {
   name: string;
   alias?: string;
 }
@@ -9,9 +9,9 @@ export default function javascriptCodegenMixin<TBase extends Constructor>(
   Base: TBase
 ) {
   return class JavascriptCodegenHelpers extends Base {
-    protected indentSpaces: number = 2;
+    public indentSpaces: number = 2;
 
-    protected sanitizeStringForJSVariableName(str: string) {
+    public sanitizeStringForJSVariableName(str: string) {
       return str.replace(/[^a-zA-Z0-9]/g, "_");
     }
 
@@ -20,7 +20,7 @@ export default function javascriptCodegenMixin<TBase extends Constructor>(
      * @param modules array of { name: string, alias?: string }, each named import
      * @returns a string of comma-separated module names/aliases, sorted
      */
-    protected formatNamedModules(modules: NamedImport[]) {
+    public formatNamedModules(modules: NamedImport[]) {
       return modules
         .map((m) => {
           if (m.alias) {
@@ -39,7 +39,7 @@ export default function javascriptCodegenMixin<TBase extends Constructor>(
      * @param moduleName the name of the file or package to import from
      * @returns i.e `import { foo, bar as name } from "./file";`
      */
-    protected codegenNamedImport(modules: NamedImport[], moduleName: string) {
+    public codegenNamedImport(modules: NamedImport[], moduleName: string) {
       const formattedModules = this.formatNamedModules(modules);
 
       return `import { ${formattedModules} } from "${moduleName}";\n`;
@@ -52,7 +52,7 @@ export default function javascriptCodegenMixin<TBase extends Constructor>(
      * @param moduleName the name of the file or package to import from
      * @returns i.e `const { foo, bar as name } = require("./file");`
      */
-    protected codegenNamedRequire(modules: NamedImport[], moduleName: string) {
+    public codegenNamedRequire(modules: NamedImport[], moduleName: string) {
       const formattedModules = this.formatNamedModules(modules);
 
       return `const { ${formattedModules} } = require("${moduleName}");\n`;
@@ -64,7 +64,7 @@ export default function javascriptCodegenMixin<TBase extends Constructor>(
      * @param moduleName the name of the file or package to import from
      * @returns i.e codegenDefaultImport("item", "./file") => `import item from "./file";`
      */
-    protected codegenDefaultImport(module: string, moduleName: string) {
+    public codegenDefaultImport(module: string, moduleName: string) {
       return `import ${module} from "${moduleName}";\n`;
     }
 
@@ -74,7 +74,7 @@ export default function javascriptCodegenMixin<TBase extends Constructor>(
      * @param moduleName the name of the file or package to import from
      * @returns i.e codegenDefaultRequire("item", "./file") => `const item = require("./file)";`
      */
-    protected codegenDefaultRequire(module: string, moduleName: string) {
+    public codegenDefaultRequire(module: string, moduleName: string) {
       return `const ${module} = require("${moduleName}");\n`;
     }
 
@@ -83,7 +83,7 @@ export default function javascriptCodegenMixin<TBase extends Constructor>(
      * @param module the name of the module to export
      * @returns i.e codegenDefaultExport("item") => "export default item;"
      */
-    protected codegenDefaultExport(module: string) {
+    public codegenDefaultExport(module: string) {
       return `export default ${module};`;
     }
 
@@ -92,11 +92,11 @@ export default function javascriptCodegenMixin<TBase extends Constructor>(
      * @param module the name of the module to export
      * @returns i.e codegenModuleExports("item") => "module.exports = item;"
      */
-    protected codegenCommonJSModuleExports(module: string) {
+    public codegenCommonJSModuleExports(module: string) {
       return `module.exports = ${module};`;
     }
 
-    protected codegenPad(depth: number) {
+    public codegenPad(depth: number) {
       return " ".repeat(depth * this.indentSpaces);
     }
   };

lib/src/http/scan.ts

@@ -0,0 +1,60 @@
+import axios, { AxiosError } from "axios";
+import getHttpClient from "./client";
+import { IInitiateScanResponse, ZInitiateScanResponse } from "./types";
+import { DittoScanCandidate } from "../scan/types";
+import { Blob } from "buffer";
+
+export async function initiateScan(
+  path: string
+): Promise<IInitiateScanResponse> {
+  try {
+    const httpClient = getHttpClient({});
+    const response = await httpClient.post("/v2/scan", { path });
+    return ZInitiateScanResponse.parse(response.data);
+  } catch (e) {
+    if (!(e instanceof AxiosError)) {
+      throw new Error(
+        "Sorry! We're having trouble reaching the Ditto API. Please try again later."
+      );
+    }
+    throw e;
+  }
+}
+
+export async function initiateClassify(scanId: string): Promise<void> {
+  try {
+    const httpClient = getHttpClient({});
+    await httpClient.post(`/v2/scan/${scanId}/classify`, {});
+  } catch (e) {
+    if (!(e instanceof AxiosError)) {
+      throw new Error(
+        "Sorry! We're having trouble reaching the Ditto API. Please try again later."
+      );
+    }
+    throw e;
+  }
+}
+
+export async function uploadCandidatesToS3(
+  candidates: DittoScanCandidate[],
+  signedUrl: string
+) {
+  // Convert TypeScript array to NDJSON format (JSON objects separated by \n)
+  const ndjsonString = candidates
+    .map((candidate) => JSON.stringify(candidate))
+    .join("\n");
+
+  // Create a Blob from the NDJSON string
+  const blob = new Blob([ndjsonString], { type: "application/x-ndjson" });
+
+  // Upload via PUT request to the pre-signed URL
+  // Needs to be done with a non-ditto http client so we don't pass our auth token
+  const httpClient = axios.create({});
+  const response = await httpClient.put(signedUrl, blob, {
+    headers: {
+      // S3 requires the Content-Length header for PUT requests
+      "Content-Length": blob.size,
+    },
+  });
+  return response;
+}

lib/src/http/types.ts

@@ -177,3 +177,12 @@ export const ZExportSwiftFileRequest = z.object({
 });
 
 export type IExportSwiftFileRequest = z.infer<typeof ZExportSwiftFileRequest>;
+
+export const ZInitiateScanBodySchema = z.object({
+  path: z.string(),
+});
+export const ZInitiateScanResponse = z.object({
+  record: z.object({ _id: z.string() }),
+  candidatesSignedS3Url: z.string(),
+});
+export type IInitiateScanResponse = z.infer<typeof ZInitiateScanResponse>;