[DIT-13028] Connect CLI to the public API scan endpoints

lib/src/commands/scan.ts

@@ -3,11 +3,16 @@ import fs from "fs/promises";
 import path from "path";
 
 import logger from "../utils/logger";
-import { quit } from "../utils/quit";
 import { DittoScanExtractSummary, runExtract } from "../scan/extract";
-import { DittoScanCandidate, DittoScanResult } from "../scan/types";
-import { preClassify } from "../scan/preClassify";
-import { DittoScanClassifyResult, runClassify } from "../scan/classify";
+import { DittoScanCandidate } from "../scan/types";
+import { quit } from "../utils/quit";
+import initAPIToken from "../services/apiToken/initAPIToken";
+import appContext from "../utils/appContext";
+import {
+  initiateClassify,
+  initiateScan,
+  uploadCandidatesToS3,
+} from "../http/scan";
 
 // Yarn sets INIT_CWD to the directory the user invoked yarn from, which
 // matters when we proxy via `cd product-text-detection && yarn ptd`.
@@ -23,12 +28,6 @@ function buildOutputPaths(outDir: string, prefix?: string) {
   const p = prefix ? `${prefix}-` : "";
   return {
     candidates: path.join(outDir, `${p}candidates.ndjson`),
-    results: path.join(outDir, `${p}results.ndjson`),
-    summary: path.join(outDir, `${p}summary.json`),
-    verify: path.join(outDir, `${p}resultsToVerify.ndjson`),
-    analysis: path.join(outDir, `${p}analysis.json`),
-    schema: path.join(outDir, `${p}schema.json`),
-    stagings: path.join(outDir, `${p}stagings.ndjson`),
   };
 }
 
@@ -49,7 +48,7 @@ async function writeCandidatesNdjson(
 // per-detection-kind candidate counts, and the output path.
 function logExtractSummary(
   summary: DittoScanExtractSummary,
-  outputPath: string
+  outputPath?: string
 ): void {
   process.stderr.write(
     `[ditto-cli scan][extract] framework: ${
@@ -64,15 +63,13 @@ function logExtractSummary(
   if (summary.i18nFileDiscovery) {
     const d = summary.i18nFileDiscovery;
     process.stderr.write(
-      `[ditto-cli scan][extract] llm file discovery (${d.task}): ${
-        d.llmConfirmed + d.autoIncluded
+      `[ditto-cli scan][extract] i18n file discovery (${d.task}): ${
+        d.heuristicConfirmed + d.autoIncluded
       }/${d.totalCandidates} files matched in ${d.elapsedMs}ms (preFiltered=${
         d.preFiltered
-      }, autoIncluded=${d.autoIncluded}, llmConsidered=${
-        d.llmConsidered
-      }, llmConfirmed=${d.llmConfirmed}, tokensIn=${
-        d.promptTokens
-      }, tokensOut=${d.completionTokens}, calls=${d.llmCalls})\n`
+      }, autoIncluded=${d.autoIncluded}, heuristicConsidered=${
+        d.heuristicConsidered
+      }, heuristicConfirmed=${d.heuristicConfirmed})\n`
     );
   }
   for (const [kind, count] of Object.entries(summary.filesByKind)) {
@@ -84,104 +81,71 @@ function logExtractSummary(
     );
   }
   process.stderr.write(
-    `[ditto-cli scan][extract] emitted ${summary.candidatesEmitted} candidates -> ${outputPath}\n`
+    `[ditto-cli scan][extract] emitted ${
+      summary.candidatesEmitted
+    } candidates -> ${outputPath ?? "Ditto"}\n`
   );
   for (const [kind, count] of Object.entries(summary.candidatesByKind)) {
     if (count > 0) process.stderr.write(`  candidates.${kind}: ${count}\n`);
   }
-  const { accept, reject, llm } = summary.candidatesByVerdict;
-  if (accept > 0 || reject > 0 || llm > 0) {
-    process.stderr.write(
-      `[ditto-cli scan][extract] rule verdicts: accept=${accept}, reject=${reject}, llm=${llm}\n`
-    );
-    for (const [name, count] of Object.entries(summary.ruleHits)) {
-      process.stderr.write(`  rule.${name}: ${count}\n`);
-    }
-  }
 }
 
-// Telemetry for the classify phase — per-status counts, LLM call/token
-// totals, and the paths to the results and summary artifacts.
-function logClassifySummary(
-  result: DittoScanClassifyResult,
-  summaryPath: string
-): void {
-  process.stderr.write(
-    `[ditto-cli scan][classify] classified ${result.summary.candidate_total} candidates\n`
-  );
-  for (const [status, count] of Object.entries(result.summary.by_status)) {
-    process.stderr.write(`  ${status}: ${count}\n`);
-  }
-  process.stderr.write(
-    `[ditto-cli scan][classify] llm_calls: ${result.summary.llm_calls}, tokens_in: ${result.summary.llm_tokens_in}, tokens_out: ${result.summary.llm_tokens_out}\n`
-  );
-  process.stderr.write(
-    `[ditto-cli scan][classify] to_manually_validate: ${result.summary.num_results_to_manually_verify}\n`
-  );
-  process.stderr.write(
-    `[ditto-cli scan][classify] pct_results_needing_manual_verification: ${result.summary.pct_results_needing_manual_verification}\n`
-  );
-  for (const [status, filePath] of Object.entries(
-    result.writtenPaths.resultsByStatus
-  )) {
-    process.stderr.write(`[ptd classify] results.${status} -> ${filePath}\n`);
-  }
-  process.stderr.write(`[ptd classify] summary  -> ${summaryPath}\n`);
-  if (result.writtenPaths.verify) {
-    process.stderr.write(
-      `[ditto-cli scan][classify] verify   -> ${result.writtenPaths.verify}\n`
-    );
-  }
+interface ISyncOptions {
+  local: boolean;
+  outDir?: string;
+  prefix?: string;
 }
-
 export const scan = async (
   path: string,
-  outDir: string = "./out",
-  prefix: string = ""
+  { local, outDir = "", prefix = "" }: ISyncOptions
 ) => {
-  const apiKey = process.env.GEMINI_API_KEY;
-  if (!apiKey || apiKey.trim().length === 0) {
+  if (local && !outDir) {
     return await quit(
-      logger.errorText("GEMINI_API_KEY is not set. Aborting."),
+      logger.errorText(
+        "Must specify --out-dir if outputting candidates locally"
+      ),
       2
     );
   }
 
   const resolvedInput = resolveUserPath(path);
-  const resolvedOutDir = resolveUserPath(outDir);
-  await fs.mkdir(resolvedOutDir, { recursive: true });
 
-  const {
-    candidates: candidatesPath,
-    results: resultsPath,
-    summary: summaryPath,
-    verify: verifyPath,
-    schema: schemaPath,
-    stagings: stagingsPath,
-  } = buildOutputPaths(resolvedOutDir, prefix);
-
-  const {
-    candidates,
-    verdicts,
-    summary: extractSummary,
-  } = await runExtract({ inputPath: resolvedInput });
-  await writeCandidatesNdjson(candidates, candidatesPath);
-  logExtractSummary(extractSummary, candidatesPath);
+  const { candidates, summary: extractSummary } = await runExtract({
+    inputPath: resolvedInput,
+  });
 
   if (candidates.length === 0) {
     logger.warnText(
       `[ditto scan] no candidates extracted; writing empty classify output\n`
     );
   }
 
-  const { llmCandidates, preClassified } = preClassify(candidates, verdicts);
+  if (local) {
+    const resolvedOutDir = resolveUserPath(outDir);
+    await fs.mkdir(resolvedOutDir, { recursive: true });
 
-  const classifyResult = await runClassify({
-    candidates: llmCandidates,
-    preClassified,
-    outputPath: resultsPath,
-    summaryPath,
-    verifyPath,
-  });
-  logClassifySummary(classifyResult, summaryPath);
+    const { candidates: candidatesPath } = buildOutputPaths(
+      resolvedOutDir,
+      prefix
+    );
+
+    await writeCandidatesNdjson(candidates, candidatesPath);
+    logExtractSummary(extractSummary, candidatesPath);
+  } else {
+    const token = await initAPIToken();
+    appContext.setApiToken(token);
+    const {
+      candidatesSignedS3Url,
+      record: { _id: recordId },
+    } = await initiateScan(path);
+    await uploadCandidatesToS3(candidates, candidatesSignedS3Url);
+    await initiateClassify(recordId);
+    logExtractSummary(extractSummary);
+    await quit(
+      logger.info(
+        `Scan initiated! Visit https://app.dittowords.com/scan/${recordId} to view progress and see results.`
+      ),
+      0
+    );
+  }
 };

lib/src/http/scan.ts

@@ -0,0 +1,60 @@
+import axios, { AxiosError } from "axios";
+import getHttpClient from "./client";
+import { IInitiateScanResponse, ZInitiateScanResponse } from "./types";
+import { DittoScanCandidate } from "../scan/types";
+import { Blob } from "buffer";
+
+export async function initiateScan(
+  path: string
+): Promise<IInitiateScanResponse> {
+  try {
+    const httpClient = getHttpClient({});
+    const response = await httpClient.post("/v2/scan", { path });
+    return ZInitiateScanResponse.parse(response.data);
+  } catch (e) {
+    if (!(e instanceof AxiosError)) {
+      throw new Error(
+        "Sorry! We're having trouble reaching the Ditto API. Please try again later."
+      );
+    }
+    throw e;
+  }
+}
+
+export async function initiateClassify(scanId: string): Promise<void> {
+  try {
+    const httpClient = getHttpClient({});
+    await httpClient.post(`/v2/scan/${scanId}/classify`, {});
+  } catch (e) {
+    if (!(e instanceof AxiosError)) {
+      throw new Error(
+        "Sorry! We're having trouble reaching the Ditto API. Please try again later."
+      );
+    }
+    throw e;
+  }
+}
+
+export async function uploadCandidatesToS3(
+  candidates: DittoScanCandidate[],
+  signedUrl: string
+) {
+  // Convert TypeScript array to NDJSON format (JSON objects separated by \n)
+  const ndjsonString = candidates
+    .map((candidate) => JSON.stringify(candidate))
+    .join("\n");
+
+  // Create a Blob from the NDJSON string
+  const blob = new Blob([ndjsonString], { type: "application/x-ndjson" });
+
+  // Upload via PUT request to the pre-signed URL
+  // Needs to be done with a non-ditto http client so we don't pass our auth token
+  const httpClient = axios.create({});
+  const response = await httpClient.put(signedUrl, blob, {
+    headers: {
+      // S3 requires the Content-Length header for PUT requests
+      "Content-Length": blob.size,
+    },
+  });
+  return response;
+}

lib/src/http/types.ts

@@ -177,3 +177,12 @@ export const ZExportSwiftFileRequest = z.object({
 });
 
 export type IExportSwiftFileRequest = z.infer<typeof ZExportSwiftFileRequest>;
+
+export const ZInitiateScanBodySchema = z.object({
+  path: z.string(),
+});
+export const ZInitiateScanResponse = z.object({
+  record: z.object({ _id: z.string() }),
+  candidatesSignedS3Url: z.string(),
+});
+export type IInitiateScanResponse = z.infer<typeof ZInitiateScanResponse>;

lib/src/index.ts

@@ -10,7 +10,6 @@ import logger from "./utils/logger";
 import initAPIToken from "./services/apiToken/initAPIToken";
 import { initProjectConfig } from "./services/projectConfig";
 import appContext from "./utils/appContext";
-import type commander from "commander";
 import { ErrorType, isDittoError, isDittoErrorType } from "./utils/DittoError";
 import processCommandMetaFlag from "./utils/processCommandMetaFlag";
 
@@ -77,12 +76,20 @@ const appEntry = async () => {
     .description(
       "Run extract + classify + infer end-to-end; emits schema.json and stagings.ndjson to --out-dir"
     )
-    .option("--out-dir <dir>", "output directory", "./out")
+    .option(
+      "--local",
+      "outputs the candidates file locally to the out-dir with the given prefix",
+      false
+    )
+    .option("--out-dir <dir>", "output directory", "")
     .option("--prefix <prefix>", "prefix for output files", "")
     .action(
-      async (inputPath: string, opts: { outDir: string; prefix?: string }) => {
+      async (
+        inputPath: string,
+        opts: { local: boolean; outDir: string; prefix?: string }
+      ) => {
         try {
-          return await scan(inputPath, opts.outDir, opts.prefix);
+          return await scan(inputPath, opts);
         } catch (error) {
           handleCommandError(error);
         }