diff --git a/lib/src/commands/scan.ts b/lib/src/commands/scan.ts index 3d39e56..e2ea439 100644 --- a/lib/src/commands/scan.ts +++ b/lib/src/commands/scan.ts @@ -3,11 +3,16 @@ import fs from "fs/promises"; import path from "path"; import logger from "../utils/logger"; -import { quit } from "../utils/quit"; import { DittoScanExtractSummary, runExtract } from "../scan/extract"; -import { DittoScanCandidate, DittoScanResult } from "../scan/types"; -import { preClassify } from "../scan/preClassify"; -import { DittoScanClassifyResult, runClassify } from "../scan/classify"; +import { DittoScanCandidate } from "../scan/types"; +import { quit } from "../utils/quit"; +import initAPIToken from "../services/apiToken/initAPIToken"; +import appContext from "../utils/appContext"; +import { + initiateClassify, + initiateScan, + uploadCandidatesToS3, +} from "../http/scan"; // Yarn sets INIT_CWD to the directory the user invoked yarn from, which // matters when we proxy via `cd product-text-detection && yarn ptd`. @@ -23,12 +28,6 @@ function buildOutputPaths(outDir: string, prefix?: string) { const p = prefix ? `${prefix}-` : ""; return { candidates: path.join(outDir, `${p}candidates.ndjson`), - results: path.join(outDir, `${p}results.ndjson`), - summary: path.join(outDir, `${p}summary.json`), - verify: path.join(outDir, `${p}resultsToVerify.ndjson`), - analysis: path.join(outDir, `${p}analysis.json`), - schema: path.join(outDir, `${p}schema.json`), - stagings: path.join(outDir, `${p}stagings.ndjson`), }; } @@ -49,7 +48,7 @@ async function writeCandidatesNdjson( // per-detection-kind candidate counts, and the output path. function logExtractSummary( summary: DittoScanExtractSummary, - outputPath: string + outputPath?: string ): void { process.stderr.write( `[ditto-cli scan][extract] framework: ${ @@ -64,15 +63,13 @@ function logExtractSummary( if (summary.i18nFileDiscovery) { const d = summary.i18nFileDiscovery; process.stderr.write( - `[ditto-cli scan][extract] llm file discovery (${d.task}): ${ - d.llmConfirmed + d.autoIncluded + `[ditto-cli scan][extract] i18n file discovery (${d.task}): ${ + d.heuristicConfirmed + d.autoIncluded }/${d.totalCandidates} files matched in ${d.elapsedMs}ms (preFiltered=${ d.preFiltered - }, autoIncluded=${d.autoIncluded}, llmConsidered=${ - d.llmConsidered - }, llmConfirmed=${d.llmConfirmed}, tokensIn=${ - d.promptTokens - }, tokensOut=${d.completionTokens}, calls=${d.llmCalls})\n` + }, autoIncluded=${d.autoIncluded}, heuristicConsidered=${ + d.heuristicConsidered + }, heuristicConfirmed=${d.heuristicConfirmed})\n` ); } for (const [kind, count] of Object.entries(summary.filesByKind)) { @@ -84,89 +81,38 @@ function logExtractSummary( ); } process.stderr.write( - `[ditto-cli scan][extract] emitted ${summary.candidatesEmitted} candidates -> ${outputPath}\n` + `[ditto-cli scan][extract] emitted ${ + summary.candidatesEmitted + } candidates -> ${outputPath ?? "Ditto"}\n` ); for (const [kind, count] of Object.entries(summary.candidatesByKind)) { if (count > 0) process.stderr.write(` candidates.${kind}: ${count}\n`); } - const { accept, reject, llm } = summary.candidatesByVerdict; - if (accept > 0 || reject > 0 || llm > 0) { - process.stderr.write( - `[ditto-cli scan][extract] rule verdicts: accept=${accept}, reject=${reject}, llm=${llm}\n` - ); - for (const [name, count] of Object.entries(summary.ruleHits)) { - process.stderr.write(` rule.${name}: ${count}\n`); - } - } } -// Telemetry for the classify phase — per-status counts, LLM call/token -// totals, and the paths to the results and summary artifacts. -function logClassifySummary( - result: DittoScanClassifyResult, - summaryPath: string -): void { - process.stderr.write( - `[ditto-cli scan][classify] classified ${result.summary.candidate_total} candidates\n` - ); - for (const [status, count] of Object.entries(result.summary.by_status)) { - process.stderr.write(` ${status}: ${count}\n`); - } - process.stderr.write( - `[ditto-cli scan][classify] llm_calls: ${result.summary.llm_calls}, tokens_in: ${result.summary.llm_tokens_in}, tokens_out: ${result.summary.llm_tokens_out}\n` - ); - process.stderr.write( - `[ditto-cli scan][classify] to_manually_validate: ${result.summary.num_results_to_manually_verify}\n` - ); - process.stderr.write( - `[ditto-cli scan][classify] pct_results_needing_manual_verification: ${result.summary.pct_results_needing_manual_verification}\n` - ); - for (const [status, filePath] of Object.entries( - result.writtenPaths.resultsByStatus - )) { - process.stderr.write(`[ptd classify] results.${status} -> ${filePath}\n`); - } - process.stderr.write(`[ptd classify] summary -> ${summaryPath}\n`); - if (result.writtenPaths.verify) { - process.stderr.write( - `[ditto-cli scan][classify] verify -> ${result.writtenPaths.verify}\n` - ); - } +interface ISyncOptions { + local: boolean; + outDir?: string; + prefix?: string; } - export const scan = async ( path: string, - outDir: string = "./out", - prefix: string = "" + { local, outDir = "", prefix = "" }: ISyncOptions ) => { - const apiKey = process.env.GEMINI_API_KEY; - if (!apiKey || apiKey.trim().length === 0) { + if (local && !outDir) { return await quit( - logger.errorText("GEMINI_API_KEY is not set. Aborting."), + logger.errorText( + "Must specify --out-dir if outputting candidates locally" + ), 2 ); } const resolvedInput = resolveUserPath(path); - const resolvedOutDir = resolveUserPath(outDir); - await fs.mkdir(resolvedOutDir, { recursive: true }); - const { - candidates: candidatesPath, - results: resultsPath, - summary: summaryPath, - verify: verifyPath, - schema: schemaPath, - stagings: stagingsPath, - } = buildOutputPaths(resolvedOutDir, prefix); - - const { - candidates, - verdicts, - summary: extractSummary, - } = await runExtract({ inputPath: resolvedInput }); - await writeCandidatesNdjson(candidates, candidatesPath); - logExtractSummary(extractSummary, candidatesPath); + const { candidates, summary: extractSummary } = await runExtract({ + inputPath: resolvedInput, + }); if (candidates.length === 0) { logger.warnText( @@ -174,14 +120,32 @@ export const scan = async ( ); } - const { llmCandidates, preClassified } = preClassify(candidates, verdicts); + if (local) { + const resolvedOutDir = resolveUserPath(outDir); + await fs.mkdir(resolvedOutDir, { recursive: true }); - const classifyResult = await runClassify({ - candidates: llmCandidates, - preClassified, - outputPath: resultsPath, - summaryPath, - verifyPath, - }); - logClassifySummary(classifyResult, summaryPath); + const { candidates: candidatesPath } = buildOutputPaths( + resolvedOutDir, + prefix + ); + + await writeCandidatesNdjson(candidates, candidatesPath); + logExtractSummary(extractSummary, candidatesPath); + } else { + const token = await initAPIToken(); + appContext.setApiToken(token); + const { + candidatesSignedS3Url, + record: { _id: recordId }, + } = await initiateScan(path); + await uploadCandidatesToS3(candidates, candidatesSignedS3Url); + await initiateClassify(recordId); + logExtractSummary(extractSummary); + await quit( + logger.info( + `Scan initiated! Visit https://app.dittowords.com/scan/${recordId} to view progress and see results.` + ), + 0 + ); + } }; diff --git a/lib/src/http/scan.ts b/lib/src/http/scan.ts new file mode 100644 index 0000000..3cbc7b6 --- /dev/null +++ b/lib/src/http/scan.ts @@ -0,0 +1,60 @@ +import axios, { AxiosError } from "axios"; +import getHttpClient from "./client"; +import { IInitiateScanResponse, ZInitiateScanResponse } from "./types"; +import { DittoScanCandidate } from "../scan/types"; +import { Blob } from "buffer"; + +export async function initiateScan( + path: string +): Promise { + try { + const httpClient = getHttpClient({}); + const response = await httpClient.post("/v2/scan", { path }); + return ZInitiateScanResponse.parse(response.data); + } catch (e) { + if (!(e instanceof AxiosError)) { + throw new Error( + "Sorry! We're having trouble reaching the Ditto API. Please try again later." + ); + } + throw e; + } +} + +export async function initiateClassify(scanId: string): Promise { + try { + const httpClient = getHttpClient({}); + await httpClient.post(`/v2/scan/${scanId}/classify`, {}); + } catch (e) { + if (!(e instanceof AxiosError)) { + throw new Error( + "Sorry! We're having trouble reaching the Ditto API. Please try again later." + ); + } + throw e; + } +} + +export async function uploadCandidatesToS3( + candidates: DittoScanCandidate[], + signedUrl: string +) { + // Convert TypeScript array to NDJSON format (JSON objects separated by \n) + const ndjsonString = candidates + .map((candidate) => JSON.stringify(candidate)) + .join("\n"); + + // Create a Blob from the NDJSON string + const blob = new Blob([ndjsonString], { type: "application/x-ndjson" }); + + // Upload via PUT request to the pre-signed URL + // Needs to be done with a non-ditto http client so we don't pass our auth token + const httpClient = axios.create({}); + const response = await httpClient.put(signedUrl, blob, { + headers: { + // S3 requires the Content-Length header for PUT requests + "Content-Length": blob.size, + }, + }); + return response; +} diff --git a/lib/src/http/types.ts b/lib/src/http/types.ts index 96f6deb..e56b714 100644 --- a/lib/src/http/types.ts +++ b/lib/src/http/types.ts @@ -177,3 +177,12 @@ export const ZExportSwiftFileRequest = z.object({ }); export type IExportSwiftFileRequest = z.infer; + +export const ZInitiateScanBodySchema = z.object({ + path: z.string(), +}); +export const ZInitiateScanResponse = z.object({ + record: z.object({ _id: z.string() }), + candidatesSignedS3Url: z.string(), +}); +export type IInitiateScanResponse = z.infer; diff --git a/lib/src/index.ts b/lib/src/index.ts index e186d08..294e512 100644 --- a/lib/src/index.ts +++ b/lib/src/index.ts @@ -10,7 +10,6 @@ import logger from "./utils/logger"; import initAPIToken from "./services/apiToken/initAPIToken"; import { initProjectConfig } from "./services/projectConfig"; import appContext from "./utils/appContext"; -import type commander from "commander"; import { ErrorType, isDittoError, isDittoErrorType } from "./utils/DittoError"; import processCommandMetaFlag from "./utils/processCommandMetaFlag"; @@ -77,12 +76,20 @@ const appEntry = async () => { .description( "Run extract + classify + infer end-to-end; emits schema.json and stagings.ndjson to --out-dir" ) - .option("--out-dir ", "output directory", "./out") + .option( + "--local", + "outputs the candidates file locally to the out-dir with the given prefix", + false + ) + .option("--out-dir ", "output directory", "") .option("--prefix ", "prefix for output files", "") .action( - async (inputPath: string, opts: { outDir: string; prefix?: string }) => { + async ( + inputPath: string, + opts: { local: boolean; outDir: string; prefix?: string } + ) => { try { - return await scan(inputPath, opts.outDir, opts.prefix); + return await scan(inputPath, opts); } catch (error) { handleCommandError(error); } diff --git a/lib/src/scan/classify.ts b/lib/src/scan/classify.ts deleted file mode 100644 index 3ddae06..0000000 --- a/lib/src/scan/classify.ts +++ /dev/null @@ -1,367 +0,0 @@ -import { createHash } from "crypto"; -import fs from "fs/promises"; -import path from "path"; -import { z } from "zod"; - -import { callGemini } from "../services/gemini"; - -import { - DittoScanCandidate, - DittoScanStatusSchema, - type DittoScanResult, - type DittoScanStatus, - type DittoScanSummary, -} from "./types"; -import DittoError, { ErrorType } from "../utils/DittoError"; - -export interface DittoScanClassifyOptions { - candidates?: DittoScanCandidate[]; - // Results already decided by the deterministic rule classifier in - // `pre-classify`. Merged into the output file ahead of LLM-decided - // results. Their `decided_by` is "rule". - preClassified?: DittoScanResult[]; - inputPath?: string; - outputPath: string; - summaryPath: string; - verifyPath: string; -} - -interface DittoScanClassifyData { - results: DittoScanResult[]; - summary: DittoScanSummary; -} - -export interface DittoScanClassifyResult extends DittoScanClassifyData { - writtenPaths: { - resultsByStatus: Partial>; - verify: string | null; - }; -} - -const MIN_CONFIDENCE_LEVEL = 0.9; -const BATCH_SIZE = 10; -const PARALLEL_BATCHES = 5; -const CLASSIFY_MODEL = "gemini-2.5-flash"; - -const SYSTEM_PROMPT = `You are classifying string literals extracted from an application codebase to determine whether they are user-facing (visible to end users in the UI) or not. The application type (web, mobile, desktop, etc.) can be inferred from the framework field on each candidate. - -For each candidate examine: -- value_raw: the actual string literal -- detection_kind: markup_text (JSX/HTML text content), markup_attr (HTML attribute value), resource_value (value inside a localization resource file), or other (any other code position) -- context_identifiers: attribute names or identifiers enclosing the string (e.g. "placeholder", "aria-label", "classname", "textid"); for resource_value this is the localization key and any variant labels (e.g. ["greeting"] or ["items", "one"]) -- language: the file's language (tsx, jsx, typescript, javascript, vue, kotlin, swift, ios_strings, android_resources, etc.) — helps interpret the source_context syntax -- framework: UI frameworks and platforms detected in the project (e.g. ["react"], ["vue"], ["android"], ["ios"]) — a React/Vue/iOS/Android file is far more likely to contain user-facing strings than a plain Node.js utility -- source_context: surrounding lines of source code with line numbers -- usage_evidence (if present): how the string constant is used elsewhere in the codebase - -Classification statuses: -- "user-facing": text visible to end users — button labels, form placeholders, error messages shown in the UI, modal titles, headings, aria-labels, tooltip text, etc. -- "not-user-facing": internal strings — CSS class names, import paths, localStorage keys, API route prefixes, internal error codes used for programmatic comparisons, keyboard key names for event matching (e.g. "Enter"), enum-like string values, Ditto textId references (e.g. "text_630d31..."), etc. -- "unsure": genuinely ambiguous; use sparingly - -Use confidence >= ${MIN_CONFIDENCE_LEVEL} when evidence clearly points one way, 0.6 - ${MIN_CONFIDENCE_LEVEL} when leaning one way with some ambiguity, < 0.6 for truly unclear cases. -Set info_needed to null unless specific missing context would meaningfully change the verdict.`; - -const ZBatchItem = z.object({ - id: z.string(), - status: DittoScanStatusSchema, - reason: z.string(), - confidence: z.number().min(0).max(1), - info_needed: z.string().nullable(), -}); - -const ZBatchResponse = z.object({ - items: z.array(ZBatchItem), -}); - -interface Stats { - llmCalls: number; - tokensIn: number; - tokensOut: number; -} - -function formatCandidate(c: DittoScanCandidate, index: number): string { - const parts = [ - `[Candidate ${index + 1}]`, - `id: ${JSON.stringify(c.id)}`, - `value_raw: ${JSON.stringify(c.value_raw)}`, - `detection_kind: ${c.detection_kind}`, - `context_identifiers: [${c.context_identifiers - .map((x) => JSON.stringify(x)) - .join(", ")}]`, - `language: ${c.language}`, - `framework: [${c.framework.map((f) => JSON.stringify(f)).join(", ")}]`, - `source_context:\n${c.source_context}`, - ]; - if (c.usage_evidence?.length) { - parts.push( - `usage_evidence:\n${c.usage_evidence - .map((e) => ` ${e.file}:${e.line}: ${e.excerpt}`) - .join("\n")}` - ); - } - parts.push(`[/Candidate ${index + 1}]`); - return parts.join("\n"); -} - -async function classifyBatch( - batch: DittoScanCandidate[], - stats: Stats -): Promise[] | null> { - const userPrompt = [ - `Classify the following ${batch.length} string literal candidates.`, - `Return a JSON object with an "items" array containing one classification per candidate, in the same order.`, - "", - batch.map((c, i) => formatCandidate(c, i)).join("\n\n"), - ].join("\n"); - - try { - const result = await callGemini(userPrompt, ZBatchResponse, SYSTEM_PROMPT); - - if (!result) return null; - - stats.llmCalls++; - stats.tokensIn += - result.metadata.geminiUsageMetadata?.promptTokenCount ?? 0; - stats.tokensOut += - result.metadata.geminiUsageMetadata?.candidatesTokenCount ?? 0; - - if (result.data.items.length !== batch.length) { - process.stderr.write( - `[classify] response had ${result.data.items.length} items for ${batch.length} candidates\n` - ); - return null; - } - return result.data.items; - } catch (e: any) { - process.stderr.write( - `[classify/gemini] error: ${e.message ?? String(e)}\n` - ); - return null; - } -} - -async function classifyWithRecovery( - batch: DittoScanCandidate[], - stats: Stats -): Promise { - let items = await classifyBatch(batch, stats); - - if (!items) { - process.stderr.write(`[classify] retrying batch of ${batch.length}...\n`); - items = await classifyBatch(batch, stats); - } - - if (items) { - const byId = new Map(items.map((item) => [item.id, item])); - return batch.flatMap((c) => { - const item = byId.get(c.id); - if (!item) return []; - return [ - { - ...c, - status: item.status, - reason: item.reason, - confidence: item.confidence, - info_needed: item.info_needed, - decided_by: "llm" as const, - }, - ]; - }); - } - - if (batch.length > 1) { - process.stderr.write( - `[classify] splitting batch of ${batch.length} after failure\n` - ); - const mid = Math.ceil(batch.length / 2); - const left = await classifyWithRecovery(batch.slice(0, mid), stats); - const right = await classifyWithRecovery(batch.slice(mid), stats); - return [...left, ...right]; - } - - process.stderr.write( - `[classify] marking candidate ${batch[0].id} as error\n` - ); - return [ - { - ...batch[0], - status: "error" as DittoScanStatus, - reason: "LLM classification failed after retries", - confidence: 0, - info_needed: null, - decided_by: "llm" as const, - }, - ]; -} - -async function classify( - candidates: DittoScanCandidate[], - preClassified: DittoScanResult[] = [] -): Promise { - const startedAt = new Date().toISOString(); - const promptHash = createHash("sha1") - .update(SYSTEM_PROMPT) - .digest("hex") - .slice(0, 12); - const stats: Stats = { llmCalls: 0, tokensIn: 0, tokensOut: 0 }; - const llmResults: DittoScanResult[] = []; - - const allBatches: DittoScanCandidate[][] = []; - for (let i = 0; i < candidates.length; i += BATCH_SIZE) { - allBatches.push(candidates.slice(i, i + BATCH_SIZE)); - } - - for (let i = 0; i < allBatches.length; i += PARALLEL_BATCHES) { - const chunk = allBatches.slice(i, i + PARALLEL_BATCHES); - const chunkResults = await Promise.all( - chunk.map((batch, j) => { - const batchIdx = i + j + 1; - process.stderr.write( - `[classify] batch ${batchIdx}/${allBatches.length} (${batch.length} candidates)\n` - ); - return classifyWithRecovery(batch, stats); - }) - ); - for (const batchResult of chunkResults) { - llmResults.push(...batchResult); - } - } - - // Rule-decided results lead so they appear first in the output file. - // by_status / counts reflect the union. - const results: DittoScanResult[] = [...preClassified, ...llmResults]; - - const byStatus: Partial> = { - unsure: 0, - error: 0, - }; - // by_rule keys are `${language}:${reason}` so rule names that appear in - // multiple languages (e.g. `logger_call` in JS / Kotlin / Swift) stay - // distinguishable. Each rule file owns one logical concept per language; - // collapsing across languages would hide which implementation is doing - // the work. - const byRule: Record = {}; - let ruleCount = 0; - let llmCount = 0; - for (const r of results) { - byStatus[r.status] = (byStatus[r.status] ?? 0) + 1; - if (r.decided_by === "rule") { - ruleCount++; - const key = `${r.language}:${r.reason}`; - byRule[key] = (byRule[key] ?? 0) + 1; - } else { - llmCount++; - } - } - - const numResultsToManuallyValidate = results.filter( - (r) => r.confidence < MIN_CONFIDENCE_LEVEL - ).length; - - const finishedAt = new Date().toISOString(); - const summary: DittoScanSummary = { - run_id: createHash("sha1") - .update(`${startedAt}:${promptHash}`) - .digest("hex") - .slice(0, 16), - started_at: startedAt, - finished_at: finishedAt, - model: CLASSIFY_MODEL, - prompt_hash: promptHash, - candidate_total: results.length, - by_status: byStatus, - num_results_to_manually_verify: numResultsToManuallyValidate, - pct_results_needing_manual_verification: - results.length > 0 - ? (numResultsToManuallyValidate / results.length) * 100 - : 0, - llm_calls: stats.llmCalls, - llm_tokens_in: stats.tokensIn, - llm_tokens_out: stats.tokensOut, - decided_by: { rule: ruleCount, llm: llmCount }, - by_rule: byRule, - }; - - return { results, summary }; -} - -export async function runClassify( - options: DittoScanClassifyOptions -): Promise { - let candidates: DittoScanCandidate[]; - - if (options.candidates) { - candidates = options.candidates; - } else if (options.inputPath) { - const raw = await fs.readFile(options.inputPath, "utf8"); - candidates = raw - .split("\n") - .filter(Boolean) - .map((line) => JSON.parse(line) as DittoScanCandidate); - } else { - const errorMessage = - "Invalid inputs. Missing both 'candidates' and 'inputPath'"; - throw new DittoError({ - type: ErrorType.ScanError, - data: { rawErrorMessage: errorMessage }, - message: errorMessage, - }); - } - - const result = await classify(candidates, options.preClassified); - - const parsedOutput = path.parse(options.outputPath); - const ext = parsedOutput.ext; - const base = path.join(parsedOutput.dir, parsedOutput.name); - - const allStatuses: DittoScanStatus[] = [ - "user-facing", - "not-user-facing", - "unsure", - "error", - ]; - const resultsByStatus: Partial> = {}; - await Promise.all( - allStatuses.map(async (status) => { - const statusResults = result.results.filter((r) => r.status === status); - const filePath = `${base}-${status}${ext}`; - if (statusResults.length === 0) { - await fs.rm(filePath, { force: true }); - return; - } - await fs.writeFile( - filePath, - statusResults.map((r) => JSON.stringify(r)).join("\n") + "\n", - "utf8" - ); - resultsByStatus[status] = filePath; - }) - ); - - await fs.writeFile( - options.summaryPath, - JSON.stringify(result.summary, null, 2) + "\n", - "utf8" - ); - - const toVerify = result.results.filter( - (r) => r.confidence < MIN_CONFIDENCE_LEVEL - ); - let writtenVerifyPath: string | null = null; - if (toVerify.length > 0) { - await fs.writeFile( - options.verifyPath, - toVerify.map((r) => JSON.stringify(r)).join("\n") + "\n", - "utf8" - ); - writtenVerifyPath = options.verifyPath; - } else { - await fs.rm(options.verifyPath, { force: true }); - } - - return { - ...result, - writtenPaths: { resultsByStatus, verify: writtenVerifyPath }, - }; -} diff --git a/lib/src/scan/extract.ts b/lib/src/scan/extract.ts index 1c9148f..3289d3f 100644 --- a/lib/src/scan/extract.ts +++ b/lib/src/scan/extract.ts @@ -8,8 +8,8 @@ import { type DittoScanDetectionKind, } from "./types"; import { createHash } from "crypto"; -import type { LlmFileTaskStats } from "./lang/llm-file-discovery"; -import { getClassificationVerdict, shouldEmit, type Verdict } from "./rules"; +import type { FileDiscoveryStats } from "./lang/file-discovery"; +import { shouldEmit } from "./rules"; import { walkCodebase } from "./walk"; export interface DittoScanExtractOptions { @@ -18,7 +18,6 @@ export interface DittoScanExtractOptions { export interface DittoScanExtractResult { candidates: DittoScanCandidate[]; - verdicts: Map; summary: DittoScanExtractSummary; } @@ -28,11 +27,9 @@ export interface DittoScanExtractSummary { filesSkippedMinified: number; candidatesEmitted: number; candidatesByKind: Record; - candidatesByVerdict: { accept: number; reject: number; llm: number }; - ruleHits: Record; framework: string[]; elapsedMs: number; - i18nFileDiscovery: LlmFileTaskStats | null; + i18nFileDiscovery: FileDiscoveryStats | null; } const CONTEXT_LINES = 3; @@ -249,9 +246,6 @@ export async function runExtract( const candidatesByKind = zeroKindCounts(); const candidates: DittoScanCandidate[] = []; - const verdicts = new Map(); - const candidatesByVerdict = { accept: 0, reject: 0, llm: 0 }; - const ruleHits: Record = {}; for (const file of files) { const lang = file.language; @@ -296,32 +290,17 @@ export async function runExtract( }; candidates.push(candidate); candidatesByKind[hit.context.parentRole]++; - - const verdict = getClassificationVerdict({ - candidate, - context: hit.context, - }); - verdicts.set(candidate.id, verdict); - if (verdict.kind === "accept" || verdict.kind === "reject") { - candidatesByVerdict[verdict.kind]++; - ruleHits[verdict.rule] = (ruleHits[verdict.rule] ?? 0) + 1; - } else { - candidatesByVerdict.llm++; - } } } return { candidates, - verdicts, summary: { filesScanned: files.length, filesByKind, filesSkippedMinified, candidatesEmitted: candidates.length, candidatesByKind, - candidatesByVerdict, - ruleHits, framework, elapsedMs: Date.now() - t0, i18nFileDiscovery, diff --git a/lib/src/scan/lang/file-discovery.ts b/lib/src/scan/lang/file-discovery.ts new file mode 100644 index 0000000..65651c4 --- /dev/null +++ b/lib/src/scan/lang/file-discovery.ts @@ -0,0 +1,146 @@ +import fs from "fs/promises"; +import { globby } from "globby"; +import path from "path"; + +const DEFAULT_PREVIEW_CHARS = 400; + +export interface FileDiscoveryTask { + taskName: string; + globs: string[]; + ignore?: string[]; + // Synchronous reject before heuristic to skip obvious non-matches. + preFilter?: (basename: string, relPath: string) => boolean; + // Confirmed without the heuristic (e.g. single-purpose extensions). + autoInclude?: (relPath: string) => boolean; + // Called with the relative path and a content preview for every file + // that survived preFilter/autoInclude. Return true to include the file. + heuristicMatch: (relPath: string, preview: string) => boolean; + previewChars?: number; +} + +export interface FileDiscoveryStats { + task: string; + totalCandidates: number; + preFiltered: number; + autoIncluded: number; + heuristicConsidered: number; + heuristicConfirmed: number; + elapsedMs: number; +} + +export interface FileDiscoveryResult { + paths: Set; + stats: FileDiscoveryStats; +} + +// Keep legacy aliases so callers can migrate at their own pace. +/** @deprecated Use FileDiscoveryTask */ +export type LlmFileTask = FileDiscoveryTask; +/** @deprecated Use FileDiscoveryStats */ +export type LlmFileTaskStats = FileDiscoveryStats; +/** @deprecated Use FileDiscoveryResult */ +export type LlmFileTaskResult = FileDiscoveryResult; + +const DEFAULT_IGNORE = [ + "**/node_modules/**", + "**/.git/**", + "**/dist/**", + "**/build/**", + "**/coverage/**", + "**/.next/**", + "**/.nuxt/**", + "**/vendor/**", +]; + +export async function runFileDiscoveryTask( + rootPath: string, + task: FileDiscoveryTask +): Promise { + const t0 = Date.now(); + const root = path.resolve(rootPath); + const stats: FileDiscoveryStats = { + task: task.taskName, + totalCandidates: 0, + preFiltered: 0, + autoIncluded: 0, + heuristicConsidered: 0, + heuristicConfirmed: 0, + elapsedMs: 0, + }; + + const allPaths = await globby(task.globs, { + cwd: root, + gitignore: true, + onlyFiles: true, + followSymbolicLinks: false, + suppressErrors: true, + ignore: [...DEFAULT_IGNORE, ...(task.ignore ?? [])], + }); + stats.totalCandidates = allPaths.length; + + const confirmed = new Set(); + const toCheck: string[] = []; + + for (const relPath of allPaths) { + const base = path.basename(relPath); + if (task.preFilter && task.preFilter(base, relPath)) { + stats.preFiltered++; + continue; + } + if (task.autoInclude && task.autoInclude(relPath)) { + confirmed.add(relPath); + stats.autoIncluded++; + continue; + } + toCheck.push(relPath); + } + stats.heuristicConsidered = toCheck.length; + + const previewChars = sanitizePositiveInt(task.previewChars, DEFAULT_PREVIEW_CHARS); + + await Promise.all( + toCheck.map(async (relPath) => { + const preview = await readPreview(path.join(root, relPath), previewChars); + if (preview !== null && task.heuristicMatch(relPath, preview)) { + confirmed.add(relPath); + } + }) + ); + + stats.heuristicConfirmed = confirmed.size - stats.autoIncluded; + stats.elapsedMs = Date.now() - t0; + return { paths: confirmed, stats }; +} + +/** @deprecated Use runFileDiscoveryTask */ +export const runLlmFileTask = runFileDiscoveryTask; + +async function readPreview( + absPath: string, + previewChars: number +): Promise { + let handle: fs.FileHandle | null = null; + try { + handle = await fs.open(absPath, "r"); + // UTF-8 expansion is at most 4 bytes/char; one extra byte lets us + // detect overflow without rereading. + const buf = new Uint8Array(previewChars * 4 + 1); + const { bytesRead } = await handle.read(buf, 0, buf.length, 0); + const decoded = Buffer.from(buf.buffer, buf.byteOffset, bytesRead).toString( + "utf8" + ); + return decoded.slice(0, previewChars); + } catch { + return null; + } finally { + await handle?.close(); + } +} + +function sanitizePositiveInt( + value: number | undefined, + fallback: number +): number { + const n = Number(value ?? fallback); + return Math.max(1, Math.floor(Number.isFinite(n) ? n : fallback)); +} diff --git a/lib/src/scan/lang/i18n-file-discovery.ts b/lib/src/scan/lang/i18n-file-discovery.ts index 01fb31c..6df4a5a 100644 --- a/lib/src/scan/lang/i18n-file-discovery.ts +++ b/lib/src/scan/lang/i18n-file-discovery.ts @@ -1,4 +1,8 @@ -import { runLlmFileTask, type LlmFileTask, type LlmFileTaskResult } from "./llm-file-discovery"; +import { + runFileDiscoveryTask, + type FileDiscoveryTask, + type FileDiscoveryResult, +} from "./file-discovery"; const NEVER_FILENAMES: ReadonlySet = new Set([ "package.json", @@ -10,34 +14,15 @@ const NEVER_FILENAMES: ReadonlySet = new Set([ "bun.lockb", ]); -const NEVER_PATTERNS: readonly RegExp[] = [/^tsconfig\.[^/]+\.json$/i, /\.schema\.json$/i, /\.lock$/i]; - -const SYSTEM_PROMPT = `You identify internationalization (i18n) files in a codebase. - -An i18n file stores user-facing UI copy (button labels, error messages, headings, etc.) keyed by identifiers, intended to be read by an i18n library (i18next, react-intl, vue-i18n, Lingui, gettext, Ditto, Rails i18n, Spring/Java ResourceBundle, …). Translations of the same content in other locales also count. - -NOT i18n files (do not include): -- Build / tool configs (package.json, tsconfig, eslint, prettier, babel, vite, webpack, vercel, nx, turbo) -- Type-checker configs (pyrightconfig, mypy.ini) -- Plugin / app manifests (plugin.json, hak.json, manifest.json, capacitor.config.json, app.json) -- Schema files (*.schema.json, JSON Schema documents, OpenAPI / Swagger specs) -- Database / infrastructure configs (database.yml, cable.yml, docker-compose.yml, helm Chart.yaml) -- API request/response fixtures, test data, mock responses, eval prompts -- LLM prompt fragments, model configs -- CMS exports / content snapshots that happen to be all strings -- Plain config files (auth_config.json, bruno.json, theme files) -- API coverage reports, benchmark scenarios -- Java/JVM application config: application.properties, log4j.properties / log4j2.properties, gradle.properties, build.properties, version.properties. These are KEY=VALUE settings (host names, ports, thread pools, log levels) — not user-facing copy - -Java .properties files ARE i18n when they hold UI strings keyed by message id (messages_en.properties, labels.properties, ApplicationResources_fr.properties, the content reads like sentences/labels rather than config values). - -Use both the file path and the content preview. The path is the strongest signal — files under \`locales/\`, \`i18n/\`, \`translations/\`, \`messages/\`, \`lang/\`, etc., or with names like \`en.json\` / \`fr.yaml\` / \`messages.en.po\` / \`messages_de.properties\`, are almost always i18n files. Custom layouts (e.g. \`___.json\`) are also valid when the content matches. - -Return ONLY the paths that ARE i18n files.`; +const NEVER_PATTERNS: readonly RegExp[] = [ + /^tsconfig\.[^/]+\.json$/i, + /\.schema\.json$/i, + /\.lock$/i, +]; // All extensions the i18n discovery pass considers. The walker uses the same -// set to decide which files are LLM-gated vs. dispatched straight to the -// language registry, so this is the single source of truth. +// set to decide which files are dispatched straight to the language registry, +// so this is the single source of truth. export const I18N_FILE_EXTENSIONS: ReadonlySet = new Set([ ".json", ".yaml", @@ -51,20 +36,172 @@ export const I18N_FILE_EXTENSIONS: ReadonlySet = new Set([ ".resw", ]); -// Subset that's single-purpose enough to skip the LLM round trip. -const AUTO_INCLUDE_EXTENSIONS: ReadonlySet = new Set([".po", ".arb", ".xliff", ".xlf", ".resx", ".resw"]); +// Subset that's single-purpose enough to skip the heuristic round trip. +const AUTO_INCLUDE_EXTENSIONS: ReadonlySet = new Set([ + ".po", + ".arb", + ".xliff", + ".xlf", + ".resx", + ".resw", +]); + +// --------------------------------------------------------------------------- +// Heuristic i18n detection +// --------------------------------------------------------------------------- + +// Directory names that strongly indicate a file is part of an i18n system. +const I18N_DIR_SEGMENTS: ReadonlySet = new Set([ + "locales", + "locale", + "i18n", + "translations", + "translation", + "messages", + "lang", + "languages", + "strings", + "nls", + "intl", + "l10n", +]); + +// Exact ISO 639-1 two-letter language codes. Using a set rather than a regex +// avoids false positives for common technical abbreviations (db, ui, go, ai…) +// that also happen to be two letters but are not valid locale codes. +const ISO_639_1: ReadonlySet = new Set([ + "af", "sq", "am", "ar", "hy", "az", "eu", "be", "bn", "bs", "bg", "my", + "ca", "zh", "hr", "cs", "da", "nl", "en", "et", "fi", "fr", "ka", "de", + "el", "gu", "ht", "ha", "he", "hi", "hu", "is", "id", "ga", "it", "ja", + "kn", "kk", "km", "ko", "ku", "ky", "lo", "lv", "lt", "lb", "mk", "mg", + "ms", "ml", "mt", "mi", "mr", "mn", "ne", "nb", "nn", "no", "or", "ps", + "fa", "pl", "pt", "pa", "ro", "ru", "sm", "sr", "sn", "si", "sk", "sl", + "so", "es", "sw", "sv", "tl", "tg", "ta", "tt", "te", "th", "bo", "tr", + "tk", "uk", "ur", "ug", "uz", "vi", "cy", "xh", "yi", "yo", "zu", +]); + +// BCP-47 subtag: separator + 2-4 char region/script (e.g. -DE, _Latn). +// Only used when the token already contains a separator, so no bare-code +// false positives are possible here. +const LOCALE_SUBTAG_RE = /^[a-z]{2}[_-][a-zA-Z]{2,4}$/i; + +function isLocaleToken(s: string): boolean { + if (s.includes("-") || s.includes("_")) { + return LOCALE_SUBTAG_RE.test(s) && ISO_639_1.has(s.slice(0, 2).toLowerCase()); + } + return ISO_639_1.has(s.toLowerCase()); +} + +// Locale as an underscore-delimited suffix in a filename stem, e.g. +// messages_en, ApplicationResources_fr, labels_de-DE. +const LOCALE_UNDERSCORE_SUFFIX_RE = /^.+_([a-z]{2}([_-][a-zA-Z]{2,4})?)$/i; + +// JSON keys that appear in non-i18n config files (package.json, etc.). +// A file containing any of these at the root level is almost certainly not i18n. +const JSON_CONFIG_KEY_RE = + /"(?:version|dependencies|devDependencies|scripts|name|peerDependencies|engines|repository|license)"\s*:/; + +function filenameHasLocaleToken(filename: string): boolean { + // Split by dots, remove the final extension, then check each dot-segment. + // We use dots (not underscores) as primary delimiters so that we only + // match whole tokens: "messages.en.json" → check "messages" and "en". + const parts = filename.split("."); + if (parts.length < 2) return false; + parts.pop(); // drop the file extension + + for (const part of parts) { + // Whole dot-segment is a locale: "en", "de-DE", "zh_CN" + if (isLocaleToken(part)) return true; + // Whole dot-segment ends with _: "messages_en", "labels_fr" + const m = part.match(LOCALE_UNDERSCORE_SUFFIX_RE); + if (m && isLocaleToken(m[1])) return true; + } + return false; +} + +// Content-based checks — used only when path/filename give no clear signal. + +function looksLikeI18nJson(preview: string): boolean { + if (!preview.trimStart().startsWith("{")) return false; + if (JSON_CONFIG_KEY_RE.test(preview)) return false; + + // Count "key": "string value" vs "key": + const stringPairs = (preview.match(/"[^"\n]+"\s*:\s*"[^"\n]*"/g) ?? []) + .length; + const nonStringPairs = ( + preview.match(/"[^"\n]+"\s*:\s*(?:\d|true|false|null|\[|\{)/g) ?? [] + ).length; + + if (stringPairs < 3 || stringPairs <= nonStringPairs) return false; + // Require at least one value that reads like a sentence/label (>10 chars). + return (preview.match(/"[^"\n]+"\s*:\s*"[^"\n]{10,}"/g) ?? []).length > 0; +} + +function looksLikeI18nYaml(preview: string): boolean { + // Reject clear infrastructure/config YAML patterns. + if ( + /^(?:host|port|database|username|password|server|adapter|pool)\s*:/im.test( + preview + ) + ) + return false; + // Require several lines that look like "key: sentence value". + const sentenceLines = ( + preview.match(/^\s*[\w.]+\s*:\s*["']?[A-Za-z][^{\[\n]{8,}/gm) ?? [] + ).length; + return sentenceLines >= 3; +} + +function looksLikeI18nProperties(preview: string): boolean { + const propLines = preview.match(/^[\w.]+\s*[=:]\s*.+$/gm) ?? []; + if (propLines.length < 3) return false; + // Require at least 2 values that contain spaces (natural language, not config). + const sentenceLike = propLines.filter((line) => { + const sep = line.search(/[=:]/); + const value = line.slice(sep + 1).trim(); + return value.includes(" ") && value.length > 8; + }); + return sentenceLike.length >= 2; +} + +function isLikelyI18nFile(relPath: string, preview: string): boolean { + const parts = relPath.split("/"); + const filename = parts[parts.length - 1]; + const extDot = filename.lastIndexOf("."); + const ext = extDot === -1 ? "" : filename.slice(extDot).toLowerCase(); + + // 1. File lives under a known i18n directory segment. + for (const seg of parts.slice(0, -1)) { + if (I18N_DIR_SEGMENTS.has(seg.toLowerCase())) return true; + } + + // 2. Any dot-separated token in the filename is a locale code. + if (filenameHasLocaleToken(filename)) return true; + + // 3. Content-based fallback for ambiguous filenames. + if (ext === ".json") return looksLikeI18nJson(preview); + if (ext === ".yaml" || ext === ".yml") return looksLikeI18nYaml(preview); + if (ext === ".properties") return looksLikeI18nProperties(preview); + return false; +} -export const I18N_FILES_TASK: LlmFileTask = { +export const I18N_FILES_TASK: FileDiscoveryTask = { taskName: "i18n files", globs: [...I18N_FILE_EXTENSIONS].map((ext) => `**/*${ext}`), - preFilter: (base) => NEVER_FILENAMES.has(base) || NEVER_PATTERNS.some((re) => re.test(base)), + preFilter: (base) => + NEVER_FILENAMES.has(base) || NEVER_PATTERNS.some((re) => re.test(base)), autoInclude: (relPath) => { const dot = relPath.lastIndexOf("."); - return dot !== -1 && AUTO_INCLUDE_EXTENSIONS.has(relPath.slice(dot).toLowerCase()); + return ( + dot !== -1 && + AUTO_INCLUDE_EXTENSIONS.has(relPath.slice(dot).toLowerCase()) + ); }, - systemPrompt: SYSTEM_PROMPT, + heuristicMatch: isLikelyI18nFile, }; -export async function findI18nFiles(rootPath: string): Promise { - return runLlmFileTask(rootPath, I18N_FILES_TASK); +export async function findI18nFiles( + rootPath: string +): Promise { + return runFileDiscoveryTask(rootPath, I18N_FILES_TASK); } diff --git a/lib/src/scan/lang/llm-file-discovery.ts b/lib/src/scan/lang/llm-file-discovery.ts deleted file mode 100644 index f2b7b80..0000000 --- a/lib/src/scan/lang/llm-file-discovery.ts +++ /dev/null @@ -1,213 +0,0 @@ -import fs from "fs/promises"; -import { globby } from "globby"; -import path from "path"; -import { z } from "zod"; - -import { callGemini } from "../../services/gemini"; - -// Generic LLM file classifier. Each task supplies globs + a system -// prompt; Gemini picks the matching subset. i18n discovery is one -// task; future passes (exclusion lists, OpenAPI specs, …) plug in the -// same way. - -const DEFAULT_PREVIEW_CHARS = 400; -const DEFAULT_MAX_FILES_PER_CALL = 250; - -export interface LlmFileTask { - taskName: string; - globs: string[]; - ignore?: string[]; - // Synchronous reject before the LLM call to save tokens. - preFilter?: (basename: string, relPath: string) => boolean; - // Confirmed without the LLM (e.g. single-purpose extensions). - autoInclude?: (relPath: string) => boolean; - systemPrompt: string; - previewChars?: number; - maxFilesPerCall?: number; -} - -export interface LlmFileTaskStats { - task: string; - totalCandidates: number; - preFiltered: number; - autoIncluded: number; - llmConsidered: number; - llmConfirmed: number; - promptTokens: number; - completionTokens: number; - llmCalls: number; - elapsedMs: number; -} - -export interface LlmFileTaskResult { - paths: Set; - stats: LlmFileTaskStats; -} - -const DEFAULT_IGNORE = [ - "**/node_modules/**", - "**/.git/**", - "**/dist/**", - "**/build/**", - "**/coverage/**", - "**/.next/**", - "**/.nuxt/**", - "**/vendor/**", -]; - -const ZResponse = z.object({ - matching_paths: z.array(z.string()), -}); - -export async function runLlmFileTask( - rootPath: string, - task: LlmFileTask -): Promise { - const t0 = Date.now(); - const root = path.resolve(rootPath); - const stats: LlmFileTaskStats = { - task: task.taskName, - totalCandidates: 0, - preFiltered: 0, - autoIncluded: 0, - llmConsidered: 0, - llmConfirmed: 0, - promptTokens: 0, - completionTokens: 0, - llmCalls: 0, - elapsedMs: 0, - }; - - const allPaths = await globby(task.globs, { - cwd: root, - gitignore: true, - onlyFiles: true, - followSymbolicLinks: false, - suppressErrors: true, - ignore: [...DEFAULT_IGNORE, ...(task.ignore ?? [])], - }); - stats.totalCandidates = allPaths.length; - - const confirmed = new Set(); - const toAsk: string[] = []; - - for (const relPath of allPaths) { - const base = path.basename(relPath); - if (task.preFilter && task.preFilter(base, relPath)) { - stats.preFiltered++; - continue; - } - if (task.autoInclude && task.autoInclude(relPath)) { - confirmed.add(relPath); - stats.autoIncluded++; - continue; - } - toAsk.push(relPath); - } - stats.llmConsidered = toAsk.length; - - if (toAsk.length === 0) { - stats.elapsedMs = Date.now() - t0; - return { paths: confirmed, stats }; - } - - const previewChars = sanitizePositiveInt( - task.previewChars, - DEFAULT_PREVIEW_CHARS - ); - const batchSize = sanitizePositiveInt( - task.maxFilesPerCall, - DEFAULT_MAX_FILES_PER_CALL - ); - - for (let i = 0; i < toAsk.length; i += batchSize) { - const batch = toAsk.slice(i, i + batchSize); - const confirmedFromBatch = await runBatch({ - root, - batch, - systemPrompt: task.systemPrompt, - previewChars, - stats, - }); - for (const p of confirmedFromBatch) confirmed.add(p); - } - - stats.llmConfirmed = confirmed.size - stats.autoIncluded; - stats.elapsedMs = Date.now() - t0; - return { paths: confirmed, stats }; -} - -async function runBatch(args: { - root: string; - batch: string[]; - systemPrompt: string; - previewChars: number; - stats: LlmFileTaskStats; -}): Promise { - const { root, batch, systemPrompt, previewChars, stats } = args; - const samples = await Promise.all( - batch.map(async (relPath) => { - let handle: fs.FileHandle | null = null; - try { - handle = await fs.open(path.join(root, relPath), "r"); - // UTF-8 expansion is at most 4 bytes/char; one extra byte lets us - // detect overflow without rereading. - const buf = new Uint8Array(previewChars * 4 + 1); - const { bytesRead } = await handle.read(buf, 0, buf.length, 0); - const decoded = Buffer.from( - buf.buffer, - buf.byteOffset, - bytesRead - ).toString("utf8"); - const preview = decoded.slice(0, previewChars); - const truncated = - decoded.length > previewChars || bytesRead === buf.length; - return { - relPath, - preview: preview + (truncated ? "…(truncated)" : ""), - }; - } catch { - return null; - } finally { - await handle?.close(); - } - }) - ); - const valid = samples.filter( - (s): s is { relPath: string; preview: string } => s !== null - ); - if (valid.length === 0) return []; - - const userPrompt = [ - "For each of the following files, decide whether it matches the task in the system instructions.", - 'Return ONLY the paths that match, exactly as given, in the "matching_paths" array.', - "", - ...valid.map( - (f, i) => - `[File ${i + 1}]\npath: ${f.relPath}\npreview:\n${f.preview}\n[/File ${ - i + 1 - }]` - ), - ].join("\n"); - - stats.llmCalls++; - const result = await callGemini(userPrompt, ZResponse, systemPrompt); - if (!result) return []; - - stats.promptTokens += - result.metadata.geminiUsageMetadata?.promptTokenCount ?? 0; - stats.completionTokens += - result.metadata.geminiUsageMetadata?.candidatesTokenCount ?? 0; - - // Drop any path the LLM returned that wasn't in the batch. - const seen = new Set(valid.map((v) => v.relPath)); - return result.data.matching_paths.filter((p) => seen.has(p)); -} - -function sanitizePositiveInt( - value: number | undefined, - fallback: number -): number { - const n = Number(value ?? fallback); - return Math.max(1, Math.floor(Number.isFinite(n) ? n : fallback)); -} diff --git a/lib/src/scan/preClassify.ts b/lib/src/scan/preClassify.ts deleted file mode 100644 index 24fe90f..0000000 --- a/lib/src/scan/preClassify.ts +++ /dev/null @@ -1,50 +0,0 @@ -import type { DittoScanCandidate, DittoScanResult } from "./types"; -import type { Verdict } from "./rules"; - -export interface DittoScanPreClassifyResult { - // Candidates the rule classifier did not decide on. These flow to the - // LLM stage. - llmCandidates: DittoScanCandidate[]; - // Candidates the rule classifier decided. Each is materialized as a - // Result with confidence=1 and decided_by="rule" so it interleaves with - // LLM-decided results in the final output. - preClassified: DittoScanResult[]; -} - -// Bridge between `runExtract` (which produces Candidates + Verdicts) and -// `runClassify` (which only wants LLM-bound Candidates). Splits the -// extract output and materializes rule-decided Results. -export function preClassify( - candidates: DittoScanCandidate[], - verdicts: Map -): DittoScanPreClassifyResult { - const llmCandidates: DittoScanCandidate[] = []; - const preClassified: DittoScanResult[] = []; - - for (const c of candidates) { - const verdict = verdicts.get(c.id) ?? { kind: "llm" as const }; - if (verdict.kind === "accept") { - preClassified.push({ - ...c, - status: "user-facing", - reason: `rule:${verdict.rule}`, - confidence: 1, - info_needed: null, - decided_by: "rule", - }); - } else if (verdict.kind === "reject") { - preClassified.push({ - ...c, - status: "not-user-facing", - reason: `rule:${verdict.rule}`, - confidence: 1, - info_needed: null, - decided_by: "rule", - }); - } else { - llmCandidates.push(c); - } - } - - return { llmCandidates, preClassified }; -} diff --git a/lib/src/scan/rules/drop.test.ts b/lib/src/scan/rules.test.ts similarity index 97% rename from lib/src/scan/rules/drop.test.ts rename to lib/src/scan/rules.test.ts index e78dcf4..05467cb 100644 --- a/lib/src/scan/rules/drop.test.ts +++ b/lib/src/scan/rules.test.ts @@ -1,6 +1,6 @@ -import type { DittoScanEnclosingContext } from "../types"; +import type { DittoScanEnclosingContext } from "./types"; -import { shouldEmit } from "./drop"; +import { shouldEmit } from "./rules"; const other: DittoScanEnclosingContext = { parentRole: "other", diff --git a/lib/src/scan/rules/drop.ts b/lib/src/scan/rules.ts similarity index 99% rename from lib/src/scan/rules/drop.ts rename to lib/src/scan/rules.ts index bc5e840..f976c85 100644 --- a/lib/src/scan/rules/drop.ts +++ b/lib/src/scan/rules.ts @@ -1,7 +1,7 @@ import type { DittoScanDetectionKind, DittoScanEnclosingContext, -} from "../types"; +} from "./types"; // Unambiguous non-text value shapes. A pattern belongs here only if there // is effectively zero chance a real UI string could match it. diff --git a/lib/src/scan/rules/accept/common.ts b/lib/src/scan/rules/accept/common.ts deleted file mode 100644 index 97307d7..0000000 --- a/lib/src/scan/rules/accept/common.ts +++ /dev/null @@ -1,100 +0,0 @@ -import type { Rule } from "../types"; - -// HTML/JSX/Vue attribute names whose values are exclusively human-readable -// copy. Symmetric counterpart to NON_TEXT_ATTR_NAMES in drop.ts: those are -// always-non-text attrs (dropped); these are always-copy attrs (accepted). -const COPY_ATTR_NAMES: ReadonlySet = new Set([ - "placeholder", - "alt", - "aria-label", - "aria-placeholder", - "aria-roledescription", - "aria-valuetext", - "title", - "summary", -]); - -// HTML elements whose text content is unambiguously user-facing copy. -// Allowlist (rather than denylist) so component tags like `` and -// generic containers like `
` fall through to the LLM — components -// can have arbitrary semantics (e.g. `` in react-i18next treats -// children as a translation key), and `
` is too generic. -// -// `text` is included to cover two distinct cases that happen to share -// the same lowercased tag name: -// - React Native's `...` (the canonical RN copy holder) -// - SVG's `...` (renders the inner text inside the SVG) -// Both are unambiguous copy. -const COPY_TAGS: ReadonlySet = new Set([ - // Headings + sectioning - "h1", - "h2", - "h3", - "h4", - "h5", - "h6", - "p", - "blockquote", - // Lists / tables - "li", - "dd", - "dt", - "td", - "th", - "caption", - // Forms / interactive - "button", - "a", - "label", - "option", - "legend", - "summary", - // Figures - "figcaption", - // Inline phrasing - "span", - "text", - "em", - "strong", - "b", - "i", - "small", - "mark", - "q", - "cite", - "dfn", - "abbr", - "time", - "sub", - "sup", - "ins", - "del", -]); - -export const COMMON_ACCEPT_RULES: Rule[] = [ - { - name: "resource_value", - // Every entry in a localization resource file (.strings, .stringsdict, - // .xcstrings, strings.xml) is intentional product copy, including the - // entries the developer marked `translatable="false"`. Those are still - // user-facing strings (app names, brand strings, accessibility labels); - // the flag just opts them out of localization, not out of the UI. - match: ({ candidate }) => candidate.detection_kind === "resource_value", - }, - { - name: "markup_attr_copy", - match: ({ candidate }) => - candidate.detection_kind === "markup_attr" && - candidate.context_identifiers.length > 0 && - COPY_ATTR_NAMES.has(candidate.context_identifiers[0].toLowerCase()), - }, - { - name: "markup_text_safe", - // JSX/Vue text content (``, `

...

`) is - // unambiguous copy when the wrapping tag is a known text-bearing - // HTML element. Components, fragments, generic containers (`
`), - // and code-bearing tags fall through to the LLM. - match: ({ context }) => - context.parentRole === "markup_text" && context.parentTag !== undefined && COPY_TAGS.has(context.parentTag), - }, -]; diff --git a/lib/src/scan/rules/accept/index.ts b/lib/src/scan/rules/accept/index.ts deleted file mode 100644 index 4474d07..0000000 --- a/lib/src/scan/rules/accept/index.ts +++ /dev/null @@ -1,16 +0,0 @@ -import type { Rule, RulesByLanguage } from "../types"; - -import { COMMON_ACCEPT_RULES } from "./common"; -import { KOTLIN_ACCEPT_RULES } from "./kotlin"; -import { SWIFT_ACCEPT_RULES } from "./swift"; - -// Accept rules keyed by Candidate.language. Language-specific rules run -// before cross-language ones; the first match wins. The map is typed -// against `RulesByLanguage` (= `{ [K in RuleLanguage]?: Rule[] }`) so an -// unknown or misspelled key is a compile error. -export const ACCEPT_RULES_BY_LANGUAGE: RulesByLanguage = { - kotlin: KOTLIN_ACCEPT_RULES, - swift: SWIFT_ACCEPT_RULES, -}; - -export const ACCEPT_RULES_COMMON: Rule[] = COMMON_ACCEPT_RULES; diff --git a/lib/src/scan/rules/accept/kotlin.ts b/lib/src/scan/rules/accept/kotlin.ts deleted file mode 100644 index c242761..0000000 --- a/lib/src/scan/rules/accept/kotlin.ts +++ /dev/null @@ -1,45 +0,0 @@ -import type { Rule } from "../types"; - -// `(receiver, method)` pairs whose first string argument is user-facing -// copy. These are the standard Android UI surfaces. Receiver match is -// strict (callee + calleeMember, bare identifier) so an unrelated -// variable named `Toast` doesn't fire. The factory signatures -// (`Toast.makeText(ctx, text, duration)`, `Snackbar.make(view, text, duration)`) -// place the only String-typed parameter at arg index 1, so any string -// literal at these call sites is necessarily the message. -const ANDROID_UI_FACTORY_CALLS: ReadonlyArray = [ - ["Toast", "makeText"], - ["Snackbar", "make"], -]; - -export const KOTLIN_ACCEPT_RULES: Rule[] = [ - { - name: "compose_text", - // Jetpack Compose's `Text(...)` composable takes a String literal that - // is rendered directly as UI. Unlike SwiftUI's Text (which auto-looks - // up against Localizable bundles via LocalizedStringKey), Compose - // performs no resource lookup — the literal IS the displayed copy. - // - // Gated on the `android` framework token so a non-Android Kotlin - // project that happens to define its own `Text(...)` function doesn't - // get every constructor argument auto-classified as UI copy. - // - // We match only the bare-identifier call `Text("...")`, not - // `Text(stringResource(...))` (which doesn't put a literal in the - // first arg position) and not method calls like `myWidget.Text(...)`. - match: ({ candidate, context }) => - candidate.framework.includes("android") && context.callee === "Text" && context.calleeMember === undefined, - }, - { - name: "android_ui_factory", - // `Toast.makeText(ctx, "Saved", LENGTH_SHORT)`, `Snackbar.make(view, "...", ...)`. - // Receiver is a bare type identifier; the method is a known factory. - match: ({ context }) => { - if (!context.callee || !context.calleeMember) return false; - for (const [receiver, method] of ANDROID_UI_FACTORY_CALLS) { - if (context.callee === receiver && context.calleeMember === method) return true; - } - return false; - }, - }, -]; diff --git a/lib/src/scan/rules/accept/swift.ts b/lib/src/scan/rules/accept/swift.ts deleted file mode 100644 index 3f0247e..0000000 --- a/lib/src/scan/rules/accept/swift.ts +++ /dev/null @@ -1,5 +0,0 @@ -import type { Rule } from "../types"; - -// SwiftUI string arguments are often user-facing, but can also be -// localization keys. Keep those ambiguous cases on the LLM path. -export const SWIFT_ACCEPT_RULES: Rule[] = []; diff --git a/lib/src/scan/rules/index.ts b/lib/src/scan/rules/index.ts deleted file mode 100644 index 93ef9c3..0000000 --- a/lib/src/scan/rules/index.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { ACCEPT_RULES_BY_LANGUAGE, ACCEPT_RULES_COMMON } from "./accept"; -import { REJECT_RULES_BY_LANGUAGE, REJECT_RULES_COMMON } from "./reject"; -import type { Rule, RuleInput, RuleLanguage, RulesByLanguage, Verdict } from "./types"; - -export { shouldEmit } from "./drop"; -export type { Rule, RuleInput, Verdict } from "./types"; - -// Guarded lookup for language-keyed rule maps. `candidate.language` is a -// free-form string at runtime (the extractor emits a wider set than rules -// dispatch on), so a value like "constructor" would otherwise surface an -// inherited prototype property and crash the `for...of` below. -function getLanguageRules(rules: RulesByLanguage, lang: string): Rule[] { - return Object.prototype.hasOwnProperty.call(rules, lang) ? rules[lang as RuleLanguage] ?? [] : []; -} - -// Apply the deterministic rule classifier to a candidate that has already -// cleared `shouldEmit`. Returns `{ kind: "llm" }` when no rule matches; -// the caller then sends the candidate to the LLM stage. -// -// Dispatch order, first match wins: -// 1. Reject before accept. A reject carve-out like Android -// `translatable="false"` needs to win over a broad accept rule -// (`resource_value`) that would otherwise fire on the same candidate. -// 2. Within reject (and within accept): language-specific before common. -// Lets a per-language rule disambiguate a shape that the cross-language -// list would otherwise treat generically. -// -// The candidate's `language` is a free string at the type level (the -// extractor emits a wider set than rules dispatch on, including -// regex-fallback extensions). The lookups below treat any non-RuleLanguage -// tag as "no language-specific rules", so the candidate flows naturally -// through common-then-llm. -export function getClassificationVerdict(input: RuleInput): Verdict { - const lang = input.candidate.language; - const rejectLangRules = getLanguageRules(REJECT_RULES_BY_LANGUAGE, lang); - const acceptLangRules = getLanguageRules(ACCEPT_RULES_BY_LANGUAGE, lang); - - for (const rule of rejectLangRules) { - if (rule.match(input)) return { kind: "reject", rule: rule.name }; - } - for (const rule of REJECT_RULES_COMMON) { - if (rule.match(input)) return { kind: "reject", rule: rule.name }; - } - for (const rule of acceptLangRules) { - if (rule.match(input)) return { kind: "accept", rule: rule.name }; - } - for (const rule of ACCEPT_RULES_COMMON) { - if (rule.match(input)) return { kind: "accept", rule: rule.name }; - } - return { kind: "llm" }; -} diff --git a/lib/src/scan/rules/reject/common.ts b/lib/src/scan/rules/reject/common.ts deleted file mode 100644 index e7cd710..0000000 --- a/lib/src/scan/rules/reject/common.ts +++ /dev/null @@ -1,9 +0,0 @@ -import type { Rule } from "../types"; - -// No cross-language reject rules at present. We previously rejected -// Android `` entries on the theory that -// they're developer-only configuration values. In practice many of -// those entries are real UI copy that the developer just chose not to -// localize (app names, brand strings, accessibility labels), so they -// flow through as `resource_value` accepts like everything else. -export const COMMON_REJECT_RULES: Rule[] = []; diff --git a/lib/src/scan/rules/reject/index.ts b/lib/src/scan/rules/reject/index.ts deleted file mode 100644 index aa86292..0000000 --- a/lib/src/scan/rules/reject/index.ts +++ /dev/null @@ -1,24 +0,0 @@ -import type { Rule, RulesByLanguage } from "../types"; - -import { COMMON_REJECT_RULES } from "./common"; -import { JAVASCRIPT_REJECT_RULES } from "./javascript"; -import { KOTLIN_REJECT_RULES } from "./kotlin"; -import { SWIFT_REJECT_RULES } from "./swift"; - -// Reject rules keyed by Candidate.language. Language-specific rules run -// before cross-language ones; the first match wins. The map is typed -// against `RulesByLanguage` (= `{ [K in RuleLanguage]?: Rule[] }`) so an -// unknown or misspelled key is a compile error. -export const REJECT_RULES_BY_LANGUAGE: RulesByLanguage = { - javascript: JAVASCRIPT_REJECT_RULES, - typescript: JAVASCRIPT_REJECT_RULES, - jsx: JAVASCRIPT_REJECT_RULES, - tsx: JAVASCRIPT_REJECT_RULES, - // Vue files run their `