diff --git a/package-lock.json b/package-lock.json index f5fa89b..ac5ef3e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -47,6 +47,7 @@ "recharts": "^2.15.4", "sonner": "^2.0.7", "tailwind-merge": "^3.4.0", + "xlsx": "^0.18.5", "zod": "^4.3.6", "zustand": "^5.0.10" }, @@ -7848,6 +7849,15 @@ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, + "node_modules/adler-32": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/adler-32/-/adler-32-1.3.1.tgz", + "integrity": "sha512-ynZ4w/nUUv5rrsR8UUGoe1VC9hZj6V5hU9Qw1HlMDJGEJw5S7TfTErWTjMys6M7vr0YWcPqs3qAr4ss0nDfP+A==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.8" + } + }, "node_modules/agent-base": { "version": "7.1.4", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", @@ -8556,6 +8566,19 @@ "url": "https://github.com/sponsors/mesqueeb" } }, + "node_modules/cfb": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/cfb/-/cfb-1.2.2.tgz", + "integrity": "sha512-KfdUZsSOw19/ObEWasvBP/Ac4reZvAGauZhs6S/gqNhXhI7cKwvlH7ulj+dOEYnca4bm4SGo8C1bTAQvnTjgQA==", + "license": "Apache-2.0", + "dependencies": { + "adler-32": "~1.3.0", + "crc-32": "~1.2.0" + }, + "engines": { + "node": ">=0.8" + } + }, "node_modules/chai": { "version": "5.3.3", "resolved": "https://registry.npmjs.org/chai/-/chai-5.3.3.tgz", @@ -8685,6 +8708,15 @@ "react-dom": "^18 || ^19 || ^19.0.0-rc" } }, + "node_modules/codepage": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/codepage/-/codepage-1.15.0.tgz", + "integrity": "sha512-3g6NUTPd/YtuuGrhMnOMRjFc+LJw/bnMp3+0r/Wcz3IXUuCosKRJvMphm5+Q+bvTVGcJJuRvVLuYba+WojaFaA==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.8" + } + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -10322,6 +10354,15 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/frac": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/frac/-/frac-1.1.2.tgz", + "integrity": "sha512-w/XBfkibaTl3YDqASwfDUqkna4Z2p9cFSr1aHDt0WoMTECnRfBOv2WArlZILlqgWlmdIlALXGpM2AOhEk5W3IA==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.8" + } + }, "node_modules/fs-constants": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", @@ -14226,6 +14267,18 @@ "node": ">=0.10.0" } }, + "node_modules/ssf": { + "version": "0.11.2", + "resolved": "https://registry.npmjs.org/ssf/-/ssf-0.11.2.tgz", + "integrity": "sha512-+idbmIXoYET47hH+d7dfm2epdOMUDjqcB4648sTZ+t2JwoyBFL/insLfB/racrDmsKB3diwsDA696pZMieAC5g==", + "license": "Apache-2.0", + "dependencies": { + "frac": "~1.1.2" + }, + "engines": { + "node": ">=0.8" + } + }, "node_modules/stable-hash": { "version": "0.0.5", "resolved": "https://registry.npmjs.org/stable-hash/-/stable-hash-0.0.5.tgz", @@ -15756,6 +15809,24 @@ "node": ">=8" } }, + "node_modules/wmf": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wmf/-/wmf-1.0.2.tgz", + "integrity": "sha512-/p9K7bEh0Dj6WbXg4JG0xvLQmIadrner1bi45VMJTfnbVHsc7yIajZyoSoK60/dtVBs12Fm6WkUI5/3WAVsNMw==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.8" + } + }, + "node_modules/word": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/word/-/word-0.3.0.tgz", + "integrity": "sha512-OELeY0Q61OXpdUfTp+oweA/vtLVg5VDOXh+3he3PNzLGG/y0oylSOC1xRVj0+l4vQ3tj/bB1HVHv1ocXkQceFA==", + "license": "Apache-2.0", + "engines": { + "node": ">=0.8" + } + }, "node_modules/word-wrap": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", @@ -15830,6 +15901,27 @@ } } }, + "node_modules/xlsx": { + "version": "0.18.5", + "resolved": "https://registry.npmjs.org/xlsx/-/xlsx-0.18.5.tgz", + "integrity": "sha512-dmg3LCjBPHZnQp5/F/+nnTa+miPJxUXB6vtk42YjBBKayDNagxGEeIdWApkYPOf3Z3pm3k62Knjzp7lMeTEtFQ==", + "license": "Apache-2.0", + "dependencies": { + "adler-32": "~1.3.0", + "cfb": "~1.2.1", + "codepage": "~1.15.0", + "crc-32": "~1.2.1", + "ssf": "~0.11.2", + "wmf": "~1.0.1", + "word": "~0.3.0" + }, + "bin": { + "xlsx": "bin/xlsx.njs" + }, + "engines": { + "node": ">=0.8" + } + }, "node_modules/xml-name-validator": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-5.0.0.tgz", diff --git a/package.json b/package.json index f7137dd..224690b 100644 --- a/package.json +++ b/package.json @@ -56,6 +56,7 @@ "recharts": "^2.15.4", "sonner": "^2.0.7", "tailwind-merge": "^3.4.0", + "xlsx": "^0.18.5", "zod": "^4.3.6", "zustand": "^5.0.10" }, diff --git a/src/components/finance/costing/params-only-import-dialog.tsx b/src/components/finance/costing/params-only-import-dialog.tsx index 187a12c..d32557f 100644 --- a/src/components/finance/costing/params-only-import-dialog.tsx +++ b/src/components/finance/costing/params-only-import-dialog.tsx @@ -11,7 +11,6 @@ import { XCircle, } from "lucide-react" import { useRouter } from "next/navigation" -import { useMutation, useQuery } from "@tanstack/react-query" import { toast } from "sonner" import { @@ -29,282 +28,342 @@ import { downloadParamsOnlyTemplate, getImportJob, } from "@/services/finance/cost-import-api" -import { costImportKeys } from "@/hooks/finance/use-cost-import" -import type { CostImportJob } from "@/types/finance/cost-import" +import { + chunkExcelFile, + PARAMS_ONLY_CONFIG, + type ChunkResult, +} from "@/lib/excel/chunked-importer" interface Props { open: boolean onOpenChange: (open: boolean) => void } -type Step = "upload" | "submitting" | "polling" | "done" | "failed" +type Step = "upload" | "parsing" | "ready" | "importing" | "done" | "failed" + +interface ChunkStatus { + index: number + productCount: number + rowCount: number + status: "pending" | "uploading" | "polling" | "done" | "partial" | "failed" + jobId?: number + inserted?: number +} + +const POLL_MS = 3_000 +const MAX_POLL_ERRORS = 5 -/** - * ParamsOnlyImportDialog — imports product_parameters + product_applicable_params - * from a file that does NOT include a product_master sheet. Mirrors the - * ImportDialog UX: file picker, background job creation, real-time polling. - * - * File is sent as multipart/form-data (binary bytes) to avoid the ~3× JSON - * inflation that would occur with Array.from(Uint8Array). - */ export function ParamsOnlyImportDialog({ open, onOpenChange }: Props) { const fileRef = useRef(null) + const router = useRouter() + const abortRef = useRef(false) + const [file, setFile] = useState(null) + const [parseStage, setParseStage] = useState("") + const [parsed, setParsed] = useState(null) const [step, setStep] = useState("upload") - const [jobId, setJobId] = useState(null) + const [chunks, setChunks] = useState([]) + const [currentIdx, setCurrentIdx] = useState(0) + const [totalInserted, setTotalInserted] = useState(0) + const [failedChunks, setFailedChunks] = useState([]) const [templateLoading, setTemplateLoading] = useState(false) - const router = useRouter() useEffect(() => { - if (open) { - setFile(null) - setStep("upload") - setJobId(null) - if (fileRef.current) fileRef.current.value = "" - } + if (open) reset() + // eslint-disable-next-line react-hooks/exhaustive-deps }, [open]) - // Submit mutation - const submitMutation = useMutation({ - mutationFn: (f: File) => bulkImportParamsOnly(f), - onSuccess: ({ jobId: id }) => { - setJobId(id) - setStep("polling") - }, - onError: (e) => { - toast.error(`Failed to start import: ${String(e)}`) + function reset() { + abortRef.current = false + setFile(null) + setParsed(null) + setStep("upload") + setChunks([]) + setCurrentIdx(0) + setTotalInserted(0) + setFailedChunks([]) + setParseStage("") + if (fileRef.current) fileRef.current.value = "" + } + + async function handleParse(f: File) { + setFile(f) + setStep("parsing") + try { + const result = await chunkExcelFile(f, PARAMS_ONLY_CONFIG, setParseStage) + setParsed(result) + setChunks( + result.chunks.map((c) => ({ + index: c.index, + productCount: c.keyCount, + rowCount: c.rowCount, + status: "pending", + })) + ) + setStep("ready") + } catch (e) { + toast.error(`Failed to parse file: ${String(e)}`) setStep("upload") - }, - }) - - // Poll job status - const { data: job } = useQuery({ - queryKey: costImportKeys.job(jobId ?? 0), - queryFn: () => getImportJob(jobId!), - enabled: step === "polling" && jobId !== null, - refetchInterval: (query) => { - const status = query.state.data?.status - return status === "DONE" || status === "FAILED" || status === "PARTIAL" ? false : 3000 - }, - }) + } + } - useEffect(() => { - if (!job) return - if (job.status === "DONE") { - setStep("done") - toast.success(`Params import complete: ${job.success} rows imported.`) - } else if (job.status === "PARTIAL") { - setStep("done") - toast.warning(`Import completed with ${job.failed} errors — check error report.`) - } else if (job.status === "FAILED") { - setStep("failed") + async function startImport(fromChunk = 0) { + if (!parsed) return + abortRef.current = false + setStep("importing") + const failed: number[] = [] + + for (let i = fromChunk; i < parsed.chunks.length; i++) { + if (abortRef.current) break + setCurrentIdx(i) + updateChunk(i, { status: "uploading" }) + + const chunk = parsed.chunks[i] + const chunkFile = new File([chunk.blob], chunk.fileName, { + type: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + }) + + let jobId: number + try { + const res = await bulkImportParamsOnly(chunkFile) + jobId = res.jobId + updateChunk(i, { status: "polling", jobId }) + } catch (e) { + toast.error(`Chunk ${i + 1} upload failed: ${String(e)}`) + updateChunk(i, { status: "failed" }) + failed.push(i) + continue + } + + let pollErrors = 0 + let done = false + while (!abortRef.current && !done) { + await delay(POLL_MS) + try { + const job = await getImportJob(jobId) + if (job.status === "DONE" || job.status === "PARTIAL" || job.status === "FAILED") { + const ok = job.status !== "FAILED" + updateChunk(i, { + status: ok ? (job.status === "DONE" ? "done" : "partial") : "failed", + inserted: job.success, + }) + if (ok) setTotalInserted((n) => n + (job.success ?? 0)) + else failed.push(i) + done = true + } + } catch { + pollErrors++ + if (pollErrors >= MAX_POLL_ERRORS) { + updateChunk(i, { status: "failed" }) + failed.push(i) + done = true + } + } + } } - }, [job?.status]) // eslint-disable-line react-hooks/exhaustive-deps - async function handleDownloadTemplate() { + setFailedChunks(failed) + setStep(failed.length === 0 ? "done" : "failed") + } + + function updateChunk(index: number, patch: Partial) { + setChunks((prev) => + prev.map((c) => (c.index === index ? { ...c, ...patch } : c)) + ) + } + + const doneCount = chunks.filter((c) => c.status === "done" || c.status === "partial").length + const progressPct = parsed ? Math.round((doneCount / parsed.chunks.length) * 100) : 0 + + async function handleTemplate() { setTemplateLoading(true) - try { - await downloadParamsOnlyTemplate() - } catch (e) { - toast.error(`Template download failed: ${String(e)}`) - } finally { - setTemplateLoading(false) - } + try { await downloadParamsOnlyTemplate() } + catch (e) { toast.error(`Template download failed: ${String(e)}`) } + finally { setTemplateLoading(false) } } function handleClose() { - setFile(null) - setStep("upload") - setJobId(null) - submitMutation.reset() - if (fileRef.current) fileRef.current.value = "" + abortRef.current = true + reset() onOpenChange(false) } - const isPolling = step === "polling" - const isDone = step === "done" - const isFailed = step === "failed" - const isSubmitting = submitMutation.isPending - - const progressPercent = - job && job.totalRows > 0 ? Math.round((job.processed / job.totalRows) * 100) : 0 - return ( - - + { if (!v) handleClose() }}> + Import Params Only (Bulk)
- {/* Template card */} - {!isPolling && !isDone && !isFailed && ( -
-
- -
-

Params-Only Template

-

- 2 sheets: product_parameters + product_applicable_params + + {/* Upload */} + {step === "upload" && ( + <> +

+
+ +
+

Params-Only Template

+

+ product_parameters + product_applicable_params +

+
+
+ +
+ +
+ +
fileRef.current?.click()} + > + { const f = e.target.files?.[0]; if (f) void handleParse(f) }} /> + +

Click to select — any size

+

+ File split automatically into {PARAMS_ONLY_CONFIG.chunkSize ?? 300}-product chunks

- -
+ +
+ + + Large files are split and uploaded chunk-by-chunk. Each chunk runs + independently — failed chunks can be retried without restarting. + +
+ )} - {/* File picker */} - {!isPolling && !isDone && !isFailed && ( -
- -
fileRef.current?.click()} - > - setFile(e.target.files?.[0] ?? null)} - /> - {file ? ( -
- - {file.name} - - ({(file.size / 1024 / 1024).toFixed(1)} MB) - -
- ) : ( - <> - -

- Click to select or drag and drop an Excel file -

-

- Supports split part-sheets (_p1, _p2, …) · Sent as binary (large files OK) -

- - )} + {/* Parsing */} + {step === "parsing" && ( +
+ +
+

Parsing file in browser…

+

{parseStage}

+

Large files may take 30–60 s

+ {file &&

{file.name} — {(file.size / 1024 / 1024).toFixed(1)} MB

}
)} - {/* Warning */} - {!isPolling && !isDone && !isFailed && ( -
- - - All-or-nothing validation. If any param_code is - unknown, the entire job fails and an error report (with{" "} - missing_param_codes tab) is generated. Products must already - exist from a prior bulk import. - -
+ {/* Ready */} + {step === "ready" && parsed && ( + <> +
+

File parsed

+
+

Products

{parsed.totalKeys.toLocaleString()}

+

Total rows

{parsed.totalRows.toLocaleString()}

+

Chunks

{parsed.chunks.length}

+
+

+ ~{PARAMS_ONLY_CONFIG.chunkSize ?? 300} products per chunk · ~{Math.round(parsed.totalRows / parsed.chunks.length).toLocaleString()} rows each · est. 3–8 min per chunk +

+
+
+ + Products must already exist from a prior bulk routing import. +
+ )} - {/* Polling progress */} - {isPolling && job && ( + {/* Importing */} + {step === "importing" && parsed && (
- Processing params… - - {job.processed.toLocaleString()} / {job.totalRows.toLocaleString()} rows - + Chunk {currentIdx + 1} / {parsed.chunks.length} + {totalInserted.toLocaleString()} rows imported +
+ +

{doneCount}/{parsed.chunks.length} complete ({progressPct}%)

+
+
+ {chunks.map((c) => ( + + ))} +
- {job.totalRows > 0 && ( - - )} -

- Running in background — you can close this dialog. -

-
- )} - - {isPolling && !job && ( -
- - Queuing import job…
)} - {/* Done state */} - {isDone && job && ( -
- + {/* Done */} + {step === "done" && ( +
+

- {job.status === "DONE" - ? `Import complete — ${job.success.toLocaleString()} rows imported` - : `Partial import — ${job.failed.toLocaleString()} errors`} + Import complete — {totalInserted.toLocaleString()} rows imported

- {job.errorFileUrl && ( -

- Error report available in Import Jobs. -

- )} -
)} - {/* Failed state */} - {isFailed && job && ( -
- -
-

Import failed

-

- {"See error report in Import Jobs for details."} -

- + {/* Failed */} + {step === "failed" && ( +
+
+ +
+

+ {failedChunks.length} chunk{failedChunks.length > 1 ? "s" : ""} failed + {totalInserted > 0 && ` — ${totalInserted.toLocaleString()} rows already imported`} +

+

+ Failed: chunks {failedChunks.map((i) => i + 1).join(", ")} +

+
+
+
+
+ {chunks.map((c) => )} +
)}
- - - {!isPolling && !isDone && !isFailed && ( - + )} + {step === "ready" && ( + <> + + + + )} + {step === "importing" && ( + + )} + {step === "failed" && failedChunks.length > 0 && ( + )} @@ -312,3 +371,29 @@ export function ParamsOnlyImportDialog({ open, onOpenChange }: Props) {
) } + +function ChunkBadge({ chunk, current }: { chunk: ChunkStatus; current: boolean }) { + const color: Record = { + pending: "bg-muted text-muted-foreground", + uploading: "bg-blue-100 text-blue-700 dark:bg-blue-900 dark:text-blue-200", + polling: "bg-yellow-100 text-yellow-700 dark:bg-yellow-900 dark:text-yellow-200", + done: "bg-green-100 text-green-700 dark:bg-green-900 dark:text-green-200", + partial: "bg-orange-100 text-orange-700 dark:bg-orange-900 dark:text-orange-200", + failed: "bg-red-100 text-red-700 dark:bg-red-900 dark:text-red-200", + } + return ( + + {chunk.index + 1} + {current && (chunk.status === "uploading" || chunk.status === "polling") && ( + + )} + + ) +} + +function delay(ms: number) { + return new Promise((resolve) => setTimeout(resolve, ms)) +} diff --git a/src/lib/excel/chunked-importer.ts b/src/lib/excel/chunked-importer.ts new file mode 100644 index 0000000..79384d3 --- /dev/null +++ b/src/lib/excel/chunked-importer.ts @@ -0,0 +1,175 @@ +/** + * chunked-importer.ts — Generic browser-side Excel chunker. + * + * Uses a Web Worker (chunked-importer.worker.ts) to parse large Excel files + * off the main thread so the UI stays responsive. + * + * ### Usage + * + * ```ts + * import { chunkExcelFile, PARAMS_ONLY_CONFIG } from "@/lib/excel/chunked-importer" + * + * const result = await chunkExcelFile(file, PARAMS_ONLY_CONFIG, (stage) => + * console.log(stage) + * ) + * // result.chunks is ParsedChunk[] ready to upload + * ``` + * + * ### Adding a new import format + * + * Create a new `ChunkerConfig` constant — no code changes required: + * + * ```ts + * export const MY_FORMAT_CONFIG: ChunkerConfig = { + * sheetGroups: [ + * { baseName: "my_sheet", outputName: "my_sheet" }, + * ], + * keyColumn: "my_id_column", + * chunkSize: 500, + * } + * ``` + */ + +export type { SheetGroupConfig } from "./chunked-importer.worker" + +export interface ChunkerConfig { + /** Sheet groups to parse and merge. */ + sheetGroups: import("./chunked-importer.worker").SheetGroupConfig[] + /** Column name used to group rows per unique key (e.g. "legacy_oracle_sys_id"). */ + keyColumn: string + /** Number of unique key values per output chunk. Default: 300. */ + chunkSize?: number +} + +export interface ParsedChunk { + index: number + blob: Blob + fileName: string + /** Number of unique key values in this chunk. */ + keyCount: number + rowCount: number +} + +export interface ChunkResult { + totalKeys: number + totalRows: number + chunks: ParsedChunk[] +} + +// ── Built-in configurations ──────────────────────────────────────────────── + +/** Params-only import: product_parameters + product_applicable_params. */ +export const PARAMS_ONLY_CONFIG: ChunkerConfig = { + sheetGroups: [ + { baseName: "product_parameters", outputName: "product_parameters" }, + { baseName: "product_applicable_params", outputName: "product_applicable_params" }, + ], + keyColumn: "legacy_oracle_sys_id", + chunkSize: 300, +} + +/** Bulk product + routing import (product_master + route sheets). */ +export const BULK_ROUTING_CONFIG: ChunkerConfig = { + sheetGroups: [ + { baseName: "product_master", outputName: "product_master" }, + { baseName: "product_parameters", outputName: "product_parameters" }, + { baseName: "product_applicable_params", outputName: "product_applicable_params" }, + { baseName: "route_head", outputName: "route_head" }, + { baseName: "route_sequences", outputName: "route_sequences" }, + { baseName: "route_rms", outputName: "route_rms" }, + ], + keyColumn: "legacy_oracle_sys_id", + chunkSize: 200, // smaller — 6 sheets per product +} + +// ── Core function ────────────────────────────────────────────────────────── + +const XLSX_MIME = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + +/** + * Parse an Excel file using a Web Worker and split into upload-ready chunks. + * + * @param file The Excel file selected by the user. + * @param config Which sheets to parse, which column is the key, chunk size. + * @param onProgress Optional callback for stage strings ("Parsing…", "Indexing…"). + * @returns Resolved when all chunks are built; rejects on parse error. + */ +export function chunkExcelFile( + file: File, + config: ChunkerConfig, + onProgress?: (stage: string) => void +): Promise { + return new Promise((resolve, reject) => { + const worker = new Worker( + new URL("./chunked-importer.worker", import.meta.url) + ) + + const chunks: ParsedChunk[] = [] + let totalKeys = 0 + let totalRows = 0 + let expectedChunks = 0 + + worker.onmessage = (e: MessageEvent) => { + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + const msg = e.data + + switch (msg.type) { + case "progress": + onProgress?.(msg.stage as string) + break + + case "ready": + totalKeys = msg.totalKeys as number + totalRows = msg.totalRows as number + expectedChunks = msg.totalChunks as number + onProgress?.(`Building chunks (0/${expectedChunks})…`) + worker.postMessage({ type: "next" }) + break + + case "chunk": { + const blob = new Blob([msg.buffer as ArrayBuffer], { type: XLSX_MIME }) + chunks.push({ + index: msg.index as number, + blob, + fileName: msg.fileName as string, + keyCount: msg.keyCount as number, + rowCount: msg.rowCount as number, + }) + onProgress?.(`Building chunks (${chunks.length}/${expectedChunks})…`) + worker.postMessage({ type: "next" }) + break + } + + case "done": + worker.terminate() + resolve({ totalKeys, totalRows, chunks }) + break + + case "error": + worker.terminate() + reject(new Error(msg.message as string)) + break + } + } + + worker.onerror = (err) => { + worker.terminate() + reject(new Error(`Worker error: ${err.message}`)) + } + + // Transfer the ArrayBuffer — zero-copy, buffer is detached in main thread. + file.arrayBuffer().then((buffer) => { + worker.postMessage( + { + type: "start", + buffer, + keyColumn: config.keyColumn, + sheetGroups: config.sheetGroups, + chunkSize: config.chunkSize ?? 300, + baseName: file.name.replace(/\.xlsx?$/i, ""), + }, + [buffer] + ) + }).catch(reject) + }) +} diff --git a/src/lib/excel/chunked-importer.worker.ts b/src/lib/excel/chunked-importer.worker.ts new file mode 100644 index 0000000..c1f03cf --- /dev/null +++ b/src/lib/excel/chunked-importer.worker.ts @@ -0,0 +1,253 @@ +/** + * chunked-importer.worker.ts — Generic Web Worker for chunking any Excel import file. + * + * Accepts a flexible SheetGroupConfig so the same worker binary handles any + * import format: params-only, product-routing, RM pricing, etc. + * + * Protocol (all messages are structured-cloneable): + * + * Main → Worker StartMessage — start parsing + * Worker → Main ProgressMsg — stage updates (multiple) + * Worker → Main ReadyMsg — parsing done; N chunks ready + * Main → Worker NextMessage — request next chunk + * Worker → Main ChunkMsg — one chunk (ArrayBuffer transferred, zero-copy) + * Worker → Main DoneMsg — all chunks sent + * Worker → Main ErrorMsg — fatal error + */ +import * as XLSX from "xlsx" + +// ── Public message types (re-exported via chunked-importer.ts) ───────────── + +export interface SheetGroupConfig { + /** Base name to match — exact match wins, otherwise contains-match (case-insensitive). */ + baseName: string + /** Sheet name written into each output chunk file. */ + outputName: string +} + +export interface StartMessage { + type: "start" + /** Raw ArrayBuffer of the .xlsx file — transferred (zero-copy). */ + buffer: ArrayBuffer + /** Column name used to group rows into per-key-value buckets. */ + keyColumn: string + /** Sheet groups to parse and merge. */ + sheetGroups: SheetGroupConfig[] + /** Number of unique key values per chunk. */ + chunkSize: number + /** Stem used to generate chunk file names (e.g. "params_2026"). */ + baseName: string +} + +export interface NextMessage { + type: "next" +} + +export interface ProgressMsg { + type: "progress" + stage: string +} + +export interface ReadyMsg { + type: "ready" + totalKeys: number + totalRows: number + totalChunks: number +} + +export interface ChunkMsg { + type: "chunk" + index: number + total: number + /** Transferred ArrayBuffer — detached from worker heap after postMessage. */ + buffer: ArrayBuffer + fileName: string + keyCount: number + rowCount: number +} + +export interface DoneMsg { + type: "done" +} + +export interface ErrorMsg { + type: "error" + message: string +} + +export type WorkerOutMsg = ProgressMsg | ReadyMsg | ChunkMsg | DoneMsg | ErrorMsg +type WorkerInMsg = StartMessage | NextMessage + +// ── Internal state ───────────────────────────────────────────────────────── + +interface ParsedGroup { + headers: unknown[] + rowsByKey: Map + outputName: string +} + +let groups: ParsedGroup[] = [] +let masterKeys: string[] = [] +let currentChunkSize = 300 +let currentBaseName = "" +let remaining: number[] = [] // chunk indices yet to be sent + +// ── Message handler ──────────────────────────────────────────────────────── + +self.onmessage = (e: MessageEvent) => { + if (e.data.type === "start") { + handleStart(e.data).catch((err: unknown) => { + emit({ type: "error", message: String(err) }) + }) + } else if (e.data.type === "next") { + sendNextChunk() + } +} + +// ── Parse ────────────────────────────────────────────────────────────────── + +async function handleStart(msg: StartMessage) { + currentChunkSize = msg.chunkSize + currentBaseName = msg.baseName + groups = [] + masterKeys = [] + + emit({ type: "progress", stage: "Parsing Excel file… (please wait)" }) + + const wb = XLSX.read(new Uint8Array(msg.buffer), { type: "array", dense: true }) + + const keySet = new Set() + let grandTotal = 0 + + for (const groupCfg of msg.sheetGroups) { + emit({ type: "progress", stage: `Indexing "${groupCfg.baseName}"…` }) + + const matching = findMatchingSheets(wb, groupCfg.baseName) + if (matching.length === 0) continue + + let headers: unknown[] = [] + let keyColIdx = -1 + const rowsByKey = new Map() + + for (let si = 0; si < matching.length; si++) { + const ws = wb.Sheets[matching[si]] + const rows = XLSX.utils.sheet_to_json(ws, { header: 1, defval: null }) + + if (si === 0) { + headers = (rows[0] as (string | null)[]).map((h) => + h != null ? String(h).trim() : "" + ) + keyColIdx = (headers as string[]).indexOf(msg.keyColumn) + if (keyColIdx === -1) { + emit({ + type: "error", + message: `Sheet "${matching[0]}" does not contain required key column "${msg.keyColumn}". Found: ${headers.filter(Boolean).join(", ")}`, + }) + return + } + } + + const startRow = si === 0 ? 1 : 1 // skip header in all sheets + for (let r = startRow; r < rows.length; r++) { + const row = rows[r] as unknown[] + if (!row || row.every((v) => v == null || v === "")) continue + const rawKey = row[keyColIdx] + if (rawKey == null || rawKey === "") continue + const key = String(rawKey).trim() + if (!rowsByKey.has(key)) rowsByKey.set(key, []) + rowsByKey.get(key)!.push(row) + grandTotal++ + if (!keySet.has(key)) { + keySet.add(key) + masterKeys.push(key) + } + } + } + + groups.push({ headers, rowsByKey, outputName: groupCfg.outputName }) + } + + if (groups.length === 0) { + emit({ type: "error", message: "No matching sheets found. Check sheet names." }) + return + } + + const totalChunks = Math.ceil(masterKeys.length / currentChunkSize) + remaining = Array.from({ length: totalChunks }, (_, i) => i) + + emit({ + type: "ready", + totalKeys: masterKeys.length, + totalRows: grandTotal, + totalChunks, + }) +} + +// ── Chunk generation ─────────────────────────────────────────────────────── + +function sendNextChunk() { + if (remaining.length === 0) { + emit({ type: "done" }) + return + } + + const idx = remaining.shift()! + const start = idx * currentChunkSize + const batchKeys = masterKeys.slice(start, start + currentChunkSize) + const total = Math.ceil(masterKeys.length / currentChunkSize) + + const chunkWb = XLSX.utils.book_new() + let chunkRows = 0 + + for (const group of groups) { + const data: unknown[][] = [group.headers] + for (const key of batchKeys) { + const rows = group.rowsByKey.get(key) ?? [] + data.push(...rows) + chunkRows += rows.length + } + XLSX.utils.book_append_sheet(chunkWb, XLSX.utils.aoa_to_sheet(data), group.outputName) + } + + const rawArr = XLSX.write(chunkWb, { + type: "array", + bookType: "xlsx", + compression: true, + }) as number[] + + // Copy into a plain ArrayBuffer so it is transferable (zero-copy to main thread). + const ab = new Uint8Array(rawArr).buffer as ArrayBuffer + + const msg: ChunkMsg = { + type: "chunk", + index: idx, + total, + buffer: ab, + fileName: `${currentBaseName}_chunk_${String(idx + 1).padStart(3, "0")}.xlsx`, + keyCount: batchKeys.length, + rowCount: chunkRows, + } + + // Transfer ab — removed from worker heap, no copy on the wire. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + ;(self as any).postMessage(msg, [ab]) +} + +// ── Helpers ──────────────────────────────────────────────────────────────── + +function findMatchingSheets(wb: XLSX.WorkBook, baseName: string): string[] { + const lower = baseName.toLowerCase() + const exact = wb.SheetNames.filter((n) => n === baseName) + if (exact.length > 0) { + // Include the exact match PLUS any continuation sheets (e.g. "params_2") + const extras = wb.SheetNames.filter( + (n) => n !== baseName && n.toLowerCase().includes(lower) + ).sort() + return [...exact, ...extras] + } + return wb.SheetNames.filter((n) => n.toLowerCase().includes(lower)).sort() +} + +function emit(msg: WorkerOutMsg) { + self.postMessage(msg) +}