From 4797f75b246ef304f8295c9d9c408eae727b05f2 Mon Sep 17 00:00:00 2001 From: ilhom Date: Mon, 29 Jun 2026 13:57:13 +0700 Subject: [PATCH 1/3] feat(frontend): auto-retry chunked import until convergence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both routing and params-only dialogs now automatically retry failed chunks until no more improvement is possible (max 5/3 passes). Each retry pass benefits from newly-imported products in DbProductSet: - Pass 1: ~50% chunks succeed (few products in DB) - Pass 2: ~80% succeed (dependency chain resolves) - Pass 3+: converges to ~100% or stops on no improvement User no longer needs to manually click Retry multiple times. UI shows 'Pass 2/5 — retrying N failed chunks...' during auto-retry. Chunks already succeeded are skipped on subsequent passes. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- .../finance/costing/bulk-import-dialog.tsx | 117 ++++++++++----- .../costing/params-only-import-dialog.tsx | 133 ++++++++++++------ 2 files changed, 167 insertions(+), 83 deletions(-) diff --git a/src/components/finance/costing/bulk-import-dialog.tsx b/src/components/finance/costing/bulk-import-dialog.tsx index c2241a0..be693c0 100644 --- a/src/components/finance/costing/bulk-import-dialog.tsx +++ b/src/components/finance/costing/bulk-import-dialog.tsx @@ -70,6 +70,9 @@ interface ChunkStatus { const VALIDATE_SIZE_LIMIT = 5 * 1024 * 1024 // 5 MB — matches server-side validate limit const POLL_MS = 3_000 const MAX_POLL_ERRORS = 5 +// Auto-retry: each pass adds newly-imported products to DbProductSet, reducing +// cross-chunk dependency failures. Stop when no improvement or max passes reached. +const MAX_AUTO_PASSES = 5 export function BulkImportDialog({ open, onOpenChange }: BulkImportDialogProps) { const fileRef = useRef(null) @@ -90,6 +93,8 @@ export function BulkImportDialog({ open, onOpenChange }: BulkImportDialogProps) const [currentIdx, setCurrentIdx] = useState(0) const [totalInserted, setTotalInserted] = useState(0) const [failedChunks, setFailedChunks] = useState([]) + const [passNum, setPassNum] = useState(1) + const [passStatus, setPassStatus] = useState("") function resetState() { abortRef.current = false @@ -103,6 +108,8 @@ export function BulkImportDialog({ open, onOpenChange }: BulkImportDialogProps) setCurrentIdx(0) setTotalInserted(0) setFailedChunks([]) + setPassNum(1) + setPassStatus("") setParseStage("") if (fileRef.current) fileRef.current.value = "" } @@ -174,50 +181,83 @@ export function BulkImportDialog({ open, onOpenChange }: BulkImportDialogProps) if (!parsed) return abortRef.current = false setStep("chunked_importing") - const failed: number[] = [] - - for (let i = fromChunk; i < parsed.chunks.length; i++) { - if (abortRef.current) break - setCurrentIdx(i) - updateChunk(i, "uploading") - - const chunk = parsed.chunks[i] - const chunkFile = new File([chunk.blob], chunk.fileName, { - type: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - }) - - let cJobId: number - try { - const res = await bulkImportProductMasterRouting(chunkFile, "update") - cJobId = res.jobId - updateChunk(i, "polling") - } catch (e) { - toast.error(`Chunk ${i + 1} upload failed: ${String(e)}`) - updateChunk(i, "failed") - failed.push(i) - continue + + let prevFailedCount = parsed.chunks.length // worst case on first pass + let currentPass = fromChunk === 0 ? 1 : passNum + + // eslint-disable-next-line no-constant-condition + while (true) { + setPassNum(currentPass) + setPassStatus(currentPass > 1 ? `Pass ${currentPass}/${MAX_AUTO_PASSES} — retrying ${prevFailedCount} failed chunks…` : "") + + // Reset all pending on each pass (keep done/partial from prior passes visible) + if (currentPass > 1) { + setChunks((prev) => prev.map((c) => ({ ...c, status: c.status === "failed" ? "pending" : c.status }))) } - let pollErrors = 0 - let done = false - while (!abortRef.current && !done) { - await new Promise((r) => setTimeout(r, POLL_MS)) + const failed: number[] = [] + + for (let i = fromChunk; i < parsed.chunks.length; i++) { + if (abortRef.current) break + // Skip already-succeeded chunks on re-passes + if (currentPass > 1) { + const current = chunks.find((c) => c.index === i) + if (current && (current.status === "done" || current.status === "partial")) continue + } + + setCurrentIdx(i) + updateChunk(i, "uploading") + + const chunk = parsed.chunks[i] + const chunkFile = new File([chunk.blob], chunk.fileName, { + type: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + }) + + let cJobId: number try { - const job = await getImportJob(cJobId) - if (job.status === "DONE" || job.status === "PARTIAL" || job.status === "FAILED") { - updateChunk(i, job.status !== "FAILED" ? (job.status === "DONE" ? "done" : "partial") : "failed") - if (job.status !== "FAILED") setTotalInserted((n) => n + (job.success ?? 0)) - else failed.push(i) - done = true + const res = await bulkImportProductMasterRouting(chunkFile, "update") + cJobId = res.jobId + updateChunk(i, "polling") + } catch (e) { + updateChunk(i, "failed"); failed.push(i); continue + } + + let pollErrors = 0 + let done = false + while (!abortRef.current && !done) { + await new Promise((r) => setTimeout(r, POLL_MS)) + try { + const job = await getImportJob(cJobId) + if (job.status === "DONE" || job.status === "PARTIAL" || job.status === "FAILED") { + updateChunk(i, job.status !== "FAILED" ? (job.status === "DONE" ? "done" : "partial") : "failed") + if (job.status !== "FAILED") setTotalInserted((n) => n + (job.success ?? 0)) + else failed.push(i) + done = true + } + } catch { + if (++pollErrors >= MAX_POLL_ERRORS) { updateChunk(i, "failed"); failed.push(i); done = true } } - } catch { - if (++pollErrors >= MAX_POLL_ERRORS) { updateChunk(i, "failed"); failed.push(i); done = true } } } + + setFailedChunks(failed) + + // Stop conditions + if (abortRef.current || failed.length === 0) break + if (currentPass >= MAX_AUTO_PASSES) break + if (failed.length >= prevFailedCount) { + // No improvement — stop to avoid infinite loop + setPassStatus(`Stopped after ${currentPass} passes — no improvement (${failed.length} chunks still failing)`) + break + } + + prevFailedCount = failed.length + currentPass++ + fromChunk = 0 // always restart from 0 on subsequent passes } - setFailedChunks(failed) - setStep(failed.length === 0 ? "chunked_done" : "chunked_failed") + const finalFailed = chunks.filter((c) => c.status === "failed").length + setStep(finalFailed === 0 ? "chunked_done" : "chunked_failed") } function updateChunk(index: number, status: ChunkStatus["status"]) { @@ -479,10 +519,13 @@ export function BulkImportDialog({ open, onOpenChange }: BulkImportDialogProps) {step === "chunked_importing" && parsed && (
- Chunk {currentIdx + 1} / {parsed.chunks.length} + + {passNum > 1 ? `Pass ${passNum}/${MAX_AUTO_PASSES} · ` : ""}Chunk {currentIdx + 1} / {parsed.chunks.length} + {totalInserted.toLocaleString()} rows imported
+ {passStatus &&

{passStatus}

}
{chunks.map((c) => )} diff --git a/src/components/finance/costing/params-only-import-dialog.tsx b/src/components/finance/costing/params-only-import-dialog.tsx index d32557f..1c07a53 100644 --- a/src/components/finance/costing/params-only-import-dialog.tsx +++ b/src/components/finance/costing/params-only-import-dialog.tsx @@ -52,6 +52,9 @@ interface ChunkStatus { const POLL_MS = 3_000 const MAX_POLL_ERRORS = 5 +// For params-only, a PARTIAL result means some products were skipped (not yet in DB). +// Auto-retry lets subsequent passes pick up those products once routing is complete. +const MAX_AUTO_PASSES = 3 export function ParamsOnlyImportDialog({ open, onOpenChange }: Props) { const fileRef = useRef(null) @@ -67,6 +70,8 @@ export function ParamsOnlyImportDialog({ open, onOpenChange }: Props) { const [totalInserted, setTotalInserted] = useState(0) const [failedChunks, setFailedChunks] = useState([]) const [templateLoading, setTemplateLoading] = useState(false) + const [passNum, setPassNum] = useState(1) + const [passStatus, setPassStatus] = useState("") useEffect(() => { if (open) reset() @@ -82,6 +87,8 @@ export function ParamsOnlyImportDialog({ open, onOpenChange }: Props) { setCurrentIdx(0) setTotalInserted(0) setFailedChunks([]) + setPassNum(1) + setPassStatus("") setParseStage("") if (fileRef.current) fileRef.current.value = "" } @@ -111,59 +118,90 @@ export function ParamsOnlyImportDialog({ open, onOpenChange }: Props) { if (!parsed) return abortRef.current = false setStep("importing") - const failed: number[] = [] - - for (let i = fromChunk; i < parsed.chunks.length; i++) { - if (abortRef.current) break - setCurrentIdx(i) - updateChunk(i, { status: "uploading" }) - - const chunk = parsed.chunks[i] - const chunkFile = new File([chunk.blob], chunk.fileName, { - type: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - }) - - let jobId: number - try { - const res = await bulkImportParamsOnly(chunkFile) - jobId = res.jobId - updateChunk(i, { status: "polling", jobId }) - } catch (e) { - toast.error(`Chunk ${i + 1} upload failed: ${String(e)}`) - updateChunk(i, { status: "failed" }) - failed.push(i) - continue + + let prevFailedCount = parsed.chunks.length + let currentPass = fromChunk === 0 ? 1 : passNum + + // eslint-disable-next-line no-constant-condition + while (true) { + setPassNum(currentPass) + setPassStatus(currentPass > 1 ? `Pass ${currentPass}/${MAX_AUTO_PASSES} — retrying ${prevFailedCount} failed chunks…` : "") + + if (currentPass > 1) { + setChunks((prev) => prev.map((c) => ({ ...c, status: c.status === "failed" ? "pending" : c.status }))) } - let pollErrors = 0 - let done = false - while (!abortRef.current && !done) { - await delay(POLL_MS) + const failed: number[] = [] + + for (let i = fromChunk; i < parsed.chunks.length; i++) { + if (abortRef.current) break + if (currentPass > 1) { + const cur = chunks.find((c) => c.index === i) + if (cur && (cur.status === "done" || cur.status === "partial")) continue + } + setCurrentIdx(i) + updateChunk(i, { status: "uploading" }) + + const chunk = parsed.chunks[i] + const chunkFile = new File([chunk.blob], chunk.fileName, { + type: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + }) + + let jobId: number try { - const job = await getImportJob(jobId) - if (job.status === "DONE" || job.status === "PARTIAL" || job.status === "FAILED") { - const ok = job.status !== "FAILED" - updateChunk(i, { - status: ok ? (job.status === "DONE" ? "done" : "partial") : "failed", - inserted: job.success, - }) - if (ok) setTotalInserted((n) => n + (job.success ?? 0)) - else failed.push(i) - done = true - } - } catch { - pollErrors++ - if (pollErrors >= MAX_POLL_ERRORS) { - updateChunk(i, { status: "failed" }) - failed.push(i) - done = true + const res = await bulkImportParamsOnly(chunkFile) + jobId = res.jobId + updateChunk(i, { status: "polling", jobId }) + } catch (e) { + toast.error(`Chunk ${i + 1} upload failed: ${String(e)}`) + updateChunk(i, { status: "failed" }) + failed.push(i) + continue + } + + let pollErrors = 0 + let done = false + while (!abortRef.current && !done) { + await delay(POLL_MS) + try { + const job = await getImportJob(jobId) + if (job.status === "DONE" || job.status === "PARTIAL" || job.status === "FAILED") { + const ok = job.status !== "FAILED" + updateChunk(i, { + status: ok ? (job.status === "DONE" ? "done" : "partial") : "failed", + inserted: job.success, + }) + if (ok) setTotalInserted((n) => n + (job.success ?? 0)) + else failed.push(i) + done = true + } + } catch { + pollErrors++ + if (pollErrors >= MAX_POLL_ERRORS) { + updateChunk(i, { status: "failed" }) + failed.push(i) + done = true + } } } } + + setFailedChunks(failed) + + if (abortRef.current || failed.length === 0) break + if (currentPass >= MAX_AUTO_PASSES) break + if (failed.length >= prevFailedCount) { + setPassStatus(`Stopped after ${currentPass} passes — ${failed.length} chunks could not be resolved`) + break + } + + prevFailedCount = failed.length + currentPass++ + fromChunk = 0 } - setFailedChunks(failed) - setStep(failed.length === 0 ? "done" : "failed") + const finalFailed = chunks.filter((c) => c.status === "failed").length + setStep(finalFailed === 0 ? "done" : "failed") } function updateChunk(index: number, patch: Partial) { @@ -280,10 +318,13 @@ export function ParamsOnlyImportDialog({ open, onOpenChange }: Props) { {step === "importing" && parsed && (
- Chunk {currentIdx + 1} / {parsed.chunks.length} + + {passNum > 1 ? `Pass ${passNum}/${MAX_AUTO_PASSES} · ` : ""}Chunk {currentIdx + 1} / {parsed.chunks.length} + {totalInserted.toLocaleString()} rows imported
+ {passStatus &&

{passStatus}

}

{doneCount}/{parsed.chunks.length} complete ({progressPct}%)

From b515a9e5f8019a5bcbd0af20057d1220e90ad9e5 Mon Sep 17 00:00:00 2001 From: ilhom Date: Mon, 29 Jun 2026 14:04:57 +0700 Subject: [PATCH 2/3] feat(frontend): two-phase chunked routing import eliminates circular deps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 1: import product_master + parameters (no cross-product refs) Phase 2: import routing AFTER all products are in DB (no deadlocks) Previously, chunk A could fail because it referenced products in chunk B, while chunk B failed because it referenced products in chunk A. Auto-retry could not resolve circular dependencies. The two-phase approach eliminates this entirely: - Phase 1 has no cross-product routing references → always converges - Phase 2 runs after Phase 1 completes → all products in DbProductSet Also adds BULK_PRODUCTS_ONLY_CONFIG and BULK_ROUTING_ONLY_CONFIG to chunked-importer.ts for use in the two-phase dialog. Co-Authored-By: Claude Sonnet 4.6 (1M context) --- .../finance/costing/bulk-import-dialog.tsx | 132 ++++++++++-------- src/lib/excel/chunked-importer.ts | 33 ++++- 2 files changed, 109 insertions(+), 56 deletions(-) diff --git a/src/components/finance/costing/bulk-import-dialog.tsx b/src/components/finance/costing/bulk-import-dialog.tsx index be693c0..f805ec4 100644 --- a/src/components/finance/costing/bulk-import-dialog.tsx +++ b/src/components/finance/costing/bulk-import-dialog.tsx @@ -44,7 +44,8 @@ import { } from "@/services/finance/cost-import-api" import { chunkExcelFile, - BULK_ROUTING_CONFIG, + BULK_PRODUCTS_ONLY_CONFIG, + BULK_ROUTING_ONLY_CONFIG, type ChunkResult, } from "@/lib/excel/chunked-importer" import type { BulkValidationResult } from "@/types/finance/cost-import" @@ -88,7 +89,11 @@ export function BulkImportDialog({ open, onOpenChange }: BulkImportDialogProps) // Chunked state const [parseStage, setParseStage] = useState("") - const [parsed, setParsed] = useState(null) + const [parsedPhase1, setParsedPhase1] = useState(null) + const [parsedPhase2, setParsedPhase2] = useState(null) + // Active phase (1=products, 2=routing) and its chunks + const [activePhase, setActivePhase] = useState(1) + const parsed = activePhase === 1 ? parsedPhase1 : parsedPhase2 const [chunks, setChunks] = useState([]) const [currentIdx, setCurrentIdx] = useState(0) const [totalInserted, setTotalInserted] = useState(0) @@ -103,7 +108,9 @@ export function BulkImportDialog({ open, onOpenChange }: BulkImportDialogProps) setValidation(null) setExpandedSheets(new Set()) setJobId(null) - setParsed(null) + setParsedPhase1(null) + setParsedPhase2(null) + setActivePhase(1) setChunks([]) setCurrentIdx(0) setTotalInserted(0) @@ -167,9 +174,17 @@ export function BulkImportDialog({ open, onOpenChange }: BulkImportDialogProps) if (!file) return setStep("parsing") try { - const result = await chunkExcelFile(file, BULK_ROUTING_CONFIG, setParseStage) - setParsed(result) - setChunks(result.chunks.map((c) => ({ index: c.index, keyCount: c.keyCount, status: "pending" }))) + setParseStage("Parsing product sheets (Phase 1)…") + const phase1 = await chunkExcelFile(file, BULK_PRODUCTS_ONLY_CONFIG, (s) => setParseStage(`Phase 1: ${s}`)) + setParsedPhase1(phase1) + + setParseStage("Parsing routing sheets (Phase 2)…") + const phase2 = await chunkExcelFile(file, BULK_ROUTING_ONLY_CONFIG, (s) => setParseStage(`Phase 2: ${s}`)) + setParsedPhase2(phase2) + + // Show Phase 1 chunks initially + setActivePhase(1) + setChunks(phase1.chunks.map((c) => ({ index: c.index, keyCount: c.keyCount, status: "pending" }))) setStep("ready") } catch (e) { toast.error(`Parse failed: ${String(e)}`) @@ -177,53 +192,49 @@ export function BulkImportDialog({ open, onOpenChange }: BulkImportDialogProps) } } - async function startChunkedImport(fromChunk = 0) { - if (!parsed) return - abortRef.current = false - setStep("chunked_importing") - - let prevFailedCount = parsed.chunks.length // worst case on first pass - let currentPass = fromChunk === 0 ? 1 : passNum + async function runPhase(phaseParsed: ChunkResult, phaseNum: 1 | 2): Promise { + // Returns count of remaining failed chunks after auto-retry. + const totalChunks = phaseParsed.chunks.length + setChunks(phaseParsed.chunks.map((c) => ({ index: c.index, keyCount: c.keyCount, status: "pending" }))) + let prevFailedCount = totalChunks + let currentPass = 1 // eslint-disable-next-line no-constant-condition while (true) { setPassNum(currentPass) - setPassStatus(currentPass > 1 ? `Pass ${currentPass}/${MAX_AUTO_PASSES} — retrying ${prevFailedCount} failed chunks…` : "") - - // Reset all pending on each pass (keep done/partial from prior passes visible) + const phaseLabel = `Phase ${phaseNum}/2` + setPassStatus( + currentPass > 1 + ? `${phaseLabel} · Pass ${currentPass}/${MAX_AUTO_PASSES} — retrying ${prevFailedCount} chunks…` + : `${phaseLabel} — ${totalChunks} chunks` + ) if (currentPass > 1) { setChunks((prev) => prev.map((c) => ({ ...c, status: c.status === "failed" ? "pending" : c.status }))) } const failed: number[] = [] - - for (let i = fromChunk; i < parsed.chunks.length; i++) { + for (let i = 0; i < totalChunks; i++) { if (abortRef.current) break - // Skip already-succeeded chunks on re-passes if (currentPass > 1) { - const current = chunks.find((c) => c.index === i) - if (current && (current.status === "done" || current.status === "partial")) continue + const cur = chunks.find((c) => c.index === i) + if (cur && (cur.status === "done" || cur.status === "partial")) continue } - setCurrentIdx(i) updateChunk(i, "uploading") - - const chunk = parsed.chunks[i] + const chunk = phaseParsed.chunks[i] const chunkFile = new File([chunk.blob], chunk.fileName, { type: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", }) - let cJobId: number try { const res = await bulkImportProductMasterRouting(chunkFile, "update") cJobId = res.jobId updateChunk(i, "polling") } catch (e) { + toast.error(`Chunk ${i + 1} upload failed: ${String(e)}`) updateChunk(i, "failed"); failed.push(i); continue } - - let pollErrors = 0 - let done = false + let pollErrors = 0; let done = false while (!abortRef.current && !done) { await new Promise((r) => setTimeout(r, POLL_MS)) try { @@ -239,25 +250,33 @@ export function BulkImportDialog({ open, onOpenChange }: BulkImportDialogProps) } } } - setFailedChunks(failed) - - // Stop conditions - if (abortRef.current || failed.length === 0) break - if (currentPass >= MAX_AUTO_PASSES) break - if (failed.length >= prevFailedCount) { - // No improvement — stop to avoid infinite loop - setPassStatus(`Stopped after ${currentPass} passes — no improvement (${failed.length} chunks still failing)`) - break - } - + if (abortRef.current || failed.length === 0) return failed.length + if (currentPass >= MAX_AUTO_PASSES || failed.length >= prevFailedCount) return failed.length prevFailedCount = failed.length currentPass++ - fromChunk = 0 // always restart from 0 on subsequent passes } + } + + async function startChunkedImport(fromChunk = 0) { + if (!parsedPhase1 || !parsedPhase2) return + abortRef.current = false + setStep("chunked_importing") - const finalFailed = chunks.filter((c) => c.status === "failed").length - setStep(finalFailed === 0 ? "chunked_done" : "chunked_failed") + // Unused parameter kept for API compatibility with retry button + void fromChunk + + // ── Phase 1: products + params (no cross-product deps → converges quickly) ─ + setActivePhase(1) + const p1Failed = await runPhase(parsedPhase1, 1) + if (abortRef.current) { setStep("chunked_failed"); return } + + // ── Phase 2: routing only — all products now in DbProductSet → no deadlock ─ + setActivePhase(2) + setPassNum(1) + const p2Failed = await runPhase(parsedPhase2, 2) + + setStep(p1Failed + p2Failed === 0 ? "chunked_done" : "chunked_failed") } function updateChunk(index: number, status: ChunkStatus["status"]) { @@ -496,21 +515,24 @@ export function BulkImportDialog({ open, onOpenChange }: BulkImportDialogProps) )} {/* Chunked: ready */} - {step === "ready" && parsed && ( -
-

File parsed

-
-

Products

{parsed.totalKeys.toLocaleString()}

-

Total rows

{parsed.totalRows.toLocaleString()}

-

Chunks

{parsed.chunks.length}

+ {step === "ready" && parsedPhase1 && parsedPhase2 && ( +
+

File parsed — Two-Phase Import

+
+
+

Phase 1 — Products

+

{parsedPhase1.totalKeys.toLocaleString()} products · {parsedPhase1.chunks.length} chunks

+

product_master + parameters

+
+
+

Phase 2 — Routing

+

{parsedPhase2.totalKeys.toLocaleString()} routes · {parsedPhase2.chunks.length} chunks

+

route_head + sequences + RMs

+
-

- ~{BULK_ROUTING_CONFIG.chunkSize} products per chunk · 6 sheets each -

-
+
- If routing references intermediate products from other chunks, those rows - will show as "missing product" warnings (non-fatal). + Phase 1 imports all products first. Phase 2 imports routing after — no circular dependency issues.
)} diff --git a/src/lib/excel/chunked-importer.ts b/src/lib/excel/chunked-importer.ts index 5873180..396e4cb 100644 --- a/src/lib/excel/chunked-importer.ts +++ b/src/lib/excel/chunked-importer.ts @@ -69,11 +69,42 @@ export const PARAMS_ONLY_CONFIG: ChunkerConfig = { } /** - * Bulk product + routing import (product_master + route sheets). + * Phase 1 of a two-phase bulk routing import: products + params only. + * All product_master rows are imported before any routing data, so that + * Phase 2 route_sequences can reference ANY product without circular-dependency failures. + */ +export const BULK_PRODUCTS_ONLY_CONFIG: ChunkerConfig = { + sheetGroups: [ + { baseName: "product_master", outputName: "product_master" }, + { baseName: "product_parameters", outputName: "product_parameters" }, + { baseName: "product_applicable_params", outputName: "product_applicable_params" }, + ], + keyColumn: "legacy_oracle_sys_id", + chunkSize: 300, +} + +/** + * Phase 2 of a two-phase bulk routing import: routing data only. + * Run AFTER BULK_PRODUCTS_ONLY_CONFIG so all products are in the DB and + * node_product_legacy_product_id cross-references always resolve. * * route_sequences and route_rms use "route_head_legacy_product_id" as their * key column (same value as legacy_oracle_sys_id — the root product ID). */ +export const BULK_ROUTING_ONLY_CONFIG: ChunkerConfig = { + sheetGroups: [ + { baseName: "route_head", outputName: "route_head" }, + { baseName: "route_sequences", outputName: "route_sequences", keyColumn: "route_head_legacy_product_id" }, + { baseName: "route_rms", outputName: "route_rms", keyColumn: "route_head_legacy_product_id" }, + ], + keyColumn: "legacy_oracle_sys_id", + chunkSize: 300, +} + +/** + * Legacy single-phase config — kept for backward compatibility. + * Prefer BULK_PRODUCTS_ONLY_CONFIG + BULK_ROUTING_ONLY_CONFIG for large files. + */ export const BULK_ROUTING_CONFIG: ChunkerConfig = { sheetGroups: [ { baseName: "product_master", outputName: "product_master" }, From e513f7517901adf0c460abc26e33cfcb09128a8b Mon Sep 17 00:00:00 2001 From: ilhom Date: Mon, 29 Jun 2026 19:33:52 +0700 Subject: [PATCH 3/3] feat(finance): presigned-upload bulk import dialog Replace the browser-side chunked importer (SheetJS web worker + multi-pass auto-retry) with the v2 ETL flow: getImportUploadURL -> PUT the file directly to MinIO (with progress) -> startCostingImport -> poll the job to DONE/PARTIAL with an error-report link. A single unified bulk-import dialog drives both datasets via a `kind` prop (product_routing | params_only). Remove the chunked-importer + worker, the params-only dialog, and the old bytes/validate BFF routes; add the upload-url + start BFF routes. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../product-master-page-client.tsx | 6 +- .../costing/import/bulk_params_only/route.ts | 73 -- .../import/bulk_product_routing/route.ts | 46 - .../v1/finance/costing/import/start/route.ts | 59 ++ .../costing/import/upload-url/route.ts | 56 ++ .../validate/bulk_product_routing/route.ts | 61 -- .../finance/costing/bulk-import-dialog.tsx | 801 +++++------------- .../costing/params-only-import-dialog.tsx | 440 ---------- src/lib/excel/chunked-importer.ts | 211 ----- src/lib/excel/chunked-importer.worker.ts | 260 ------ src/services/finance/cost-import-api.ts | 147 ++-- src/types/finance/cost-import.ts | 28 +- src/types/generated/finance/v1/cost_import.ts | 527 +++++++++++- 13 files changed, 924 insertions(+), 1791 deletions(-) delete mode 100644 src/app/api/v1/finance/costing/import/bulk_params_only/route.ts delete mode 100644 src/app/api/v1/finance/costing/import/bulk_product_routing/route.ts create mode 100644 src/app/api/v1/finance/costing/import/start/route.ts create mode 100644 src/app/api/v1/finance/costing/import/upload-url/route.ts delete mode 100644 src/app/api/v1/finance/costing/validate/bulk_product_routing/route.ts delete mode 100644 src/components/finance/costing/params-only-import-dialog.tsx delete mode 100644 src/lib/excel/chunked-importer.ts delete mode 100644 src/lib/excel/chunked-importer.worker.ts diff --git a/src/app/(dashboard)/finance/product-master/product-master-page-client.tsx b/src/app/(dashboard)/finance/product-master/product-master-page-client.tsx index d8c8cab..48800ad 100644 --- a/src/app/(dashboard)/finance/product-master/product-master-page-client.tsx +++ b/src/app/(dashboard)/finance/product-master/product-master-page-client.tsx @@ -25,7 +25,6 @@ import { ProductMasterTable, } from "@/components/finance/cost-product-master" import { BulkImportDialog } from "@/components/finance/costing/bulk-import-dialog" -import { ParamsOnlyImportDialog } from "@/components/finance/costing/params-only-import-dialog" import { ImportDialog } from "@/components/finance/costing/import-dialog" import { ProductTypeCombobox } from "@/components/finance/comboboxes" import { DataTablePagination } from "@/components/shared" @@ -55,6 +54,7 @@ export default function ProductMasterPageClient() { const [importOpen, setImportOpen] = useState(false) const [bulkImportOpen, setBulkImportOpen] = useState(false) const [paramsImportOpen, setParamsImportOpen] = useState(false) + // Both bulk menu items open the same unified ETL dialog, differing only by kind. const [editing, setEditing] = useState(null) const [bulkExportLoading, setBulkExportLoading] = useState(false) @@ -232,8 +232,8 @@ export default function ProductMasterPageClient() { onOpenChange={setDeactivateOpen} product={editing} /> - - + +
) } diff --git a/src/app/api/v1/finance/costing/import/bulk_params_only/route.ts b/src/app/api/v1/finance/costing/import/bulk_params_only/route.ts deleted file mode 100644 index 6cee208..0000000 --- a/src/app/api/v1/finance/costing/import/bulk_params_only/route.ts +++ /dev/null @@ -1,73 +0,0 @@ -// POST /api/v1/finance/costing/import/bulk_params_only -// Accepts multipart/form-data ("file" field) to avoid JSON number-array inflation: -// a 50 MB binary file as JSON becomes ~200 MB — this sends the actual binary bytes. -// Falls back to JSON body (small files only) for backward compatibility. -// Returns: { base, data: { jobId: number } } -import { NextRequest, NextResponse } from "next/server" -import { - getCostDataImportClient, - createMetadataFromRequest, - isGrpcError, - handleGrpcError, -} from "@/lib/grpc" - -export async function POST(request: NextRequest) { - try { - const metadata = createMetadataFromRequest(request) - const client = getCostDataImportClient() - - let fileContent: Uint8Array - let fileName: string - - const contentType = request.headers.get("content-type") ?? "" - - if (contentType.includes("multipart/form-data")) { - const formData = await request.formData() - const file = formData.get("file") as File | null - if (!file) { - return NextResponse.json( - { - base: { - isSuccess: false, - statusCode: "400", - message: "Missing 'file' field in form data", - validationErrors: [], - }, - }, - { status: 400 }, - ) - } - fileContent = new Uint8Array(await file.arrayBuffer()) - fileName = file.name - } else { - // Legacy JSON fallback - const body = await request.json() - fileContent = new Uint8Array(body.fileContent as number[]) - fileName = body.fileName ?? "" - } - - const res = await client.importBulkParamsOnly( - { fileContent, fileName }, - metadata, - ) - - return NextResponse.json({ - base: res.base, - data: { jobId: res.jobId, status: res.status }, - }) - } catch (error) { - if (isGrpcError(error)) return handleGrpcError(error) - console.error("Error importing bulk params:", error) - return NextResponse.json( - { - base: { - isSuccess: false, - statusCode: "500", - message: "Failed to import params", - validationErrors: [], - }, - }, - { status: 500 }, - ) - } -} diff --git a/src/app/api/v1/finance/costing/import/bulk_product_routing/route.ts b/src/app/api/v1/finance/costing/import/bulk_product_routing/route.ts deleted file mode 100644 index 8c66313..0000000 --- a/src/app/api/v1/finance/costing/import/bulk_product_routing/route.ts +++ /dev/null @@ -1,46 +0,0 @@ -// POST /api/v1/finance/costing/import/bulk_product_routing -// Import product master + routing data from an Excel file. -// Accepts JSON body: { fileContent: number[], fileName: string, duplicateAction?: string } -// Returns: { base, data: { jobId: number } } - -import { NextRequest, NextResponse } from "next/server" -import { - getCostDataImportClient, - createMetadataFromRequest, - isGrpcError, - handleGrpcError, -} from "@/lib/grpc" - -export async function POST(request: NextRequest) { - try { - const body = await request.json() - const metadata = createMetadataFromRequest(request) - const fileContent = new Uint8Array(body.fileContent as number[]) - const fileName: string = body.fileName ?? "" - const duplicateAction: string = body.duplicateAction ?? "update" - - const res = await getCostDataImportClient().importBulkProductRouting( - { fileContent, fileName, duplicateAction }, - metadata, - ) - - return NextResponse.json({ - base: res.base, - data: { jobId: res.jobId, status: res.status }, - }) - } catch (error) { - if (isGrpcError(error)) return handleGrpcError(error) - console.error("Error importing bulk product routing:", error) - return NextResponse.json( - { - base: { - isSuccess: false, - statusCode: "500", - message: "Failed to import bulk product routing", - validationErrors: [], - }, - }, - { status: 500 }, - ) - } -} diff --git a/src/app/api/v1/finance/costing/import/start/route.ts b/src/app/api/v1/finance/costing/import/start/route.ts new file mode 100644 index 0000000..9bf3457 --- /dev/null +++ b/src/app/api/v1/finance/costing/import/start/route.ts @@ -0,0 +1,59 @@ +// POST /api/v1/finance/costing/import/start +// Starts an async ETL import job for an already-uploaded object (from the +// presigned upload). Accepts JSON body: +// { kind: "product_routing" | "params_only", objectKey: string, fileName?: string } + +import { NextRequest, NextResponse } from "next/server" +import { + getCostDataImportClient, + createMetadataFromRequest, + isGrpcError, + handleGrpcError, +} from "@/lib/grpc" +import { ImportKind } from "@/types/generated/finance/v1/cost_import" + +function toImportKind(kind: string): ImportKind { + if (kind === "product_routing") return ImportKind.IMPORT_KIND_PRODUCT_ROUTING + if (kind === "params_only") return ImportKind.IMPORT_KIND_PARAMS_ONLY + return ImportKind.IMPORT_KIND_UNSPECIFIED +} + +export async function POST(request: NextRequest) { + try { + const body = await request.json() + const metadata = createMetadataFromRequest(request) + + const res = await getCostDataImportClient().startCostingImport( + { + kind: toImportKind(body.kind), + objectKey: body.objectKey ?? "", + fileName: body.fileName ?? "", + duplicateAction: body.duplicateAction ?? "", + }, + metadata, + ) + + if (res.base && !res.base.isSuccess) { + return NextResponse.json({ base: res.base }, { status: 422 }) + } + + return NextResponse.json({ + base: res.base, + data: { jobId: res.jobId, status: res.status }, + }) + } catch (error) { + if (isGrpcError(error)) return handleGrpcError(error) + console.error("Error starting costing import:", error) + return NextResponse.json( + { + base: { + isSuccess: false, + statusCode: "500", + message: "Failed to start import", + validationErrors: [], + }, + }, + { status: 500 }, + ) + } +} diff --git a/src/app/api/v1/finance/costing/import/upload-url/route.ts b/src/app/api/v1/finance/costing/import/upload-url/route.ts new file mode 100644 index 0000000..d4aa638 --- /dev/null +++ b/src/app/api/v1/finance/costing/import/upload-url/route.ts @@ -0,0 +1,56 @@ +// POST /api/v1/finance/costing/import/upload-url +// Returns a presigned PUT URL so the browser can upload a bulk-import file +// (xlsx or zipped CSV) directly to object storage, bypassing the BFF/gRPC path. +// Accepts JSON body: { kind: "product_routing" | "params_only", fileName: string } + +import { NextRequest, NextResponse } from "next/server" +import { + getCostDataImportClient, + createMetadataFromRequest, + isGrpcError, + handleGrpcError, +} from "@/lib/grpc" +import { ImportKind } from "@/types/generated/finance/v1/cost_import" + +function toImportKind(kind: string): ImportKind { + if (kind === "product_routing") return ImportKind.IMPORT_KIND_PRODUCT_ROUTING + if (kind === "params_only") return ImportKind.IMPORT_KIND_PARAMS_ONLY + return ImportKind.IMPORT_KIND_UNSPECIFIED +} + +export async function POST(request: NextRequest) { + try { + const body = await request.json() + const metadata = createMetadataFromRequest(request) + + const res = await getCostDataImportClient().getImportUploadURL( + { kind: toImportKind(body.kind), fileName: body.fileName ?? "" }, + metadata, + ) + + if (res.base && !res.base.isSuccess) { + return NextResponse.json({ base: res.base }, { status: 422 }) + } + + return NextResponse.json({ + base: res.base, + uploadUrl: res.uploadUrl, + objectKey: res.objectKey, + expiresInSeconds: res.expiresInSeconds, + }) + } catch (error) { + if (isGrpcError(error)) return handleGrpcError(error) + console.error("Error getting import upload URL:", error) + return NextResponse.json( + { + base: { + isSuccess: false, + statusCode: "500", + message: "Failed to get upload URL", + validationErrors: [], + }, + }, + { status: 500 }, + ) + } +} diff --git a/src/app/api/v1/finance/costing/validate/bulk_product_routing/route.ts b/src/app/api/v1/finance/costing/validate/bulk_product_routing/route.ts deleted file mode 100644 index 698a289..0000000 --- a/src/app/api/v1/finance/costing/validate/bulk_product_routing/route.ts +++ /dev/null @@ -1,61 +0,0 @@ -// POST /api/v1/finance/costing/validate/bulk_product_routing -// Validate a bulk product routing Excel file before importing. -// Accepts JSON body: { fileContent: number[], fileName: string } -// Returns: { isValid: boolean, sheets: BulkSheetValidationResult[] } - -import { NextRequest, NextResponse } from "next/server" -import { - getCostDataImportClient, - createMetadataFromRequest, - isGrpcError, - handleGrpcError, -} from "@/lib/grpc" - -export async function POST(request: NextRequest) { - try { - const body = await request.json() - const metadata = createMetadataFromRequest(request) - const fileContent = new Uint8Array(body.fileContent as number[]) - const fileName: string = body.fileName ?? "" - - const res = await getCostDataImportClient().validateBulkProductRoutingFile( - { fileContent, fileName }, - metadata, - ) - - // Surface backend errors (e.g. timeout on very large files) as a non-2xx response - // so the frontend can display the message rather than showing an empty result. - if (res.base && !res.base.isSuccess) { - return NextResponse.json({ base: res.base }, { status: 422 }) - } - - return NextResponse.json({ - isValid: res.isValid, - sheets: res.sheets.map((sheet) => ({ - sheetName: sheet.sheetName, - totalRows: sheet.totalRows, - errorCount: sheet.errorCount, - warningCount: sheet.warningCount, - sampleErrors: sheet.sampleErrors.map((e) => ({ - rowNumber: e.rowNumber, - field: e.field, - message: e.message, - })), - })), - }) - } catch (error) { - if (isGrpcError(error)) return handleGrpcError(error) - console.error("Error validating bulk product routing file:", error) - return NextResponse.json( - { - base: { - isSuccess: false, - statusCode: "500", - message: "Failed to validate bulk product routing file", - validationErrors: [], - }, - }, - { status: 500 }, - ) - } -} diff --git a/src/components/finance/costing/bulk-import-dialog.tsx b/src/components/finance/costing/bulk-import-dialog.tsx index f805ec4..d3b5ce0 100644 --- a/src/components/finance/costing/bulk-import-dialog.tsx +++ b/src/components/finance/costing/bulk-import-dialog.tsx @@ -1,12 +1,10 @@ "use client" -import React, { useEffect, useRef, useState } from "react" +import { useEffect, useRef, useState } from "react" import { - AlertCircle, CheckCircle2, - ChevronDown, - ChevronRight, Download, + ExternalLink, FileSpreadsheet, Loader2, Upload, @@ -15,684 +13,315 @@ import { import { useRouter } from "next/navigation" import { toast } from "sonner" -import { Dialog } from "@/components/ui/dialog" +import { + Dialog, + DialogContent, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog" import { Button } from "@/components/ui/button" -import { Badge } from "@/components/ui/badge" import { Label } from "@/components/ui/label" import { Progress } from "@/components/ui/progress" +import { StatusBadge } from "@/components/common/status-badge" import { - Table, - TableBody, - TableCell, - TableHead, - TableHeader, - TableRow, -} from "@/components/ui/table" -import { - ScrollableDialogContent, - ScrollableDialogHeader, - ScrollableDialogBody, - ScrollableDialogFooter, -} from "@/components/common/scrollable-dialog" -import { DialogTitle } from "@/components/ui/dialog" -import { - bulkImportProductMasterRouting, - downloadBulkProductRoutingTemplate, - exportBulkProductRouting, - validateBulkProductRoutingFile, + getImportUploadURL, + putToPresignedUrl, + startCostingImport, getImportJob, + downloadBulkProductRoutingTemplate, + downloadParamsOnlyTemplate, } from "@/services/finance/cost-import-api" -import { - chunkExcelFile, - BULK_PRODUCTS_ONLY_CONFIG, - BULK_ROUTING_ONLY_CONFIG, - type ChunkResult, -} from "@/lib/excel/chunked-importer" -import type { BulkValidationResult } from "@/types/finance/cost-import" +import type { CostImportJob, ImportKindKey } from "@/types/finance/cost-import" export interface BulkImportDialogProps { open: boolean onOpenChange: (open: boolean) => void + /** Which dataset to import. Defaults to product_routing. */ + kind?: ImportKindKey } -// "validate" flow: upload → validating → validated → submitting → done -// "chunked" flow: upload → parsing → ready → chunked_importing → chunked_done | chunked_failed -type Step = - | "upload" - | "validating" | "validated" | "submitting" | "done" - | "parsing" | "ready" | "chunked_importing" | "chunked_done" | "chunked_failed" +type Step = "upload" | "uploading" | "starting" | "processing" | "done" | "partial" | "failed" + +interface KindConfig { + title: string + accept: string + fileHint: string + templateLabel: string + downloadTemplate: () => Promise +} -interface ChunkStatus { - index: number - keyCount: number - status: "pending" | "uploading" | "polling" | "done" | "partial" | "failed" +const KIND_CONFIG: Record = { + product_routing: { + title: "Bulk Import — Product Master & Routing", + accept: ".xlsx,.zip", + fileHint: "Format: .xlsx (6 sheet) atau .zip berisi CSV per sheet", + templateLabel: "Template (xlsx)", + downloadTemplate: downloadBulkProductRoutingTemplate, + }, + params_only: { + title: "Bulk Import — Params Only", + accept: ".zip", + fileHint: "Format: .zip berisi product_parameters.csv + applicable_params.csv", + templateLabel: "Template (xlsx)", + downloadTemplate: downloadParamsOnlyTemplate, + }, } -const VALIDATE_SIZE_LIMIT = 5 * 1024 * 1024 // 5 MB — matches server-side validate limit const POLL_MS = 3_000 const MAX_POLL_ERRORS = 5 -// Auto-retry: each pass adds newly-imported products to DbProductSet, reducing -// cross-chunk dependency failures. Stop when no improvement or max passes reached. -const MAX_AUTO_PASSES = 5 -export function BulkImportDialog({ open, onOpenChange }: BulkImportDialogProps) { +export function BulkImportDialog({ open, onOpenChange, kind = "product_routing" }: BulkImportDialogProps) { const fileRef = useRef(null) const router = useRouter() const abortRef = useRef(false) + const cfg = KIND_CONFIG[kind] const [file, setFile] = useState(null) const [step, setStep] = useState("upload") - const [validation, setValidation] = useState(null) - const [expandedSheets, setExpandedSheets] = useState>(new Set()) - const [jobId, setJobId] = useState(null) + const [uploadPct, setUploadPct] = useState(0) + const [job, setJob] = useState(null) const [templateLoading, setTemplateLoading] = useState(false) - // Chunked state - const [parseStage, setParseStage] = useState("") - const [parsedPhase1, setParsedPhase1] = useState(null) - const [parsedPhase2, setParsedPhase2] = useState(null) - // Active phase (1=products, 2=routing) and its chunks - const [activePhase, setActivePhase] = useState(1) - const parsed = activePhase === 1 ? parsedPhase1 : parsedPhase2 - const [chunks, setChunks] = useState([]) - const [currentIdx, setCurrentIdx] = useState(0) - const [totalInserted, setTotalInserted] = useState(0) - const [failedChunks, setFailedChunks] = useState([]) - const [passNum, setPassNum] = useState(1) - const [passStatus, setPassStatus] = useState("") - function resetState() { abortRef.current = false setFile(null) setStep("upload") - setValidation(null) - setExpandedSheets(new Set()) - setJobId(null) - setParsedPhase1(null) - setParsedPhase2(null) - setActivePhase(1) - setChunks([]) - setCurrentIdx(0) - setTotalInserted(0) - setFailedChunks([]) - setPassNum(1) - setPassStatus("") - setParseStage("") + setUploadPct(0) + setJob(null) if (fileRef.current) fileRef.current.value = "" } - // eslint-disable-next-line react-hooks/exhaustive-deps - useEffect(() => { if (open) resetState() }, [open]) + useEffect(() => { if (open) resetState() }, [open, kind]) function handleFileChange(e: React.ChangeEvent) { - const selected = e.target.files?.[0] ?? null - setFile(selected) - setValidation(null) - setStep("upload") + setFile(e.target.files?.[0] ?? null) } async function handleDownloadTemplate() { setTemplateLoading(true) try { - await downloadBulkProductRoutingTemplate() + await cfg.downloadTemplate() } catch (e) { - toast.error(`Template download failed: ${String(e)}`) + toast.error(`Template download gagal: ${String(e)}`) } finally { setTemplateLoading(false) } } - async function handleValidate() { - if (!file) return - setStep("validating") - try { - const result = await validateBulkProductRoutingFile(file) - setValidation(result) - setStep("validated") - } catch (e) { - toast.error(`Validation failed: ${String(e)}`) - setStep("upload") + async function pollUntilDone(jobId: number) { + let pollErrors = 0 + while (!abortRef.current) { + await new Promise((r) => setTimeout(r, POLL_MS)) + if (abortRef.current) return + try { + const j = await getImportJob(jobId) + setJob(j) + if (j.status === "DONE") { setStep("done"); return } + if (j.status === "PARTIAL") { setStep("partial"); return } + if (j.status === "FAILED") { setStep("failed"); return } + } catch { + if (++pollErrors >= MAX_POLL_ERRORS) { + toast.error("Gagal memantau status job — cek halaman Import Jobs.") + setStep("failed") + return + } + } } } async function handleImport() { if (!file) return - setStep("submitting") + abortRef.current = false try { - const result = await bulkImportProductMasterRouting(file, "update") - setJobId(result.jobId) - setStep("done") - toast.success(`Import queued — Job #${result.jobId}`) + setStep("uploading") + setUploadPct(0) + const { uploadUrl, objectKey } = await getImportUploadURL(kind, file.name) + await putToPresignedUrl(uploadUrl, file, setUploadPct) + + setStep("starting") + const { jobId } = await startCostingImport(kind, objectKey, file.name) + toast.success(`Import dijadwalkan — Job #${jobId}`) + + setStep("processing") + setJob({ jobId } as CostImportJob) + await pollUntilDone(jobId) } catch (e) { - toast.error(`Import failed: ${String(e)}`) - setStep("validated") - } - } - - // ── Chunked import flow ──────────────────────────────────────────────────── - async function handleChunkedImport() { - if (!file) return - setStep("parsing") - try { - setParseStage("Parsing product sheets (Phase 1)…") - const phase1 = await chunkExcelFile(file, BULK_PRODUCTS_ONLY_CONFIG, (s) => setParseStage(`Phase 1: ${s}`)) - setParsedPhase1(phase1) - - setParseStage("Parsing routing sheets (Phase 2)…") - const phase2 = await chunkExcelFile(file, BULK_ROUTING_ONLY_CONFIG, (s) => setParseStage(`Phase 2: ${s}`)) - setParsedPhase2(phase2) - - // Show Phase 1 chunks initially - setActivePhase(1) - setChunks(phase1.chunks.map((c) => ({ index: c.index, keyCount: c.keyCount, status: "pending" }))) - setStep("ready") - } catch (e) { - toast.error(`Parse failed: ${String(e)}`) + toast.error(`Import gagal: ${String(e)}`) setStep("upload") } } - async function runPhase(phaseParsed: ChunkResult, phaseNum: 1 | 2): Promise { - // Returns count of remaining failed chunks after auto-retry. - const totalChunks = phaseParsed.chunks.length - setChunks(phaseParsed.chunks.map((c) => ({ index: c.index, keyCount: c.keyCount, status: "pending" }))) - let prevFailedCount = totalChunks - let currentPass = 1 - - // eslint-disable-next-line no-constant-condition - while (true) { - setPassNum(currentPass) - const phaseLabel = `Phase ${phaseNum}/2` - setPassStatus( - currentPass > 1 - ? `${phaseLabel} · Pass ${currentPass}/${MAX_AUTO_PASSES} — retrying ${prevFailedCount} chunks…` - : `${phaseLabel} — ${totalChunks} chunks` - ) - if (currentPass > 1) { - setChunks((prev) => prev.map((c) => ({ ...c, status: c.status === "failed" ? "pending" : c.status }))) - } - - const failed: number[] = [] - for (let i = 0; i < totalChunks; i++) { - if (abortRef.current) break - if (currentPass > 1) { - const cur = chunks.find((c) => c.index === i) - if (cur && (cur.status === "done" || cur.status === "partial")) continue - } - setCurrentIdx(i) - updateChunk(i, "uploading") - const chunk = phaseParsed.chunks[i] - const chunkFile = new File([chunk.blob], chunk.fileName, { - type: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - }) - let cJobId: number - try { - const res = await bulkImportProductMasterRouting(chunkFile, "update") - cJobId = res.jobId - updateChunk(i, "polling") - } catch (e) { - toast.error(`Chunk ${i + 1} upload failed: ${String(e)}`) - updateChunk(i, "failed"); failed.push(i); continue - } - let pollErrors = 0; let done = false - while (!abortRef.current && !done) { - await new Promise((r) => setTimeout(r, POLL_MS)) - try { - const job = await getImportJob(cJobId) - if (job.status === "DONE" || job.status === "PARTIAL" || job.status === "FAILED") { - updateChunk(i, job.status !== "FAILED" ? (job.status === "DONE" ? "done" : "partial") : "failed") - if (job.status !== "FAILED") setTotalInserted((n) => n + (job.success ?? 0)) - else failed.push(i) - done = true - } - } catch { - if (++pollErrors >= MAX_POLL_ERRORS) { updateChunk(i, "failed"); failed.push(i); done = true } - } - } - } - setFailedChunks(failed) - if (abortRef.current || failed.length === 0) return failed.length - if (currentPass >= MAX_AUTO_PASSES || failed.length >= prevFailedCount) return failed.length - prevFailedCount = failed.length - currentPass++ - } - } - - async function startChunkedImport(fromChunk = 0) { - if (!parsedPhase1 || !parsedPhase2) return - abortRef.current = false - setStep("chunked_importing") - - // Unused parameter kept for API compatibility with retry button - void fromChunk - - // ── Phase 1: products + params (no cross-product deps → converges quickly) ─ - setActivePhase(1) - const p1Failed = await runPhase(parsedPhase1, 1) - if (abortRef.current) { setStep("chunked_failed"); return } - - // ── Phase 2: routing only — all products now in DbProductSet → no deadlock ─ - setActivePhase(2) - setPassNum(1) - const p2Failed = await runPhase(parsedPhase2, 2) - - setStep(p1Failed + p2Failed === 0 ? "chunked_done" : "chunked_failed") - } - - function updateChunk(index: number, status: ChunkStatus["status"]) { - setChunks((prev) => prev.map((c) => (c.index === index ? { ...c, status } : c))) - } - function handleClose() { abortRef.current = true resetState() onOpenChange(false) } - function toggleSheet(sheetName: string) { - setExpandedSheets((prev) => { - const next = new Set(prev) - if (next.has(sheetName)) { - next.delete(sheetName) - } else { - next.add(sheetName) - } - return next - }) - } - - const hasErrors = validation?.sheets.some((s) => s.errorCount > 0) ?? false - const isValidating = step === "validating" - const isSubmitting = step === "submitting" - const isDone = step === "done" - const canImport = step === "validated" && !hasErrors && (validation?.isValid ?? false) - const isChunkedFlow = ["parsing","ready","chunked_importing","chunked_done","chunked_failed"].includes(step) - const fileTooLargeForValidate = (file?.size ?? 0) > VALIDATE_SIZE_LIMIT - const doneCount = chunks.filter((c) => c.status === "done" || c.status === "partial").length - const progressPct = parsed ? Math.round((doneCount / parsed.chunks.length) * 100) : 0 + const busy = step === "uploading" || step === "starting" || step === "processing" + const terminal = step === "done" || step === "partial" || step === "failed" + const processedPct = + job && job.totalRows > 0 ? Math.round((job.processed / job.totalRows) * 100) : 0 return ( - - - - Bulk Import — Product Master & Routing - - - - {/* Template Download — hidden once job submitted */} - {!isDone && ( -
-
- -
-

Import Template

-

- Download template Excel dengan 6 sheet yang diperlukan -

-
-
- -
- )} - - {/* File Upload — hidden once job submitted */} - {!isDone && ( -
- -
fileRef.current?.click()} - > - - {file ? ( -
- - {file.name} - - ({(file.size / 1024).toFixed(1)} KB) - + { if (!v) handleClose() }}> + + + {cfg.title} + + +
+ {/* Template download + file picker — only before upload starts */} + {step === "upload" && ( + <> +
+
+ +
+

Import Template

+

Unduh template & isi sesuai kolom

- ) : ( - <> - -

- Klik untuk memilih atau drag & drop file Excel -

-

- Format: .xlsx -

- - )} -
-
- )} - - {/* Validating spinner */} - {isValidating && ( -
- - Memvalidasi file… -
- )} - - {/* Validation results table */} - {validation && !isValidating && ( -
-
- {validation.isValid ? ( - - ) : ( - - )} - - {validation.isValid - ? "File valid — siap diimport" - : "Validasi gagal — perbaiki error sebelum import"} - +
+
-
- - - - Sheet - Row - Error - Warning - - - - - {validation.sheets.map((sheet) => ( - - 0 ? "bg-destructive/5" : undefined} - > - {sheet.sheetName} - {sheet.totalRows} - - {sheet.errorCount > 0 ? ( - - {sheet.errorCount} - - ) : ( - 0 - )} - - - {sheet.warningCount > 0 ? ( - - {sheet.warningCount} - - ) : ( - 0 - )} - - - {sheet.sampleErrors.length > 0 && ( - - )} - - - - {/* Sample errors */} - {expandedSheets.has(sheet.sheetName) && - sheet.sampleErrors.map((err, i) => ( - - - Row {err.rowNumber}{err.field ? ` [${err.field}]` : ""}: {err.message} - - - ))} - - ))} - -
+
+ +
fileRef.current?.click()} + > + + {file ? ( +
+ + {file.name} + + ({(file.size / 1024 / 1024).toFixed(1)} MB) + +
+ ) : ( + <> + +

Klik untuk memilih file

+

{cfg.fileHint}

+ + )} +
+

+ File diunggah langsung ke storage lalu diproses di server (streaming) — ukuran besar tidak masalah. +

-
+ )} - {/* Done (single-shot) */} - {isDone && jobId !== null && ( -
- - Import dijadwalkan sebagai Job #{jobId}. Notifikasi akan dikirim saat selesai. -
- )} - - {/* File too large warning */} - {file && step === "upload" && fileTooLargeForValidate && ( -
- - - File {(file.size / 1024 / 1024).toFixed(1)} MB melebihi batas validasi 5 MB. - Gunakan Chunked Import untuk melanjutkan. - -
- )} - - {/* Chunked: parsing */} - {step === "parsing" && ( -
- -
-

Parsing file…

-

{parseStage}

+ {/* Uploading progress */} + {step === "uploading" && ( +
+
+ Mengunggah file… + {uploadPct}%
+
)} - {/* Chunked: ready */} - {step === "ready" && parsedPhase1 && parsedPhase2 && ( -
-

File parsed — Two-Phase Import

-
-
-

Phase 1 — Products

-

{parsedPhase1.totalKeys.toLocaleString()} products · {parsedPhase1.chunks.length} chunks

-

product_master + parameters

-
-
-

Phase 2 — Routing

-

{parsedPhase2.totalKeys.toLocaleString()} routes · {parsedPhase2.chunks.length} chunks

-

route_head + sequences + RMs

-
-
-
- - Phase 1 imports all products first. Phase 2 imports routing after — no circular dependency issues. -
+ {/* Starting */} + {step === "starting" && ( +
+ Menjadwalkan job…
)} - {/* Chunked: importing */} - {step === "chunked_importing" && parsed && ( -
+ {/* Processing — poll job */} + {step === "processing" && ( +
- - {passNum > 1 ? `Pass ${passNum}/${MAX_AUTO_PASSES} · ` : ""}Chunk {currentIdx + 1} / {parsed.chunks.length} + + Memproses + {job?.jobId ? ` — Job #${job.jobId}` : ""} - {totalInserted.toLocaleString()} rows imported -
- - {passStatus &&

{passStatus}

} -
-
- {chunks.map((c) => )} -
-
-
- )} - - {/* Chunked: done */} - {step === "chunked_done" && ( -
- -
-

- Import complete — {totalInserted.toLocaleString()} rows imported -

- + {job && }
-
- )} - - {/* Chunked: failed */} - {step === "chunked_failed" && ( -
-
- -
-

- {failedChunks.length} chunk{failedChunks.length > 1 ? "s" : ""} failed - {totalInserted > 0 && ` — ${totalInserted.toLocaleString()} rows already imported`} + {job && job.totalRows > 0 && ( + <> + +

+ {job.processed.toLocaleString()} / {job.totalRows.toLocaleString()} baris diproses

-
-
-
-
- {chunks.map((c) => )} -
-
+ + )} +

+ Dialog boleh ditutup — notifikasi dikirim saat selesai. +

)} - - - - - {/* Validate flow buttons */} - {file && step === "upload" && !fileTooLargeForValidate && ( - - )} - {canImport && ( - + {/* Terminal states */} + {terminal && job && ( + )} +
- {/* Chunked flow buttons */} - {file && step === "upload" && ( - - )} - {step === "ready" && ( - + {step === "upload" && file && ( + )} - {step === "chunked_importing" && ( - - )} - {step === "chunked_failed" && failedChunks.length > 0 && ( - )} - - + +
) } -/** - * Standalone "Export All" button that queues an async bulk export job. - */ -export function BulkExportButton({ - productTypeCodes, -}: { - productTypeCodes?: string[] -}) { - const [loading, setLoading] = useState(false) - - async function handleExport() { - setLoading(true) - try { - const result = await exportBulkProductRouting({ productTypeCodes }) - toast.success(`Export queued — Job #${result.jobId}`) - } catch (e) { - toast.error(`Export failed: ${String(e)}`) - } finally { - setLoading(false) - } - } - - return ( - - ) -} - -function RoutingChunkBadge({ chunk, current }: { chunk: ChunkStatus; current: boolean }) { - const color: Record = { - pending: "bg-muted text-muted-foreground", - uploading: "bg-blue-100 text-blue-700 dark:bg-blue-900 dark:text-blue-200", - polling: "bg-yellow-100 text-yellow-700 dark:bg-yellow-900 dark:text-yellow-200", - done: "bg-green-100 text-green-700 dark:bg-green-900 dark:text-green-200", - partial: "bg-orange-100 text-orange-700 dark:bg-orange-900 dark:text-orange-200", - failed: "bg-red-100 text-red-700 dark:bg-red-900 dark:text-red-200", +function ResultPanel({ step, job }: { step: Step; job: CostImportJob }) { + if (step === "failed") { + return ( +
+ +
+

Import gagal — Job #{job.jobId}

+

Buka halaman Import Jobs untuk detail.

+
+
+ ) } + const isPartial = step === "partial" return ( - - {chunk.index + 1} - {current && (chunk.status === "uploading" || chunk.status === "polling") && ( - - )} - +
+ +
+

+ {isPartial ? "Selesai dengan sebagian error" : "Import selesai"} — Job #{job.jobId} +

+

+ {job.success.toLocaleString()} sukses + {job.failed > 0 && ` · ${job.failed.toLocaleString()} gagal`} + {job.skipped > 0 && ` · ${job.skipped.toLocaleString()} dilewati`} +

+ {job.errorFileUrl && ( + + Unduh laporan error + + )} +
+
) } - -export type { BulkValidationResult } diff --git a/src/components/finance/costing/params-only-import-dialog.tsx b/src/components/finance/costing/params-only-import-dialog.tsx deleted file mode 100644 index 1c07a53..0000000 --- a/src/components/finance/costing/params-only-import-dialog.tsx +++ /dev/null @@ -1,440 +0,0 @@ -"use client" - -import { useEffect, useRef, useState } from "react" -import { - AlertCircle, - CheckCircle2, - Download, - FileSpreadsheet, - Loader2, - Upload, - XCircle, -} from "lucide-react" -import { useRouter } from "next/navigation" -import { toast } from "sonner" - -import { - Dialog, - DialogContent, - DialogFooter, - DialogHeader, - DialogTitle, -} from "@/components/ui/dialog" -import { Button } from "@/components/ui/button" -import { Label } from "@/components/ui/label" -import { Progress } from "@/components/ui/progress" -import { - bulkImportParamsOnly, - downloadParamsOnlyTemplate, - getImportJob, -} from "@/services/finance/cost-import-api" -import { - chunkExcelFile, - PARAMS_ONLY_CONFIG, - type ChunkResult, -} from "@/lib/excel/chunked-importer" - -interface Props { - open: boolean - onOpenChange: (open: boolean) => void -} - -type Step = "upload" | "parsing" | "ready" | "importing" | "done" | "failed" - -interface ChunkStatus { - index: number - productCount: number - rowCount: number - status: "pending" | "uploading" | "polling" | "done" | "partial" | "failed" - jobId?: number - inserted?: number -} - -const POLL_MS = 3_000 -const MAX_POLL_ERRORS = 5 -// For params-only, a PARTIAL result means some products were skipped (not yet in DB). -// Auto-retry lets subsequent passes pick up those products once routing is complete. -const MAX_AUTO_PASSES = 3 - -export function ParamsOnlyImportDialog({ open, onOpenChange }: Props) { - const fileRef = useRef(null) - const router = useRouter() - const abortRef = useRef(false) - - const [file, setFile] = useState(null) - const [parseStage, setParseStage] = useState("") - const [parsed, setParsed] = useState(null) - const [step, setStep] = useState("upload") - const [chunks, setChunks] = useState([]) - const [currentIdx, setCurrentIdx] = useState(0) - const [totalInserted, setTotalInserted] = useState(0) - const [failedChunks, setFailedChunks] = useState([]) - const [templateLoading, setTemplateLoading] = useState(false) - const [passNum, setPassNum] = useState(1) - const [passStatus, setPassStatus] = useState("") - - useEffect(() => { - if (open) reset() - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [open]) - - function reset() { - abortRef.current = false - setFile(null) - setParsed(null) - setStep("upload") - setChunks([]) - setCurrentIdx(0) - setTotalInserted(0) - setFailedChunks([]) - setPassNum(1) - setPassStatus("") - setParseStage("") - if (fileRef.current) fileRef.current.value = "" - } - - async function handleParse(f: File) { - setFile(f) - setStep("parsing") - try { - const result = await chunkExcelFile(f, PARAMS_ONLY_CONFIG, setParseStage) - setParsed(result) - setChunks( - result.chunks.map((c) => ({ - index: c.index, - productCount: c.keyCount, - rowCount: c.rowCount, - status: "pending", - })) - ) - setStep("ready") - } catch (e) { - toast.error(`Failed to parse file: ${String(e)}`) - setStep("upload") - } - } - - async function startImport(fromChunk = 0) { - if (!parsed) return - abortRef.current = false - setStep("importing") - - let prevFailedCount = parsed.chunks.length - let currentPass = fromChunk === 0 ? 1 : passNum - - // eslint-disable-next-line no-constant-condition - while (true) { - setPassNum(currentPass) - setPassStatus(currentPass > 1 ? `Pass ${currentPass}/${MAX_AUTO_PASSES} — retrying ${prevFailedCount} failed chunks…` : "") - - if (currentPass > 1) { - setChunks((prev) => prev.map((c) => ({ ...c, status: c.status === "failed" ? "pending" : c.status }))) - } - - const failed: number[] = [] - - for (let i = fromChunk; i < parsed.chunks.length; i++) { - if (abortRef.current) break - if (currentPass > 1) { - const cur = chunks.find((c) => c.index === i) - if (cur && (cur.status === "done" || cur.status === "partial")) continue - } - setCurrentIdx(i) - updateChunk(i, { status: "uploading" }) - - const chunk = parsed.chunks[i] - const chunkFile = new File([chunk.blob], chunk.fileName, { - type: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - }) - - let jobId: number - try { - const res = await bulkImportParamsOnly(chunkFile) - jobId = res.jobId - updateChunk(i, { status: "polling", jobId }) - } catch (e) { - toast.error(`Chunk ${i + 1} upload failed: ${String(e)}`) - updateChunk(i, { status: "failed" }) - failed.push(i) - continue - } - - let pollErrors = 0 - let done = false - while (!abortRef.current && !done) { - await delay(POLL_MS) - try { - const job = await getImportJob(jobId) - if (job.status === "DONE" || job.status === "PARTIAL" || job.status === "FAILED") { - const ok = job.status !== "FAILED" - updateChunk(i, { - status: ok ? (job.status === "DONE" ? "done" : "partial") : "failed", - inserted: job.success, - }) - if (ok) setTotalInserted((n) => n + (job.success ?? 0)) - else failed.push(i) - done = true - } - } catch { - pollErrors++ - if (pollErrors >= MAX_POLL_ERRORS) { - updateChunk(i, { status: "failed" }) - failed.push(i) - done = true - } - } - } - } - - setFailedChunks(failed) - - if (abortRef.current || failed.length === 0) break - if (currentPass >= MAX_AUTO_PASSES) break - if (failed.length >= prevFailedCount) { - setPassStatus(`Stopped after ${currentPass} passes — ${failed.length} chunks could not be resolved`) - break - } - - prevFailedCount = failed.length - currentPass++ - fromChunk = 0 - } - - const finalFailed = chunks.filter((c) => c.status === "failed").length - setStep(finalFailed === 0 ? "done" : "failed") - } - - function updateChunk(index: number, patch: Partial) { - setChunks((prev) => - prev.map((c) => (c.index === index ? { ...c, ...patch } : c)) - ) - } - - const doneCount = chunks.filter((c) => c.status === "done" || c.status === "partial").length - const progressPct = parsed ? Math.round((doneCount / parsed.chunks.length) * 100) : 0 - - async function handleTemplate() { - setTemplateLoading(true) - try { await downloadParamsOnlyTemplate() } - catch (e) { toast.error(`Template download failed: ${String(e)}`) } - finally { setTemplateLoading(false) } - } - - function handleClose() { - abortRef.current = true - reset() - onOpenChange(false) - } - - return ( - { if (!v) handleClose() }}> - - - Import Params Only (Bulk) - - -
- - {/* Upload */} - {step === "upload" && ( - <> -
-
- -
-

Params-Only Template

-

- product_parameters + product_applicable_params -

-
-
- -
- -
- -
fileRef.current?.click()} - > - { const f = e.target.files?.[0]; if (f) void handleParse(f) }} /> - -

Click to select — any size

-

- File split automatically into {PARAMS_ONLY_CONFIG.chunkSize ?? 300}-product chunks -

-
-
- -
- - - Large files are split and uploaded chunk-by-chunk. Each chunk runs - independently — failed chunks can be retried without restarting. - -
- - )} - - {/* Parsing */} - {step === "parsing" && ( -
- -
-

Parsing file in browser…

-

{parseStage}

-

Large files may take 30–60 s

-
- {file &&

{file.name} — {(file.size / 1024 / 1024).toFixed(1)} MB

} -
- )} - - {/* Ready */} - {step === "ready" && parsed && ( - <> -
-

File parsed

-
-

Products

{parsed.totalKeys.toLocaleString()}

-

Total rows

{parsed.totalRows.toLocaleString()}

-

Chunks

{parsed.chunks.length}

-
-

- ~{PARAMS_ONLY_CONFIG.chunkSize ?? 300} products per chunk · ~{Math.round(parsed.totalRows / parsed.chunks.length).toLocaleString()} rows each · est. 3–8 min per chunk -

-
-
- - Products must already exist from a prior bulk routing import. -
- - )} - - {/* Importing */} - {step === "importing" && parsed && ( -
-
- - {passNum > 1 ? `Pass ${passNum}/${MAX_AUTO_PASSES} · ` : ""}Chunk {currentIdx + 1} / {parsed.chunks.length} - - {totalInserted.toLocaleString()} rows imported -
- - {passStatus &&

{passStatus}

} -

{doneCount}/{parsed.chunks.length} complete ({progressPct}%)

-
-
- {chunks.map((c) => ( - - ))} -
-
-
- )} - - {/* Done */} - {step === "done" && ( -
- -
-

- Import complete — {totalInserted.toLocaleString()} rows imported -

-

- {parsed?.chunks.length} chunks processed - {chunks.filter((c) => c.status === "partial").length > 0 && - ` · ${chunks.filter((c) => c.status === "partial").length} with partial errors`} -

- -
-
- )} - - {/* Failed */} - {step === "failed" && ( -
-
- -
-

- {failedChunks.length} chunk{failedChunks.length > 1 ? "s" : ""} failed - {totalInserted > 0 && ` — ${totalInserted.toLocaleString()} rows already imported`} -

-

- Failed: chunks {failedChunks.map((i) => i + 1).join(", ")} -

-
-
-
-
- {chunks.map((c) => )} -
-
-
- )} -
- - - {(step === "upload" || step === "done" || step === "failed") && ( - - )} - {step === "ready" && ( - <> - - - - )} - {step === "importing" && ( - - )} - {step === "failed" && failedChunks.length > 0 && ( - - )} - - - - ) -} - -function ChunkBadge({ chunk, current }: { chunk: ChunkStatus; current: boolean }) { - const color: Record = { - pending: "bg-muted text-muted-foreground", - uploading: "bg-blue-100 text-blue-700 dark:bg-blue-900 dark:text-blue-200", - polling: "bg-yellow-100 text-yellow-700 dark:bg-yellow-900 dark:text-yellow-200", - done: "bg-green-100 text-green-700 dark:bg-green-900 dark:text-green-200", - partial: "bg-orange-100 text-orange-700 dark:bg-orange-900 dark:text-orange-200", - failed: "bg-red-100 text-red-700 dark:bg-red-900 dark:text-red-200", - } - return ( - - {chunk.index + 1} - {current && (chunk.status === "uploading" || chunk.status === "polling") && ( - - )} - - ) -} - -function delay(ms: number) { - return new Promise((resolve) => setTimeout(resolve, ms)) -} diff --git a/src/lib/excel/chunked-importer.ts b/src/lib/excel/chunked-importer.ts deleted file mode 100644 index 396e4cb..0000000 --- a/src/lib/excel/chunked-importer.ts +++ /dev/null @@ -1,211 +0,0 @@ -/** - * chunked-importer.ts — Generic browser-side Excel chunker. - * - * Uses a Web Worker (chunked-importer.worker.ts) to parse large Excel files - * off the main thread so the UI stays responsive. - * - * ### Usage - * - * ```ts - * import { chunkExcelFile, PARAMS_ONLY_CONFIG } from "@/lib/excel/chunked-importer" - * - * const result = await chunkExcelFile(file, PARAMS_ONLY_CONFIG, (stage) => - * console.log(stage) - * ) - * // result.chunks is ParsedChunk[] ready to upload - * ``` - * - * ### Adding a new import format - * - * Create a new `ChunkerConfig` constant — no code changes required: - * - * ```ts - * export const MY_FORMAT_CONFIG: ChunkerConfig = { - * sheetGroups: [ - * { baseName: "my_sheet", outputName: "my_sheet" }, - * ], - * keyColumn: "my_id_column", - * chunkSize: 500, - * } - * ``` - */ - -export type { SheetGroupConfig } from "./chunked-importer.worker" - -export interface ChunkerConfig { - /** Sheet groups to parse and merge. */ - sheetGroups: import("./chunked-importer.worker").SheetGroupConfig[] - /** Column name used to group rows per unique key (e.g. "legacy_oracle_sys_id"). */ - keyColumn: string - /** Number of unique key values per output chunk. Default: 300. */ - chunkSize?: number -} - -export interface ParsedChunk { - index: number - blob: Blob - fileName: string - /** Number of unique key values in this chunk. */ - keyCount: number - rowCount: number -} - -export interface ChunkResult { - totalKeys: number - totalRows: number - chunks: ParsedChunk[] -} - -// ── Built-in configurations ──────────────────────────────────────────────── - -/** Params-only import: product_parameters + product_applicable_params. */ -export const PARAMS_ONLY_CONFIG: ChunkerConfig = { - sheetGroups: [ - { baseName: "product_parameters", outputName: "product_parameters" }, - { baseName: "product_applicable_params", outputName: "product_applicable_params" }, - ], - keyColumn: "legacy_oracle_sys_id", - chunkSize: 300, -} - -/** - * Phase 1 of a two-phase bulk routing import: products + params only. - * All product_master rows are imported before any routing data, so that - * Phase 2 route_sequences can reference ANY product without circular-dependency failures. - */ -export const BULK_PRODUCTS_ONLY_CONFIG: ChunkerConfig = { - sheetGroups: [ - { baseName: "product_master", outputName: "product_master" }, - { baseName: "product_parameters", outputName: "product_parameters" }, - { baseName: "product_applicable_params", outputName: "product_applicable_params" }, - ], - keyColumn: "legacy_oracle_sys_id", - chunkSize: 300, -} - -/** - * Phase 2 of a two-phase bulk routing import: routing data only. - * Run AFTER BULK_PRODUCTS_ONLY_CONFIG so all products are in the DB and - * node_product_legacy_product_id cross-references always resolve. - * - * route_sequences and route_rms use "route_head_legacy_product_id" as their - * key column (same value as legacy_oracle_sys_id — the root product ID). - */ -export const BULK_ROUTING_ONLY_CONFIG: ChunkerConfig = { - sheetGroups: [ - { baseName: "route_head", outputName: "route_head" }, - { baseName: "route_sequences", outputName: "route_sequences", keyColumn: "route_head_legacy_product_id" }, - { baseName: "route_rms", outputName: "route_rms", keyColumn: "route_head_legacy_product_id" }, - ], - keyColumn: "legacy_oracle_sys_id", - chunkSize: 300, -} - -/** - * Legacy single-phase config — kept for backward compatibility. - * Prefer BULK_PRODUCTS_ONLY_CONFIG + BULK_ROUTING_ONLY_CONFIG for large files. - */ -export const BULK_ROUTING_CONFIG: ChunkerConfig = { - sheetGroups: [ - { baseName: "product_master", outputName: "product_master" }, - { baseName: "product_parameters", outputName: "product_parameters" }, - { baseName: "product_applicable_params", outputName: "product_applicable_params" }, - { baseName: "route_head", outputName: "route_head" }, - { baseName: "route_sequences", outputName: "route_sequences", keyColumn: "route_head_legacy_product_id" }, - { baseName: "route_rms", outputName: "route_rms", keyColumn: "route_head_legacy_product_id" }, - ], - keyColumn: "legacy_oracle_sys_id", - chunkSize: 200, -} - -// ── Core function ────────────────────────────────────────────────────────── - -const XLSX_MIME = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" - -/** - * Parse an Excel file using a Web Worker and split into upload-ready chunks. - * - * @param file The Excel file selected by the user. - * @param config Which sheets to parse, which column is the key, chunk size. - * @param onProgress Optional callback for stage strings ("Parsing…", "Indexing…"). - * @returns Resolved when all chunks are built; rejects on parse error. - */ -export function chunkExcelFile( - file: File, - config: ChunkerConfig, - onProgress?: (stage: string) => void -): Promise { - return new Promise((resolve, reject) => { - const worker = new Worker( - new URL("./chunked-importer.worker", import.meta.url) - ) - - const chunks: ParsedChunk[] = [] - let totalKeys = 0 - let totalRows = 0 - let expectedChunks = 0 - - worker.onmessage = (e: MessageEvent) => { - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - const msg = e.data - - switch (msg.type) { - case "progress": - onProgress?.(msg.stage as string) - break - - case "ready": - totalKeys = msg.totalKeys as number - totalRows = msg.totalRows as number - expectedChunks = msg.totalChunks as number - onProgress?.(`Building chunks (0/${expectedChunks})…`) - worker.postMessage({ type: "next" }) - break - - case "chunk": { - const blob = new Blob([msg.buffer as ArrayBuffer], { type: XLSX_MIME }) - chunks.push({ - index: msg.index as number, - blob, - fileName: msg.fileName as string, - keyCount: msg.keyCount as number, - rowCount: msg.rowCount as number, - }) - onProgress?.(`Building chunks (${chunks.length}/${expectedChunks})…`) - worker.postMessage({ type: "next" }) - break - } - - case "done": - worker.terminate() - resolve({ totalKeys, totalRows, chunks }) - break - - case "error": - worker.terminate() - reject(new Error(msg.message as string)) - break - } - } - - worker.onerror = (err) => { - worker.terminate() - reject(new Error(`Worker error: ${err.message}`)) - } - - // Transfer the ArrayBuffer — zero-copy, buffer is detached in main thread. - file.arrayBuffer().then((buffer) => { - worker.postMessage( - { - type: "start", - buffer, - keyColumn: config.keyColumn, - sheetGroups: config.sheetGroups, - chunkSize: config.chunkSize ?? 300, - baseName: file.name.replace(/\.xlsx?$/i, ""), - }, - [buffer] - ) - }).catch(reject) - }) -} diff --git a/src/lib/excel/chunked-importer.worker.ts b/src/lib/excel/chunked-importer.worker.ts deleted file mode 100644 index 03ea1d5..0000000 --- a/src/lib/excel/chunked-importer.worker.ts +++ /dev/null @@ -1,260 +0,0 @@ -/** - * chunked-importer.worker.ts — Generic Web Worker for chunking any Excel import file. - * - * Accepts a flexible SheetGroupConfig so the same worker binary handles any - * import format: params-only, product-routing, RM pricing, etc. - * - * Protocol (all messages are structured-cloneable): - * - * Main → Worker StartMessage — start parsing - * Worker → Main ProgressMsg — stage updates (multiple) - * Worker → Main ReadyMsg — parsing done; N chunks ready - * Main → Worker NextMessage — request next chunk - * Worker → Main ChunkMsg — one chunk (ArrayBuffer transferred, zero-copy) - * Worker → Main DoneMsg — all chunks sent - * Worker → Main ErrorMsg — fatal error - */ -import * as XLSX from "xlsx" - -// ── Public message types (re-exported via chunked-importer.ts) ───────────── - -export interface SheetGroupConfig { - /** Base name to match — exact match wins, otherwise contains-match (case-insensitive). */ - baseName: string - /** Sheet name written into each output chunk file. */ - outputName: string - /** - * Per-sheet key column override. Falls back to StartMessage.keyColumn when omitted. - * Use when a sheet uses a different column name for the same logical key - * (e.g. route_sequences uses "route_head_legacy_product_id" instead of "legacy_oracle_sys_id"). - */ - keyColumn?: string -} - -export interface StartMessage { - type: "start" - /** Raw ArrayBuffer of the .xlsx file — transferred (zero-copy). */ - buffer: ArrayBuffer - /** Column name used to group rows into per-key-value buckets. */ - keyColumn: string - /** Sheet groups to parse and merge. */ - sheetGroups: SheetGroupConfig[] - /** Number of unique key values per chunk. */ - chunkSize: number - /** Stem used to generate chunk file names (e.g. "params_2026"). */ - baseName: string -} - -export interface NextMessage { - type: "next" -} - -export interface ProgressMsg { - type: "progress" - stage: string -} - -export interface ReadyMsg { - type: "ready" - totalKeys: number - totalRows: number - totalChunks: number -} - -export interface ChunkMsg { - type: "chunk" - index: number - total: number - /** Transferred ArrayBuffer — detached from worker heap after postMessage. */ - buffer: ArrayBuffer - fileName: string - keyCount: number - rowCount: number -} - -export interface DoneMsg { - type: "done" -} - -export interface ErrorMsg { - type: "error" - message: string -} - -export type WorkerOutMsg = ProgressMsg | ReadyMsg | ChunkMsg | DoneMsg | ErrorMsg -type WorkerInMsg = StartMessage | NextMessage - -// ── Internal state ───────────────────────────────────────────────────────── - -interface ParsedGroup { - headers: unknown[] - rowsByKey: Map - outputName: string -} - -let groups: ParsedGroup[] = [] -let masterKeys: string[] = [] -let currentChunkSize = 300 -let currentBaseName = "" -let remaining: number[] = [] // chunk indices yet to be sent - -// ── Message handler ──────────────────────────────────────────────────────── - -self.onmessage = (e: MessageEvent) => { - if (e.data.type === "start") { - handleStart(e.data).catch((err: unknown) => { - emit({ type: "error", message: String(err) }) - }) - } else if (e.data.type === "next") { - sendNextChunk() - } -} - -// ── Parse ────────────────────────────────────────────────────────────────── - -async function handleStart(msg: StartMessage) { - currentChunkSize = msg.chunkSize - currentBaseName = msg.baseName - groups = [] - masterKeys = [] - - emit({ type: "progress", stage: "Parsing Excel file… (please wait)" }) - - const wb = XLSX.read(new Uint8Array(msg.buffer), { type: "array", dense: true }) - - const keySet = new Set() - let grandTotal = 0 - - for (const groupCfg of msg.sheetGroups) { - emit({ type: "progress", stage: `Indexing "${groupCfg.baseName}"…` }) - - const matching = findMatchingSheets(wb, groupCfg.baseName) - if (matching.length === 0) continue - - let headers: unknown[] = [] - let keyColIdx = -1 - const rowsByKey = new Map() - - for (let si = 0; si < matching.length; si++) { - const ws = wb.Sheets[matching[si]] - const rows = XLSX.utils.sheet_to_json(ws, { header: 1, defval: null }) - - if (si === 0) { - headers = (rows[0] as (string | null)[]).map((h) => - h != null ? String(h).trim() : "" - ) - const effectiveKeyCol = groupCfg.keyColumn ?? msg.keyColumn - keyColIdx = (headers as string[]).indexOf(effectiveKeyCol) - if (keyColIdx === -1) { - emit({ - type: "error", - message: `Sheet "${matching[0]}" does not contain key column "${effectiveKeyCol}". Found: ${headers.filter(Boolean).join(", ")}`, - }) - return - } - } - - const startRow = si === 0 ? 1 : 1 // skip header in all sheets - for (let r = startRow; r < rows.length; r++) { - const row = rows[r] as unknown[] - if (!row || row.every((v) => v == null || v === "")) continue - const rawKey = row[keyColIdx] - if (rawKey == null || rawKey === "") continue - const key = String(rawKey).trim() - if (!rowsByKey.has(key)) rowsByKey.set(key, []) - rowsByKey.get(key)!.push(row) - grandTotal++ - if (!keySet.has(key)) { - keySet.add(key) - masterKeys.push(key) - } - } - } - - groups.push({ headers, rowsByKey, outputName: groupCfg.outputName }) - } - - if (groups.length === 0) { - emit({ type: "error", message: "No matching sheets found. Check sheet names." }) - return - } - - const totalChunks = Math.ceil(masterKeys.length / currentChunkSize) - remaining = Array.from({ length: totalChunks }, (_, i) => i) - - emit({ - type: "ready", - totalKeys: masterKeys.length, - totalRows: grandTotal, - totalChunks, - }) -} - -// ── Chunk generation ─────────────────────────────────────────────────────── - -function sendNextChunk() { - if (remaining.length === 0) { - emit({ type: "done" }) - return - } - - const idx = remaining.shift()! - const start = idx * currentChunkSize - const batchKeys = masterKeys.slice(start, start + currentChunkSize) - const total = Math.ceil(masterKeys.length / currentChunkSize) - - const chunkWb = XLSX.utils.book_new() - let chunkRows = 0 - - for (const group of groups) { - const data: unknown[][] = [group.headers] - for (const key of batchKeys) { - const rows = group.rowsByKey.get(key) ?? [] - data.push(...rows) - chunkRows += rows.length - } - XLSX.utils.book_append_sheet(chunkWb, XLSX.utils.aoa_to_sheet(data), group.outputName) - } - - const rawArr = XLSX.write(chunkWb, { - type: "array", - bookType: "xlsx", - compression: true, - }) as number[] - - // Copy into a plain ArrayBuffer so it is transferable (zero-copy to main thread). - const ab = new Uint8Array(rawArr).buffer as ArrayBuffer - - const msg: ChunkMsg = { - type: "chunk", - index: idx, - total, - buffer: ab, - fileName: `${currentBaseName}_chunk_${String(idx + 1).padStart(3, "0")}.xlsx`, - keyCount: batchKeys.length, - rowCount: chunkRows, - } - - // Transfer ab — removed from worker heap, no copy on the wire. - // eslint-disable-next-line @typescript-eslint/no-explicit-any - ;(self as any).postMessage(msg, [ab]) -} - -// ── Helpers ──────────────────────────────────────────────────────────────── - -function findMatchingSheets(wb: XLSX.WorkBook, baseName: string): string[] { - const lower = baseName.toLowerCase() - const exact = wb.SheetNames.filter((n) => n === baseName) - if (exact.length > 0) { - // Include the exact match PLUS any continuation sheets (e.g. "params_2") - const extras = wb.SheetNames.filter( - (n) => n !== baseName && n.toLowerCase().includes(lower) - ).sort() - return [...exact, ...extras] - } - return wb.SheetNames.filter((n) => n.toLowerCase().includes(lower)).sort() -} - -function emit(msg: WorkerOutMsg) { - self.postMessage(msg) -} diff --git a/src/services/finance/cost-import-api.ts b/src/services/finance/cost-import-api.ts index 2ecd952..2b62ad3 100644 --- a/src/services/finance/cost-import-api.ts +++ b/src/services/finance/cost-import-api.ts @@ -5,9 +5,7 @@ import type { SyncImportResult, AsyncImportResponse, ImportEntity, - BulkValidationResult, - BulkSheetValidationResult, - BulkRowError, + ImportKindKey, } from "@/types/finance/cost-import" const BASE = "/api/v1/finance/costing" @@ -145,53 +143,78 @@ export async function listImportJobs( } } -// ── Bulk Product Routing ──────────────────────────────────────────────────── +// ── Bulk ETL import (v2 — presigned upload) ───────────────────────────────── /** - * Queue a bulk import of product master + routing data from an Excel file. - * Returns a job ID that can be polled with getImportJob(). + * Step 1 of the ETL import flow: ask the backend for a presigned PUT URL so the + * browser can upload the file directly to object storage (bypassing the BFF and + * gRPC message path entirely — no size inflation, no OOM). */ -export async function bulkImportProductMasterRouting( - file: File, - duplicateAction?: string, -): Promise<{ jobId: number; status: string }> { - const fileContent = Array.from(new Uint8Array(await file.arrayBuffer())) - const res = await fetch(`${BASE}/import/bulk_product_routing`, { +export async function getImportUploadURL( + kind: ImportKindKey, + fileName: string, +): Promise<{ uploadUrl: string; objectKey: string; expiresInSeconds: number }> { + const res = await fetch(`${BASE}/import/upload-url`, { method: "POST", headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ - fileContent, - fileName: file.name, - duplicateAction: duplicateAction ?? "update", - }), + body: JSON.stringify({ kind, fileName }), }) - if (!res.ok) throw new Error(`Bulk import failed: ${res.status}`) - const json = await res.json() + const json = await res.json().catch(() => null) + if (!res.ok || json?.base?.isSuccess === false) { + throw new Error(json?.base?.message || `Failed to get upload URL: ${res.status}`) + } return { - jobId: json.data?.jobId ?? json.data?.job_id ?? 0, - status: json.data?.status ?? "", + uploadUrl: json.uploadUrl ?? json.upload_url ?? "", + objectKey: json.objectKey ?? json.object_key ?? "", + expiresInSeconds: Number(json.expiresInSeconds ?? json.expires_in_seconds ?? 0), } } /** - * Queue a params-only bulk import (product_parameters + product_applicable_params). - * Products must already exist from a prior bulk_product_routing import. - * Uses multipart/form-data to send binary bytes without JSON inflation overhead. - * Returns a job ID that can be polled with getImportJob(). + * Step 2 of the ETL import flow: PUT the file straight to the presigned MinIO URL + * with upload progress. Uses XMLHttpRequest because fetch() cannot report upload + * progress. Requires MinIO CORS to allow PUT from the app origin. */ -export async function bulkImportParamsOnly( +export function putToPresignedUrl( + uploadUrl: string, file: File, + onProgress?: (percent: number) => void, +): Promise { + return new Promise((resolve, reject) => { + const xhr = new XMLHttpRequest() + xhr.open("PUT", uploadUrl, true) + xhr.upload.onprogress = (e) => { + if (e.lengthComputable && onProgress) { + onProgress(Math.round((e.loaded / e.total) * 100)) + } + } + xhr.onload = () => { + if (xhr.status >= 200 && xhr.status < 300) resolve() + else reject(new Error(`Upload failed: ${xhr.status} ${xhr.statusText}`)) + } + xhr.onerror = () => + reject(new Error("Upload failed — check network / storage CORS configuration")) + xhr.send(file) + }) +} + +/** + * Step 3 of the ETL import flow: tell the backend to start the async ETL job for + * the already-uploaded object. Returns the job ID to poll with getImportJob(). + */ +export async function startCostingImport( + kind: ImportKindKey, + objectKey: string, + fileName: string, ): Promise<{ jobId: number; status: string }> { - const formData = new FormData() - formData.append("file", file) - const res = await fetch(`${BASE}/import/bulk_params_only`, { + const res = await fetch(`${BASE}/import/start`, { method: "POST", - body: formData, // binary multipart — no Content-Type header needed (browser sets boundary) + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ kind, objectKey, fileName }), }) - if (!res.ok) throw new Error(`Params-only import failed: ${res.status}`) - const json = await res.json() - if (!json.base?.isSuccess) { - throw new Error(json.base?.message || `Import failed`) + const json = await res.json().catch(() => null) + if (!res.ok || json?.base?.isSuccess === false) { + throw new Error(json?.base?.message || `Failed to start import: ${res.status}`) } return { jobId: json.data?.jobId ?? json.data?.job_id ?? 0, @@ -214,62 +237,6 @@ export async function downloadParamsOnlyTemplate(): Promise { URL.revokeObjectURL(url) } -/** - * Validate a bulk product routing Excel file without importing. - * Returns a per-sheet summary with error/warning counts and sample errors. - */ -export async function validateBulkProductRoutingFile( - file: File, -): Promise { - const fileContent = Array.from(new Uint8Array(await file.arrayBuffer())) - const res = await fetch(`${BASE}/validate/bulk_product_routing`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ fileContent, fileName: file.name }), - }) - if (!res.ok) { - const errJson = await res.json().catch(() => null) - throw new Error(errJson?.base?.message || `Validation failed: ${res.status}`) - } - const json = await res.json() - - if (!json.base?.isSuccess && json.base?.isSuccess !== undefined) { - throw new Error(json.base?.message || "Validation request failed") - } - - // Normalise: handle both camelCase (gRPC-gateway) and snake_case (raw proto) - const isValid: boolean = json.isValid ?? json.is_valid ?? false - const rawSheets: unknown[] = Array.isArray(json.sheets) ? json.sheets : [] - - const sheets: BulkSheetValidationResult[] = rawSheets.map((s) => { - const sheet = s as Record - const rawErrors: unknown[] = Array.isArray(sheet.sampleErrors) - ? (sheet.sampleErrors as unknown[]) - : Array.isArray(sheet.sample_errors) - ? (sheet.sample_errors as unknown[]) - : [] - - const sampleErrors: BulkRowError[] = rawErrors.map((e) => { - const err = e as Record - return { - rowNumber: Number(err.rowNumber ?? err.row_number ?? 0), - field: String(err.field ?? ""), - message: String(err.message ?? ""), - } - }) - - return { - sheetName: String(sheet.sheetName ?? sheet.sheet_name ?? ""), - totalRows: Number(sheet.totalRows ?? sheet.total_rows ?? 0), - errorCount: Number(sheet.errorCount ?? sheet.error_count ?? 0), - warningCount: Number(sheet.warningCount ?? sheet.warning_count ?? 0), - sampleErrors, - } - }) - - return { isValid, sheets } -} - /** * Queue an async export of product master + routing data to MinIO. * When productSysIds is provided, exports only those products and their full diff --git a/src/types/finance/cost-import.ts b/src/types/finance/cost-import.ts index 38b92f5..32691bc 100644 --- a/src/types/finance/cost-import.ts +++ b/src/types/finance/cost-import.ts @@ -101,24 +101,12 @@ export const ENTITY_BULK_PRODUCT_ROUTING = "bulk_product_routing" /** Entity constant for the bulk export job (product master + routing). */ export const ENTITY_BULK_PRODUCT_ROUTING_EXPORT = "bulk_product_routing_export" -/** Row-level error from a single sheet during bulk validation. */ -export interface BulkRowError { - rowNumber: number - field: string - message: string -} +// ── v2 ETL presigned-upload import ────────────────────────────────────────── -/** Per-sheet result from ValidateBulkProductRoutingFile. */ -export interface BulkSheetValidationResult { - sheetName: string - totalRows: number - errorCount: number - warningCount: number - sampleErrors: BulkRowError[] -} - -/** Full result from ValidateBulkProductRoutingFile. */ -export interface BulkValidationResult { - isValid: boolean - sheets: BulkSheetValidationResult[] -} +/** + * ImportKindKey is the client-facing key for the dataset an ETL import carries. + * It maps 1:1 to the backend proto `ImportKind` enum: + * "product_routing" → IMPORT_KIND_PRODUCT_ROUTING (xlsx or zipped CSV) + * "params_only" → IMPORT_KIND_PARAMS_ONLY (zip of CSV files) + */ +export type ImportKindKey = "product_routing" | "params_only" diff --git a/src/types/generated/finance/v1/cost_import.ts b/src/types/generated/finance/v1/cost_import.ts index 5d39441..30b7834 100644 --- a/src/types/generated/finance/v1/cost_import.ts +++ b/src/types/generated/finance/v1/cost_import.ts @@ -10,6 +10,49 @@ import { BaseResponse, PaginationRequest, PaginationResponse } from "../../commo export const protobufPackage = "finance.v1"; +/** ImportKind identifies which dataset a presigned upload / import job carries. */ +export enum ImportKind { + /** IMPORT_KIND_UNSPECIFIED - Default — invalid, must be set by the caller. */ + IMPORT_KIND_UNSPECIFIED = 0, + /** IMPORT_KIND_PRODUCT_ROUTING - Product master + routing dataset (xlsx or zip CSV). */ + IMPORT_KIND_PRODUCT_ROUTING = 1, + /** IMPORT_KIND_PARAMS_ONLY - Params-only dataset (zip of CSV files). */ + IMPORT_KIND_PARAMS_ONLY = 2, + UNRECOGNIZED = -1, +} + +export function importKindFromJSON(object: any): ImportKind { + switch (object) { + case 0: + case "IMPORT_KIND_UNSPECIFIED": + return ImportKind.IMPORT_KIND_UNSPECIFIED; + case 1: + case "IMPORT_KIND_PRODUCT_ROUTING": + return ImportKind.IMPORT_KIND_PRODUCT_ROUTING; + case 2: + case "IMPORT_KIND_PARAMS_ONLY": + return ImportKind.IMPORT_KIND_PARAMS_ONLY; + case -1: + case "UNRECOGNIZED": + default: + return ImportKind.UNRECOGNIZED; + } +} + +export function importKindToJSON(object: ImportKind): string { + switch (object) { + case ImportKind.IMPORT_KIND_UNSPECIFIED: + return "IMPORT_KIND_UNSPECIFIED"; + case ImportKind.IMPORT_KIND_PRODUCT_ROUTING: + return "IMPORT_KIND_PRODUCT_ROUTING"; + case ImportKind.IMPORT_KIND_PARAMS_ONLY: + return "IMPORT_KIND_PARAMS_ONLY"; + case ImportKind.UNRECOGNIZED: + default: + return "UNRECOGNIZED"; + } +} + export interface CostImportJob { jobId: number; entity: string; @@ -137,7 +180,7 @@ export interface ImportBulkProductRoutingResponse { * ImportBulkParamsOnlyRequest imports product_parameters + product_applicable_params * from a file that does NOT contain a product_master sheet. Products must already * exist in the database from a prior bulk import. Supports split part-sheets - * (e.g. "product_parameters_p1" + "_p2"). Max 50 MB. + * (e.g. "product_parameters_p1" + "_p2"). Max 200 MB. */ export interface ImportBulkParamsOnlyRequest { fileContent: Uint8Array; @@ -181,6 +224,44 @@ export interface ExportBulkProductRoutingResponse { status: string; } +/** + * GetImportUploadURLRequest asks for a presigned PUT URL to upload an import + * file directly to object storage (bypassing the BFF and gRPC message path). + */ +export interface GetImportUploadURLRequest { + kind: ImportKind; + fileName: string; +} + +/** + * GetImportUploadURLResponse returns the presigned PUT URL plus the object key + * to pass back to StartCostingImport once the upload completes. + */ +export interface GetImportUploadURLResponse { + base: BaseResponse | undefined; + uploadUrl: string; + objectKey: string; + expiresInSeconds: number; +} + +/** + * StartCostingImportRequest starts an async ETL import job for an already + * uploaded object (identified by object_key from GetImportUploadURL). + */ +export interface StartCostingImportRequest { + kind: ImportKind; + objectKey: string; + fileName: string; + duplicateAction: string; +} + +/** StartCostingImportResponse returns the created async job ID and its status. */ +export interface StartCostingImportResponse { + base: BaseResponse | undefined; + jobId: number; + status: string; +} + function createBaseCostImportJob(): CostImportJob { return { jobId: 0, @@ -2859,6 +2940,426 @@ export const ExportBulkProductRoutingResponse: MessageFns = { + encode(message: GetImportUploadURLRequest, writer: BinaryWriter = new BinaryWriter()): BinaryWriter { + if (message.kind !== 0) { + writer.uint32(8).int32(message.kind); + } + if (message.fileName !== "") { + writer.uint32(18).string(message.fileName); + } + return writer; + }, + + decode(input: BinaryReader | Uint8Array, length?: number): GetImportUploadURLRequest { + const reader = input instanceof BinaryReader ? input : new BinaryReader(input); + const end = length === undefined ? reader.len : reader.pos + length; + const message = createBaseGetImportUploadURLRequest(); + while (reader.pos < end) { + const tag = reader.uint32(); + switch (tag >>> 3) { + case 1: { + if (tag !== 8) { + break; + } + + message.kind = reader.int32() as any; + continue; + } + case 2: { + if (tag !== 18) { + break; + } + + message.fileName = reader.string(); + continue; + } + } + if ((tag & 7) === 4 || tag === 0) { + break; + } + reader.skip(tag & 7); + } + return message; + }, + + fromJSON(object: any): GetImportUploadURLRequest { + return { + kind: isSet(object.kind) ? importKindFromJSON(object.kind) : 0, + fileName: isSet(object.fileName) + ? globalThis.String(object.fileName) + : isSet(object.file_name) + ? globalThis.String(object.file_name) + : "", + }; + }, + + toJSON(message: GetImportUploadURLRequest): unknown { + const obj: any = {}; + if (message.kind !== 0) { + obj.kind = importKindToJSON(message.kind); + } + if (message.fileName !== "") { + obj.fileName = message.fileName; + } + return obj; + }, + + create(base?: DeepPartial): GetImportUploadURLRequest { + return GetImportUploadURLRequest.fromPartial(base ?? {}); + }, + fromPartial(object: DeepPartial): GetImportUploadURLRequest { + const message = createBaseGetImportUploadURLRequest(); + message.kind = object.kind ?? 0; + message.fileName = object.fileName ?? ""; + return message; + }, +}; + +function createBaseGetImportUploadURLResponse(): GetImportUploadURLResponse { + return { base: undefined, uploadUrl: "", objectKey: "", expiresInSeconds: 0 }; +} + +export const GetImportUploadURLResponse: MessageFns = { + encode(message: GetImportUploadURLResponse, writer: BinaryWriter = new BinaryWriter()): BinaryWriter { + if (message.base !== undefined) { + BaseResponse.encode(message.base, writer.uint32(10).fork()).join(); + } + if (message.uploadUrl !== "") { + writer.uint32(18).string(message.uploadUrl); + } + if (message.objectKey !== "") { + writer.uint32(26).string(message.objectKey); + } + if (message.expiresInSeconds !== 0) { + writer.uint32(32).int64(message.expiresInSeconds); + } + return writer; + }, + + decode(input: BinaryReader | Uint8Array, length?: number): GetImportUploadURLResponse { + const reader = input instanceof BinaryReader ? input : new BinaryReader(input); + const end = length === undefined ? reader.len : reader.pos + length; + const message = createBaseGetImportUploadURLResponse(); + while (reader.pos < end) { + const tag = reader.uint32(); + switch (tag >>> 3) { + case 1: { + if (tag !== 10) { + break; + } + + message.base = BaseResponse.decode(reader, reader.uint32()); + continue; + } + case 2: { + if (tag !== 18) { + break; + } + + message.uploadUrl = reader.string(); + continue; + } + case 3: { + if (tag !== 26) { + break; + } + + message.objectKey = reader.string(); + continue; + } + case 4: { + if (tag !== 32) { + break; + } + + message.expiresInSeconds = longToNumber(reader.int64()); + continue; + } + } + if ((tag & 7) === 4 || tag === 0) { + break; + } + reader.skip(tag & 7); + } + return message; + }, + + fromJSON(object: any): GetImportUploadURLResponse { + return { + base: isSet(object.base) ? BaseResponse.fromJSON(object.base) : undefined, + uploadUrl: isSet(object.uploadUrl) + ? globalThis.String(object.uploadUrl) + : isSet(object.upload_url) + ? globalThis.String(object.upload_url) + : "", + objectKey: isSet(object.objectKey) + ? globalThis.String(object.objectKey) + : isSet(object.object_key) + ? globalThis.String(object.object_key) + : "", + expiresInSeconds: isSet(object.expiresInSeconds) + ? globalThis.Number(object.expiresInSeconds) + : isSet(object.expires_in_seconds) + ? globalThis.Number(object.expires_in_seconds) + : 0, + }; + }, + + toJSON(message: GetImportUploadURLResponse): unknown { + const obj: any = {}; + if (message.base !== undefined) { + obj.base = BaseResponse.toJSON(message.base); + } + if (message.uploadUrl !== "") { + obj.uploadUrl = message.uploadUrl; + } + if (message.objectKey !== "") { + obj.objectKey = message.objectKey; + } + if (message.expiresInSeconds !== 0) { + obj.expiresInSeconds = Math.round(message.expiresInSeconds); + } + return obj; + }, + + create(base?: DeepPartial): GetImportUploadURLResponse { + return GetImportUploadURLResponse.fromPartial(base ?? {}); + }, + fromPartial(object: DeepPartial): GetImportUploadURLResponse { + const message = createBaseGetImportUploadURLResponse(); + message.base = (object.base !== undefined && object.base !== null) + ? BaseResponse.fromPartial(object.base) + : undefined; + message.uploadUrl = object.uploadUrl ?? ""; + message.objectKey = object.objectKey ?? ""; + message.expiresInSeconds = object.expiresInSeconds ?? 0; + return message; + }, +}; + +function createBaseStartCostingImportRequest(): StartCostingImportRequest { + return { kind: 0, objectKey: "", fileName: "", duplicateAction: "" }; +} + +export const StartCostingImportRequest: MessageFns = { + encode(message: StartCostingImportRequest, writer: BinaryWriter = new BinaryWriter()): BinaryWriter { + if (message.kind !== 0) { + writer.uint32(8).int32(message.kind); + } + if (message.objectKey !== "") { + writer.uint32(18).string(message.objectKey); + } + if (message.fileName !== "") { + writer.uint32(26).string(message.fileName); + } + if (message.duplicateAction !== "") { + writer.uint32(34).string(message.duplicateAction); + } + return writer; + }, + + decode(input: BinaryReader | Uint8Array, length?: number): StartCostingImportRequest { + const reader = input instanceof BinaryReader ? input : new BinaryReader(input); + const end = length === undefined ? reader.len : reader.pos + length; + const message = createBaseStartCostingImportRequest(); + while (reader.pos < end) { + const tag = reader.uint32(); + switch (tag >>> 3) { + case 1: { + if (tag !== 8) { + break; + } + + message.kind = reader.int32() as any; + continue; + } + case 2: { + if (tag !== 18) { + break; + } + + message.objectKey = reader.string(); + continue; + } + case 3: { + if (tag !== 26) { + break; + } + + message.fileName = reader.string(); + continue; + } + case 4: { + if (tag !== 34) { + break; + } + + message.duplicateAction = reader.string(); + continue; + } + } + if ((tag & 7) === 4 || tag === 0) { + break; + } + reader.skip(tag & 7); + } + return message; + }, + + fromJSON(object: any): StartCostingImportRequest { + return { + kind: isSet(object.kind) ? importKindFromJSON(object.kind) : 0, + objectKey: isSet(object.objectKey) + ? globalThis.String(object.objectKey) + : isSet(object.object_key) + ? globalThis.String(object.object_key) + : "", + fileName: isSet(object.fileName) + ? globalThis.String(object.fileName) + : isSet(object.file_name) + ? globalThis.String(object.file_name) + : "", + duplicateAction: isSet(object.duplicateAction) + ? globalThis.String(object.duplicateAction) + : isSet(object.duplicate_action) + ? globalThis.String(object.duplicate_action) + : "", + }; + }, + + toJSON(message: StartCostingImportRequest): unknown { + const obj: any = {}; + if (message.kind !== 0) { + obj.kind = importKindToJSON(message.kind); + } + if (message.objectKey !== "") { + obj.objectKey = message.objectKey; + } + if (message.fileName !== "") { + obj.fileName = message.fileName; + } + if (message.duplicateAction !== "") { + obj.duplicateAction = message.duplicateAction; + } + return obj; + }, + + create(base?: DeepPartial): StartCostingImportRequest { + return StartCostingImportRequest.fromPartial(base ?? {}); + }, + fromPartial(object: DeepPartial): StartCostingImportRequest { + const message = createBaseStartCostingImportRequest(); + message.kind = object.kind ?? 0; + message.objectKey = object.objectKey ?? ""; + message.fileName = object.fileName ?? ""; + message.duplicateAction = object.duplicateAction ?? ""; + return message; + }, +}; + +function createBaseStartCostingImportResponse(): StartCostingImportResponse { + return { base: undefined, jobId: 0, status: "" }; +} + +export const StartCostingImportResponse: MessageFns = { + encode(message: StartCostingImportResponse, writer: BinaryWriter = new BinaryWriter()): BinaryWriter { + if (message.base !== undefined) { + BaseResponse.encode(message.base, writer.uint32(10).fork()).join(); + } + if (message.jobId !== 0) { + writer.uint32(16).int64(message.jobId); + } + if (message.status !== "") { + writer.uint32(26).string(message.status); + } + return writer; + }, + + decode(input: BinaryReader | Uint8Array, length?: number): StartCostingImportResponse { + const reader = input instanceof BinaryReader ? input : new BinaryReader(input); + const end = length === undefined ? reader.len : reader.pos + length; + const message = createBaseStartCostingImportResponse(); + while (reader.pos < end) { + const tag = reader.uint32(); + switch (tag >>> 3) { + case 1: { + if (tag !== 10) { + break; + } + + message.base = BaseResponse.decode(reader, reader.uint32()); + continue; + } + case 2: { + if (tag !== 16) { + break; + } + + message.jobId = longToNumber(reader.int64()); + continue; + } + case 3: { + if (tag !== 26) { + break; + } + + message.status = reader.string(); + continue; + } + } + if ((tag & 7) === 4 || tag === 0) { + break; + } + reader.skip(tag & 7); + } + return message; + }, + + fromJSON(object: any): StartCostingImportResponse { + return { + base: isSet(object.base) ? BaseResponse.fromJSON(object.base) : undefined, + jobId: isSet(object.jobId) + ? globalThis.Number(object.jobId) + : isSet(object.job_id) + ? globalThis.Number(object.job_id) + : 0, + status: isSet(object.status) ? globalThis.String(object.status) : "", + }; + }, + + toJSON(message: StartCostingImportResponse): unknown { + const obj: any = {}; + if (message.base !== undefined) { + obj.base = BaseResponse.toJSON(message.base); + } + if (message.jobId !== 0) { + obj.jobId = Math.round(message.jobId); + } + if (message.status !== "") { + obj.status = message.status; + } + return obj; + }, + + create(base?: DeepPartial): StartCostingImportResponse { + return StartCostingImportResponse.fromPartial(base ?? {}); + }, + fromPartial(object: DeepPartial): StartCostingImportResponse { + const message = createBaseStartCostingImportResponse(); + message.base = (object.base !== undefined && object.base !== null) + ? BaseResponse.fromPartial(object.base) + : undefined; + message.jobId = object.jobId ?? 0; + message.status = object.status ?? ""; + return message; + }, +}; + export type CostDataImportServiceDefinition = typeof CostDataImportServiceDefinition; export const CostDataImportServiceDefinition = { name: "CostDataImportService", @@ -2964,6 +3465,30 @@ export const CostDataImportServiceDefinition = { responseStream: false, options: {}, }, + /** + * GetImportUploadURL returns a presigned PUT URL for direct browser upload of + * an import file to object storage. + */ + getImportUploadURL: { + name: "GetImportUploadURL", + requestType: GetImportUploadURLRequest, + requestStream: false, + responseType: GetImportUploadURLResponse, + responseStream: false, + options: {}, + }, + /** + * StartCostingImport starts an async ETL import job for a previously uploaded + * object (referenced by object_key). + */ + startCostingImport: { + name: "StartCostingImport", + requestType: StartCostingImportRequest, + requestStream: false, + responseType: StartCostingImportResponse, + responseStream: false, + options: {}, + }, }, } as const;