From 25d4ffc2a4c0c776a9b4bb9abb70ac036c25cb75 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 6 Nov 2025 14:01:25 +0000 Subject: [PATCH 1/5] Add followBlueprintOrder option to export-workbook plugin - Add followBlueprintOrder option to PluginOptions interface - When enabled, export columns in sheet.config.fields order instead of API values order - Handle duplicate transformed labels by disambiguating with field keys - Support includeRecordIds option with blueprint ordering - Append non-blueprint fields at the end of the export - Pass explicit header array to json_to_sheet for deterministic ordering This resolves the issue where exported column order differs from the review screen order, which follows sheet.config.fields. The option defaults to false to maintain backward compatibility. Co-Authored-By: christopher.harrison@flatfile.io --- plugins/export-workbook/src/options.ts | 2 + plugins/export-workbook/src/plugin.ts | 167 ++++++++++++++++++------- 2 files changed, 125 insertions(+), 44 deletions(-) diff --git a/plugins/export-workbook/src/options.ts b/plugins/export-workbook/src/options.ts index aa09799d..a049957a 100644 --- a/plugins/export-workbook/src/options.ts +++ b/plugins/export-workbook/src/options.ts @@ -43,6 +43,7 @@ export type ColumnNameTransformerCallback = ( * @property {boolean} debug - show helpful messages useful for debugging (use intended for development). * @property {Record} sheetOptions - map of sheet slug to ExportSheetOptions. * @property {ColumnNameTransformerCallback} columnNameTransformer - callback to transform column names. + * @property {boolean} followBlueprintOrder - if true, export columns in sheet.config.fields order instead of API values order. Handles duplicate labels by disambiguating with field keys. */ export interface PluginOptions { readonly jobName?: string @@ -56,4 +57,5 @@ export interface PluginOptions { readonly debug?: boolean readonly sheetOptions?: Record readonly columnNameTransformer?: ColumnNameTransformerCallback + readonly followBlueprintOrder?: boolean } diff --git a/plugins/export-workbook/src/plugin.ts b/plugins/export-workbook/src/plugin.ts index f5ed3303..b6e873ce 100644 --- a/plugins/export-workbook/src/plugin.ts +++ b/plugins/export-workbook/src/plugin.ts @@ -67,56 +67,130 @@ export const exportRecords = async ( : async (name: string) => name try { + let headerMapping: Map | null = null + let orderedHeaders: string[] | null = null + + if (options.followBlueprintOrder) { + const blueprintKeys = sheet.config.fields + .filter((field) => !options.excludeFields?.includes(field.key)) + .map((field) => field.key) + + const transformedLabels = await Promise.all( + blueprintKeys.map(async (key) => { + const label = await columnNameTransformer(key, event) + return { key, label } + }) + ) + + const labelCounts = new Map() + headerMapping = new Map() + orderedHeaders = [] + + for (const { key, label } of transformedLabels) { + const count = labelCounts.get(label) || 0 + labelCounts.set(label, count + 1) + + const uniqueLabel = count > 0 ? `${label} (${key})` : label + headerMapping.set(key, uniqueLabel) + orderedHeaders.push(uniqueLabel) + } + + if (options.includeRecordIds) { + orderedHeaders.unshift('recordId') + } + } + let results = await processRecords( sheet.id, async (records): Promise => { const processedRecords = await Promise.all( records.map(async (record: Flatfile.RecordWithLinks) => { const { id: recordId, values: row } = record - const rowEntries = await Promise.all( - Object.keys(row).map(async (colName: string) => { - if (options.excludeFields?.includes(colName)) { - return null - } - const formatCell = (cellValue: Flatfile.CellValue) => { - const { value, messages } = cellValue - const cell: XLSX.CellObject = { - t: 's', - v: Array.isArray(value) ? value.join(', ') : value, - c: [], - } - if (options.excludeMessages) { - cell.c = [] - } else if (messages.length > 0) { - cell.c = messages.map((m) => ({ - a: 'Flatfile', - t: `[${m.type.toUpperCase()}]: ${m.message}`, - T: true, - })) - cell.c.hidden = true - } - - return cell - } - const transformedColName = await columnNameTransformer( - colName, + const formatCell = ( + cellValue: Flatfile.CellValue | undefined + ) => { + if (!cellValue) { + return { + t: 's', + v: '', + c: [], + } as XLSX.CellObject + } + const { value, messages } = cellValue + const cell: XLSX.CellObject = { + t: 's', + v: Array.isArray(value) ? value.join(', ') : value, + c: [], + } + if (options.excludeMessages) { + cell.c = [] + } else if (messages.length > 0) { + cell.c = messages.map((m) => ({ + a: 'Flatfile', + t: `[${m.type.toUpperCase()}]: ${m.message}`, + T: true, + })) + cell.c.hidden = true + } + + return cell + } + + if (options.followBlueprintOrder && headerMapping) { + const rowValue: Record = {} + + for (const [key, uniqueLabel] of headerMapping.entries()) { + rowValue[uniqueLabel] = formatCell(row[key]) + } + + const blueprintKeys = new Set(headerMapping.keys()) + const additionalKeys = Object.keys(row).filter( + (key) => + !blueprintKeys.has(key) && + !options.excludeFields?.includes(key) + ) + + for (const key of additionalKeys) { + const transformedLabel = await columnNameTransformer( + key, event ) - return [transformedColName, formatCell(row[colName])] - }) - ) - - const rowValue = Object.fromEntries( - rowEntries.filter((entry) => entry !== null) - ) - - return options?.includeRecordIds - ? { - recordId, - ...rowValue, - } - : rowValue + rowValue[transformedLabel] = formatCell(row[key]) + } + + return options.includeRecordIds + ? { + recordId, + ...rowValue, + } + : rowValue + } else { + const rowEntries = await Promise.all( + Object.keys(row).map(async (colName: string) => { + if (options.excludeFields?.includes(colName)) { + return null + } + + const transformedColName = await columnNameTransformer( + colName, + event + ) + return [transformedColName, formatCell(row[colName])] + }) + ) + + const rowValue = Object.fromEntries( + rowEntries.filter((entry) => entry !== null) + ) + + return options?.includeRecordIds + ? { + recordId, + ...rowValue, + } + : rowValue + } }) ) return processedRecords @@ -143,11 +217,16 @@ export const exportRecords = async ( } const rows = results.flat() - const worksheet = XLSX.utils.json_to_sheet( - rows, - createXLSXSheetOptions(options.sheetOptions?.[sheet.config.slug]) + const sheetOptions = createXLSXSheetOptions( + options.sheetOptions?.[sheet.config.slug] ) + if (options.followBlueprintOrder && orderedHeaders) { + sheetOptions.header = orderedHeaders + } + + const worksheet = XLSX.utils.json_to_sheet(rows, sheetOptions) + XLSX.utils.book_append_sheet( xlsxWorkbook, worksheet, From 3bff9896b20e1f33b8ca8504befac2f03c2c1150 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 6 Nov 2025 14:26:39 +0000 Subject: [PATCH 2/5] Fix critical bug: collect and append additional field headers When followBlueprintOrder is enabled, additional fields (not in blueprint) are now properly collected and appended to the header array so they appear in the export. Changes: - Track labelCounts, extrasKeyToUniqueLabel, and extrasHeaderOrder - During record processing, collect additional field keys and transform them - Disambiguate duplicate labels using the same labelCounts as blueprint fields - Append extrasHeaderOrder to orderedHeaders before passing to json_to_sheet This ensures additional fields appear at the end of the export instead of being silently dropped. Co-Authored-By: christopher.harrison@flatfile.io --- plugins/export-workbook/src/plugin.ts | 34 +++++++++++++++++++++------ 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/plugins/export-workbook/src/plugin.ts b/plugins/export-workbook/src/plugin.ts index b6e873ce..53d5b88b 100644 --- a/plugins/export-workbook/src/plugin.ts +++ b/plugins/export-workbook/src/plugin.ts @@ -69,6 +69,9 @@ export const exportRecords = async ( try { let headerMapping: Map | null = null let orderedHeaders: string[] | null = null + let labelCounts: Map | null = null + let extrasKeyToUniqueLabel: Map | null = null + let extrasHeaderOrder: string[] | null = null if (options.followBlueprintOrder) { const blueprintKeys = sheet.config.fields @@ -82,9 +85,11 @@ export const exportRecords = async ( }) ) - const labelCounts = new Map() + labelCounts = new Map() headerMapping = new Map() orderedHeaders = [] + extrasKeyToUniqueLabel = new Map() + extrasHeaderOrder = [] for (const { key, label } of transformedLabels) { const count = labelCounts.get(label) || 0 @@ -137,7 +142,13 @@ export const exportRecords = async ( return cell } - if (options.followBlueprintOrder && headerMapping) { + if ( + options.followBlueprintOrder && + headerMapping && + labelCounts && + extrasKeyToUniqueLabel && + extrasHeaderOrder + ) { const rowValue: Record = {} for (const [key, uniqueLabel] of headerMapping.entries()) { @@ -152,11 +163,17 @@ export const exportRecords = async ( ) for (const key of additionalKeys) { - const transformedLabel = await columnNameTransformer( - key, - event - ) - rowValue[transformedLabel] = formatCell(row[key]) + let uniqueLabel = extrasKeyToUniqueLabel.get(key) + if (!uniqueLabel) { + const baseLabel = await columnNameTransformer(key, event) + const count = labelCounts.get(baseLabel) || 0 + labelCounts.set(baseLabel, count + 1) + uniqueLabel = + count > 0 ? `${baseLabel} (${key})` : baseLabel + extrasKeyToUniqueLabel.set(key, uniqueLabel) + extrasHeaderOrder.push(uniqueLabel) + } + rowValue[uniqueLabel] = formatCell(row[key]) } return options.includeRecordIds @@ -222,6 +239,9 @@ export const exportRecords = async ( ) if (options.followBlueprintOrder && orderedHeaders) { + if (extrasHeaderOrder && extrasHeaderOrder.length > 0) { + orderedHeaders = orderedHeaders.concat(extrasHeaderOrder) + } sheetOptions.header = orderedHeaders } From 4247b40c1c5f38fb43c434daba732d23e8d401af Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 6 Nov 2025 14:54:55 +0000 Subject: [PATCH 3/5] Address ChatGPT feedback: add memoization, Set optimization, number types, and determinism - Add transformer memoization to avoid redundant async calls for same keys - Convert excludeFields to Set for O(1) lookups instead of array.includes() - Preserve number types in cells (t: 'n' for numbers, t: 's' for strings) - Sort extrasHeaderOrder before appending to ensure deterministic header order - Update all columnNameTransformer calls to use memoizedTransform - Update all excludeFields checks to use excludeFieldsSet Note: SheetJS comment handling unchanged - our xlsx 0.20.2 types confirm T?: boolean (thread flag) and hidden on Comments array are both valid. Co-Authored-By: christopher.harrison@flatfile.io --- plugins/export-workbook/src/plugin.ts | 68 +++++++++++++++++---------- 1 file changed, 44 insertions(+), 24 deletions(-) diff --git a/plugins/export-workbook/src/plugin.ts b/plugins/export-workbook/src/plugin.ts index 53d5b88b..3e497b4d 100644 --- a/plugins/export-workbook/src/plugin.ts +++ b/plugins/export-workbook/src/plugin.ts @@ -67,6 +67,18 @@ export const exportRecords = async ( : async (name: string) => name try { + const excludeFieldsSet = new Set(options.excludeFields ?? []) + + const transformCache = new Map() + const memoizedTransform = async (key: string) => { + if (transformCache.has(key)) { + return transformCache.get(key)! + } + const result = await columnNameTransformer(key, event) + transformCache.set(key, result) + return result + } + let headerMapping: Map | null = null let orderedHeaders: string[] | null = null let labelCounts: Map | null = null @@ -75,12 +87,12 @@ export const exportRecords = async ( if (options.followBlueprintOrder) { const blueprintKeys = sheet.config.fields - .filter((field) => !options.excludeFields?.includes(field.key)) + .filter((field) => !excludeFieldsSet.has(field.key)) .map((field) => field.key) const transformedLabels = await Promise.all( blueprintKeys.map(async (key) => { - const label = await columnNameTransformer(key, event) + const label = await memoizedTransform(key) return { key, label } }) ) @@ -114,32 +126,42 @@ export const exportRecords = async ( const formatCell = ( cellValue: Flatfile.CellValue | undefined - ) => { + ): XLSX.CellObject => { if (!cellValue) { return { t: 's', v: '', c: [], - } as XLSX.CellObject + } } const { value, messages } = cellValue - const cell: XLSX.CellObject = { - t: 's', - v: Array.isArray(value) ? value.join(', ') : value, - c: [], - } - if (options.excludeMessages) { - cell.c = [] - } else if (messages.length > 0) { - cell.c = messages.map((m) => ({ + const rawValue = Array.isArray(value) + ? value.join(', ') + : value + + let c: any = [] + if (!options.excludeMessages && messages.length > 0) { + c = messages.map((m) => ({ a: 'Flatfile', t: `[${m.type.toUpperCase()}]: ${m.message}`, T: true, })) - cell.c.hidden = true + c.hidden = true } - return cell + if (typeof rawValue === 'number') { + return { + t: 'n', + v: rawValue, + c, + } + } + + return { + t: 's', + v: rawValue, + c, + } } if ( @@ -158,14 +180,13 @@ export const exportRecords = async ( const blueprintKeys = new Set(headerMapping.keys()) const additionalKeys = Object.keys(row).filter( (key) => - !blueprintKeys.has(key) && - !options.excludeFields?.includes(key) + !blueprintKeys.has(key) && !excludeFieldsSet.has(key) ) for (const key of additionalKeys) { let uniqueLabel = extrasKeyToUniqueLabel.get(key) if (!uniqueLabel) { - const baseLabel = await columnNameTransformer(key, event) + const baseLabel = await memoizedTransform(key) const count = labelCounts.get(baseLabel) || 0 labelCounts.set(baseLabel, count + 1) uniqueLabel = @@ -185,14 +206,12 @@ export const exportRecords = async ( } else { const rowEntries = await Promise.all( Object.keys(row).map(async (colName: string) => { - if (options.excludeFields?.includes(colName)) { + if (excludeFieldsSet.has(colName)) { return null } - const transformedColName = await columnNameTransformer( - colName, - event - ) + const transformedColName = + await memoizedTransform(colName) return [transformedColName, formatCell(row[colName])] }) ) @@ -225,7 +244,7 @@ export const exportRecords = async ( const emptyRowEntries = await Promise.all( sheet.config.fields.map(async (field) => [ - await columnNameTransformer(field.key, event), + await memoizedTransform(field.key), emptyCell, ]) ) @@ -240,6 +259,7 @@ export const exportRecords = async ( if (options.followBlueprintOrder && orderedHeaders) { if (extrasHeaderOrder && extrasHeaderOrder.length > 0) { + extrasHeaderOrder.sort() orderedHeaders = orderedHeaders.concat(extrasHeaderOrder) } sheetOptions.header = orderedHeaders From 90c3b369fa0c70a6d210b383a2f000282eb47f65 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 6 Nov 2025 15:24:26 +0000 Subject: [PATCH 4/5] Address user feedback: fix edge cases and add correctness improvements 1. Filter emptyRowEntries with excludeFieldsSet to prevent header/row misalignment 2. Handle includeRecordIds for zero-row exports (add recordId to empty row and header) 3. Hoist blueprintKeys set outside record loop to avoid rebuilding per record 4. Sort extras by key (not label) for stability across transformer changes 5. Add comment shape test to guard against xlsx upgrades (T flag and hidden on array) 6. Update JSDoc to document recordId placement behavior (first column in blueprint mode, unspecified in non-blueprint mode) All fixes verified with lint and unit tests (8/8 passing including new comment test). Co-Authored-By: christopher.harrison@flatfile.io --- plugins/export-workbook/src/options.ts | 4 +-- plugins/export-workbook/src/plugin.ts | 34 ++++++++++++++++------- plugins/export-workbook/src/utils.spec.ts | 29 +++++++++++++++++++ 3 files changed, 55 insertions(+), 12 deletions(-) diff --git a/plugins/export-workbook/src/options.ts b/plugins/export-workbook/src/options.ts index a049957a..8858ae5a 100644 --- a/plugins/export-workbook/src/options.ts +++ b/plugins/export-workbook/src/options.ts @@ -37,13 +37,13 @@ export type ColumnNameTransformerCallback = ( * @property {string[]} excludeFields - list of field names to exclude from the exported data. This applies to all sheets. * @property {boolean} excludeMessages - exclude record messages from the exported data. * @property {Flatfile.Filter} recordFilter - filter to apply to the records before exporting. - * @property {boolean} includeRecordIds - include record ids in the exported data. + * @property {boolean} includeRecordIds - include record ids in the exported data. When followBlueprintOrder is true, recordId appears as the first column. In non-blueprint mode, recordId position is unspecified (depends on object key order). * @property {boolean} autoDownload - auto download the file after exporting * @property {string} filename - filename to use for the exported file. * @property {boolean} debug - show helpful messages useful for debugging (use intended for development). * @property {Record} sheetOptions - map of sheet slug to ExportSheetOptions. * @property {ColumnNameTransformerCallback} columnNameTransformer - callback to transform column names. - * @property {boolean} followBlueprintOrder - if true, export columns in sheet.config.fields order instead of API values order. Handles duplicate labels by disambiguating with field keys. + * @property {boolean} followBlueprintOrder - if true, export columns in sheet.config.fields order instead of API values order. Handles duplicate labels by disambiguating with field keys. Additional fields not in blueprint are sorted by key and appended at the end. */ export interface PluginOptions { readonly jobName?: string diff --git a/plugins/export-workbook/src/plugin.ts b/plugins/export-workbook/src/plugin.ts index 3e497b4d..8d92cdec 100644 --- a/plugins/export-workbook/src/plugin.ts +++ b/plugins/export-workbook/src/plugin.ts @@ -84,6 +84,7 @@ export const exportRecords = async ( let labelCounts: Map | null = null let extrasKeyToUniqueLabel: Map | null = null let extrasHeaderOrder: string[] | null = null + let blueprintKeysSet: Set = new Set() if (options.followBlueprintOrder) { const blueprintKeys = sheet.config.fields @@ -112,6 +113,8 @@ export const exportRecords = async ( orderedHeaders.push(uniqueLabel) } + blueprintKeysSet = new Set(headerMapping.keys()) + if (options.includeRecordIds) { orderedHeaders.unshift('recordId') } @@ -177,10 +180,9 @@ export const exportRecords = async ( rowValue[uniqueLabel] = formatCell(row[key]) } - const blueprintKeys = new Set(headerMapping.keys()) const additionalKeys = Object.keys(row).filter( (key) => - !blueprintKeys.has(key) && !excludeFieldsSet.has(key) + !blueprintKeysSet.has(key) && !excludeFieldsSet.has(key) ) for (const key of additionalKeys) { @@ -243,13 +245,20 @@ export const exportRecords = async ( } const emptyRowEntries = await Promise.all( - sheet.config.fields.map(async (field) => [ - await memoizedTransform(field.key), - emptyCell, - ]) + sheet.config.fields + .filter((f) => !excludeFieldsSet.has(f.key)) + .map(async (field) => [ + await memoizedTransform(field.key), + emptyCell, + ]) ) - results = [[Object.fromEntries(emptyRowEntries)]] + const emptyRow = Object.fromEntries(emptyRowEntries) + if (options.includeRecordIds) { + results = [[{ recordId: '', ...emptyRow }]] + } else { + results = [[emptyRow]] + } } const rows = results.flat() @@ -258,9 +267,14 @@ export const exportRecords = async ( ) if (options.followBlueprintOrder && orderedHeaders) { - if (extrasHeaderOrder && extrasHeaderOrder.length > 0) { - extrasHeaderOrder.sort() - orderedHeaders = orderedHeaders.concat(extrasHeaderOrder) + if (extrasKeyToUniqueLabel && extrasKeyToUniqueLabel.size > 0) { + const extrasKeysSorted = Array.from( + extrasKeyToUniqueLabel.keys() + ).sort() + const extrasOrderedLabels = extrasKeysSorted.map( + (k) => extrasKeyToUniqueLabel.get(k)! + ) + orderedHeaders = orderedHeaders.concat(extrasOrderedLabels) } sheetOptions.header = orderedHeaders } diff --git a/plugins/export-workbook/src/utils.spec.ts b/plugins/export-workbook/src/utils.spec.ts index 62fa41af..ebce4fe9 100644 --- a/plugins/export-workbook/src/utils.spec.ts +++ b/plugins/export-workbook/src/utils.spec.ts @@ -1,6 +1,7 @@ import { createXLSXSheetOptions } from './utils' import type { ExportSheetOptions } from './options' import type { JSON2SheetOpts } from 'xlsx' +import * as XLSX from 'xlsx' describe('createXLSXSheetOptions', () => { it.each([ @@ -21,3 +22,31 @@ describe('createXLSXSheetOptions', () => { } ) }) + +describe('XLSX comment shape', () => { + it('should preserve hidden comments with T flag across write/read cycle', () => { + const wb = XLSX.utils.book_new() + const ws = XLSX.utils.aoa_to_sheet([['Test']]) + + const comments: any = [ + { + a: 'Flatfile', + t: '[INFO]: Test message', + T: true, + }, + ] + comments.hidden = true + + ws['A1'].c = comments + + XLSX.utils.book_append_sheet(wb, ws, 'Sheet1') + + const xlsxData = XLSX.write(wb, { type: 'buffer', bookType: 'xlsx' }) + const readWb = XLSX.read(xlsxData, { type: 'buffer' }) + const readWs = readWb.Sheets['Sheet1'] + + expect(readWs['A1'].c).toBeDefined() + expect(readWs['A1'].c.length).toBeGreaterThan(0) + expect(readWs['A1'].c.hidden).toBe(true) + }) +}) From 0b1af48531bb67d94aa5132e5ac8fd4586981c50 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 6 Nov 2025 17:09:54 +0000 Subject: [PATCH 5/5] Revert number typing change to preserve backward compatibility Remove automatic number type detection (t: 'n' for numbers) to avoid breaking existing exports. All values now remain as strings (t: 's') by default. This makes the PR fully backward compatible - only the opt-in followBlueprintOrder option changes behavior. Existing exports continue to work exactly as before. The number typing feature can be added later as a separate opt-in option if needed. Co-Authored-By: christopher.harrison@flatfile.io --- plugins/export-workbook/src/plugin.ts | 8 -------- 1 file changed, 8 deletions(-) diff --git a/plugins/export-workbook/src/plugin.ts b/plugins/export-workbook/src/plugin.ts index 8d92cdec..3d6091e5 100644 --- a/plugins/export-workbook/src/plugin.ts +++ b/plugins/export-workbook/src/plugin.ts @@ -152,14 +152,6 @@ export const exportRecords = async ( c.hidden = true } - if (typeof rawValue === 'number') { - return { - t: 'n', - v: rawValue, - c, - } - } - return { t: 's', v: rawValue,