Add followBlueprintOrder option to export-workbook plugin

plugins/export-workbook/src/options.ts

@@ -37,12 +37,13 @@ export type ColumnNameTransformerCallback = (
  * @property {string[]} excludeFields - list of field names to exclude from the exported data. This applies to all sheets.
  * @property {boolean} excludeMessages - exclude record messages from the exported data.
  * @property {Flatfile.Filter} recordFilter - filter to apply to the records before exporting.
- * @property {boolean} includeRecordIds - include record ids in the exported data.
+ * @property {boolean} includeRecordIds - include record ids in the exported data. When followBlueprintOrder is true, recordId appears as the first column. In non-blueprint mode, recordId position is unspecified (depends on object key order).
  * @property {boolean} autoDownload - auto download the file after exporting
  * @property {string} filename - filename to use for the exported file.
  * @property {boolean} debug - show helpful messages useful for debugging (use intended for development).
  * @property {Record<string, ExportSheetOptions>} sheetOptions - map of sheet slug to ExportSheetOptions.
  * @property {ColumnNameTransformerCallback} columnNameTransformer - callback to transform column names.
+ * @property {boolean} followBlueprintOrder - if true, export columns in sheet.config.fields order instead of API values order. Handles duplicate labels by disambiguating with field keys. Additional fields not in blueprint are sorted by key and appended at the end.
  */
 export interface PluginOptions {
   readonly jobName?: string
@@ -56,4 +57,5 @@ export interface PluginOptions {
   readonly debug?: boolean
   readonly sheetOptions?: Record<string, ExportSheetOptions>
   readonly columnNameTransformer?: ColumnNameTransformerCallback
+  readonly followBlueprintOrder?: boolean
 }

plugins/export-workbook/src/plugin.ts

@@ -67,56 +67,160 @@ export const exportRecords = async (
         : async (name: string) => name
 
       try {
+        const excludeFieldsSet = new Set(options.excludeFields ?? [])
+
+        const transformCache = new Map<string, string>()
+        const memoizedTransform = async (key: string) => {
+          if (transformCache.has(key)) {
+            return transformCache.get(key)!
+          }
+          const result = await columnNameTransformer(key, event)
+          transformCache.set(key, result)
+          return result
+        }
+
+        let headerMapping: Map<string, string> | null = null
+        let orderedHeaders: string[] | null = null
+        let labelCounts: Map<string, number> | null = null
+        let extrasKeyToUniqueLabel: Map<string, string> | null = null
+        let extrasHeaderOrder: string[] | null = null
+        let blueprintKeysSet: Set<string> = new Set()
+
+        if (options.followBlueprintOrder) {
+          const blueprintKeys = sheet.config.fields
+            .filter((field) => !excludeFieldsSet.has(field.key))
+            .map((field) => field.key)
+
+          const transformedLabels = await Promise.all(
+            blueprintKeys.map(async (key) => {
+              const label = await memoizedTransform(key)
+              return { key, label }
+            })
+          )
+
+          labelCounts = new Map<string, number>()
+          headerMapping = new Map<string, string>()
+          orderedHeaders = []
+          extrasKeyToUniqueLabel = new Map<string, string>()
+          extrasHeaderOrder = []
+
+          for (const { key, label } of transformedLabels) {
+            const count = labelCounts.get(label) || 0
+            labelCounts.set(label, count + 1)
+
+            const uniqueLabel = count > 0 ? `${label} (${key})` : label
+            headerMapping.set(key, uniqueLabel)
+            orderedHeaders.push(uniqueLabel)
+          }
+
+          blueprintKeysSet = new Set(headerMapping.keys())
+
+          if (options.includeRecordIds) {
+            orderedHeaders.unshift('recordId')
+          }
+        }
+
         let results = await processRecords(
           sheet.id,
           async (records): Promise<Flatfile.RecordWithLinks[]> => {
             const processedRecords = await Promise.all(
               records.map(async (record: Flatfile.RecordWithLinks) => {
                 const { id: recordId, values: row } = record
-                const rowEntries = await Promise.all(
-                  Object.keys(row).map(async (colName: string) => {
-                    if (options.excludeFields?.includes(colName)) {
-                      return null
+
+                const formatCell = (
+                  cellValue: Flatfile.CellValue | undefined
+                ): XLSX.CellObject => {
+                  if (!cellValue) {
+                    return {
+                      t: 's',
+                      v: '',
+                      c: [],
+                    }
+                  }
+                  const { value, messages } = cellValue
+                  const rawValue = Array.isArray(value)
+                    ? value.join(', ')
+                    : value
+
+                  let c: any = []
+                  if (!options.excludeMessages && messages.length > 0) {
+                    c = messages.map((m) => ({
+                      a: 'Flatfile',
+                      t: `[${m.type.toUpperCase()}]: ${m.message}`,
+                      T: true,
+                    }))
+                    c.hidden = true
+                  }
+
+                  return {
+                    t: 's',
+                    v: rawValue,
+                    c,
+                  }
+                }
+
+                if (
+                  options.followBlueprintOrder &&
+                  headerMapping &&
+                  labelCounts &&
+                  extrasKeyToUniqueLabel &&
+                  extrasHeaderOrder
+                ) {
+                  const rowValue: Record<string, XLSX.CellObject> = {}
+
+                  for (const [key, uniqueLabel] of headerMapping.entries()) {
+                    rowValue[uniqueLabel] = formatCell(row[key])
+                  }
+
+                  const additionalKeys = Object.keys(row).filter(
+                    (key) =>
+                      !blueprintKeysSet.has(key) && !excludeFieldsSet.has(key)
+                  )
+
+                  for (const key of additionalKeys) {
+                    let uniqueLabel = extrasKeyToUniqueLabel.get(key)
+                    if (!uniqueLabel) {
+                      const baseLabel = await memoizedTransform(key)
+                      const count = labelCounts.get(baseLabel) || 0
+                      labelCounts.set(baseLabel, count + 1)
+                      uniqueLabel =
+                        count > 0 ? `${baseLabel} (${key})` : baseLabel
+                      extrasKeyToUniqueLabel.set(key, uniqueLabel)
+                      extrasHeaderOrder.push(uniqueLabel)
                     }
-                    const formatCell = (cellValue: Flatfile.CellValue) => {
-                      const { value, messages } = cellValue
-                      const cell: XLSX.CellObject = {
-                        t: 's',
-                        v: Array.isArray(value) ? value.join(', ') : value,
-                        c: [],
+                    rowValue[uniqueLabel] = formatCell(row[key])
+                  }
+
+                  return options.includeRecordIds
+                    ? {
+                        recordId,
+                        ...rowValue,
                       }
-                      if (options.excludeMessages) {
-                        cell.c = []
-                      } else if (messages.length > 0) {
-                        cell.c = messages.map((m) => ({
-                          a: 'Flatfile',
-                          t: `[${m.type.toUpperCase()}]: ${m.message}`,
-                          T: true,
-                        }))
-                        cell.c.hidden = true
+                    : rowValue
+                } else {
+                  const rowEntries = await Promise.all(
+                    Object.keys(row).map(async (colName: string) => {
+                      if (excludeFieldsSet.has(colName)) {
+                        return null
                       }
 
-                      return cell
-                    }
+                      const transformedColName =
+                        await memoizedTransform(colName)
+                      return [transformedColName, formatCell(row[colName])]
+                    })
+                  )
 
-                    const transformedColName = await columnNameTransformer(
-                      colName,
-                      event
-                    )
-                    return [transformedColName, formatCell(row[colName])]
-                  })
-                )
-
-                const rowValue = Object.fromEntries(
-                  rowEntries.filter((entry) => entry !== null)
-                )
-
-                return options?.includeRecordIds
-                  ? {
-                      recordId,
-                      ...rowValue,
-                    }
-                  : rowValue
+                  const rowValue = Object.fromEntries(
+                    rowEntries.filter((entry) => entry !== null)
+                  )
+
+                  return options?.includeRecordIds
+                    ? {
+                        recordId,
+                        ...rowValue,
+                      }
+                    : rowValue
+                }
               })
             )
             return processedRecords
@@ -133,21 +237,42 @@ export const exportRecords = async (
           }
 
           const emptyRowEntries = await Promise.all(
-            sheet.config.fields.map(async (field) => [
-              await columnNameTransformer(field.key, event),
-              emptyCell,
-            ])
+            sheet.config.fields
+              .filter((f) => !excludeFieldsSet.has(f.key))
+              .map(async (field) => [
+                await memoizedTransform(field.key),
+                emptyCell,
+              ])
           )
 
-          results = [[Object.fromEntries(emptyRowEntries)]]
+          const emptyRow = Object.fromEntries(emptyRowEntries)
+          if (options.includeRecordIds) {
+            results = [[{ recordId: '', ...emptyRow }]]
+          } else {
+            results = [[emptyRow]]
+          }
         }
         const rows = results.flat()
 
-        const worksheet = XLSX.utils.json_to_sheet(
-          rows,
-          createXLSXSheetOptions(options.sheetOptions?.[sheet.config.slug])
+        const sheetOptions = createXLSXSheetOptions(
+          options.sheetOptions?.[sheet.config.slug]
         )
 
+        if (options.followBlueprintOrder && orderedHeaders) {
+          if (extrasKeyToUniqueLabel && extrasKeyToUniqueLabel.size > 0) {
+            const extrasKeysSorted = Array.from(
+              extrasKeyToUniqueLabel.keys()
+            ).sort()
+            const extrasOrderedLabels = extrasKeysSorted.map(
+              (k) => extrasKeyToUniqueLabel.get(k)!
+            )
+            orderedHeaders = orderedHeaders.concat(extrasOrderedLabels)
+          }
+          sheetOptions.header = orderedHeaders
+        }
+
+        const worksheet = XLSX.utils.json_to_sheet(rows, sheetOptions)
+
         XLSX.utils.book_append_sheet(
           xlsxWorkbook,
           worksheet,

plugins/export-workbook/src/utils.spec.ts

@@ -1,6 +1,7 @@
 import { createXLSXSheetOptions } from './utils'
 import type { ExportSheetOptions } from './options'
 import type { JSON2SheetOpts } from 'xlsx'
+import * as XLSX from 'xlsx'
 
 describe('createXLSXSheetOptions', () => {
   it.each([
@@ -21,3 +22,31 @@ describe('createXLSXSheetOptions', () => {
     }
   )
 })
+
+describe('XLSX comment shape', () => {
+  it('should preserve hidden comments with T flag across write/read cycle', () => {
+    const wb = XLSX.utils.book_new()
+    const ws = XLSX.utils.aoa_to_sheet([['Test']])
+
+    const comments: any = [
+      {
+        a: 'Flatfile',
+        t: '[INFO]: Test message',
+        T: true,
+      },
+    ]
+    comments.hidden = true
+
+    ws['A1'].c = comments
+
+    XLSX.utils.book_append_sheet(wb, ws, 'Sheet1')
+
+    const xlsxData = XLSX.write(wb, { type: 'buffer', bookType: 'xlsx' })
+    const readWb = XLSX.read(xlsxData, { type: 'buffer' })
+    const readWs = readWb.Sheets['Sheet1']
+
+    expect(readWs['A1'].c).toBeDefined()
+    expect(readWs['A1'].c.length).toBeGreaterThan(0)
+    expect(readWs['A1'].c.hidden).toBe(true)
+  })
+})