feat: Implement delimited formats support

.changeset/sad-chairs-pull.md

@@ -0,0 +1,16 @@
+---
+"@dpkit/datahub": minor
+"@dpkit/folder": minor
+"@dpkit/github": minor
+"@dpkit/inline": minor
+"@dpkit/zenodo": minor
+"dpkit": minor
+"@dpkit/table": minor
+"@dpkit/ckan": minor
+"@dpkit/core": minor
+"@dpkit/file": minor
+"@dpkit/csv": minor
+"@dpkit/zip": minor
+---
+
+Support delimited formats

ckan/package/save.ts

@@ -8,7 +8,7 @@ import {
 } from "@dpkit/core"
 import {
   getPackageBasepath,
-  readFileStream,
+  loadFileStream,
   saveResourceFiles,
 } from "@dpkit/file"
 import { makeCkanApiRequest } from "../general/index.js"
@@ -67,7 +67,7 @@ export async function savePackageToCkan(
 
           const upload = {
             name: props.denormalizedPath,
-            data: await blob(await readFileStream(props.normalizedPath)),
+            data: await blob(await loadFileStream(props.normalizedPath)),
           }
 
           const result = await makeCkanApiRequest<CkanResource>({

ckan/plugin.ts

@@ -4,15 +4,17 @@ import { loadPackageFromCkan } from "./package/load.js"
 
 export class CkanPlugin implements Plugin {
   async loadPackage(source: string) {
-    const isRemote = isRemotePath(source)
-    if (!isRemote) return undefined
+    const isCkan = getIsCkan(source)
+    if (!isCkan) return undefined
 
-    const withDataset = source.includes("/dataset/")
-    if (!withDataset) return undefined
-
-    const cleanup = async () => {}
     const dataPackage = await loadPackageFromCkan(source)
-
-    return { dataPackage, cleanup }
+    return dataPackage
   }
 }
+
+function getIsCkan(path: string) {
+  const isRemote = isRemotePath(path)
+  if (!isRemote) return false
+
+  return path.includes("/dataset/")
+}

core/package/Package.ts

@@ -7,6 +7,11 @@ import type { Contributor } from "./Contributor.js"
  * @see https://datapackage.org/standard/data-package/
  */
 export interface Package extends Metadata {
+  /**
+   * Data resources in this package (required)
+   */
+  resources: Resource[]
+
   /**
    * Unique package identifier
    * Should use lowercase alphanumeric characters, periods, hyphens, and underscores
@@ -65,11 +70,6 @@ export interface Package extends Metadata {
    */
   created?: string
 
-  /**
-   * Data resources in this package (required)
-   */
-  resources: Resource[]
-
   /**
    * Package image
    */

core/plugin.ts

@@ -1,11 +1,7 @@
 import type { Package } from "./package/index.js"
 
 export interface Plugin {
-  loadPackage?(
-    source: string,
-  ): Promise<
-    undefined | { dataPackage: Package; cleanup?: () => Promise<void> }
-  >
+  loadPackage?(source: string): Promise<Package | undefined>
 
   savePackage?(
     dataPackage: Package,

core/resource/index.ts

@@ -1,4 +1,5 @@
 export type { Resource } from "./Resource.js"
+export { inferFormat } from "./infer.js"
 export { assertResource } from "./assert.js"
 export { loadResourceDescriptor } from "./load.js"
 export { saveResourceDescriptor } from "./save.js"

core/resource/infer.ts

@@ -0,0 +1,20 @@
+import { getFilename, getFormat } from "../general/index.js"
+import type { Resource } from "./Resource.js"
+
+export function inferFormat(resource: Partial<Resource>) {
+  let format = resource.format
+
+  if (!format) {
+    if (resource.path) {
+      const path = Array.isArray(resource.path)
+        ? resource.path[0]
+        : resource.path
+      if (path) {
+        const filename = getFilename(path)
+        format = getFormat(filename)
+      }
+    }
+  }
+
+  return format
+}

csv/OVERVIEW.md

@@ -0,0 +1,140 @@
+# @dpkit/csv
+
+Comprehensive CSV and TSV file handling with automatic format detection, advanced header processing, and high-performance data operations.
+
+## Introduction
+
+The CSV plugin is a part of the [dpkit](https://github.com/datisthq/dpkit) ecosystem providing these capabilities:
+
+- `loadCsvTable`
+- `saveCsvTable`
+- `inferCsvDialect`
+
+These functions are low-level and handles only CSV files on the IO and dialect level. So, for example, `loadCsvTable` will always return all the fields having a string type.
+
+For having both loading and processing of CSV files, the [dpkit](https://github.com/datisthq/dpkit) ecosystem provides the `readTable` function which is a high-level function that handles both loading and processing of CSV files.
+
+The CSV plugin automatically handles `.csv` and `.tsv` files when using dpkit:
+
+```typescript
+import { readTable } from "dpkit"
+
+const table = await readTable({path: "table.csv"})
+// the field types will be automatically inferred
+// or you can provide a Table Schema
+```
+
+## Basic Usage
+
+### Loading CSV Files
+
+```typescript
+import { loadCsvTable } from "@dpkit/csv"
+
+// Load a simple CSV file
+const table = await loadCsvTable({ path: "data.csv" })
+
+// Load with custom dialect
+const table = await loadCsvTable({
+  path: "data.csv",
+  dialect: {
+    delimiter: ";",
+    header: true,
+    skipInitialSpace: true
+  }
+})
+
+// Load multiple CSV files (concatenated)
+const table = await loadCsvTable({
+  path: ["part1.csv", "part2.csv", "part3.csv"]
+})
+```
+
+### Saving CSV Files
+
+```typescript
+import { saveCsvTable } from "@dpkit/csv"
+
+// Save with default options
+await saveCsvTable(table, { path: "output.csv" })
+
+// Save with custom dialect
+await saveCsvTable(table, {
+  path: "output.csv",
+  dialect: {
+    delimiter: "\t",
+    quoteChar: "'"
+  }
+})
+```
+
+### Dialect Detection
+
+```typescript
+import { inferCsvDialect } from "@dpkit/csv"
+
+// Automatically detect CSV format
+const dialect = await inferCsvDialect({ path: "unknown-dialect.csv" })
+console.log(dialect) // { delimiter: ",", header: true, quoteChar: '"' }
+
+// Use detected dialect to load
+const table = await loadCsvTable({
+  path: "unknown-dialect.csv",
+  dialect
+})
+```
+
+## Advanced Features
+
+### Multi-Header Row Processing
+
+```typescript
+// CSV with multiple header rows:
+// Year,2023,2023,2024,2024
+// Quarter,Q1,Q2,Q1,Q2
+// Revenue,100,120,110,130
+
+const table = await loadCsvTable({
+  path: "multi-header.csv",
+  dialect: {
+    headerRows: [1, 2],
+    headerJoin: "_"
+  }
+})
+// Resulting columns: ["Year_Quarter", "2023_Q1", "2023_Q2", "2024_Q1", "2024_Q2"]
+```
+
+### Comment Row Handling
+
+```typescript
+// CSV with comment rows:
+// # This is a comment
+// # Generated on 2024-01-01
+// Name,Age,City
+// John,25,NYC
+
+const table = await loadCsvTable({
+  path: "with-comments.csv",
+  dialect: {
+    commentRows: [1, 2],
+    header: true
+  }
+})
+```
+
+### Remote File Loading
+
+```typescript
+// Load from URL
+const table = await loadCsvTable({
+  path: "https://example.com/data.csv"
+})
+
+// Load multiple remote files
+const table = await loadCsvTable({
+  path: [
+    "https://api.example.com/data-2023.csv",
+    "https://api.example.com/data-2024.csv"
+  ]
+})
+```

csv/csv-sniffer.d.ts

@@ -0,0 +1 @@
+declare module "csv-sniffer"

csv/dialect/index.ts

@@ -0,0 +1 @@
+export { inferCsvDialect } from "./infer.js"

csv/dialect/infer.spec.ts

@@ -0,0 +1,66 @@
+import { writeTempFile } from "@dpkit/file"
+import { describe, expect, it } from "vitest"
+import { inferCsvDialect } from "./infer.js"
+
+describe("inferCsvDialect", () => {
+  it("should infer a simple CSV file", async () => {
+    const path = await writeTempFile("id,name\n1,english\n2,中文")
+    const dialect = await inferCsvDialect({ path })
+
+    expect(dialect).toEqual({
+      delimiter: ",",
+    })
+  })
+
+  it("should infer quoteChar", async () => {
+    const path = await writeTempFile('id,name\n1,"John Doe"\n2,"Jane Smith"')
+    const dialect = await inferCsvDialect({ path })
+
+    expect(dialect).toEqual({
+      delimiter: ",",
+      quoteChar: '"',
+    })
+  })
+
+  it("should infer quoteChar with single quotes", async () => {
+    const path = await writeTempFile("id,name\n1,'John Doe'\n2,'Jane Smith'")
+    const dialect = await inferCsvDialect({ path })
+
+    expect(dialect).toEqual({
+      delimiter: ",",
+      quoteChar: "'",
+    })
+  })
+
+  it("should infer header false when no header present", async () => {
+    const path = await writeTempFile("1,english\n2,中文\n3,español")
+    const dialect = await inferCsvDialect({ path })
+
+    expect(dialect).toEqual({
+      delimiter: ",",
+      header: false,
+    })
+  })
+
+  it("should not set header when header is present", async () => {
+    const path = await writeTempFile("id,name\n1,english\n2,中文")
+    const dialect = await inferCsvDialect({ path })
+
+    expect(dialect).toEqual({
+      delimiter: ",",
+    })
+  })
+
+  // TODO: recover if possible with csv-sniffer
+  it.skip("should infer complex CSV with quotes and header", async () => {
+    const path = await writeTempFile(
+      'name,description\n"Product A","A great product with, commas"\n"Product B","Another product"',
+    )
+
+    const dialect = await inferCsvDialect({ path })
+    expect(dialect).toEqual({
+      delimiter: ",",
+      quoteChar: '"',
+    })
+  })
+})

csv/dialect/infer.ts

@@ -0,0 +1,45 @@
+import { text } from "node:stream/consumers"
+import type { Dialect, Resource } from "@dpkit/core"
+import { loadFileStream } from "@dpkit/file"
+import type { InferDialectOptions } from "@dpkit/table"
+import { default as CsvSnifferFactory } from "csv-sniffer"
+
+const POSSIBLE_DELIMITERS = [",", ";", ":", "|", "\t", "^", "*", "&"]
+
+export async function inferCsvDialect(
+  resource: Partial<Resource>,
+  options?: InferDialectOptions,
+) {
+  const { sampleBytes = 10_000 } = options ?? {}
+  const dialect: Dialect = {}
+
+  if (resource.path) {
+    const stream = await loadFileStream(resource.path, {
+      maxBytes: sampleBytes,
+    })
+
+    const sample = await text(stream)
+
+    const CsvSniffer = CsvSnifferFactory()
+    const sniffer = new CsvSniffer(POSSIBLE_DELIMITERS)
+    const result = sniffer.sniff(sample)
+
+    if (result.delimiter) {
+      dialect.delimiter = result.delimiter
+    }
+
+    if (result.quoteChar) {
+      dialect.quoteChar = result.quoteChar
+    }
+
+    //if (result.lineTerminator) {
+    //  dialect.lineTerminator = result.lineTerminator
+    //}
+
+    if (!result.hasHeader) {
+      dialect.header = false
+    }
+  }
+
+  return dialect
+}

csv/index.ts

@@ -0,0 +1,3 @@
+export * from "./dialect/index.js"
+export * from "./table/index.js"
+export * from "./plugin.js"