From 426507932cc4346da8e9e9b960a5cc317ddb73dd Mon Sep 17 00:00:00 2001 From: roll Date: Mon, 30 Jun 2025 17:32:39 +0100 Subject: [PATCH] feat: implement delimited formats --- .changeset/sad-chairs-pull.md | 16 + ckan/package/save.ts | 4 +- ckan/plugin.ts | 18 +- core/package/Package.ts | 10 +- core/plugin.ts | 6 +- core/resource/index.ts | 1 + core/resource/infer.ts | 20 + csv/OVERVIEW.md | 140 +++++++ csv/csv-sniffer.d.ts | 1 + csv/dialect/index.ts | 1 + csv/dialect/infer.spec.ts | 66 ++++ csv/dialect/infer.ts | 45 +++ csv/index.ts | 3 + csv/package.json | 11 + csv/plugin.ts | 37 ++ .../recording.har | 290 ++++++++++++++ .../recording.har | 156 ++++++++ csv/table/index.ts | 2 + csv/table/load.spec.ts | 374 ++++++++++++++++++ csv/table/load.ts | 165 ++++++++ csv/table/save.spec.ts | 74 ++++ csv/table/save.ts | 15 + csv/typedoc.json | 5 + datahub/plugin.ts | 16 +- docs/astro.config.ts | 3 +- docs/components/Header/SocialIcons.astro | 8 +- docs/content/docs/overview/getting-started.md | 40 +- docs/examples/getting-started-4.ts | 15 +- docs/package.json | 1 + dpkit/dialect/index.ts | 1 + dpkit/dialect/infer.ts | 21 + dpkit/general/index.ts | 1 - dpkit/index.ts | 5 +- dpkit/package.json | 1 + dpkit/package/load.ts | 5 +- dpkit/package/save.ts | 2 +- dpkit/{general/dpkit.ts => plugin.ts} | 13 +- dpkit/table/index.ts | 4 + dpkit/table/infer.ts | 28 ++ dpkit/table/load.ts | 14 + dpkit/table/read.ts | 9 + dpkit/table/save.ts | 13 + dpkit/table/validate.ts | 11 + file/file/fetch.ts | 17 + file/file/index.ts | 4 + file/{general => file}/path.ts | 0 file/file/save.ts | 10 + file/file/temp.ts | 27 ++ file/general/file.ts | 10 - file/general/index.ts | 4 - file/general/stream/read.ts | 37 -- file/index.ts | 3 +- file/package.json | 1 + file/stream/index.ts | 2 + file/stream/load.ts | 69 ++++ .../stream/write.ts => stream/save.ts} | 2 +- folder/folder/index.ts | 1 + folder/folder/temp.ts | 17 + folder/index.ts | 1 + folder/package.json | 3 +- folder/plugin.ts | 16 +- github/package/save.ts | 4 +- github/plugin.ts | 16 +- inline/index.ts | 1 + inline/plugin.ts | 16 + inline/table/index.ts | 3 +- inline/table/{read.spec.ts => load.spec.ts} | 42 +- inline/table/{read.ts => load.ts} | 15 +- inline/table/validate.ts | 8 - pnpm-lock.yaml | 51 ++- table/OVERVIEW.md | 13 +- table/field/checks/enum.spec.ts | 12 +- table/field/checks/maxLength.spec.ts | 8 +- table/field/checks/maximum.spec.ts | 10 +- table/field/checks/minLength.spec.ts | 8 +- table/field/checks/minimum.spec.ts | 10 +- table/field/checks/pattern.spec.ts | 8 +- table/field/checks/required.spec.ts | 6 +- table/field/checks/type.spec.ts | 40 +- table/field/checks/unique.spec.ts | 12 +- table/field/index.ts | 2 +- .../{validate.spec.ts => inspect.spec.ts} | 112 +++--- table/field/{validate.ts => inspect.ts} | 14 +- table/field/types/array.spec.ts | 4 +- table/field/types/array.ts | 1 - table/field/types/boolean.ts | 1 - table/field/types/date.ts | 1 - table/field/types/datetime.ts | 1 - table/field/types/duration.ts | 1 - table/field/types/geojson.spec.ts | 4 +- table/field/types/geojson.ts | 1 - table/field/types/geopoint.ts | 1 - table/field/types/integer.spec.ts | 6 +- table/field/types/integer.ts | 1 - table/field/types/list.ts | 1 - table/field/types/number.ts | 1 - table/field/types/object.spec.ts | 4 +- table/field/types/object.ts | 1 - table/field/types/string.ts | 1 - table/field/types/time.ts | 1 - table/field/types/year.spec.ts | 4 +- table/field/types/year.ts | 1 - table/field/types/yearmonth.ts | 1 - table/index.ts | 1 + table/plugin.ts | 19 + table/row/checks/unique.spec.ts | 18 +- table/row/index.ts | 2 +- table/row/{validate.ts => inspect.ts} | 2 +- table/schema/index.ts | 2 +- table/schema/infer.ts | 21 +- table/table/index.ts | 2 +- .../{validate.spec.ts => inspect.spec.ts} | 79 ++-- table/table/{validate.ts => inspect.ts} | 43 +- table/table/process.ts | 12 +- zenodo/package/save.ts | 4 +- zenodo/plugin.ts | 16 +- zip/package.json | 1 + zip/package/load.ts | 16 +- zip/package/save.ts | 4 +- zip/plugin.ts | 13 +- 120 files changed, 2118 insertions(+), 480 deletions(-) create mode 100644 .changeset/sad-chairs-pull.md create mode 100644 core/resource/infer.ts create mode 100644 csv/OVERVIEW.md create mode 100644 csv/csv-sniffer.d.ts create mode 100644 csv/dialect/index.ts create mode 100644 csv/dialect/infer.spec.ts create mode 100644 csv/dialect/infer.ts create mode 100644 csv/plugin.ts create mode 100644 csv/table/fixtures/generated/should-load-remote-file-multipart_2595965849/recording.har create mode 100644 csv/table/fixtures/generated/should-load-remote-file_1137408454/recording.har create mode 100644 csv/table/index.ts create mode 100644 csv/table/load.spec.ts create mode 100644 csv/table/load.ts create mode 100644 csv/table/save.spec.ts create mode 100644 csv/table/save.ts create mode 100644 csv/typedoc.json create mode 100644 dpkit/dialect/index.ts create mode 100644 dpkit/dialect/infer.ts delete mode 100644 dpkit/general/index.ts rename dpkit/{general/dpkit.ts => plugin.ts} (63%) create mode 100644 dpkit/table/index.ts create mode 100644 dpkit/table/infer.ts create mode 100644 dpkit/table/load.ts create mode 100644 dpkit/table/read.ts create mode 100644 dpkit/table/save.ts create mode 100644 dpkit/table/validate.ts create mode 100644 file/file/fetch.ts create mode 100644 file/file/index.ts rename file/{general => file}/path.ts (100%) create mode 100644 file/file/save.ts create mode 100644 file/file/temp.ts delete mode 100644 file/general/file.ts delete mode 100644 file/general/index.ts delete mode 100644 file/general/stream/read.ts create mode 100644 file/stream/index.ts create mode 100644 file/stream/load.ts rename file/{general/stream/write.ts => stream/save.ts} (92%) create mode 100644 folder/folder/temp.ts create mode 100644 inline/plugin.ts rename inline/table/{read.spec.ts => load.spec.ts} (71%) rename inline/table/{read.ts => load.ts} (84%) delete mode 100644 inline/table/validate.ts rename table/field/{validate.spec.ts => inspect.spec.ts} (69%) rename table/field/{validate.ts => inspect.ts} (87%) create mode 100644 table/plugin.ts rename table/row/{validate.ts => inspect.ts} (82%) rename table/table/{validate.spec.ts => inspect.spec.ts} (74%) rename table/table/{validate.ts => inspect.ts} (81%) diff --git a/.changeset/sad-chairs-pull.md b/.changeset/sad-chairs-pull.md new file mode 100644 index 00000000..d756e412 --- /dev/null +++ b/.changeset/sad-chairs-pull.md @@ -0,0 +1,16 @@ +--- +"@dpkit/datahub": minor +"@dpkit/folder": minor +"@dpkit/github": minor +"@dpkit/inline": minor +"@dpkit/zenodo": minor +"dpkit": minor +"@dpkit/table": minor +"@dpkit/ckan": minor +"@dpkit/core": minor +"@dpkit/file": minor +"@dpkit/csv": minor +"@dpkit/zip": minor +--- + +Support delimited formats diff --git a/ckan/package/save.ts b/ckan/package/save.ts index 47ba79b2..13b5b544 100644 --- a/ckan/package/save.ts +++ b/ckan/package/save.ts @@ -8,7 +8,7 @@ import { } from "@dpkit/core" import { getPackageBasepath, - readFileStream, + loadFileStream, saveResourceFiles, } from "@dpkit/file" import { makeCkanApiRequest } from "../general/index.js" @@ -67,7 +67,7 @@ export async function savePackageToCkan( const upload = { name: props.denormalizedPath, - data: await blob(await readFileStream(props.normalizedPath)), + data: await blob(await loadFileStream(props.normalizedPath)), } const result = await makeCkanApiRequest({ diff --git a/ckan/plugin.ts b/ckan/plugin.ts index 3e81e54d..594214b6 100644 --- a/ckan/plugin.ts +++ b/ckan/plugin.ts @@ -4,15 +4,17 @@ import { loadPackageFromCkan } from "./package/load.js" export class CkanPlugin implements Plugin { async loadPackage(source: string) { - const isRemote = isRemotePath(source) - if (!isRemote) return undefined + const isCkan = getIsCkan(source) + if (!isCkan) return undefined - const withDataset = source.includes("/dataset/") - if (!withDataset) return undefined - - const cleanup = async () => {} const dataPackage = await loadPackageFromCkan(source) - - return { dataPackage, cleanup } + return dataPackage } } + +function getIsCkan(path: string) { + const isRemote = isRemotePath(path) + if (!isRemote) return false + + return path.includes("/dataset/") +} diff --git a/core/package/Package.ts b/core/package/Package.ts index 238a1fd7..2582263d 100644 --- a/core/package/Package.ts +++ b/core/package/Package.ts @@ -7,6 +7,11 @@ import type { Contributor } from "./Contributor.js" * @see https://datapackage.org/standard/data-package/ */ export interface Package extends Metadata { + /** + * Data resources in this package (required) + */ + resources: Resource[] + /** * Unique package identifier * Should use lowercase alphanumeric characters, periods, hyphens, and underscores @@ -65,11 +70,6 @@ export interface Package extends Metadata { */ created?: string - /** - * Data resources in this package (required) - */ - resources: Resource[] - /** * Package image */ diff --git a/core/plugin.ts b/core/plugin.ts index f81ffd8f..02e33725 100644 --- a/core/plugin.ts +++ b/core/plugin.ts @@ -1,11 +1,7 @@ import type { Package } from "./package/index.js" export interface Plugin { - loadPackage?( - source: string, - ): Promise< - undefined | { dataPackage: Package; cleanup?: () => Promise } - > + loadPackage?(source: string): Promise savePackage?( dataPackage: Package, diff --git a/core/resource/index.ts b/core/resource/index.ts index 0a2b13e3..7b3d3229 100644 --- a/core/resource/index.ts +++ b/core/resource/index.ts @@ -1,4 +1,5 @@ export type { Resource } from "./Resource.js" +export { inferFormat } from "./infer.js" export { assertResource } from "./assert.js" export { loadResourceDescriptor } from "./load.js" export { saveResourceDescriptor } from "./save.js" diff --git a/core/resource/infer.ts b/core/resource/infer.ts new file mode 100644 index 00000000..2edb9d98 --- /dev/null +++ b/core/resource/infer.ts @@ -0,0 +1,20 @@ +import { getFilename, getFormat } from "../general/index.js" +import type { Resource } from "./Resource.js" + +export function inferFormat(resource: Partial) { + let format = resource.format + + if (!format) { + if (resource.path) { + const path = Array.isArray(resource.path) + ? resource.path[0] + : resource.path + if (path) { + const filename = getFilename(path) + format = getFormat(filename) + } + } + } + + return format +} diff --git a/csv/OVERVIEW.md b/csv/OVERVIEW.md new file mode 100644 index 00000000..a8128bb2 --- /dev/null +++ b/csv/OVERVIEW.md @@ -0,0 +1,140 @@ +# @dpkit/csv + +Comprehensive CSV and TSV file handling with automatic format detection, advanced header processing, and high-performance data operations. + +## Introduction + +The CSV plugin is a part of the [dpkit](https://github.com/datisthq/dpkit) ecosystem providing these capabilities: + +- `loadCsvTable` +- `saveCsvTable` +- `inferCsvDialect` + +These functions are low-level and handles only CSV files on the IO and dialect level. So, for example, `loadCsvTable` will always return all the fields having a string type. + +For having both loading and processing of CSV files, the [dpkit](https://github.com/datisthq/dpkit) ecosystem provides the `readTable` function which is a high-level function that handles both loading and processing of CSV files. + +The CSV plugin automatically handles `.csv` and `.tsv` files when using dpkit: + +```typescript +import { readTable } from "dpkit" + +const table = await readTable({path: "table.csv"}) +// the field types will be automatically inferred +// or you can provide a Table Schema +``` + +## Basic Usage + +### Loading CSV Files + +```typescript +import { loadCsvTable } from "@dpkit/csv" + +// Load a simple CSV file +const table = await loadCsvTable({ path: "data.csv" }) + +// Load with custom dialect +const table = await loadCsvTable({ + path: "data.csv", + dialect: { + delimiter: ";", + header: true, + skipInitialSpace: true + } +}) + +// Load multiple CSV files (concatenated) +const table = await loadCsvTable({ + path: ["part1.csv", "part2.csv", "part3.csv"] +}) +``` + +### Saving CSV Files + +```typescript +import { saveCsvTable } from "@dpkit/csv" + +// Save with default options +await saveCsvTable(table, { path: "output.csv" }) + +// Save with custom dialect +await saveCsvTable(table, { + path: "output.csv", + dialect: { + delimiter: "\t", + quoteChar: "'" + } +}) +``` + +### Dialect Detection + +```typescript +import { inferCsvDialect } from "@dpkit/csv" + +// Automatically detect CSV format +const dialect = await inferCsvDialect({ path: "unknown-dialect.csv" }) +console.log(dialect) // { delimiter: ",", header: true, quoteChar: '"' } + +// Use detected dialect to load +const table = await loadCsvTable({ + path: "unknown-dialect.csv", + dialect +}) +``` + +## Advanced Features + +### Multi-Header Row Processing + +```typescript +// CSV with multiple header rows: +// Year,2023,2023,2024,2024 +// Quarter,Q1,Q2,Q1,Q2 +// Revenue,100,120,110,130 + +const table = await loadCsvTable({ + path: "multi-header.csv", + dialect: { + headerRows: [1, 2], + headerJoin: "_" + } +}) +// Resulting columns: ["Year_Quarter", "2023_Q1", "2023_Q2", "2024_Q1", "2024_Q2"] +``` + +### Comment Row Handling + +```typescript +// CSV with comment rows: +// # This is a comment +// # Generated on 2024-01-01 +// Name,Age,City +// John,25,NYC + +const table = await loadCsvTable({ + path: "with-comments.csv", + dialect: { + commentRows: [1, 2], + header: true + } +}) +``` + +### Remote File Loading + +```typescript +// Load from URL +const table = await loadCsvTable({ + path: "https://example.com/data.csv" +}) + +// Load multiple remote files +const table = await loadCsvTable({ + path: [ + "https://api.example.com/data-2023.csv", + "https://api.example.com/data-2024.csv" + ] +}) +``` diff --git a/csv/csv-sniffer.d.ts b/csv/csv-sniffer.d.ts new file mode 100644 index 00000000..d9eb649a --- /dev/null +++ b/csv/csv-sniffer.d.ts @@ -0,0 +1 @@ +declare module "csv-sniffer" diff --git a/csv/dialect/index.ts b/csv/dialect/index.ts new file mode 100644 index 00000000..12e94327 --- /dev/null +++ b/csv/dialect/index.ts @@ -0,0 +1 @@ +export { inferCsvDialect } from "./infer.js" diff --git a/csv/dialect/infer.spec.ts b/csv/dialect/infer.spec.ts new file mode 100644 index 00000000..84acb196 --- /dev/null +++ b/csv/dialect/infer.spec.ts @@ -0,0 +1,66 @@ +import { writeTempFile } from "@dpkit/file" +import { describe, expect, it } from "vitest" +import { inferCsvDialect } from "./infer.js" + +describe("inferCsvDialect", () => { + it("should infer a simple CSV file", async () => { + const path = await writeTempFile("id,name\n1,english\n2,中文") + const dialect = await inferCsvDialect({ path }) + + expect(dialect).toEqual({ + delimiter: ",", + }) + }) + + it("should infer quoteChar", async () => { + const path = await writeTempFile('id,name\n1,"John Doe"\n2,"Jane Smith"') + const dialect = await inferCsvDialect({ path }) + + expect(dialect).toEqual({ + delimiter: ",", + quoteChar: '"', + }) + }) + + it("should infer quoteChar with single quotes", async () => { + const path = await writeTempFile("id,name\n1,'John Doe'\n2,'Jane Smith'") + const dialect = await inferCsvDialect({ path }) + + expect(dialect).toEqual({ + delimiter: ",", + quoteChar: "'", + }) + }) + + it("should infer header false when no header present", async () => { + const path = await writeTempFile("1,english\n2,中文\n3,español") + const dialect = await inferCsvDialect({ path }) + + expect(dialect).toEqual({ + delimiter: ",", + header: false, + }) + }) + + it("should not set header when header is present", async () => { + const path = await writeTempFile("id,name\n1,english\n2,中文") + const dialect = await inferCsvDialect({ path }) + + expect(dialect).toEqual({ + delimiter: ",", + }) + }) + + // TODO: recover if possible with csv-sniffer + it.skip("should infer complex CSV with quotes and header", async () => { + const path = await writeTempFile( + 'name,description\n"Product A","A great product with, commas"\n"Product B","Another product"', + ) + + const dialect = await inferCsvDialect({ path }) + expect(dialect).toEqual({ + delimiter: ",", + quoteChar: '"', + }) + }) +}) diff --git a/csv/dialect/infer.ts b/csv/dialect/infer.ts new file mode 100644 index 00000000..c0be8726 --- /dev/null +++ b/csv/dialect/infer.ts @@ -0,0 +1,45 @@ +import { text } from "node:stream/consumers" +import type { Dialect, Resource } from "@dpkit/core" +import { loadFileStream } from "@dpkit/file" +import type { InferDialectOptions } from "@dpkit/table" +import { default as CsvSnifferFactory } from "csv-sniffer" + +const POSSIBLE_DELIMITERS = [",", ";", ":", "|", "\t", "^", "*", "&"] + +export async function inferCsvDialect( + resource: Partial, + options?: InferDialectOptions, +) { + const { sampleBytes = 10_000 } = options ?? {} + const dialect: Dialect = {} + + if (resource.path) { + const stream = await loadFileStream(resource.path, { + maxBytes: sampleBytes, + }) + + const sample = await text(stream) + + const CsvSniffer = CsvSnifferFactory() + const sniffer = new CsvSniffer(POSSIBLE_DELIMITERS) + const result = sniffer.sniff(sample) + + if (result.delimiter) { + dialect.delimiter = result.delimiter + } + + if (result.quoteChar) { + dialect.quoteChar = result.quoteChar + } + + //if (result.lineTerminator) { + // dialect.lineTerminator = result.lineTerminator + //} + + if (!result.hasHeader) { + dialect.header = false + } + } + + return dialect +} diff --git a/csv/index.ts b/csv/index.ts index e69de29b..ef2a40d5 100644 --- a/csv/index.ts +++ b/csv/index.ts @@ -0,0 +1,3 @@ +export * from "./dialect/index.js" +export * from "./table/index.js" +export * from "./plugin.js" diff --git a/csv/package.json b/csv/package.json index 5d6bebb6..1b26e93f 100644 --- a/csv/package.json +++ b/csv/package.json @@ -21,5 +21,16 @@ ], "scripts": { "build": "tsc --build" + }, + "dependencies": { + "@dpkit/core": "workspace:*", + "@dpkit/file": "workspace:*", + "@dpkit/table": "workspace:*", + "csv-sniffer": "^0.1.1", + "nodejs-polars": "^0.18.0" + }, + "devDependencies": { + "@dpkit/test": "workspace:*", + "tempy": "^3.1.0" } } diff --git a/csv/plugin.ts b/csv/plugin.ts new file mode 100644 index 00000000..a0490935 --- /dev/null +++ b/csv/plugin.ts @@ -0,0 +1,37 @@ +import type { Resource } from "@dpkit/core" +import { inferFormat } from "@dpkit/core" +import type { InferDialectOptions, TablePlugin } from "@dpkit/table" +import type { SaveTableOptions, Table } from "@dpkit/table" +import { inferCsvDialect } from "./dialect/index.js" +import { loadCsvTable, saveCsvTable } from "./table/index.js" + +export class CsvPlugin implements TablePlugin { + async inferDialect( + resource: Partial, + options?: InferDialectOptions, + ) { + const isCsv = getIsCsv(resource) + if (!isCsv) return undefined + + return await inferCsvDialect(resource, options) + } + + async loadTable(resource: Partial) { + const isCsv = getIsCsv(resource) + if (!isCsv) return undefined + + return await loadCsvTable(resource) + } + + async saveTable(table: Table, options: SaveTableOptions) { + const isCsv = getIsCsv({ path: options.path }) + if (!isCsv) return undefined + + return await saveCsvTable(table, options) + } +} + +function getIsCsv(resource: Partial) { + const format = inferFormat(resource) + return ["csv", "tsv"].includes(format ?? "") +} diff --git a/csv/table/fixtures/generated/should-load-remote-file-multipart_2595965849/recording.har b/csv/table/fixtures/generated/should-load-remote-file-multipart_2595965849/recording.har new file mode 100644 index 00000000..57e9a4fa --- /dev/null +++ b/csv/table/fixtures/generated/should-load-remote-file-multipart_2595965849/recording.har @@ -0,0 +1,290 @@ +{ + "log": { + "_recordingName": "should load remote file (multipart)", + "creator": { + "comment": "persister:fs", + "name": "Polly.JS", + "version": "6.0.6" + }, + "entries": [ + { + "_id": "4c7b0fb73c46c7315d57cbb41752f90c", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [], + "headersSize": 146, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://gist.githubusercontent.com/roll/d20416f5e6dfc3fc1a7c4eef8452d581/raw/1c9cd1a020389921bf83e5a0395bb00c6b27402d/tabl1.csv" + }, + "response": { + "bodySize": 29, + "content": { + "mimeType": "text/plain; charset=utf-8", + "size": 29, + "text": "id,name\n1,english\n2,中国人" + }, + "cookies": [], + "headers": [ + { + "name": "accept-ranges", + "value": "bytes" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "cache-control", + "value": "max-age=300" + }, + { + "name": "connection", + "value": "keep-alive" + }, + { + "name": "content-length", + "value": "29" + }, + { + "name": "content-security-policy", + "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" + }, + { + "name": "content-type", + "value": "text/plain; charset=utf-8" + }, + { + "name": "cross-origin-resource-policy", + "value": "cross-origin" + }, + { + "name": "date", + "value": "Fri, 27 Jun 2025 11:08:00 GMT" + }, + { + "name": "etag", + "value": "W/\"7e7cf438cb55bf28aa7f609346bfa9dc9efadbcfe2c8e2ea5c7d093632951d62\"" + }, + { + "name": "expires", + "value": "Fri, 27 Jun 2025 11:13:00 GMT" + }, + { + "name": "source-age", + "value": "67" + }, + { + "name": "strict-transport-security", + "value": "max-age=31536000" + }, + { + "name": "vary", + "value": "Authorization,Accept-Encoding" + }, + { + "name": "via", + "value": "1.1 varnish" + }, + { + "name": "x-cache", + "value": "HIT" + }, + { + "name": "x-cache-hits", + "value": "1" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "x-fastly-request-id", + "value": "1a812923feaad90809c9c7877a640b355f7a8dc5" + }, + { + "name": "x-frame-options", + "value": "deny" + }, + { + "name": "x-github-request-id", + "value": "317B:3663E4:D636E:EF3FF:685E7B4C" + }, + { + "name": "x-served-by", + "value": "cache-lis1490022-LIS" + }, + { + "name": "x-timer", + "value": "S1751022480.239594,VS0,VE1" + }, + { + "name": "x-xss-protection", + "value": "1; mode=block" + } + ], + "headersSize": 873, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-06-27T11:08:00.063Z", + "time": 178, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 178 + } + }, + { + "_id": "26b4c877aae0bbdbc9889671caf26f73", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [], + "headersSize": 147, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://gist.githubusercontent.com/roll/ba1c358f1daa994f8094633669d60f50/raw/03e0e54f9d4df7ff08b0d0db937fe5d720d8dfe6/table2.csv" + }, + "response": { + "bodySize": 8, + "content": { + "mimeType": "text/plain; charset=utf-8", + "size": 8, + "text": "3,german" + }, + "cookies": [], + "headers": [ + { + "name": "accept-ranges", + "value": "bytes" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "cache-control", + "value": "max-age=300" + }, + { + "name": "connection", + "value": "keep-alive" + }, + { + "name": "content-length", + "value": "8" + }, + { + "name": "content-security-policy", + "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" + }, + { + "name": "content-type", + "value": "text/plain; charset=utf-8" + }, + { + "name": "cross-origin-resource-policy", + "value": "cross-origin" + }, + { + "name": "date", + "value": "Fri, 27 Jun 2025 11:08:00 GMT" + }, + { + "name": "etag", + "value": "W/\"5b8bd499e42ca79dc52ec0b203194d24d475f240c0b8ff6e0aab54d5798e008a\"" + }, + { + "name": "expires", + "value": "Fri, 27 Jun 2025 11:13:00 GMT" + }, + { + "name": "source-age", + "value": "9" + }, + { + "name": "strict-transport-security", + "value": "max-age=31536000" + }, + { + "name": "vary", + "value": "Authorization,Accept-Encoding" + }, + { + "name": "via", + "value": "1.1 varnish" + }, + { + "name": "x-cache", + "value": "HIT" + }, + { + "name": "x-cache-hits", + "value": "1" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "x-fastly-request-id", + "value": "a9c8e4b9a305628ef72ef83689b4b8140501b803" + }, + { + "name": "x-frame-options", + "value": "deny" + }, + { + "name": "x-github-request-id", + "value": "89C9:33FD66:D620E:EF85E:685E7B85" + }, + { + "name": "x-served-by", + "value": "cache-lis1490040-LIS" + }, + { + "name": "x-timer", + "value": "S1751022480.244457,VS0,VE1" + }, + { + "name": "x-xss-protection", + "value": "1; mode=block" + } + ], + "headersSize": 871, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-06-27T11:08:00.063Z", + "time": 180, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 180 + } + } + ], + "pages": [], + "version": "1.2" + } +} diff --git a/csv/table/fixtures/generated/should-load-remote-file_1137408454/recording.har b/csv/table/fixtures/generated/should-load-remote-file_1137408454/recording.har new file mode 100644 index 00000000..e3adcae7 --- /dev/null +++ b/csv/table/fixtures/generated/should-load-remote-file_1137408454/recording.har @@ -0,0 +1,156 @@ +{ + "log": { + "_recordingName": "should load remote file", + "creator": { + "comment": "persister:fs", + "name": "Polly.JS", + "version": "6.0.6" + }, + "entries": [ + { + "_id": "c00cb7e4f4b92fa8d3f3fc16f382ce35", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [], + "headersSize": 115, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/core/package/fixtures/table.csv" + }, + "response": { + "bodySize": 51, + "content": { + "mimeType": "text/plain; charset=utf-8", + "size": 51, + "text": "id,name\n1,english\n2,中国人\n" + }, + "cookies": [], + "headers": [ + { + "name": "accept-ranges", + "value": "bytes" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "cache-control", + "value": "max-age=300" + }, + { + "name": "connection", + "value": "keep-alive" + }, + { + "name": "content-encoding", + "value": "gzip" + }, + { + "name": "content-length", + "value": "51" + }, + { + "name": "content-security-policy", + "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" + }, + { + "name": "content-type", + "value": "text/plain; charset=utf-8" + }, + { + "name": "cross-origin-resource-policy", + "value": "cross-origin" + }, + { + "name": "date", + "value": "Fri, 27 Jun 2025 11:00:42 GMT" + }, + { + "name": "etag", + "value": "W/\"0172a2fd99319bed82fe7cccbd7a44b27a77f7200caf0d04b7f23cbb6b81026d\"" + }, + { + "name": "expires", + "value": "Fri, 27 Jun 2025 11:05:42 GMT" + }, + { + "name": "source-age", + "value": "0" + }, + { + "name": "strict-transport-security", + "value": "max-age=31536000" + }, + { + "name": "vary", + "value": "Authorization,Accept-Encoding" + }, + { + "name": "via", + "value": "1.1 varnish" + }, + { + "name": "x-cache", + "value": "HIT" + }, + { + "name": "x-cache-hits", + "value": "0" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "x-fastly-request-id", + "value": "ef6f299a2d5a3def740b8b3bf4b2014349750906" + }, + { + "name": "x-frame-options", + "value": "deny" + }, + { + "name": "x-github-request-id", + "value": "574A:2F9F3:2787D:2C8BD:685E6F3A" + }, + { + "name": "x-served-by", + "value": "cache-lis1490037-LIS" + }, + { + "name": "x-timer", + "value": "S1751022042.364693,VS0,VE181" + }, + { + "name": "x-xss-protection", + "value": "1; mode=block" + } + ], + "headersSize": 897, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-06-27T11:00:42.220Z", + "time": 321, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 321 + } + } + ], + "pages": [], + "version": "1.2" + } +} diff --git a/csv/table/index.ts b/csv/table/index.ts new file mode 100644 index 00000000..0181af2e --- /dev/null +++ b/csv/table/index.ts @@ -0,0 +1,2 @@ +export { loadCsvTable } from "./load.js" +export { saveCsvTable } from "./save.js" diff --git a/csv/table/load.spec.ts b/csv/table/load.spec.ts new file mode 100644 index 00000000..eb811b56 --- /dev/null +++ b/csv/table/load.spec.ts @@ -0,0 +1,374 @@ +import { writeTempFile } from "@dpkit/file" +import { useRecording } from "@dpkit/test" +import { describe, expect, it } from "vitest" +import { loadCsvTable } from "./load.js" + +describe("loadCsvTable", () => { + useRecording() + + describe("file variations", () => { + it("should load local file", async () => { + const path = await writeTempFile("id,name\n1,english\n2,中文") + const table = await loadCsvTable({ path }) + + expect((await table.collect()).toRecords()).toEqual([ + { id: "1", name: "english" }, + { id: "2", name: "中文" }, + ]) + }) + + it("should load local file (multipart)", async () => { + const path1 = await writeTempFile("id,name\n1,english") + const path2 = await writeTempFile("2,中文\n3,german") + const table = await loadCsvTable({ path: [path1, path2] }) + + expect((await table.collect()).toRecords()).toEqual([ + { id: "1", name: "english" }, + { id: "2", name: "中文" }, + { id: "3", name: "german" }, + ]) + }) + + it("should load remote file", async () => { + const table = await loadCsvTable({ + path: "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/core/package/fixtures/table.csv", + }) + + expect((await table.collect()).toRecords()).toEqual([ + { id: "1", name: "english" }, + { id: "2", name: "中国人" }, + ]) + }) + + it("should load remote file (multipart)", async () => { + const table = await loadCsvTable({ + path: [ + "https://gist.githubusercontent.com/roll/d20416f5e6dfc3fc1a7c4eef8452d581/raw/1c9cd1a020389921bf83e5a0395bb00c6b27402d/tabl1.csv", + "https://gist.githubusercontent.com/roll/ba1c358f1daa994f8094633669d60f50/raw/03e0e54f9d4df7ff08b0d0db937fe5d720d8dfe6/table2.csv", + ], + }) + + expect((await table.collect()).toRecords()).toEqual([ + { id: "1", name: "english" }, + { id: "2", name: "中国人" }, + { id: "3", name: "german" }, + ]) + }) + }) + + describe("dialect variations", () => { + it("should handle windows line terminator by default", async () => { + const path = await writeTempFile("id,name\r\n1,english\r\n2,中文") + const table = await loadCsvTable({ path }) + + expect((await table.collect()).toRecords()).toEqual([ + { id: "1", name: "english" }, + { id: "2", name: "中文" }, + ]) + }) + + it("should handle custom delimiter", async () => { + const path = await writeTempFile("id|name\n1|alice\n2|bob") + const table = await loadCsvTable({ + path, + dialect: { delimiter: "|" }, + }) + + expect((await table.collect()).toRecords()).toEqual([ + { id: "1", name: "alice" }, + { id: "2", name: "bob" }, + ]) + }) + + it("should handle files without header", async () => { + const path = await writeTempFile("1,alice\n2,bob") + const table = await loadCsvTable({ + path, + dialect: { header: false }, + }) + + const records = (await table.collect()).toRecords() + expect(records).toEqual([ + { column_1: "1", column_2: "alice" }, + { column_1: "2", column_2: "bob" }, + ]) + }) + + it.skip("should handle custom line terminator", async () => { + const path = await writeTempFile("id,name|1,alice|2,bob") + const table = await loadCsvTable({ + path, + dialect: { lineTerminator: "|" }, + }) + + expect((await table.collect()).toRecords()).toEqual([ + { id: "1", name: "alice" }, + { id: "2", name: "bob" }, + ]) + }) + + it.skip("should handle escape char", async () => { + const path = await writeTempFile( + "id,name\n1,apple|,fruits\n2,orange|,fruits", + ) + + const table = await loadCsvTable({ + path, + dialect: { escapeChar: "|" }, + }) + + expect((await table.collect()).toRecords()).toEqual([ + { id: "1", name: "apple,fruits" }, + { id: "2", name: "orange,fruits" }, + ]) + }) + + it("should handle custom quote character", async () => { + const path = await writeTempFile( + "id,name\n1,'alice smith'\n2,'bob jones'", + ) + + const table = await loadCsvTable({ + path, + dialect: { quoteChar: "'" }, + }) + + expect((await table.collect()).toRecords()).toEqual([ + { id: "1", name: "alice smith" }, + { id: "2", name: "bob jones" }, + ]) + }) + + it("should handle double quote by default", async () => { + const path = await writeTempFile( + 'id,name\n1,"alice""smith"\n2,"bob""jones"', + ) + + const table = await loadCsvTable({ + path, + dialect: { doubleQuote: true }, + }) + + expect((await table.collect()).toRecords()).toEqual([ + { id: "1", name: 'alice"smith' }, + { id: "2", name: 'bob"jones' }, + ]) + }) + + it.skip("should handle disabling double quote", async () => { + const path = await writeTempFile( + 'id,name\n1,"alice""smith"\n2,"bob""jones"', + ) + + const table = await loadCsvTable({ + path, + dialect: { doubleQuote: false }, + }) + + expect((await table.collect()).toRecords()).toEqual([ + { id: "1", name: "alicesmith" }, + { id: "2", name: "bobjones" }, + ]) + }) + + it("should handle comment character", async () => { + const path = await writeTempFile( + "# This is a comment\nid,name\n1,alice\n# Another comment\n2,bob", + ) + + const table = await loadCsvTable({ + path, + dialect: { commentChar: "#" }, + }) + + const records = (await table.collect()).toRecords() + expect(records).toEqual([ + { id: "1", name: "alice" }, + { id: "2", name: "bob" }, + ]) + }) + + it("should support headerRows", async () => { + const path = await writeTempFile("#comment\nid,name\n1,alice\n2,bob") + + const table = await loadCsvTable({ + path, + dialect: { headerRows: [2] }, + }) + + const records = (await table.collect()).toRecords() + expect(records).toEqual([ + { id: "1", name: "alice" }, + { id: "2", name: "bob" }, + ]) + }) + + it("should support headerJoin", async () => { + const path = await writeTempFile( + "#comment\nid,name\nint,str\n1,alice\n2,bob", + ) + + const table = await loadCsvTable({ + path, + dialect: { headerRows: [2, 3], headerJoin: "_" }, + }) + + const records = (await table.collect()).toRecords() + expect(records).toEqual([ + { id_int: "1", name_str: "alice" }, + { id_int: "2", name_str: "bob" }, + ]) + }) + + it("should support commentRows", async () => { + const path = await writeTempFile("id,name\n1,alice\ncomment\n2,bob") + + const table = await loadCsvTable({ + path, + dialect: { commentRows: [3] }, + }) + + const records = (await table.collect()).toRecords() + expect(records).toEqual([ + { id: "1", name: "alice" }, + { id: "2", name: "bob" }, + ]) + }) + + it("should support headerRows and commentRows", async () => { + const path = await writeTempFile( + "#comment\nid,name\n1,alice\n#comment\n2,bob", + ) + + const table = await loadCsvTable({ + path, + dialect: { headerRows: [2], commentRows: [4] }, + }) + + const records = (await table.collect()).toRecords() + expect(records).toEqual([ + { id: "1", name: "alice" }, + { id: "2", name: "bob" }, + ]) + }) + + it("should support headerJoin and commentRows", async () => { + const path = await writeTempFile( + "#comment\nid,name\nint,str\n1,alice\n#comment\n2,bob", + ) + + const table = await loadCsvTable({ + path, + dialect: { headerRows: [2, 3], headerJoin: "_", commentRows: [5] }, + }) + + const records = (await table.collect()).toRecords() + expect(records).toEqual([ + { id_int: "1", name_str: "alice" }, + { id_int: "2", name_str: "bob" }, + ]) + }) + + it("should handle null sequence", async () => { + const path = await writeTempFile( + "id,name,age\n1,alice,25\n2,N/A,30\n3,bob,N/A", + ) + const table = await loadCsvTable({ + path, + dialect: { nullSequence: "N/A" }, + }) + + expect((await table.collect()).toRecords()).toEqual([ + { id: "1", name: "alice", age: "25" }, + { id: "2", name: null, age: "30" }, + { id: "3", name: "bob", age: null }, + ]) + }) + + it("should handle skip initial space", async () => { + const path = await writeTempFile( + "id,name,category\n1, alice, fruits\n2, bob, vegetables\n3,charlie,grains", + ) + const table = await loadCsvTable({ + path, + dialect: { skipInitialSpace: true }, + }) + + expect((await table.collect()).toRecords()).toEqual([ + { id: "1", name: "alice", category: "fruits" }, + { id: "2", name: "bob", category: "vegetables" }, + { id: "3", name: "charlie", category: "grains" }, + ]) + }) + + it("should handle multiple dialect options together", async () => { + const path = await writeTempFile( + "#comment\nid|'full name'|age\n1|'alice smith'|25\n2|'bob jones'|30", + ) + const table = await loadCsvTable({ + path, + dialect: { + delimiter: "|", + quoteChar: "'", + commentChar: "#", + header: true, + }, + }) + + expect((await table.collect()).toRecords()).toEqual([ + { id: "1", "full name": "alice smith", age: "25" }, + { id: "2", "full name": "bob jones", age: "30" }, + ]) + }) + + it("should handle utf8 encoding", async () => { + const path = await writeTempFile( + Buffer.from("id,name\n1,café\n2,naïve", "utf8"), + ) + + const table = await loadCsvTable({ + path, + encoding: "utf8", + }) + + expect((await table.collect()).toRecords()).toEqual([ + { id: "1", name: "café" }, + { id: "2", name: "naïve" }, + ]) + }) + + // TODO: currently not supported by nodejs-polars + it.skip("should handle utf16 encoding", async () => { + const path = await writeTempFile( + Buffer.from("id,name\n1,café\n2,naïve", "utf16le"), + ) + + const table = await loadCsvTable({ + path, + encoding: "utf16", + }) + + expect((await table.collect()).toRecords()).toEqual([ + { id: "1", name: "café" }, + { id: "2", name: "naïve" }, + ]) + }) + + // TODO: currently not supported by nodejs-polars + it.skip("should handle latin1 encoding", async () => { + const path = await writeTempFile( + Buffer.from("id,name\n1,café\n2,résumé", "latin1"), + ) + + const table = await loadCsvTable({ + path, + encoding: "latin1", + }) + + expect((await table.collect()).toRecords()).toEqual([ + { id: "1", name: "café" }, + { id: "2", name: "résumé" }, + ]) + }) + }) +}) diff --git a/csv/table/load.ts b/csv/table/load.ts new file mode 100644 index 00000000..fda2281d --- /dev/null +++ b/csv/table/load.ts @@ -0,0 +1,165 @@ +import type { Dialect, Resource } from "@dpkit/core" +import { loadDialect } from "@dpkit/core" +import { prefetchFiles } from "@dpkit/file" +import type { Table } from "@dpkit/table" +import { DataFrame, col, scanCSV } from "nodejs-polars" +import type { ScanCsvOptions } from "nodejs-polars" +import { Utf8, concat } from "nodejs-polars" + +// TODO: Condier using sample to extract header first +// for better commentChar + headerRows/commentRows support +// (consult with the Data Package Working Group) + +export async function loadCsvTable(resource: Partial) { + const dialect = + typeof resource.dialect === "string" + ? await loadDialect(resource.dialect) + : resource.dialect + + const [firstPath, ...restPaths] = await prefetchFiles(resource.path) + const scanOptions = getScanOptions(resource, dialect) + + if (!firstPath) { + return DataFrame().lazy() + } + + let table = scanCSV(firstPath, scanOptions) + + const polarsSchema = Object.fromEntries( + table.columns.map(name => [name, Utf8]), + ) + + if (restPaths.length) { + table = concat([ + table, + ...restPaths.map(path => + scanCSV(path, { + ...scanOptions, + hasHeader: false, + schema: polarsSchema, + }), + ), + ]) + } + + if (dialect) { + table = await joinHeaderRows(table, dialect) + table = skipCommentRows(table, dialect) + table = stripInitialSpace(table, dialect) + } + + return table +} + +function getScanOptions(resource: Partial, dialect?: Dialect) { + const options: Partial = { + inferSchemaLength: 0, + truncateRaggedLines: true, + } + + if (resource.encoding) { + options.encoding = resource.encoding + } + + options.skipRows = getRowsToSkip(dialect) + options.hasHeader = dialect?.header !== false + options.sep = dialect?.delimiter ?? "," + + // TODO: enable after this polars issues is fixed + // https://github.com/pola-rs/nodejs-polars/issues/333 + //options.eolChar = dialect?.lineTerminator ?? "\n" + + // TODO: try convincing nodejs-polars to support escapeChar + // https://github.com/pola-rs/polars/issues/3074 + //options.escapeChar = dialect?.escapeChar + + options.quoteChar = dialect?.quoteChar ?? '"' + options.nullValues = dialect?.nullSequence + + // TODO: try convincing nodejs-polars to support doubleQuote + //options.doubleQuote = dialect?.doubleQuote ?? true + + // TODO: remove ts-ignore when issues is fixed + // https://github.com/pola-rs/nodejs-polars/issues/334 + // @ts-ignore + options.commentPrefix = dialect?.commentChar + + return options +} + +async function joinHeaderRows(table: Table, dialect: Dialect) { + const headerOffset = getHeaderOffset(dialect) + const headerRows = getHeaderRows(dialect) + const headerJoin = dialect?.headerJoin ?? " " + if (headerRows.length < 2) { + return table + } + + const extraLabelsFrame = await table + .withRowCount() + .withColumn(col("row_nr").add(1)) + .filter(col("row_nr").add(headerOffset).isIn(headerRows)) + .select(table.columns.map(name => col(name).str.concat(headerJoin))) + .collect() + + const labels = table.columns + const extraLabels = extraLabelsFrame.row(0) + + const mapping = Object.fromEntries( + labels.map((label, index) => [ + label, + [label, extraLabels[index]].join(headerJoin), + ]), + ) + + return table + .withRowCount() + .withColumn(col("row_nr").add(1)) + .filter(col("row_nr").add(headerOffset).isIn(headerRows).not()) + .rename(mapping) + .drop("row_nr") +} + +function skipCommentRows(table: Table, dialect: Dialect) { + const commentOffset = getCommentOffset(dialect) + if (!dialect?.commentRows) { + return table + } + + return table + .withRowCount() + .withColumn(col("row_nr").add(1)) + .filter(col("row_nr").add(commentOffset).isIn(dialect.commentRows).not()) + .drop("row_nr") +} + +function stripInitialSpace(table: Table, dialect: Dialect) { + if (!dialect?.skipInitialSpace) { + return table + } + + return table.select( + // TODO: rebase on stripCharsStart when it's fixed in polars + // https://github.com/pola-rs/nodejs-polars/issues/336 + table.columns.map(name => col(name).str.strip().as(name)), + ) +} + +function getRowsToSkip(dialect?: Dialect) { + const headerRows = getHeaderRows(dialect) + return headerRows[0] ? headerRows[0] - 1 : 0 +} + +function getHeaderOffset(dialect?: Dialect) { + const headerRows = getHeaderRows(dialect) + return headerRows.at(0) ?? 0 +} + +function getCommentOffset(dialect?: Dialect) { + const headerRows = getHeaderRows(dialect) + return headerRows.at(-1) ?? 0 +} + +function getHeaderRows(dialect?: Dialect) { + return dialect?.header !== false ? (dialect?.headerRows ?? [1]) : [] +} diff --git a/csv/table/save.spec.ts b/csv/table/save.spec.ts new file mode 100644 index 00000000..f5b28a08 --- /dev/null +++ b/csv/table/save.spec.ts @@ -0,0 +1,74 @@ +import { readFile } from "node:fs/promises" +import { DataFrame } from "nodejs-polars" +import { temporaryFileTask } from "tempy" +import { describe, expect, it } from "vitest" +import { saveCsvTable } from "./save.js" + +describe("saveCsvTable", () => { + it("should save table to CSV file", async () => { + await temporaryFileTask(async path => { + const table = DataFrame({ + id: [1.0, 2.0, 3.0], + name: ["Alice", "Bob", "Charlie"], + }).lazy() + + await saveCsvTable(table, { path }) + + const content = await readFile(path, "utf-8") + expect(content).toEqual("id,name\n1.0,Alice\n2.0,Bob\n3.0,Charlie\n") + }) + }) + + it("should save with custom delimiter", async () => { + await temporaryFileTask(async path => { + const table = DataFrame({ + id: [1.0, 2.0, 3.0], + name: ["Alice", "Bob", "Charlie"], + }).lazy() + + await saveCsvTable(table, { + path, + dialect: { delimiter: ";" }, + }) + + const content = await readFile(path, "utf-8") + expect(content).toEqual("id;name\n1.0;Alice\n2.0;Bob\n3.0;Charlie\n") + }) + }) + + it("should save without header", async () => { + await temporaryFileTask(async path => { + const table = DataFrame({ + id: [1.0, 2.0, 3.0], + name: ["Alice", "Bob", "Charlie"], + }).lazy() + + await saveCsvTable(table, { + path, + dialect: { header: false }, + }) + + const content = await readFile(path, "utf-8") + expect(content).toEqual("1.0,Alice\n2.0,Bob\n3.0,Charlie\n") + }) + }) + + it("should save with custom quote char", async () => { + await temporaryFileTask(async path => { + const table = DataFrame({ + id: [1.0, 2.0, 3.0], + name: ["Alice,Smith", "Bob,Jones", "Charlie,Brown"], + }).lazy() + + await saveCsvTable(table, { + path, + dialect: { quoteChar: "'" }, + }) + + const content = await readFile(path, "utf-8") + expect(content).toEqual( + "id,name\n1.0,'Alice,Smith'\n2.0,'Bob,Jones'\n3.0,'Charlie,Brown'\n", + ) + }) + }) +}) diff --git a/csv/table/save.ts b/csv/table/save.ts new file mode 100644 index 00000000..12ab6377 --- /dev/null +++ b/csv/table/save.ts @@ -0,0 +1,15 @@ +import type { SaveTableOptions, Table } from "@dpkit/table" + +// TODO: so currently, nodejs-polars uses sync sink/write functions?? + +export async function saveCsvTable(table: Table, options: SaveTableOptions) { + table.sinkCSV(options?.path, { + maintainOrder: true, + includeHeader: options?.dialect?.header ?? true, + separator: options?.dialect?.delimiter ?? ",", + //lineTerminator: options?.dialect?.lineTerminator ?? "\r\n", + quoteChar: options?.dialect?.quoteChar ?? '"', + }) + + return options.path +} diff --git a/csv/typedoc.json b/csv/typedoc.json new file mode 100644 index 00000000..fc33b815 --- /dev/null +++ b/csv/typedoc.json @@ -0,0 +1,5 @@ +{ + "entryPoints": ["index.ts"], + "projectDocuments": ["CHANGELOG.md", "OVERVIEW.md"], + "skipErrorChecking": true +} diff --git a/datahub/plugin.ts b/datahub/plugin.ts index 71295727..62830240 100644 --- a/datahub/plugin.ts +++ b/datahub/plugin.ts @@ -4,15 +4,17 @@ import { loadPackageFromDatahub } from "./package/index.js" export class DatahubPlugin implements Plugin { async loadPackage(source: string) { - const isRemote = isRemotePath(source) - if (!isRemote) return undefined - - const isDatahub = new URL(source).hostname === "datahub.io" + const isDatahub = getIsDatahub(source) if (!isDatahub) return undefined - const cleanup = async () => {} const dataPackage = await loadPackageFromDatahub(source) - - return { dataPackage, cleanup } + return dataPackage } } + +function getIsDatahub(path: string) { + const isRemote = isRemotePath(path) + if (!isRemote) return false + + return new URL(path).hostname === "datahub.io" +} diff --git a/docs/astro.config.ts b/docs/astro.config.ts index aa748d10..4d1fa973 100644 --- a/docs/astro.config.ts +++ b/docs/astro.config.ts @@ -8,6 +8,7 @@ const PACKAGES = { "@dpkit/camtrap": "../camtrap", "@dpkit/ckan": "../ckan", "@dpkit/core": "../core", + "@dpkit/csv": "../csv", "@dpkit/datahub": "../datahub", "@dpkit/file": "../file", "@dpkit/github": "../github", @@ -67,7 +68,7 @@ export default defineConfig({ { label: "Guides", autogenerate: { directory: "guides" } }, { label: "Packages", - //collapsed: true, + collapsed: true, items: generatePackageSidebars(), }, ], diff --git a/docs/components/Header/SocialIcons.astro b/docs/components/Header/SocialIcons.astro index e2884c5e..71137f2d 100644 --- a/docs/components/Header/SocialIcons.astro +++ b/docs/components/Header/SocialIcons.astro @@ -5,14 +5,18 @@ import type { Props } from "@astrojs/starlight/props" - + Datist - + Data Package + + Polars + +