diff --git a/.changeset/clear-colts-prove.md b/.changeset/clear-colts-prove.md new file mode 100644 index 00000000..8f05cec4 --- /dev/null +++ b/.changeset/clear-colts-prove.md @@ -0,0 +1,24 @@ +--- +"@dpkit/database": minor +"@dpkit/camtrap": minor +"@dpkit/datahub": minor +"@dpkit/parquet": minor +"@dpkit/folder": minor +"@dpkit/github": minor +"@dpkit/inline": minor +"@dpkit/zenodo": minor +"@dpkit/arrow": minor +"@dpkit/table": minor +"@dpkit/ckan": minor +"@dpkit/core": minor +"@dpkit/file": minor +"@dpkit/json": minor +"@dpkit/test": minor +"@dpkit/xlsx": minor +"@dpkit/all": minor +"@dpkit/csv": minor +"@dpkit/ods": minor +"@dpkit/zip": minor +--- + +Added database support diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..ff86c559 --- /dev/null +++ b/.env.example @@ -0,0 +1,2 @@ +DPKIT_MYSQL_URL='' +DPKIT_POSTGRESQL_URL='' diff --git a/.github/workflows/general.yaml b/.github/workflows/general.yaml index 5cf4a6fe..eeae1682 100644 --- a/.github/workflows/general.yaml +++ b/.github/workflows/general.yaml @@ -11,6 +11,26 @@ on: jobs: test: runs-on: ubuntu-latest + + services: + postgres: + image: postgres:16 + env: + POSTGRES_PASSWORD: test + POSTGRES_USER: test + POSTGRES_DB: test + ports: + - 5432:5432 + mysql: + image: mysql:8 + env: + MYSQL_ROOT_PASSWORD: test + MYSQL_DATABASE: test + MYSQL_USER: test + MYSQL_PASSWORD: test + ports: + - 3306:3306 + steps: - name: Checkout Repo uses: actions/checkout@v4 @@ -27,6 +47,9 @@ jobs: run: pnpm build - name: Test Packages run: pnpm test + env: + DPKIT_MYSQL_URL: mysql://test:test@localhost:3306/test + DPKIT_POSTGRESQL_URL: postgresql://test:test@localhost:5432/test - name: Upload Coverage uses: codecov/codecov-action@v5 with: diff --git a/.gitignore b/.gitignore index 33dea10d..83dca33e 100644 --- a/.gitignore +++ b/.gitignore @@ -64,7 +64,8 @@ compile/ dist/ .astro/ /docs/content/docs/reference/ -**/.claude/settings.local.json -AGENTS.md -CLAUDE.md .cursor/ +.claude/ +.serena/ +.mcp.json +.env diff --git a/all/README.md b/all/README.md index 476f8355..035a67a4 100644 --- a/all/README.md +++ b/all/README.md @@ -1,3 +1,3 @@ # @dpkit/all -dpkit is a fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). +dpkit is a fast data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/all/index.ts b/all/index.ts index d1e59557..90ad4b3a 100644 --- a/all/index.ts +++ b/all/index.ts @@ -14,5 +14,6 @@ export * from "@dpkit/zip" export * from "./dialect/index.ts" export * from "./package/index.ts" export * from "./resource/index.ts" +export * from "./schema/index.ts" export * from "./table/index.ts" export * from "./plugin.ts" diff --git a/all/package.json b/all/package.json index c64f7829..5bb88eeb 100644 --- a/all/package.json +++ b/all/package.json @@ -25,6 +25,7 @@ "@dpkit/ckan": "workspace:*", "@dpkit/core": "workspace:*", "@dpkit/csv": "workspace:*", + "@dpkit/database": "workspace:*", "@dpkit/datahub": "workspace:*", "@dpkit/file": "workspace:*", "@dpkit/folder": "workspace:*", diff --git a/all/plugin.ts b/all/plugin.ts index 894a847a..eb5acebe 100644 --- a/all/plugin.ts +++ b/all/plugin.ts @@ -1,6 +1,7 @@ import { ArrowPlugin } from "@dpkit/arrow" import { CkanPlugin } from "@dpkit/ckan" import { CsvPlugin } from "@dpkit/csv" +import { DatabasePlugin } from "@dpkit/database" import { DatahubPlugin } from "@dpkit/datahub" import { FolderPlugin } from "@dpkit/folder" import { GithubPlugin } from "@dpkit/github" @@ -39,3 +40,6 @@ dpkit.register(JsonPlugin) dpkit.register(OdsPlugin) dpkit.register(ParquetPlugin) dpkit.register(XlsxPlugin) + +// Mixed functions +dpkit.register(DatabasePlugin) diff --git a/all/resource/infer.ts b/all/resource/infer.ts index 6eccd130..53f1f030 100644 --- a/all/resource/infer.ts +++ b/all/resource/infer.ts @@ -2,7 +2,8 @@ import type { Resource } from "@dpkit/core" import { inferResourceFormat, inferResourceName } from "@dpkit/core" import { prefetchFile } from "@dpkit/file" import { inferFileBytes, inferFileEncoding, inferFileHash } from "@dpkit/file" -import { inferTable } from "../table/index.ts" +import { inferDialect } from "../dialect/index.ts" +import { inferSchema } from "../schema/index.ts" // TODO: Support multipart resources? (clarify on the specs level) @@ -18,6 +19,7 @@ export async function inferResource(resource: Partial) { if (typeof resource.path === "string") { const localPath = await prefetchFile(resource.path) + const localResource = { ...resource, path: localPath } if (!result.encoding) { const encoding = await inferFileEncoding(localPath) @@ -34,13 +36,15 @@ export async function inferResource(resource: Partial) { result.hash = await inferFileHash(localPath) } - if (!result.schema) { + if (!result.dialect) { try { - const localResource = { ...resource, path: localPath } - const { dialect, schema } = await inferTable(localResource) + result.dialect = await inferDialect(localResource) + } catch {} + } - result.dialect = dialect - result.schema = schema + if (!result.schema) { + try { + result.schema = await inferSchema(localResource) } catch {} } } diff --git a/all/resource/validate.ts b/all/resource/validate.ts index cffe661e..321f6afb 100644 --- a/all/resource/validate.ts +++ b/all/resource/validate.ts @@ -1,15 +1,18 @@ import type { Descriptor, Resource } from "@dpkit/core" +import { loadResourceSchema } from "@dpkit/core" import { loadDescriptor, validateResourceDescriptor } from "@dpkit/core" import { validateFile } from "@dpkit/file" -import { validateTable } from "../table/index.ts" +import { validateTable } from "@dpkit/table" +import { inferSchema } from "../schema/index.ts" +import { loadTable } from "../table/index.ts" // TODO: Support multipart resources? (clarify on the specs level) export async function validateResource( - pathOrDescriptorOrResource: string | Descriptor | Partial, + source: string | Descriptor | Partial, options?: { basepath?: string }, ) { - let descriptor = pathOrDescriptorOrResource + let descriptor = source let basepath = options?.basepath if (typeof descriptor === "string") { @@ -39,7 +42,14 @@ export async function validateResource( try { // TODO: rebase on not-rasing? // It will raise if the resource is not a table - return await validateTable(resource) + let schema = await loadResourceSchema(resource.schema) + if (!schema) schema = await inferSchema(resource) + + const table = await loadTable(resource, { denormalized: true }) + const errors = await validateTable(table, { schema }) + + const valid = errors.length === 0 + return { valid, errors } } catch {} return { valid: true, errors: [] } diff --git a/all/schema/index.ts b/all/schema/index.ts new file mode 100644 index 00000000..a02c9b7a --- /dev/null +++ b/all/schema/index.ts @@ -0,0 +1 @@ +export { inferSchema } from "./infer.ts" diff --git a/all/schema/infer.ts b/all/schema/infer.ts new file mode 100644 index 00000000..bd8b7f8b --- /dev/null +++ b/all/schema/infer.ts @@ -0,0 +1,21 @@ +import type { Resource, Schema } from "@dpkit/core" +import type { InferSchemaOptions } from "@dpkit/table" +import { inferSchemaFromTable } from "@dpkit/table" +import { dpkit } from "../plugin.ts" +import { loadTable } from "../table/index.ts" + +export async function inferSchema( + resource: Partial, + options?: InferSchemaOptions, +): Promise { + for (const plugin of dpkit.plugins) { + const schema = await plugin.inferSchema?.(resource, options) + if (schema) { + return schema + } + } + + const table = await loadTable(resource, { denormalized: true }) + const schema = await inferSchemaFromTable(table, options) + return schema +} diff --git a/all/table/index.ts b/all/table/index.ts index 5fb5a1f7..310e12ef 100644 --- a/all/table/index.ts +++ b/all/table/index.ts @@ -1,5 +1,2 @@ export { loadTable } from "./load.ts" -export { readTable } from "./read.ts" export { saveTable } from "./save.ts" -export { inferTable } from "./infer.ts" -export { validateTable } from "./validate.ts" diff --git a/all/table/infer.ts b/all/table/infer.ts index bef87aa7..42970275 100644 --- a/all/table/infer.ts +++ b/all/table/infer.ts @@ -1,7 +1,7 @@ import type { Dialect, Resource, Schema } from "@dpkit/core" import { loadResourceDialect, loadResourceSchema } from "@dpkit/core" import type { Table } from "@dpkit/table" -import { inferSchema } from "@dpkit/table" +import { inferSchemaFromTable } from "@dpkit/table" import { inferDialect } from "../dialect/index.ts" import { loadTable } from "./load.ts" @@ -15,11 +15,14 @@ export async function inferTable( dialect = await inferDialect(resource) } - const table = await loadTable({ ...resource, dialect }) + const table = await loadTable( + { ...resource, dialect }, + { denormalized: true }, + ) let schema = await loadResourceSchema(resource.schema) if (!schema) { - schema = await inferSchema(table) + schema = await inferSchemaFromTable(table) } return { dialect, schema, table } diff --git a/all/table/load.ts b/all/table/load.ts index 2674f30d..029e6dc7 100644 --- a/all/table/load.ts +++ b/all/table/load.ts @@ -1,10 +1,13 @@ import type { Resource } from "@dpkit/core" -import type { Table } from "@dpkit/table" +import type { LoadTableOptions, Table } from "@dpkit/table" import { dpkit } from "../plugin.ts" -export async function loadTable(resource: Partial): Promise { +export async function loadTable( + resource: Partial, + options?: LoadTableOptions, +): Promise
{ for (const plugin of dpkit.plugins) { - const table = await plugin.loadTable?.(resource) + const table = await plugin.loadTable?.(resource, options) if (table) { return table } diff --git a/all/table/read.ts b/all/table/read.ts deleted file mode 100644 index 3d8c0550..00000000 --- a/all/table/read.ts +++ /dev/null @@ -1,9 +0,0 @@ -import type { Resource } from "@dpkit/core" -import type { Table } from "@dpkit/table" -import { processTable } from "@dpkit/table" -import { inferTable } from "./infer.ts" - -export async function readTable(resource: Partial): Promise
{ - const { table, schema } = await inferTable(resource) - return await processTable(table, { schema }) -} diff --git a/all/table/validate.ts b/all/table/validate.ts deleted file mode 100644 index 53589a2a..00000000 --- a/all/table/validate.ts +++ /dev/null @@ -1,11 +0,0 @@ -import type { Resource } from "@dpkit/core" -import { inspectTable } from "@dpkit/table" -import { inferTable } from "./infer.ts" - -export async function validateTable(resource: Partial) { - const { table, schema } = await inferTable(resource) - const errors = await inspectTable(table, { schema }) - - const valid = errors.length === 0 - return { valid, errors } -} diff --git a/arrow/README.md b/arrow/README.md index 6a3d5f20..ef549a4a 100644 --- a/arrow/README.md +++ b/arrow/README.md @@ -1,3 +1,3 @@ # @dpkit/arrow -dpkit is a fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). +dpkit is a fast data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/arrow/package.json b/arrow/package.json index 726d241e..09c0e6d7 100644 --- a/arrow/package.json +++ b/arrow/package.json @@ -27,7 +27,7 @@ "@dpkit/file": "workspace:*", "@dpkit/table": "workspace:*", "csv-sniffer": "^0.1.1", - "nodejs-polars": "^0.21.0" + "nodejs-polars": "^0.21.1" }, "devDependencies": { "@dpkit/test": "workspace:*" diff --git a/arrow/plugin.ts b/arrow/plugin.ts index c678b6d5..0220ea31 100644 --- a/arrow/plugin.ts +++ b/arrow/plugin.ts @@ -1,19 +1,22 @@ import type { Resource } from "@dpkit/core" import { inferResourceFormat } from "@dpkit/core" +import type { LoadTableOptions } from "@dpkit/table" import type { TablePlugin } from "@dpkit/table" import type { SaveTableOptions, Table } from "@dpkit/table" import { loadArrowTable, saveArrowTable } from "./table/index.ts" export class ArrowPlugin implements TablePlugin { - async loadTable(resource: Partial) { + async loadTable(resource: Partial, options?: LoadTableOptions) { const isArrow = getIsArrow(resource) if (!isArrow) return undefined - return await loadArrowTable(resource) + return await loadArrowTable(resource, options) } async saveTable(table: Table, options: SaveTableOptions) { - const isArrow = getIsArrow(options) + const { path, format } = options + + const isArrow = getIsArrow({ path, format }) if (!isArrow) return undefined return await saveArrowTable(table, options) diff --git a/arrow/table.arrow b/arrow/table.arrow deleted file mode 100644 index 83e3baa0..00000000 Binary files a/arrow/table.arrow and /dev/null differ diff --git a/arrow/table/load.spec.ts b/arrow/table/load.spec.ts index cd78f74e..0f3ca972 100644 --- a/arrow/table/load.spec.ts +++ b/arrow/table/load.spec.ts @@ -4,9 +4,9 @@ import { DataFrame } from "nodejs-polars" import { describe, expect, it } from "vitest" import { loadArrowTable } from "./load.ts" -describe("loadArrowTable", () => { - useRecording() +useRecording() +describe("loadArrowTable", () => { describe("file variations", () => { it("should load local file", async () => { const path = getTempFilePath() diff --git a/arrow/table/load.ts b/arrow/table/load.ts index 931b0057..da0ed071 100644 --- a/arrow/table/load.ts +++ b/arrow/table/load.ts @@ -1,13 +1,18 @@ import type { Resource } from "@dpkit/core" +import { loadResourceSchema } from "@dpkit/core" import { prefetchFiles } from "@dpkit/file" +import type { LoadTableOptions } from "@dpkit/table" +import { inferSchemaFromTable, normalizeTable } from "@dpkit/table" import { concat } from "nodejs-polars" -import { DataFrame } from "nodejs-polars" import { scanIPC } from "nodejs-polars" -export async function loadArrowTable(resource: Partial) { +export async function loadArrowTable( + resource: Partial, + options?: LoadTableOptions, +) { const [firstPath, ...restPaths] = await prefetchFiles(resource.path) if (!firstPath) { - return DataFrame().lazy() + throw new Error("Resource path is not defined") } let table = scanIPC(firstPath) @@ -15,5 +20,11 @@ export async function loadArrowTable(resource: Partial) { table = concat([table, ...restPaths.map(path => scanIPC(path))]) } + if (!options?.denormalized) { + let schema = await loadResourceSchema(resource.schema) + if (!schema) schema = await inferSchemaFromTable(table, options) + table = await normalizeTable(table, schema) + } + return table } diff --git a/arrow/table/save.spec.ts b/arrow/table/save.spec.ts index 41eef947..3b479d67 100644 --- a/arrow/table/save.spec.ts +++ b/arrow/table/save.spec.ts @@ -1,5 +1,5 @@ import { getTempFilePath } from "@dpkit/file" -import { DataFrame } from "nodejs-polars" +import { DataFrame, DataType, Series } from "nodejs-polars" import { describe, expect, it } from "vitest" import { loadArrowTable } from "./load.ts" import { saveArrowTable } from "./save.ts" @@ -14,11 +14,70 @@ describe("saveArrowTable", () => { await saveArrowTable(source, { path }) - const target = await loadArrowTable({ path }) - expect((await target.collect()).toRecords()).toEqual([ + const table = await loadArrowTable({ path }) + expect((await table.collect()).toRecords()).toEqual([ { id: 1.0, name: "Alice" }, { id: 2.0, name: "Bob" }, { id: 3.0, name: "Charlie" }, ]) }) + + it("should save and load various data types", async () => { + const path = getTempFilePath() + + const source = DataFrame([ + Series("array", ["[1, 2, 3]"], DataType.String), + Series("boolean", [true], DataType.Bool), + Series("date", [new Date(Date.UTC(2025, 0, 1))], DataType.Date), + Series("datetime", [new Date(Date.UTC(2025, 0, 1))], DataType.Datetime), + Series("duration", ["P23DT23H"], DataType.String), + Series("geojson", ['{"value": 1}'], DataType.String), + Series("geopoint", [[40.0, 50.0]], DataType.List(DataType.Float32)), + Series("integer", [1], DataType.Int32), + Series("list", [[1.0, 2.0, 3.0]], DataType.List(DataType.Float32)), + Series("number", [1.1], DataType.Float64), + Series("object", ['{"value": 1}']), + Series("string", ["string"], DataType.String), + Series("time", [new Date(Date.UTC(2025, 0, 1))], DataType.Time), + Series("year", [2025], DataType.Int32), + Series("yearmonth", [[2025, 1]], DataType.List(DataType.Int16)), + ]).lazy() + + await saveArrowTable(source, { + path, + fieldTypes: { + array: "array", + geojson: "geojson", + geopoint: "geopoint", + list: "list", + object: "object", + // TODO: Remove time after: + // https://github.com/pola-rs/nodejs-polars/issues/364 + time: "time", + year: "year", + yearmonth: "yearmonth", + }, + }) + + const target = await loadArrowTable({ path }, { denormalized: true }) + expect((await target.collect()).toRecords()).toEqual([ + { + array: "[1, 2, 3]", + boolean: true, + date: "2025-01-01", + datetime: new Date(Date.UTC(2025, 0, 1)), + duration: "P23DT23H", + geojson: '{"value": 1}', + geopoint: "40.0,50.0", + integer: 1, + list: [1.0, 2.0, 3.0], + number: 1.1, + object: '{"value": 1}', + string: "string", + time: "00:00:00", + year: 2025, + yearmonth: "2025-01", + }, + ]) + }) }) diff --git a/arrow/table/save.ts b/arrow/table/save.ts index 087e5143..5417106d 100644 --- a/arrow/table/save.ts +++ b/arrow/table/save.ts @@ -1,10 +1,33 @@ +import { assertLocalPathVacant } from "@dpkit/file" +import { denormalizeTable, inferSchemaFromTable } from "@dpkit/table" import type { SaveTableOptions, Table } from "@dpkit/table" // TODO: rebase on sinkIPC when it is available // https://github.com/pola-rs/nodejs-polars/issues/353 export async function saveArrowTable(table: Table, options: SaveTableOptions) { - const { path } = options + const { path, overwrite } = options + + if (!overwrite) { + await assertLocalPathVacant(path) + } + + const schema = await inferSchemaFromTable(table, { + ...options, + keepStrings: true, + }) + + table = await denormalizeTable(table, schema, { + nativeTypes: [ + "boolean", + "datetime", + "integer", + "list", + "number", + "string", + "year", + ], + }) const df = await table.collect() df.writeIPC(path) diff --git a/camtrap/README.md b/camtrap/README.md index 25a8489c..9069fe46 100644 --- a/camtrap/README.md +++ b/camtrap/README.md @@ -1,3 +1,3 @@ # @dpkit/camtrap -dpkit is a fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). +dpkit is a fast data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/ckan/README.md b/ckan/README.md index 07d440a1..510c0d3a 100644 --- a/ckan/README.md +++ b/ckan/README.md @@ -1,3 +1,3 @@ # @dpkit/ckan -dpkit is a fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). +dpkit is a fast data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/ckan/package/process/denormalize.spec.ts b/ckan/package/denormalize.spec.ts similarity index 98% rename from ckan/package/process/denormalize.spec.ts rename to ckan/package/denormalize.spec.ts index 17ae1f3d..ecaae259 100644 --- a/ckan/package/process/denormalize.spec.ts +++ b/ckan/package/denormalize.spec.ts @@ -1,10 +1,10 @@ import type { Package } from "@dpkit/core" import { describe, expect, it } from "vitest" -import type { CkanPackage } from "../Package.ts" -import ckanPackageFixture from "../fixtures/ckan-package.json" with { +import type { CkanPackage } from "./Package.ts" +import { denormalizeCkanPackage } from "./denormalize.ts" +import ckanPackageFixture from "./fixtures/ckan-package.json" with { type: "json", } -import { denormalizeCkanPackage } from "./denormalize.ts" import { normalizeCkanPackage } from "./normalize.ts" describe("denormalizeCkanPackage", () => { diff --git a/ckan/package/process/denormalize.ts b/ckan/package/denormalize.ts similarity index 91% rename from ckan/package/process/denormalize.ts rename to ckan/package/denormalize.ts index 24e845bf..2d146995 100644 --- a/ckan/package/process/denormalize.ts +++ b/ckan/package/denormalize.ts @@ -1,9 +1,9 @@ import type { Package } from "@dpkit/core" import type { SetRequired } from "type-fest" -import type { CkanResource } from "../../resource/Resource.ts" -import { denormalizeCkanResource } from "../../resource/process/denormalize.ts" -import type { CkanPackage } from "../Package.ts" -import type { CkanTag } from "../Tag.ts" +import type { CkanResource } from "../resource/Resource.ts" +import { denormalizeCkanResource } from "../resource/index.ts" +import type { CkanPackage } from "./Package.ts" +import type { CkanTag } from "./Tag.ts" /** * Denormalizes a Frictionless Data Package to CKAN Package format diff --git a/ckan/package/load.spec.ts b/ckan/package/load.spec.ts index 2501ee43..46bc6c91 100644 --- a/ckan/package/load.spec.ts +++ b/ckan/package/load.spec.ts @@ -2,9 +2,9 @@ import { useRecording } from "@dpkit/test" import { describe, expect, it } from "vitest" import { loadPackageFromCkan } from "./load.ts" -describe("loadPackageFromCkan", () => { - useRecording() +useRecording() +describe("loadPackageFromCkan", () => { it("should load a package", async () => { const dataPackage = await loadPackageFromCkan( "https://data.nhm.ac.uk/dataset/join-the-dots-collection-level-descriptions", diff --git a/ckan/package/load.ts b/ckan/package/load.ts index d63e92e0..78a342c9 100644 --- a/ckan/package/load.ts +++ b/ckan/package/load.ts @@ -1,7 +1,7 @@ import { mergePackages } from "@dpkit/core" import { makeCkanApiRequest } from "../ckan/index.ts" import type { CkanPackage } from "./Package.ts" -import { normalizeCkanPackage } from "./process/normalize.ts" +import { normalizeCkanPackage } from "./normalize.ts" /** * Load a package from a CKAN instance diff --git a/ckan/package/process/normalize.spec.ts b/ckan/package/normalize.spec.ts similarity index 96% rename from ckan/package/process/normalize.spec.ts rename to ckan/package/normalize.spec.ts index a7b15f1a..3b1d342c 100644 --- a/ckan/package/process/normalize.spec.ts +++ b/ckan/package/normalize.spec.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest" -import type { CkanPackage } from "../Package.ts" -import ckanPackageFixture from "../fixtures/ckan-package.json" with { +import type { CkanPackage } from "./Package.ts" +import ckanPackageFixture from "./fixtures/ckan-package.json" with { type: "json", } import { normalizeCkanPackage } from "./normalize.ts" diff --git a/ckan/package/process/normalize.ts b/ckan/package/normalize.ts similarity index 94% rename from ckan/package/process/normalize.ts rename to ckan/package/normalize.ts index 2fb73e04..4feffb22 100644 --- a/ckan/package/process/normalize.ts +++ b/ckan/package/normalize.ts @@ -1,7 +1,7 @@ import type { Contributor, Package } from "@dpkit/core" import type { License } from "@dpkit/core" -import { normalizeCkanResource } from "../../resource/process/normalize.ts" -import type { CkanPackage } from "../Package.ts" +import { normalizeCkanResource } from "../resource/index.ts" +import type { CkanPackage } from "./Package.ts" /** * Normalizes a CKAN package to Frictionless Data package format diff --git a/ckan/package/save.ts b/ckan/package/save.ts index 34bc3cbe..2b426dc5 100644 --- a/ckan/package/save.ts +++ b/ckan/package/save.ts @@ -14,7 +14,7 @@ import { import { makeCkanApiRequest } from "../ckan/index.ts" import type { CkanResource } from "../resource/index.ts" import { denormalizeCkanResource } from "../resource/index.ts" -import { denormalizeCkanPackage } from "./process/denormalize.ts" +import { denormalizeCkanPackage } from "./denormalize.ts" export async function savePackageToCkan( dataPackage: Package, diff --git a/ckan/resource/process/denormalize.ts b/ckan/resource/denormalize.ts similarity index 94% rename from ckan/resource/process/denormalize.ts rename to ckan/resource/denormalize.ts index a5369387..8d1f70b6 100644 --- a/ckan/resource/process/denormalize.ts +++ b/ckan/resource/denormalize.ts @@ -1,5 +1,5 @@ import type { Resource } from "@dpkit/core" -import type { CkanResource } from "../Resource.ts" +import type { CkanResource } from "./Resource.ts" /** * Denormalizes a Frictionless Data Resource to CKAN Resource format diff --git a/ckan/resource/index.ts b/ckan/resource/index.ts index 35de3bcb..64669b60 100644 --- a/ckan/resource/index.ts +++ b/ckan/resource/index.ts @@ -1,2 +1,3 @@ export type { CkanResource } from "./Resource.ts" -export { denormalizeCkanResource } from "./process/denormalize.ts" +export { denormalizeCkanResource } from "./denormalize.ts" +export { normalizeCkanResource } from "./normalize.ts" diff --git a/ckan/resource/process/normalize.ts b/ckan/resource/normalize.ts similarity index 93% rename from ckan/resource/process/normalize.ts rename to ckan/resource/normalize.ts index ac130ec3..10fd19c0 100644 --- a/ckan/resource/process/normalize.ts +++ b/ckan/resource/normalize.ts @@ -1,7 +1,7 @@ import type { Resource } from "@dpkit/core" import { getFilename } from "@dpkit/core" -import { normalizeCkanSchema } from "../../schema/index.ts" -import type { CkanResource } from "../Resource.ts" +import { normalizeCkanSchema } from "../schema/index.ts" +import type { CkanResource } from "./Resource.ts" /** * Normalizes a CKAN resource to Frictionless Data Resource format diff --git a/ckan/schema/process/denormalize.ts b/ckan/schema/denormalize.ts similarity index 93% rename from ckan/schema/process/denormalize.ts rename to ckan/schema/denormalize.ts index 220cd6ff..cb7cf661 100644 --- a/ckan/schema/process/denormalize.ts +++ b/ckan/schema/denormalize.ts @@ -1,6 +1,6 @@ import type { Field, Schema } from "@dpkit/core" -import type { CkanField, CkanFieldInfo } from "../Field.ts" -import type { CkanSchema } from "../Schema.ts" +import type { CkanField, CkanFieldInfo } from "./Field.ts" +import type { CkanSchema } from "./Schema.ts" /** * Denormalizes a Table Schema to a CKAN schema format diff --git a/ckan/schema/index.ts b/ckan/schema/index.ts index b4084269..fc123c0f 100644 --- a/ckan/schema/index.ts +++ b/ckan/schema/index.ts @@ -1,3 +1,4 @@ export type { CkanSchema } from "./Schema.ts" export type { CkanField } from "./Field.ts" -export { normalizeCkanSchema } from "./process/normalize.ts" +export { normalizeCkanSchema } from "./normalize.ts" +export { denormalizeCkanSchema } from "./denormalize.ts" diff --git a/ckan/schema/process/normalize.ts b/ckan/schema/normalize.ts similarity index 95% rename from ckan/schema/process/normalize.ts rename to ckan/schema/normalize.ts index ddf5314b..030dfba4 100644 --- a/ckan/schema/process/normalize.ts +++ b/ckan/schema/normalize.ts @@ -10,8 +10,8 @@ import type { StringField, TimeField, } from "@dpkit/core" -import type { CkanField } from "../Field.ts" -import type { CkanSchema } from "../Schema.ts" +import type { CkanField } from "./Field.ts" +import type { CkanSchema } from "./Schema.ts" /** * Normalizes a CKAN schema to a Table Schema format diff --git a/core/README.md b/core/README.md index 5e811d9c..f7b98112 100644 --- a/core/README.md +++ b/core/README.md @@ -1,3 +1,3 @@ # @dpkit/core -dpkit is a fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). +dpkit is a fast data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/core/dialect/process/denormalize.ts b/core/dialect/denormalize.ts similarity index 57% rename from core/dialect/process/denormalize.ts rename to core/dialect/denormalize.ts index 17666b10..243a9182 100644 --- a/core/dialect/process/denormalize.ts +++ b/core/dialect/denormalize.ts @@ -1,5 +1,5 @@ -import type { Descriptor } from "../../general/index.ts" -import type { Dialect } from "../Dialect.ts" +import type { Descriptor } from "../general/index.ts" +import type { Dialect } from "./Dialect.ts" export function denormalizeDialect(dialect: Dialect) { dialect = globalThis.structuredClone(dialect) diff --git a/core/dialect/index.ts b/core/dialect/index.ts index 8ddda4d2..f71a211d 100644 --- a/core/dialect/index.ts +++ b/core/dialect/index.ts @@ -3,5 +3,5 @@ export { validateDialect } from "./validate.ts" export type { Dialect } from "./Dialect.ts" export { loadDialect } from "./load.ts" export { saveDialect } from "./save.ts" -export { normalizeDialect } from "./process/normalize.ts" -export { denormalizeDialect } from "./process/denormalize.ts" +export { normalizeDialect } from "./normalize.ts" +export { denormalizeDialect } from "./denormalize.ts" diff --git a/core/dialect/process/normalize.ts b/core/dialect/normalize.ts similarity index 91% rename from core/dialect/process/normalize.ts rename to core/dialect/normalize.ts index 8da6af89..afedfc7b 100644 --- a/core/dialect/process/normalize.ts +++ b/core/dialect/normalize.ts @@ -1,4 +1,4 @@ -import type { Descriptor } from "../../general/index.ts" +import type { Descriptor } from "../general/index.ts" export function normalizeDialect(descriptor: Descriptor) { descriptor = globalThis.structuredClone(descriptor) diff --git a/core/dialect/save.ts b/core/dialect/save.ts index bdcd89e7..f836922a 100644 --- a/core/dialect/save.ts +++ b/core/dialect/save.ts @@ -1,6 +1,6 @@ import { saveDescriptor } from "../general/index.ts" import type { Dialect } from "./Dialect.ts" -import { denormalizeDialect } from "./process/denormalize.ts" +import { denormalizeDialect } from "./denormalize.ts" const CURRENT_PROFILE = "https://datapackage.org/profiles/2.0/tabledialect.json" diff --git a/core/dialect/validate.ts b/core/dialect/validate.ts index 01ce15cf..9411ea87 100644 --- a/core/dialect/validate.ts +++ b/core/dialect/validate.ts @@ -1,7 +1,7 @@ import { type Descriptor, validateDescriptor } from "../general/index.ts" import { loadProfile } from "../general/index.ts" import type { Dialect } from "./Dialect.ts" -import { normalizeDialect } from "./process/normalize.ts" +import { normalizeDialect } from "./normalize.ts" const DEFAULT_PROFILE = "https://datapackage.org/profiles/1.0/tabledialect.json" diff --git a/core/field/Type.ts b/core/field/Type.ts new file mode 100644 index 00000000..3e881703 --- /dev/null +++ b/core/field/Type.ts @@ -0,0 +1,3 @@ +import type { Field } from "./Field.ts" + +export type FieldType = Exclude diff --git a/core/field/process/denormalize.ts b/core/field/denormalize.ts similarity index 58% rename from core/field/process/denormalize.ts rename to core/field/denormalize.ts index 264b70e9..99d18346 100644 --- a/core/field/process/denormalize.ts +++ b/core/field/denormalize.ts @@ -1,5 +1,5 @@ -import type { Descriptor } from "../../general/index.ts" -import type { Field } from "../Field.ts" +import type { Descriptor } from "../general/index.ts" +import type { Field } from "./Field.ts" export function denormalizeField(field: Field) { field = globalThis.structuredClone(field) diff --git a/core/field/index.ts b/core/field/index.ts index 2c797109..0547fecf 100644 --- a/core/field/index.ts +++ b/core/field/index.ts @@ -1,3 +1,5 @@ export type { Field } from "./Field.ts" +export type { FieldType } from "./Type.ts" export type * from "./types/index.ts" -export { normalizeField } from "./process/normalize.ts" +export { normalizeField } from "./normalize.ts" +export { denormalizeField } from "./denormalize.ts" diff --git a/core/field/process/normalize.ts b/core/field/normalize.ts similarity index 97% rename from core/field/process/normalize.ts rename to core/field/normalize.ts index af4af06a..beab5bab 100644 --- a/core/field/process/normalize.ts +++ b/core/field/normalize.ts @@ -1,4 +1,4 @@ -import type { Descriptor } from "../../general/index.ts" +import type { Descriptor } from "../general/index.ts" export function normalizeField(descriptor: Descriptor) { descriptor = globalThis.structuredClone(descriptor) diff --git a/core/field/types/String.ts b/core/field/types/String.ts index 1fa494b5..eefca2b2 100644 --- a/core/field/types/String.ts +++ b/core/field/types/String.ts @@ -11,13 +11,12 @@ export interface StringField extends BaseField { /** * Format of the string - * - default: any valid string * - email: valid email address * - uri: valid URI * - binary: base64 encoded string * - uuid: valid UUID string */ - format?: "default" | "email" | "uri" | "binary" | "uuid" | string + format?: "email" | "uri" | "binary" | "uuid" /** * Categories for enum values diff --git a/core/general/index.ts b/core/general/index.ts index 20eee670..d3d66287 100644 --- a/core/general/index.ts +++ b/core/general/index.ts @@ -16,4 +16,5 @@ export { normalizePath, denormalizePath, getFilename, + getProtocol, } from "./path.ts" diff --git a/core/general/path.ts b/core/general/path.ts index 35975ddb..6e741755 100644 --- a/core/general/path.ts +++ b/core/general/path.ts @@ -23,6 +23,15 @@ export function getName(filename?: string) { return slugify(name) } +export function getProtocol(path: string) { + try { + const url = new URL(path) + return url.protocol.replace(":", "") + } catch { + return "file" + } +} + export function getFormat(filename?: string) { return filename?.split(".").slice(-1)[0]?.toLowerCase() } diff --git a/core/package/process/denormalize.ts b/core/package/denormalize.ts similarity index 68% rename from core/package/process/denormalize.ts rename to core/package/denormalize.ts index 82954bab..522cbdf9 100644 --- a/core/package/process/denormalize.ts +++ b/core/package/denormalize.ts @@ -1,6 +1,6 @@ -import type { Descriptor } from "../../general/index.ts" -import { denormalizeResource } from "../../resource/index.ts" -import type { Package } from "../Package.ts" +import type { Descriptor } from "../general/index.ts" +import { denormalizeResource } from "../resource/index.ts" +import type { Package } from "./Package.ts" export function denormalizePackage( dataPackage: Package, diff --git a/core/package/index.ts b/core/package/index.ts index a019a487..b0b2d762 100644 --- a/core/package/index.ts +++ b/core/package/index.ts @@ -3,7 +3,7 @@ export { assertPackage } from "./assert.ts" export { loadPackageDescriptor } from "./load.ts" export { savePackageDescriptor } from "./save.ts" export { validatePackageDescriptor } from "./validate.ts" -export { normalizePackage } from "./process/normalize.ts" -export { denormalizePackage } from "./process/denormalize.ts" +export { normalizePackage } from "./normalize.ts" +export { denormalizePackage } from "./denormalize.ts" export type { Contributor } from "./Contributor.ts" export { mergePackages } from "./merge.ts" diff --git a/core/package/process/normalize.ts b/core/package/normalize.ts similarity index 90% rename from core/package/process/normalize.ts rename to core/package/normalize.ts index de87b3b3..deaf4378 100644 --- a/core/package/process/normalize.ts +++ b/core/package/normalize.ts @@ -1,5 +1,5 @@ -import type { Descriptor } from "../../general/index.ts" -import { normalizeResource } from "../../resource/index.ts" +import type { Descriptor } from "../general/index.ts" +import { normalizeResource } from "../resource/index.ts" export function normalizePackage( descriptor: Descriptor, diff --git a/core/package/save.ts b/core/package/save.ts index bed14a7b..bb8bd24a 100644 --- a/core/package/save.ts +++ b/core/package/save.ts @@ -1,6 +1,6 @@ import { getBasepath, saveDescriptor } from "../general/index.ts" import type { Package } from "./Package.ts" -import { denormalizePackage } from "./process/denormalize.ts" +import { denormalizePackage } from "./denormalize.ts" const CURRENT_PROFILE = "https://datapackage.org/profiles/2.0/datapackage.json" diff --git a/core/package/validate.ts b/core/package/validate.ts index 4df8d899..91bb7581 100644 --- a/core/package/validate.ts +++ b/core/package/validate.ts @@ -1,7 +1,7 @@ import { type Descriptor, validateDescriptor } from "../general/index.ts" import { loadProfile } from "../general/index.ts" import type { Package } from "./Package.ts" -import { normalizePackage } from "./process/normalize.ts" +import { normalizePackage } from "./normalize.ts" const DEFAULT_PROFILE = "https://datapackage.org/profiles/1.0/datapackage.json" diff --git a/core/resource/process/denormalize.ts b/core/resource/denormalize.ts similarity index 82% rename from core/resource/process/denormalize.ts rename to core/resource/denormalize.ts index 528b4f89..4b661645 100644 --- a/core/resource/process/denormalize.ts +++ b/core/resource/denormalize.ts @@ -1,8 +1,8 @@ -import { denormalizeDialect } from "../../dialect/index.ts" -import { denormalizePath } from "../../general/index.ts" -import type { Descriptor } from "../../general/index.ts" -import { denormalizeSchema } from "../../schema/index.ts" -import type { Resource } from "../Resource.ts" +import { denormalizeDialect } from "../dialect/index.ts" +import { denormalizePath } from "../general/index.ts" +import type { Descriptor } from "../general/index.ts" +import { denormalizeSchema } from "../schema/index.ts" +import type { Resource } from "./Resource.ts" export function denormalizeResource( resource: Resource, diff --git a/core/resource/index.ts b/core/resource/index.ts index 224df1d7..7037d5e1 100644 --- a/core/resource/index.ts +++ b/core/resource/index.ts @@ -4,8 +4,8 @@ export { assertResource } from "./assert.ts" export { loadResourceDescriptor } from "./load.ts" export { saveResourceDescriptor } from "./save.ts" export { validateResourceDescriptor } from "./validate.ts" -export { normalizeResource } from "./process/normalize.ts" -export { denormalizeResource } from "./process/denormalize.ts" +export { normalizeResource } from "./normalize.ts" +export { denormalizeResource } from "./denormalize.ts" export type { Source } from "./Source.ts" export type { License } from "./License.ts" export { loadResourceDialect } from "./dialect.ts" diff --git a/core/resource/infer.spec.ts b/core/resource/infer.spec.ts new file mode 100644 index 00000000..564aeec7 --- /dev/null +++ b/core/resource/infer.spec.ts @@ -0,0 +1,125 @@ +import { describe, expect, it } from "vitest" +import { inferResourceFormat, inferResourceName } from "./infer.ts" + +describe("inferResourceName", () => { + it("returns existing name when provided", () => { + const resource = { name: "existing-name" } + expect(inferResourceName(resource)).toBe("existing-name") + }) + + it("infers name from single string path", () => { + const resource = { path: "/data/users.csv" } + expect(inferResourceName(resource)).toBe("users") + }) + + it("infers name from first path in array", () => { + const resource = { path: ["/data/users.csv", "/data/backup.csv"] } + expect(inferResourceName(resource)).toBe("users") + }) + + it("infers name from URL path", () => { + const resource = { path: "https://example.com/data/products.json" } + expect(inferResourceName(resource)).toBe("products") + }) + + it("returns default name when no path or name", () => { + const resource = {} + expect(inferResourceName(resource)).toBe("resource") + }) + + it("returns default name when path has no filename", () => { + const resource = { path: "/data/folder/" } + expect(inferResourceName(resource)).toBe("resource") + }) + + it("handles complex filename with multiple dots", () => { + const resource = { path: "/data/file.backup.csv" } + expect(inferResourceName(resource)).toBe("file") + }) + + it("slugifies filename with spaces and special characters", () => { + const resource = { path: "/data/My Data File!.csv" } + expect(inferResourceName(resource)).toBe("my-data-file") + }) +}) + +describe("inferResourceFormat", () => { + it("returns existing format when provided", () => { + const resource = { format: "json" } + expect(inferResourceFormat(resource)).toBe("json") + }) + + it("infers format from single string path", () => { + const resource = { path: "/data/users.csv" } + expect(inferResourceFormat(resource)).toBe("csv") + }) + + it("infers format from first path in array", () => { + const resource = { path: ["/data/users.xlsx", "/data/backup.csv"] } + expect(inferResourceFormat(resource)).toBe("xlsx") + }) + + it("infers format from URL path", () => { + const resource = { path: "https://example.com/data/products.json" } + expect(inferResourceFormat(resource)).toBe("json") + }) + + it("returns lowercase format", () => { + const resource = { path: "/data/file.CSV" } + expect(inferResourceFormat(resource)).toBe("csv") + }) + + it("handles multiple extensions", () => { + const resource = { path: "/data/file.tar.gz" } + expect(inferResourceFormat(resource)).toBe("gz") + }) + + it("returns undefined when no path", () => { + const resource = {} + expect(inferResourceFormat(resource)).toBeUndefined() + }) + + it("returns undefined when path has no extension", () => { + const resource = { path: "/data/file" } + expect(inferResourceFormat(resource)).toBeUndefined() + }) + + it("returns undefined when filename cannot be determined", () => { + const resource = { path: "/data/folder/" } + expect(inferResourceFormat(resource)).toBeUndefined() + }) + + it("infers postgresql protocol from connection string", () => { + const resource = { + path: "postgresql://user:password@localhost:5432/database", + } + expect(inferResourceFormat(resource)).toBe("postgresql") + }) + + it("infers mysql protocol from connection string", () => { + const resource = { path: "mysql://user:password@localhost:3306/database" } + expect(inferResourceFormat(resource)).toBe("mysql") + }) + + it("infers sqlite protocol from file path", () => { + const resource = { path: "sqlite:///path/to/database.db" } + expect(inferResourceFormat(resource)).toBe("sqlite") + }) + + it("infers sqlite protocol with file scheme", () => { + const resource = { path: "sqlite://localhost/path/to/database.db" } + expect(inferResourceFormat(resource)).toBe("sqlite") + }) + + it("handles postgres protocol with ssl parameters", () => { + const resource = { + path: "postgresql://user:pass@host:5432/db?sslmode=require", + } + expect(inferResourceFormat(resource)).toBe("postgresql") + }) + + it("handles mysql protocol with options", () => { + const resource = { path: "mysql://user:pass@host:3306/db?charset=utf8" } + expect(inferResourceFormat(resource)).toBe("mysql") + }) +}) diff --git a/core/resource/infer.ts b/core/resource/infer.ts index ceecbf8a..02e06ff6 100644 --- a/core/resource/infer.ts +++ b/core/resource/infer.ts @@ -1,4 +1,9 @@ -import { getFilename, getFormat, getName } from "../general/index.ts" +import { + getFilename, + getFormat, + getName, + getProtocol, +} from "../general/index.ts" import type { Resource } from "./Resource.ts" export function inferResourceName(resource: Partial) { @@ -23,12 +28,21 @@ export function inferResourceFormat(resource: Partial) { const path = Array.isArray(resource.path) ? resource.path[0] : resource.path + if (path) { - const filename = getFilename(path) - format = getFormat(filename) + const protocol = getProtocol(path) + + if (DATABASE_PROTOCOLS.includes(protocol)) { + format = protocol + } else { + const filename = getFilename(path) + format = getFormat(filename) + } } } } return format } + +const DATABASE_PROTOCOLS = ["postgresql", "mysql", "sqlite"] diff --git a/core/resource/process/normalize.ts b/core/resource/normalize.ts similarity index 88% rename from core/resource/process/normalize.ts rename to core/resource/normalize.ts index 018ced18..4fd83fe4 100644 --- a/core/resource/process/normalize.ts +++ b/core/resource/normalize.ts @@ -1,7 +1,7 @@ -import { normalizeDialect } from "../../dialect/index.ts" -import { isDescriptor, normalizePath } from "../../general/index.ts" -import type { Descriptor } from "../../general/index.ts" -import { normalizeSchema } from "../../schema/index.ts" +import { normalizeDialect } from "../dialect/index.ts" +import { isDescriptor, normalizePath } from "../general/index.ts" +import type { Descriptor } from "../general/index.ts" +import { normalizeSchema } from "../schema/index.ts" export function normalizeResource( descriptor: Descriptor, diff --git a/core/resource/save.ts b/core/resource/save.ts index bc83ae37..255d437c 100644 --- a/core/resource/save.ts +++ b/core/resource/save.ts @@ -1,6 +1,6 @@ import { getBasepath, saveDescriptor } from "../general/index.ts" import type { Resource } from "./Resource.ts" -import { denormalizeResource } from "./process/denormalize.ts" +import { denormalizeResource } from "./denormalize.ts" const CURRENT_PROFILE = "https://datapackage.org/profiles/2.0/dataresource.json" diff --git a/core/resource/validate.ts b/core/resource/validate.ts index ca631847..d8264551 100644 --- a/core/resource/validate.ts +++ b/core/resource/validate.ts @@ -4,7 +4,7 @@ import { type Descriptor, validateDescriptor } from "../general/index.ts" import { loadProfile } from "../general/index.ts" import { loadSchema } from "../schema/index.ts" import type { Resource } from "./Resource.ts" -import { normalizeResource } from "./process/normalize.ts" +import { normalizeResource } from "./normalize.ts" const DEFAULT_PROFILE = "https://datapackage.org/profiles/1.0/dataresource.json" diff --git a/core/schema/process/denormalize.ts b/core/schema/denormalize.ts similarity index 58% rename from core/schema/process/denormalize.ts rename to core/schema/denormalize.ts index 8e1d4383..b382293a 100644 --- a/core/schema/process/denormalize.ts +++ b/core/schema/denormalize.ts @@ -1,5 +1,5 @@ -import type { Descriptor } from "../../general/index.ts" -import type { Schema } from "../Schema.ts" +import type { Descriptor } from "../general/index.ts" +import type { Schema } from "./Schema.ts" export function denormalizeSchema(schema: Schema) { schema = globalThis.structuredClone(schema) diff --git a/core/schema/index.ts b/core/schema/index.ts index 4c8341dd..92e1b7a3 100644 --- a/core/schema/index.ts +++ b/core/schema/index.ts @@ -4,5 +4,5 @@ export { assertSchema } from "./assert.ts" export { loadSchema } from "./load.ts" export { saveSchema } from "./save.ts" export { validateSchema } from "./validate.ts" -export { normalizeSchema } from "./process/normalize.ts" -export { denormalizeSchema } from "./process/denormalize.ts" +export { normalizeSchema } from "./normalize.ts" +export { denormalizeSchema } from "./denormalize.ts" diff --git a/core/schema/process/normalize.ts b/core/schema/normalize.ts similarity index 94% rename from core/schema/process/normalize.ts rename to core/schema/normalize.ts index 4b35afd3..bea81f1b 100644 --- a/core/schema/process/normalize.ts +++ b/core/schema/normalize.ts @@ -1,6 +1,6 @@ import invariant from "tiny-invariant" -import { normalizeField } from "../../field/index.ts" -import type { Descriptor } from "../../general/index.ts" +import { normalizeField } from "../field/index.ts" +import type { Descriptor } from "../general/index.ts" export function normalizeSchema(descriptor: Descriptor) { descriptor = globalThis.structuredClone(descriptor) diff --git a/core/schema/save.ts b/core/schema/save.ts index 5f6c801e..f289d3aa 100644 --- a/core/schema/save.ts +++ b/core/schema/save.ts @@ -1,6 +1,6 @@ import { saveDescriptor } from "../general/index.ts" import type { Schema } from "./Schema.ts" -import { denormalizeSchema } from "./process/denormalize.ts" +import { denormalizeSchema } from "./denormalize.ts" const CURRENT_PROFILE = "https://datapackage.org/profiles/2.0/tableschema.json" diff --git a/core/schema/validate.ts b/core/schema/validate.ts index 4016e4dc..f022a24c 100644 --- a/core/schema/validate.ts +++ b/core/schema/validate.ts @@ -1,7 +1,7 @@ import { type Descriptor, validateDescriptor } from "../general/index.ts" import { loadProfile } from "../general/index.ts" import type { Schema } from "./Schema.ts" -import { normalizeSchema } from "./process/normalize.ts" +import { normalizeSchema } from "./normalize.ts" const DEFAULT_PROFILE = "https://datapackage.org/profiles/1.0/tableschema.json" diff --git a/csv/README.md b/csv/README.md index 48dd1031..b7f1be2a 100644 --- a/csv/README.md +++ b/csv/README.md @@ -1,3 +1,3 @@ # @dpkit/csv -dpkit is a fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). +dpkit is a fast data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/csv/dialect/infer.ts b/csv/dialect/infer.ts index e2df3fb3..d5fd88d4 100644 --- a/csv/dialect/infer.ts +++ b/csv/dialect/infer.ts @@ -1,16 +1,20 @@ import { text } from "node:stream/consumers" import type { Dialect, Resource } from "@dpkit/core" import { loadFileStream } from "@dpkit/file" -import type { InferDialectOptions } from "@dpkit/table" import { default as CsvSnifferFactory } from "csv-sniffer" -const POSSIBLE_DELIMITERS = [",", ";", ":", "|", "\t", "^", "*", "&"] +const CSV_DELIMITERS = [",", ";", ":", "|", "\t", "^", "*", "&"] +const TSV_DELIMITERS = ["\t"] export async function inferCsvDialect( resource: Partial, - options?: InferDialectOptions, + options?: { + sampleBytes?: number + }, ) { const { sampleBytes = 10_000 } = options ?? {} + const isTabs = resource.format === "tsv" + const dialect: Dialect = {} if (resource.path) { @@ -19,16 +23,13 @@ export async function inferCsvDialect( }) const sample = await text(stream) + const result = sniffSample(sample, isTabs ? TSV_DELIMITERS : CSV_DELIMITERS) - const CsvSniffer = CsvSnifferFactory() - const sniffer = new CsvSniffer(POSSIBLE_DELIMITERS) - const result = sniffer.sniff(sample) - - if (result.delimiter) { + if (result?.delimiter) { dialect.delimiter = result.delimiter } - if (result.quoteChar) { + if (result?.quoteChar) { dialect.quoteChar = result.quoteChar } @@ -44,3 +45,15 @@ export async function inferCsvDialect( return dialect } + +// Sniffer can fail for some reasons +function sniffSample(sample: string, delimiters: string[]) { + try { + const CsvSniffer = CsvSnifferFactory() + const sniffer = new CsvSniffer(delimiters) + const result = sniffer.sniff(sample) + return result + } catch { + return undefined + } +} diff --git a/csv/package.json b/csv/package.json index 08d7eeb6..0b6deb41 100644 --- a/csv/package.json +++ b/csv/package.json @@ -27,7 +27,7 @@ "@dpkit/file": "workspace:*", "@dpkit/table": "workspace:*", "csv-sniffer": "^0.1.1", - "nodejs-polars": "^0.21.0" + "nodejs-polars": "^0.21.1" }, "devDependencies": { "@dpkit/test": "workspace:*" diff --git a/csv/plugin.ts b/csv/plugin.ts index a8be7407..137ba8d6 100644 --- a/csv/plugin.ts +++ b/csv/plugin.ts @@ -1,37 +1,29 @@ import type { Resource } from "@dpkit/core" import { inferResourceFormat } from "@dpkit/core" -import type { InferDialectOptions, TablePlugin } from "@dpkit/table" +import type { LoadTableOptions } from "@dpkit/table" +import type { TablePlugin } from "@dpkit/table" import type { SaveTableOptions, Table } from "@dpkit/table" -import { inferCsvDialect } from "./dialect/index.ts" import { loadCsvTable, saveCsvTable } from "./table/index.ts" export class CsvPlugin implements TablePlugin { - async inferDialect( - resource: Partial, - options?: InferDialectOptions, - ) { - const isCsv = getIsCsv(resource) - if (!isCsv) return undefined + async loadTable(resource: Partial, options?: LoadTableOptions) { + const csvFormat = getCsvFormat(resource) + if (!csvFormat) return undefined - return await inferCsvDialect(resource, options) - } - - async loadTable(resource: Partial) { - const isCsv = getIsCsv(resource) - if (!isCsv) return undefined - - return await loadCsvTable(resource) + return await loadCsvTable({ ...resource, format: csvFormat }, options) } async saveTable(table: Table, options: SaveTableOptions) { - const isCsv = getIsCsv(options) - if (!isCsv) return undefined + const { path, format } = options + + const csvFormat = getCsvFormat({ path, format }) + if (!csvFormat) return undefined - return await saveCsvTable(table, options) + return await saveCsvTable(table, { ...options, format: csvFormat }) } } -function getIsCsv(resource: Partial) { +function getCsvFormat(resource: Partial) { const format = inferResourceFormat(resource) - return ["csv", "tsv"].includes(format ?? "") + return format === "csv" || format === "tsv" ? format : undefined } diff --git a/csv/table/fixtures/generated/loadCsvTable-should-load-remote-file-multipart_959749322/recording.har b/csv/table/fixtures/generated/loadCsvTable-should-load-remote-file-multipart_959749322/recording.har new file mode 100644 index 00000000..d5fd1edc --- /dev/null +++ b/csv/table/fixtures/generated/loadCsvTable-should-load-remote-file-multipart_959749322/recording.har @@ -0,0 +1,428 @@ +{ + "log": { + "_recordingName": "loadCsvTable-should load remote file (multipart)", + "creator": { + "comment": "persister:fs", + "name": "Polly.JS", + "version": "6.0.6" + }, + "entries": [ + { + "_id": "26b4c877aae0bbdbc9889671caf26f73", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [], + "headersSize": 147, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://gist.githubusercontent.com/roll/ba1c358f1daa994f8094633669d60f50/raw/03e0e54f9d4df7ff08b0d0db937fe5d720d8dfe6/table2.csv" + }, + "response": { + "bodySize": 8, + "content": { + "mimeType": "text/plain; charset=utf-8", + "size": 8, + "text": "3,german" + }, + "cookies": [], + "headers": [ + { + "name": "accept-ranges", + "value": "bytes" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "cache-control", + "value": "max-age=300" + }, + { + "name": "connection", + "value": "keep-alive" + }, + { + "name": "content-length", + "value": "8" + }, + { + "name": "content-security-policy", + "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" + }, + { + "name": "content-type", + "value": "text/plain; charset=utf-8" + }, + { + "name": "cross-origin-resource-policy", + "value": "cross-origin" + }, + { + "name": "date", + "value": "Sat, 06 Sep 2025 10:04:07 GMT" + }, + { + "name": "etag", + "value": "W/\"5b8bd499e42ca79dc52ec0b203194d24d475f240c0b8ff6e0aab54d5798e008a\"" + }, + { + "name": "expires", + "value": "Sat, 06 Sep 2025 10:09:07 GMT" + }, + { + "name": "source-age", + "value": "0" + }, + { + "name": "strict-transport-security", + "value": "max-age=31536000" + }, + { + "name": "vary", + "value": "Authorization,Accept-Encoding" + }, + { + "name": "via", + "value": "1.1 varnish" + }, + { + "name": "x-cache", + "value": "MISS" + }, + { + "name": "x-cache-hits", + "value": "0" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "x-fastly-request-id", + "value": "6ac56b400c8e699392b52f7722022152964648e4" + }, + { + "name": "x-frame-options", + "value": "deny" + }, + { + "name": "x-github-request-id", + "value": "9CF9:39D0D:5218B4:64FEF2:68BC070C" + }, + { + "name": "x-served-by", + "value": "cache-lis1490037-LIS" + }, + { + "name": "x-timer", + "value": "S1757153047.464387,VS0,VE203" + }, + { + "name": "x-xss-protection", + "value": "1; mode=block" + } + ], + "headersSize": 875, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-09-06T10:04:07.323Z", + "time": 502, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 502 + } + }, + { + "_id": "4c7b0fb73c46c7315d57cbb41752f90c", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [], + "headersSize": 146, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://gist.githubusercontent.com/roll/d20416f5e6dfc3fc1a7c4eef8452d581/raw/1c9cd1a020389921bf83e5a0395bb00c6b27402d/tabl1.csv" + }, + "response": { + "bodySize": 29, + "content": { + "mimeType": "text/plain; charset=utf-8", + "size": 29, + "text": "id,name\n1,english\n2,中国人" + }, + "cookies": [], + "headers": [ + { + "name": "accept-ranges", + "value": "bytes" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "cache-control", + "value": "max-age=300" + }, + { + "name": "connection", + "value": "keep-alive" + }, + { + "name": "content-length", + "value": "29" + }, + { + "name": "content-security-policy", + "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" + }, + { + "name": "content-type", + "value": "text/plain; charset=utf-8" + }, + { + "name": "cross-origin-resource-policy", + "value": "cross-origin" + }, + { + "name": "date", + "value": "Sat, 06 Sep 2025 10:04:07 GMT" + }, + { + "name": "etag", + "value": "W/\"7e7cf438cb55bf28aa7f609346bfa9dc9efadbcfe2c8e2ea5c7d093632951d62\"" + }, + { + "name": "expires", + "value": "Sat, 06 Sep 2025 10:09:07 GMT" + }, + { + "name": "source-age", + "value": "0" + }, + { + "name": "strict-transport-security", + "value": "max-age=31536000" + }, + { + "name": "vary", + "value": "Authorization,Accept-Encoding" + }, + { + "name": "via", + "value": "1.1 varnish" + }, + { + "name": "x-cache", + "value": "MISS" + }, + { + "name": "x-cache-hits", + "value": "0" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "x-fastly-request-id", + "value": "7036cce4ccc1398c2666d71bb47b5cbfa8f5ba68" + }, + { + "name": "x-frame-options", + "value": "deny" + }, + { + "name": "x-github-request-id", + "value": "8FFA:1BED5D:52749A:655FAA:68BC06EC" + }, + { + "name": "x-served-by", + "value": "cache-lis1490044-LIS" + }, + { + "name": "x-timer", + "value": "S1757153047.468876,VS0,VE201" + }, + { + "name": "x-xss-protection", + "value": "1; mode=block" + } + ], + "headersSize": 877, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-09-06T10:04:07.323Z", + "time": 504, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 504 + } + }, + { + "_id": "4c7b0fb73c46c7315d57cbb41752f90c", + "_order": 1, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [], + "headersSize": 146, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://gist.githubusercontent.com/roll/d20416f5e6dfc3fc1a7c4eef8452d581/raw/1c9cd1a020389921bf83e5a0395bb00c6b27402d/tabl1.csv" + }, + "response": { + "bodySize": 29, + "content": { + "mimeType": "text/plain; charset=utf-8", + "size": 29, + "text": "id,name\n1,english\n2,中国人" + }, + "cookies": [], + "headers": [ + { + "name": "accept-ranges", + "value": "bytes" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "cache-control", + "value": "max-age=300" + }, + { + "name": "connection", + "value": "keep-alive" + }, + { + "name": "content-length", + "value": "29" + }, + { + "name": "content-security-policy", + "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" + }, + { + "name": "content-type", + "value": "text/plain; charset=utf-8" + }, + { + "name": "cross-origin-resource-policy", + "value": "cross-origin" + }, + { + "name": "date", + "value": "Sat, 06 Sep 2025 10:04:07 GMT" + }, + { + "name": "etag", + "value": "W/\"7e7cf438cb55bf28aa7f609346bfa9dc9efadbcfe2c8e2ea5c7d093632951d62\"" + }, + { + "name": "expires", + "value": "Sat, 06 Sep 2025 10:09:07 GMT" + }, + { + "name": "source-age", + "value": "0" + }, + { + "name": "strict-transport-security", + "value": "max-age=31536000" + }, + { + "name": "vary", + "value": "Authorization,Accept-Encoding" + }, + { + "name": "via", + "value": "1.1 varnish" + }, + { + "name": "x-cache", + "value": "HIT" + }, + { + "name": "x-cache-hits", + "value": "1" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "x-fastly-request-id", + "value": "b9614519cddeeea737c2902dbbbcfc083479cfe9" + }, + { + "name": "x-frame-options", + "value": "deny" + }, + { + "name": "x-github-request-id", + "value": "8FFA:1BED5D:52749A:655FAA:68BC06EC" + }, + { + "name": "x-served-by", + "value": "cache-lis1490044-LIS" + }, + { + "name": "x-timer", + "value": "S1757153048.833694,VS0,VE1" + }, + { + "name": "x-xss-protection", + "value": "1; mode=block" + } + ], + "headersSize": 874, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-09-06T10:04:07.829Z", + "time": 38, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 38 + } + } + ], + "pages": [], + "version": "1.2" + } +} diff --git a/csv/table/fixtures/generated/loadCsvTable-file-variations-should-load-remote-file-multipart_2684063187/recording.har b/csv/table/fixtures/generated/loadCsvTable-should-load-remote-file_2012170585/recording.har similarity index 74% rename from csv/table/fixtures/generated/loadCsvTable-file-variations-should-load-remote-file-multipart_2684063187/recording.har rename to csv/table/fixtures/generated/loadCsvTable-should-load-remote-file_2012170585/recording.har index b35e26df..11a275df 100644 --- a/csv/table/fixtures/generated/loadCsvTable-file-variations-should-load-remote-file-multipart_2684063187/recording.har +++ b/csv/table/fixtures/generated/loadCsvTable-should-load-remote-file_2012170585/recording.har @@ -1,6 +1,6 @@ { "log": { - "_recordingName": "loadCsvTable-file variations-should load remote file (multipart)", + "_recordingName": "loadCsvTable-should load remote file", "creator": { "comment": "persister:fs", "name": "Polly.JS", @@ -8,25 +8,25 @@ }, "entries": [ { - "_id": "26b4c877aae0bbdbc9889671caf26f73", + "_id": "ae14a4d32b5c31cc8710730c067aa383", "_order": 0, "cache": {}, "request": { "bodySize": 0, "cookies": [], "headers": [], - "headersSize": 147, + "headersSize": 126, "httpVersion": "HTTP/1.1", "method": "GET", "queryString": [], - "url": "https://gist.githubusercontent.com/roll/ba1c358f1daa994f8094633669d60f50/raw/03e0e54f9d4df7ff08b0d0db937fe5d720d8dfe6/table2.csv" + "url": "https://raw.githubusercontent.com/datisthq/dpkit-typescript/refs/heads/main/core/package/fixtures/table.csv" }, "response": { - "bodySize": 8, + "bodySize": 51, "content": { "mimeType": "text/plain; charset=utf-8", - "size": 8, - "text": "3,german" + "size": 51, + "text": "id,name\n1,english\n2,中国人\n" }, "cookies": [], "headers": [ @@ -46,9 +46,13 @@ "name": "connection", "value": "keep-alive" }, + { + "name": "content-encoding", + "value": "gzip" + }, { "name": "content-length", - "value": "8" + "value": "51" }, { "name": "content-security-policy", @@ -64,15 +68,15 @@ }, { "name": "date", - "value": "Thu, 07 Aug 2025 07:49:22 GMT" + "value": "Sat, 06 Sep 2025 10:04:07 GMT" }, { "name": "etag", - "value": "W/\"5b8bd499e42ca79dc52ec0b203194d24d475f240c0b8ff6e0aab54d5798e008a\"" + "value": "W/\"0172a2fd99319bed82fe7cccbd7a44b27a77f7200caf0d04b7f23cbb6b81026d\"" }, { "name": "expires", - "value": "Thu, 07 Aug 2025 07:54:22 GMT" + "value": "Sat, 06 Sep 2025 10:09:07 GMT" }, { "name": "source-age", @@ -104,7 +108,7 @@ }, { "name": "x-fastly-request-id", - "value": "e3f6b1cd316c828a6d26a6c7abf4eacabcd7427c" + "value": "56701f5afb8cd46af7b1bca7d05532ceae2f4641" }, { "name": "x-frame-options", @@ -112,29 +116,29 @@ }, { "name": "x-github-request-id", - "value": "C776:1DBBBF:159D488:18A1D41:68945A7A" + "value": "4A16:23FABD:51E323:64C9C4:68BC0713" }, { "name": "x-served-by", - "value": "cache-lis1490045-LIS" + "value": "cache-lis1490057-LIS" }, { "name": "x-timer", - "value": "S1754552962.188191,VS0,VE191" + "value": "S1757153047.952634,VS0,VE193" }, { "name": "x-xss-protection", "value": "1; mode=block" } ], - "headersSize": 878, + "headersSize": 901, "httpVersion": "HTTP/1.1", "redirectURL": "", "status": 200, "statusText": "OK" }, - "startedDateTime": "2025-08-07T07:49:22.095Z", - "time": 396, + "startedDateTime": "2025-09-06T10:04:06.800Z", + "time": 420, "timings": { "blocked": -1, "connect": -1, @@ -142,29 +146,29 @@ "receive": 0, "send": 0, "ssl": -1, - "wait": 396 + "wait": 420 } }, { - "_id": "4c7b0fb73c46c7315d57cbb41752f90c", - "_order": 0, + "_id": "ae14a4d32b5c31cc8710730c067aa383", + "_order": 1, "cache": {}, "request": { "bodySize": 0, "cookies": [], "headers": [], - "headersSize": 146, + "headersSize": 126, "httpVersion": "HTTP/1.1", "method": "GET", "queryString": [], - "url": "https://gist.githubusercontent.com/roll/d20416f5e6dfc3fc1a7c4eef8452d581/raw/1c9cd1a020389921bf83e5a0395bb00c6b27402d/tabl1.csv" + "url": "https://raw.githubusercontent.com/datisthq/dpkit-typescript/refs/heads/main/core/package/fixtures/table.csv" }, "response": { - "bodySize": 29, + "bodySize": 51, "content": { "mimeType": "text/plain; charset=utf-8", - "size": 29, - "text": "id,name\n1,english\n2,中国人" + "size": 51, + "text": "id,name\n1,english\n2,中国人\n" }, "cookies": [], "headers": [ @@ -184,9 +188,13 @@ "name": "connection", "value": "keep-alive" }, + { + "name": "content-encoding", + "value": "gzip" + }, { "name": "content-length", - "value": "29" + "value": "51" }, { "name": "content-security-policy", @@ -202,15 +210,15 @@ }, { "name": "date", - "value": "Thu, 07 Aug 2025 07:49:22 GMT" + "value": "Sat, 06 Sep 2025 10:04:07 GMT" }, { "name": "etag", - "value": "W/\"7e7cf438cb55bf28aa7f609346bfa9dc9efadbcfe2c8e2ea5c7d093632951d62\"" + "value": "W/\"0172a2fd99319bed82fe7cccbd7a44b27a77f7200caf0d04b7f23cbb6b81026d\"" }, { "name": "expires", - "value": "Thu, 07 Aug 2025 07:54:22 GMT" + "value": "Sat, 06 Sep 2025 10:09:07 GMT" }, { "name": "source-age", @@ -230,11 +238,11 @@ }, { "name": "x-cache", - "value": "MISS" + "value": "HIT" }, { "name": "x-cache-hits", - "value": "0" + "value": "1" }, { "name": "x-content-type-options", @@ -242,7 +250,7 @@ }, { "name": "x-fastly-request-id", - "value": "b119a2897ff927f7e523c0d19c3bd2e598e02caa" + "value": "085d4da4bf6a0830938a3b6b312f898481ecce46" }, { "name": "x-frame-options", @@ -250,29 +258,29 @@ }, { "name": "x-github-request-id", - "value": "DEC0:24DBEA:15E02C0:193FD98:68945A7D" + "value": "4A16:23FABD:51E323:64C9C4:68BC0713" }, { "name": "x-served-by", - "value": "cache-lis1490046-LIS" + "value": "cache-lis1490057-LIS" }, { "name": "x-timer", - "value": "S1754552962.178760,VS0,VE204" + "value": "S1757153047.226376,VS0,VE1" }, { "name": "x-xss-protection", "value": "1; mode=block" } ], - "headersSize": 879, + "headersSize": 898, "httpVersion": "HTTP/1.1", "redirectURL": "", "status": 200, "statusText": "OK" }, - "startedDateTime": "2025-08-07T07:49:22.095Z", - "time": 406, + "startedDateTime": "2025-09-06T10:04:07.224Z", + "time": 48, "timings": { "blocked": -1, "connect": -1, @@ -280,7 +288,7 @@ "receive": 0, "send": 0, "ssl": -1, - "wait": 406 + "wait": 48 } } ], diff --git a/csv/table/load.spec.ts b/csv/table/load.spec.ts index 0ba8bb56..dc32c372 100644 --- a/csv/table/load.spec.ts +++ b/csv/table/load.spec.ts @@ -4,372 +4,379 @@ import { useRecording } from "@dpkit/test" import { describe, expect, it } from "vitest" import { loadCsvTable } from "./load.ts" +useRecording() + describe("loadCsvTable", () => { - useRecording() + it("should load local file", async () => { + const path = await writeTempFile("id,name\n1,english\n2,中文") + const table = await loadCsvTable({ path }) + + expect((await table.collect()).toRecords()).toEqual([ + { id: 1, name: "english" }, + { id: 2, name: "中文" }, + ]) + }) - describe("file variations", () => { - it("should load local file", async () => { - const path = await writeTempFile("id,name\n1,english\n2,中文") - const table = await loadCsvTable({ path }) + it("should load local file (multipart)", async () => { + const path1 = await writeTempFile("id,name\n1,english") + const path2 = await writeTempFile("2,中文\n3,german") - expect((await table.collect()).toRecords()).toEqual([ - { id: "1", name: "english" }, - { id: "2", name: "中文" }, - ]) - }) + const table = await loadCsvTable({ path: [path1, path2] }) - it("should load local file (multipart)", async () => { - const path1 = await writeTempFile("id,name\n1,english") - const path2 = await writeTempFile("2,中文\n3,german") - const table = await loadCsvTable({ path: [path1, path2] }) + expect((await table.collect()).toRecords()).toEqual([ + { id: 1, name: "english" }, + { id: 2, name: "中文" }, + { id: 3, name: "german" }, + ]) + }) - expect((await table.collect()).toRecords()).toEqual([ - { id: "1", name: "english" }, - { id: "2", name: "中文" }, - { id: "3", name: "german" }, - ]) + it("should load remote file", async () => { + const table = await loadCsvTable({ + path: "https://raw.githubusercontent.com/datisthq/dpkit-typescript/refs/heads/main/core/package/fixtures/table.csv", }) - it("should load remote file", async () => { - const table = await loadCsvTable({ - path: "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/core/package/fixtures/table.csv", - }) + expect((await table.collect()).toRecords()).toEqual([ + { id: 1, name: "english" }, + { id: 2, name: "中国人" }, + ]) + }) - expect((await table.collect()).toRecords()).toEqual([ - { id: "1", name: "english" }, - { id: "2", name: "中国人" }, - ]) + it("should load remote file (multipart)", async () => { + const table = await loadCsvTable({ + path: [ + "https://gist.githubusercontent.com/roll/d20416f5e6dfc3fc1a7c4eef8452d581/raw/1c9cd1a020389921bf83e5a0395bb00c6b27402d/tabl1.csv", + "https://gist.githubusercontent.com/roll/ba1c358f1daa994f8094633669d60f50/raw/03e0e54f9d4df7ff08b0d0db937fe5d720d8dfe6/table2.csv", + ], }) - it("should load remote file (multipart)", async () => { - const table = await loadCsvTable({ - path: [ - "https://gist.githubusercontent.com/roll/d20416f5e6dfc3fc1a7c4eef8452d581/raw/1c9cd1a020389921bf83e5a0395bb00c6b27402d/tabl1.csv", - "https://gist.githubusercontent.com/roll/ba1c358f1daa994f8094633669d60f50/raw/03e0e54f9d4df7ff08b0d0db937fe5d720d8dfe6/table2.csv", - ], - }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: "1", name: "english" }, - { id: "2", name: "中国人" }, - { id: "3", name: "german" }, - ]) - }) + expect((await table.collect()).toRecords()).toEqual([ + { id: 1, name: "english" }, + { id: 2, name: "中国人" }, + { id: 3, name: "german" }, + ]) }) - describe("dialect variations", () => { - it("should handle windows line terminator by default", async () => { - const path = await writeTempFile("id,name\r\n1,english\r\n2,中文") - const table = await loadCsvTable({ path }) + it("should handle windows line terminator by default", async () => { + const path = await writeTempFile("id,name\r\n1,english\r\n2,中文") + const table = await loadCsvTable({ path }) - expect((await table.collect()).toRecords()).toEqual([ - { id: "1", name: "english" }, - { id: "2", name: "中文" }, - ]) - }) + expect((await table.collect()).toRecords()).toEqual([ + { id: 1, name: "english" }, + { id: 2, name: "中文" }, + ]) + }) - it("should handle custom delimiter", async () => { - const path = await writeTempFile("id|name\n1|alice\n2|bob") - const table = await loadCsvTable({ - path, - dialect: { delimiter: "|" }, - }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: "1", name: "alice" }, - { id: "2", name: "bob" }, - ]) + it("should handle custom delimiter", async () => { + const path = await writeTempFile("id|name\n1|alice\n2|bob") + const table = await loadCsvTable({ + path, + dialect: { delimiter: "|" }, }) - it("should handle files without header", async () => { - const path = await writeTempFile("1,alice\n2,bob") - const table = await loadCsvTable({ - path, - dialect: { header: false }, - }) - - const records = (await table.collect()).toRecords() - expect(records).toEqual([ - { column_1: "1", column_2: "alice" }, - { column_1: "2", column_2: "bob" }, - ]) + expect((await table.collect()).toRecords()).toEqual([ + { id: 1, name: "alice" }, + { id: 2, name: "bob" }, + ]) + }) + + it("should handle files without header", async () => { + const path = await writeTempFile("1,alice\n2,bob") + const table = await loadCsvTable({ + path, + dialect: { header: false }, }) - it.skip("should handle custom line terminator", async () => { - const path = await writeTempFile("id,name|1,alice|2,bob") - const table = await loadCsvTable({ - path, - dialect: { lineTerminator: "|" }, - }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: "1", name: "alice" }, - { id: "2", name: "bob" }, - ]) + const records = (await table.collect()).toRecords() + expect(records).toEqual([ + { column_1: 1, column_2: "alice" }, + { column_1: 2, column_2: "bob" }, + ]) + }) + + it.skip("should handle custom line terminator", async () => { + const path = await writeTempFile("id,name|1,alice|2,bob") + const table = await loadCsvTable({ + path, + dialect: { lineTerminator: "|" }, }) - it.skip("should handle escape char", async () => { - const path = await writeTempFile( - "id,name\n1,apple|,fruits\n2,orange|,fruits", - ) + expect((await table.collect()).toRecords()).toEqual([ + { id: 1, name: "alice" }, + { id: 2, name: "bob" }, + ]) + }) - const table = await loadCsvTable({ - path, - dialect: { escapeChar: "|" }, - }) + it.skip("should handle escape char", async () => { + const path = await writeTempFile( + "id,name\n1,apple|,fruits\n2,orange|,fruits", + ) - expect((await table.collect()).toRecords()).toEqual([ - { id: "1", name: "apple,fruits" }, - { id: "2", name: "orange,fruits" }, - ]) + const table = await loadCsvTable({ + path, + dialect: { escapeChar: "|" }, }) - it("should handle custom quote character", async () => { - const path = await writeTempFile( - "id,name\n1,'alice smith'\n2,'bob jones'", - ) + expect((await table.collect()).toRecords()).toEqual([ + { id: 1, name: "apple,fruits" }, + { id: 2, name: "orange,fruits" }, + ]) + }) - const table = await loadCsvTable({ - path, - dialect: { quoteChar: "'" }, - }) + it("should handle custom quote character", async () => { + const path = await writeTempFile("id,name\n1,'alice smith'\n2,'bob jones'") - expect((await table.collect()).toRecords()).toEqual([ - { id: "1", name: "alice smith" }, - { id: "2", name: "bob jones" }, - ]) + const table = await loadCsvTable({ + path, + dialect: { quoteChar: "'" }, }) - it("should handle double quote by default", async () => { - const path = await writeTempFile( - 'id,name\n1,"alice""smith"\n2,"bob""jones"', - ) + expect((await table.collect()).toRecords()).toEqual([ + { id: 1, name: "alice smith" }, + { id: 2, name: "bob jones" }, + ]) + }) - const table = await loadCsvTable({ - path, - dialect: { doubleQuote: true }, - }) + it("should handle double quote by default", async () => { + const path = await writeTempFile( + 'id,name\n1,"alice""smith"\n2,"bob""jones"', + ) - expect((await table.collect()).toRecords()).toEqual([ - { id: "1", name: 'alice"smith' }, - { id: "2", name: 'bob"jones' }, - ]) + const table = await loadCsvTable({ + path, + dialect: { doubleQuote: true }, }) - it.skip("should handle disabling double quote", async () => { - const path = await writeTempFile( - 'id,name\n1,"alice""smith"\n2,"bob""jones"', - ) + expect((await table.collect()).toRecords()).toEqual([ + { id: 1, name: 'alice"smith' }, + { id: 2, name: 'bob"jones' }, + ]) + }) - const table = await loadCsvTable({ - path, - dialect: { doubleQuote: false }, - }) + it.skip("should handle disabling double quote", async () => { + const path = await writeTempFile( + 'id,name\n1,"alice""smith"\n2,"bob""jones"', + ) - expect((await table.collect()).toRecords()).toEqual([ - { id: "1", name: "alicesmith" }, - { id: "2", name: "bobjones" }, - ]) + const table = await loadCsvTable({ + path, + dialect: { doubleQuote: false }, }) - it("should handle comment character", async () => { - const path = await writeTempFile( - "# This is a comment\nid,name\n1,alice\n# Another comment\n2,bob", - ) - - const table = await loadCsvTable({ - path, - dialect: { commentChar: "#" }, - }) - - const records = (await table.collect()).toRecords() - expect(records).toEqual([ - { id: "1", name: "alice" }, - { id: "2", name: "bob" }, - ]) + expect((await table.collect()).toRecords()).toEqual([ + { id: 1, name: "alicesmith" }, + { id: 2, name: "bobjones" }, + ]) + }) + + it("should handle comment character", async () => { + const path = await writeTempFile( + "# This is a comment\nid,name\n1,alice\n# Another comment\n2,bob", + ) + + const table = await loadCsvTable({ + path, + dialect: { commentChar: "#" }, }) - it("should support headerRows", async () => { - const path = await writeTempFile("#comment\nid,name\n1,alice\n2,bob") + const records = (await table.collect()).toRecords() + expect(records).toEqual([ + { id: 1, name: "alice" }, + { id: 2, name: "bob" }, + ]) + }) - const table = await loadCsvTable({ - path, - dialect: { headerRows: [2] }, - }) + it("should support headerRows", async () => { + const path = await writeTempFile("#comment\nid,name\n1,alice\n2,bob") - const records = (await table.collect()).toRecords() - expect(records).toEqual([ - { id: "1", name: "alice" }, - { id: "2", name: "bob" }, - ]) + const table = await loadCsvTable({ + path, + dialect: { headerRows: [2] }, }) - it("should support headerJoin", async () => { - const path = await writeTempFile( - "#comment\nid,name\nint,str\n1,alice\n2,bob", - ) - - const table = await loadCsvTable({ - path, - dialect: { headerRows: [2, 3], headerJoin: "_" }, - }) - - const records = (await table.collect()).toRecords() - expect(records).toEqual([ - { id_int: "1", name_str: "alice" }, - { id_int: "2", name_str: "bob" }, - ]) + const records = (await table.collect()).toRecords() + expect(records).toEqual([ + { id: 1, name: "alice" }, + { id: 2, name: "bob" }, + ]) + }) + + it("should support headerJoin", async () => { + const path = await writeTempFile( + "#comment\nid,name\nint,str\n1,alice\n2,bob", + ) + + const table = await loadCsvTable({ + path, + dialect: { headerRows: [2, 3], headerJoin: "_" }, }) - it("should support commentRows", async () => { - const path = await writeTempFile("id,name\n1,alice\ncomment\n2,bob") + const records = (await table.collect()).toRecords() + expect(records).toEqual([ + { id_int: 1, name_str: "alice" }, + { id_int: 2, name_str: "bob" }, + ]) + }) - const table = await loadCsvTable({ - path, - dialect: { commentRows: [3] }, - }) + it("should support commentRows", async () => { + const path = await writeTempFile("id,name\n1,alice\ncomment\n2,bob") - const records = (await table.collect()).toRecords() - expect(records).toEqual([ - { id: "1", name: "alice" }, - { id: "2", name: "bob" }, - ]) + const table = await loadCsvTable({ + path, + dialect: { commentRows: [3] }, }) - it("should support headerRows and commentRows", async () => { - const path = await writeTempFile( - "#comment\nid,name\n1,alice\n#comment\n2,bob", - ) - - const table = await loadCsvTable({ - path, - dialect: { headerRows: [2], commentRows: [4] }, - }) - - const records = (await table.collect()).toRecords() - expect(records).toEqual([ - { id: "1", name: "alice" }, - { id: "2", name: "bob" }, - ]) + const records = (await table.collect()).toRecords() + expect(records).toEqual([ + { id: 1, name: "alice" }, + { id: 2, name: "bob" }, + ]) + }) + + it("should support headerRows and commentRows", async () => { + const path = await writeTempFile( + "#comment\nid,name\n1,alice\n#comment\n2,bob", + ) + + const table = await loadCsvTable({ + path, + dialect: { headerRows: [2], commentRows: [4] }, }) - it("should support headerJoin and commentRows", async () => { - const path = await writeTempFile( - "#comment\nid,name\nint,str\n1,alice\n#comment\n2,bob", - ) - - const table = await loadCsvTable({ - path, - dialect: { headerRows: [2, 3], headerJoin: "_", commentRows: [5] }, - }) - - const records = (await table.collect()).toRecords() - expect(records).toEqual([ - { id_int: "1", name_str: "alice" }, - { id_int: "2", name_str: "bob" }, - ]) + const records = (await table.collect()).toRecords() + expect(records).toEqual([ + { id: 1, name: "alice" }, + { id: 2, name: "bob" }, + ]) + }) + + it("should support headerJoin and commentRows", async () => { + const path = await writeTempFile( + "#comment\nid,name\nint,str\n1,alice\n#comment\n2,bob", + ) + + const table = await loadCsvTable({ + path, + dialect: { headerRows: [2, 3], headerJoin: "_", commentRows: [5] }, }) - it("should handle null sequence", async () => { - const path = await writeTempFile( - "id,name,age\n1,alice,25\n2,N/A,30\n3,bob,N/A", - ) - const table = await loadCsvTable({ - path, - dialect: { nullSequence: "N/A" }, - }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: "1", name: "alice", age: "25" }, - { id: "2", name: null, age: "30" }, - { id: "3", name: "bob", age: null }, - ]) + const records = (await table.collect()).toRecords() + expect(records).toEqual([ + { id_int: 1, name_str: "alice" }, + { id_int: 2, name_str: "bob" }, + ]) + }) + + it("should handle null sequence", async () => { + const path = await writeTempFile( + "id,name,age\n1,alice,25\n2,N/A,30\n3,bob,N/A", + ) + const table = await loadCsvTable({ + path, + dialect: { nullSequence: "N/A" }, }) - it("should handle skip initial space", async () => { - const path = await writeTempFile( - "id,name,category\n1, alice, fruits\n2, bob, vegetables\n3,charlie,grains", - ) - const table = await loadCsvTable({ - path, - dialect: { skipInitialSpace: true }, - }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: "1", name: "alice", category: "fruits" }, - { id: "2", name: "bob", category: "vegetables" }, - { id: "3", name: "charlie", category: "grains" }, - ]) + expect((await table.collect()).toRecords()).toEqual([ + { id: 1, name: "alice", age: 25 }, + { id: 2, name: null, age: 30 }, + { id: 3, name: "bob", age: null }, + ]) + }) + + it("should handle skip initial space", async () => { + const path = await writeTempFile( + "id,name,category\n1, alice, fruits\n2, bob, vegetables\n3,charlie,grains", + ) + const table = await loadCsvTable({ + path, + dialect: { skipInitialSpace: true }, }) - it("should handle multiple dialect options together", async () => { - const path = await writeTempFile( - "#comment\nid|'full name'|age\n1|'alice smith'|25\n2|'bob jones'|30", - ) - const table = await loadCsvTable({ - path, - dialect: { - delimiter: "|", - quoteChar: "'", - commentChar: "#", - header: true, - }, - }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: "1", "full name": "alice smith", age: "25" }, - { id: "2", "full name": "bob jones", age: "30" }, - ]) + expect((await table.collect()).toRecords()).toEqual([ + { id: 1, name: "alice", category: "fruits" }, + { id: 2, name: "bob", category: "vegetables" }, + { id: 3, name: "charlie", category: "grains" }, + ]) + }) + + it("should handle multiple dialect options together", async () => { + const path = await writeTempFile( + "#comment\nid|'full name'|age\n1|'alice smith'|25\n2|'bob jones'|30", + ) + const table = await loadCsvTable({ + path, + dialect: { + delimiter: "|", + quoteChar: "'", + commentChar: "#", + header: true, + }, }) - it("should handle utf8 encoding", async () => { - const path = await writeTempFile( - Buffer.from("id,name\n1,café\n2,naïve", "utf8"), - ) + expect((await table.collect()).toRecords()).toEqual([ + { id: 1, "full name": "alice smith", age: 25 }, + { id: 2, "full name": "bob jones", age: 30 }, + ]) + }) - const table = await loadCsvTable({ - path, - encoding: "utf8", - }) + it("should handle utf8 encoding", async () => { + const path = await writeTempFile( + Buffer.from("id,name\n1,café\n2,naïve", "utf8"), + ) - expect((await table.collect()).toRecords()).toEqual([ - { id: "1", name: "café" }, - { id: "2", name: "naïve" }, - ]) + const table = await loadCsvTable({ + path, + encoding: "utf8", }) - // TODO: currently not supported by nodejs-polars - it.skip("should handle utf16 encoding", async () => { - const path = await writeTempFile( - Buffer.from("id,name\n1,café\n2,naïve", "utf16le"), - ) - - const table = await loadCsvTable({ - path, - encoding: "utf16", - }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: "1", name: "café" }, - { id: "2", name: "naïve" }, - ]) + expect((await table.collect()).toRecords()).toEqual([ + { id: 1, name: "café" }, + { id: 2, name: "naïve" }, + ]) + }) + + // TODO: currently not supported by nodejs-polars + it.skip("should handle utf16 encoding", async () => { + const path = await writeTempFile( + Buffer.from("id,name\n1,café\n2,naïve", "utf16le"), + ) + + const table = await loadCsvTable({ + path, + encoding: "utf16", }) - // TODO: currently not supported by nodejs-polars - it.skip("should handle latin1 encoding", async () => { - const path = await writeTempFile( - Buffer.from("id,name\n1,café\n2,résumé", "latin1"), - ) - - const table = await loadCsvTable({ - path, - encoding: "latin1", - }) - - expect((await table.collect()).toRecords()).toEqual([ - { id: "1", name: "café" }, - { id: "2", name: "résumé" }, - ]) + expect((await table.collect()).toRecords()).toEqual([ + { id: 1, name: "café" }, + { id: 2, name: "naïve" }, + ]) + }) + + // TODO: currently not supported by nodejs-polars + it.skip("should handle latin1 encoding", async () => { + const path = await writeTempFile( + Buffer.from("id,name\n1,café\n2,résumé", "latin1"), + ) + + const table = await loadCsvTable({ + path, + encoding: "latin1", }) + + expect((await table.collect()).toRecords()).toEqual([ + { id: 1, name: "café" }, + { id: 2, name: "résumé" }, + ]) + }) +}) + +describe("loadCsvTable (format=tsv)", () => { + it("should load local file", async () => { + const path = await writeTempFile("id\tname\n1\tenglish\n2\t中文") + const table = await loadCsvTable({ path, format: "tsv" }) + + expect((await table.collect()).toRecords()).toEqual([ + { id: 1, name: "english" }, + { id: 2, name: "中文" }, + ]) }) }) diff --git a/csv/table/load.ts b/csv/table/load.ts index 64a3f910..3b80ddef 100644 --- a/csv/table/load.ts +++ b/csv/table/load.ts @@ -1,24 +1,32 @@ import type { Dialect, Resource } from "@dpkit/core" -import { loadResourceDialect } from "@dpkit/core" +import { loadResourceDialect, loadResourceSchema } from "@dpkit/core" import { prefetchFiles } from "@dpkit/file" +import { inferSchemaFromTable, normalizeTable } from "@dpkit/table" import { stripInitialSpace } from "@dpkit/table" import { joinHeaderRows } from "@dpkit/table" import { skipCommentRows } from "@dpkit/table" -import { DataFrame, scanCSV } from "nodejs-polars" +import type { LoadTableOptions } from "@dpkit/table" +import { scanCSV } from "nodejs-polars" import type { ScanCsvOptions } from "nodejs-polars" import { Utf8, concat } from "nodejs-polars" +import { inferCsvDialect } from "../dialect/index.ts" // TODO: Condier using sample to extract header first // for better commentChar + headerRows/commentRows support // (consult with the Data Package Working Group) -export async function loadCsvTable(resource: Partial) { +export async function loadCsvTable( + resource: Partial & { format?: "csv" | "tsv" }, + options?: LoadTableOptions, +) { const [firstPath, ...restPaths] = await prefetchFiles(resource.path) if (!firstPath) { - return DataFrame().lazy() + throw new Error("Resource path is not defined") } - const dialect = await loadResourceDialect(resource.dialect) + let dialect = await loadResourceDialect(resource.dialect) + if (!dialect) dialect = await inferCsvDialect(resource, options) + const scanOptions = getScanOptions(resource, dialect) let table = scanCSV(firstPath, scanOptions) @@ -45,6 +53,12 @@ export async function loadCsvTable(resource: Partial) { table = stripInitialSpace(table, { dialect }) } + if (!options?.denormalized) { + let schema = await loadResourceSchema(resource.schema) + if (!schema) schema = await inferSchemaFromTable(table, options) + table = await normalizeTable(table, schema) + } + return table } diff --git a/csv/table/save.spec.ts b/csv/table/save.spec.ts index 691482f3..de76ff47 100644 --- a/csv/table/save.spec.ts +++ b/csv/table/save.spec.ts @@ -1,7 +1,8 @@ import { readFile } from "node:fs/promises" import { getTempFilePath } from "@dpkit/file" -import { DataFrame } from "nodejs-polars" +import { DataFrame, DataType, Series } from "nodejs-polars" import { describe, expect, it } from "vitest" +import { loadCsvTable } from "./load.ts" import { saveCsvTable } from "./save.ts" describe("saveCsvTable", () => { @@ -67,4 +68,78 @@ describe("saveCsvTable", () => { "id,name\n1.0,'Alice,Smith'\n2.0,'Bob,Jones'\n3.0,'Charlie,Brown'\n", ) }) + + it("should save and load various data types", async () => { + const path = getTempFilePath() + + const source = DataFrame([ + Series("array", ["[1, 2, 3]"], DataType.String), + Series("boolean", [true], DataType.Bool), + Series("date", [new Date(Date.UTC(2025, 0, 1))], DataType.Date), + Series("datetime", [new Date(Date.UTC(2025, 0, 1))], DataType.Datetime), + Series("duration", ["P23DT23H"], DataType.String), + Series("geojson", ['{"value": 1}'], DataType.String), + Series("geopoint", [[40.0, 50.0]], DataType.List(DataType.Float32)), + Series("integer", [1], DataType.Int32), + Series("list", [[1.0, 2.0, 3.0]], DataType.List(DataType.Float32)), + Series("number", [1.1], DataType.Float64), + Series("object", ['{"value": 1}']), + Series("string", ["string"], DataType.String), + Series("time", [new Date(Date.UTC(2025, 0, 1))], DataType.Time), + Series("year", [2025], DataType.Int32), + Series("yearmonth", [[2025, 1]], DataType.List(DataType.Int16)), + ]).lazy() + + await saveCsvTable(source, { + path, + fieldTypes: { + array: "array", + geojson: "geojson", + geopoint: "geopoint", + list: "list", + object: "object", + // TODO: Remove time after: + // https://github.com/pola-rs/nodejs-polars/issues/364 + time: "time", + year: "year", + yearmonth: "yearmonth", + }, + }) + + const target = await loadCsvTable({ path }, { denormalized: true }) + expect((await target.collect()).toRecords()).toEqual([ + { + array: "[1, 2, 3]", + boolean: "true", + date: "2025-01-01", + datetime: "2025-01-01T00:00:00", + duration: "P23DT23H", + geojson: '{"value": 1}', + geopoint: "40.0,50.0", + integer: "1", + list: "1.0,2.0,3.0", + number: "1.1", + object: '{"value": 1}', + string: "string", + time: "00:00:00", + year: "2025", + yearmonth: "2025-01", + }, + ]) + }) +}) + +describe("saveCsvTable (format=tsv)", () => { + it("should save table to file", async () => { + const path = getTempFilePath() + const table = DataFrame({ + id: [1.0, 2.0, 3.0], + name: ["Alice", "Bob", "Charlie"], + }).lazy() + + await saveCsvTable(table, { path, format: "tsv" }) + + const content = await readFile(path, "utf-8") + expect(content).toEqual("id\tname\n1.0\tAlice\n2.0\tBob\n3.0\tCharlie\n") + }) }) diff --git a/csv/table/save.ts b/csv/table/save.ts index bc2bdbb9..da30a317 100644 --- a/csv/table/save.ts +++ b/csv/table/save.ts @@ -1,13 +1,32 @@ +import { assertLocalPathVacant } from "@dpkit/file" import type { SaveTableOptions, Table } from "@dpkit/table" +import { denormalizeTable, inferSchemaFromTable } from "@dpkit/table" -export async function saveCsvTable(table: Table, options: SaveTableOptions) { - const { path } = options +export async function saveCsvTable( + table: Table, + options: SaveTableOptions & { format?: "csv" | "tsv" }, +) { + const { path, format, overwrite } = options + const isTabs = format === "tsv" + + if (!overwrite) { + await assertLocalPathVacant(path) + } + + const schema = await inferSchemaFromTable(table, { + ...options, + keepStrings: true, + }) + + table = await denormalizeTable(table, schema, { + nativeTypes: ["string"], + }) await table .sinkCSV(path, { maintainOrder: true, includeHeader: options.dialect?.header ?? true, - separator: options.dialect?.delimiter ?? ",", + separator: options.dialect?.delimiter ?? (isTabs ? "\t" : ","), //lineTerminator: options.dialect?.lineTerminator ?? "\r\n", quoteChar: options.dialect?.quoteChar ?? '"', }) diff --git a/db/CHANGELOG.md b/database/CHANGELOG.md similarity index 90% rename from db/CHANGELOG.md rename to database/CHANGELOG.md index c451ded3..7ff1e3f1 100644 --- a/db/CHANGELOG.md +++ b/database/CHANGELOG.md @@ -1,4 +1,4 @@ -# @dpkit/db +# @dpkit/database ## 0.3.0 diff --git a/database/README.md b/database/README.md new file mode 100644 index 00000000..eb5ed140 --- /dev/null +++ b/database/README.md @@ -0,0 +1,3 @@ +# @dpkit/database + +dpkit is a fast data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/database/adapters/base.ts b/database/adapters/base.ts new file mode 100644 index 00000000..c1d0adfd --- /dev/null +++ b/database/adapters/base.ts @@ -0,0 +1,91 @@ +import type { Field, FieldType, Schema } from "@dpkit/core" +import type { Dialect } from "kysely" +import { Kysely } from "kysely" +import { LRUCache } from "lru-cache" +import type { DatabaseField, DatabaseType } from "../field/index.ts" +import type { DatabaseSchema } from "../schema/index.ts" + +// We cache database connections (only works in serverfull environments) +const databases = new LRUCache>({ + dispose: database => database.destroy(), + max: 10, +}) + +export abstract class BaseAdapter { + abstract get nativeTypes(): FieldType[] + abstract createDialect(path: string): Dialect + abstract normalizeType(databaseType: DatabaseType): Field["type"] + abstract denormalizeType(fieldType: Field["type"]): DatabaseType + + async connectDatabase(path: string) { + const cachedDatabase = databases.get(path) + if (cachedDatabase) { + return cachedDatabase + } + + const dialect = this.createDialect(path) + const database = new Kysely({ dialect }) + databases.set(path, new Kysely({ dialect })) + + return database + } + + normalizeSchema(databaseSchema: DatabaseSchema) { + const schema: Schema = { fields: [] } + + for (const databaseField of databaseSchema.columns) { + schema.fields.push(this.normalizeField(databaseField)) + } + + return schema + } + + normalizeField(databaseField: DatabaseField) { + const field: Field = { + name: databaseField.name, + type: this.normalizeType(databaseField.dataType), + } + + if (!databaseField.isNullable) { + field.constraints ??= {} + field.constraints.required = true + } + + if (databaseField.comment) { + field.description = databaseField.comment + } + + return field + } + + denormalizeSchema(schema: Schema, tableName: string): DatabaseSchema { + const databaseSchema: DatabaseSchema = { + name: tableName, + columns: [], + isView: false, + } + + for (const field of schema.fields) { + databaseSchema.columns.push(this.denormalizeField(field)) + } + + if (schema.primaryKey) { + databaseSchema.primaryKey = schema.primaryKey + } + + return databaseSchema + } + + denormalizeField(field: Field): DatabaseField { + const databaseField: DatabaseField = { + name: field.name, + dataType: this.denormalizeType(field.type), + isNullable: !field.constraints?.required, + comment: field.description, + isAutoIncrementing: false, + hasDefaultValue: false, + } + + return databaseField + } +} diff --git a/database/adapters/create.ts b/database/adapters/create.ts new file mode 100644 index 00000000..18e7eb68 --- /dev/null +++ b/database/adapters/create.ts @@ -0,0 +1,17 @@ +import type { DatabaseFormat } from "../resource/index.ts" +import { MysqlAdapter } from "./mysql.ts" +import { PostgresqlAdapter } from "./postgresql.ts" +import { SqliteAdapter } from "./sqlite.ts" + +export function createAdapter(format: DatabaseFormat) { + switch (format) { + case "postgresql": + return new PostgresqlAdapter() + case "mysql": + return new MysqlAdapter() + case "sqlite": + return new SqliteAdapter() + default: + throw new Error(`Unsupported database format: "${format}"`) + } +} diff --git a/database/adapters/mysql.spec.ts b/database/adapters/mysql.spec.ts new file mode 100644 index 00000000..517a172b --- /dev/null +++ b/database/adapters/mysql.spec.ts @@ -0,0 +1,188 @@ +import { useRecording } from "@dpkit/test" +import { DataFrame, DataType, Series } from "nodejs-polars" +import { describe, expect, it } from "vitest" +import { loadPackageFromDatabase } from "../package/index.ts" +import { inferDatabaseSchema } from "../schema/index.ts" +import { loadDatabaseTable, saveDatabaseTable } from "../table/index.ts" +import { createAdapter } from "./create.ts" + +useRecording() + +const path = process.env.DPKIT_MYSQL_URL + +// Vitest runs in-file tests sequentially so we can use the same table +const dialect = { table: "dpkit" } +const record1 = { id: 1, name: "english" } +const record2 = { id: 2, name: "中文" } + +describe.skipIf(!path)("MysqlAdapter", () => { + if (!path) return + + it("should infer schema", async () => { + const source = DataFrame([ + Series("string", ["string"], DataType.Utf8), + Series("integer", [1], DataType.Int32), + Series("number", [1.1], DataType.Float64), + ]).lazy() + + await saveDatabaseTable(source, { + path, + dialect, + format: "mysql", + overwrite: true, + }) + + const schema = await inferDatabaseSchema({ + path, + dialect, + format: "mysql", + }) + + expect(schema).toEqual({ + fields: [ + { name: "string", type: "string" }, + { name: "integer", type: "integer" }, + { name: "number", type: "number" }, + ], + }) + }) + + it("should save/load table", async () => { + const source = DataFrame([record1, record2]).lazy() + + await saveDatabaseTable(source, { + path, + dialect, + format: "mysql", + overwrite: true, + }) + + const target = await loadDatabaseTable({ + path, + dialect, + format: "mysql", + }) + + expect((await target.collect()).toRecords()).toEqual([record1, record2]) + }) + + it("should save/load table with various data types", async () => { + const source = DataFrame([ + Series("array", ["[1, 2, 3]"], DataType.String), + Series("boolean", [true], DataType.Bool), + Series("date", [new Date(Date.UTC(2025, 0, 1))], DataType.Date), + Series("datetime", [new Date(Date.UTC(2025, 0, 1))], DataType.Datetime), + Series("duration", ["P23DT23H"], DataType.String), + Series("geojson", ['{"value": 1}'], DataType.String), + Series("geopoint", [[40.0, 50.0]], DataType.List(DataType.Float32)), + Series("integer", [1], DataType.Int32), + Series("list", [[1.0, 2.0, 3.0]], DataType.List(DataType.Float32)), + Series("number", [1.1], DataType.Float64), + Series("object", ['{"value": 1}']), + Series("string", ["string"], DataType.String), + Series("time", [new Date(Date.UTC(2025, 0, 1))], DataType.Time), + Series("year", [2025], DataType.Int32), + Series("yearmonth", [[2025, 1]], DataType.List(DataType.Int16)), + ]).lazy() + + await saveDatabaseTable(source, { + path, + dialect, + format: "mysql", + overwrite: true, + fieldTypes: { + geojson: "geojson", + geopoint: "geopoint", + list: "list", + object: "object", + // TODO: Remove time after: + // https://github.com/pola-rs/nodejs-polars/issues/364 + time: "time", + year: "year", + yearmonth: "yearmonth", + }, + }) + + const target = await loadDatabaseTable( + { path, dialect, format: "mysql" }, + { denormalized: true }, + ) + + expect((await target.collect()).toRecords()).toEqual([ + { + array: "[1, 2, 3]", + boolean: 1, + date: "2025-01-01", + datetime: new Date(Date.UTC(2025, 0, 1)), + duration: "P23DT23H", + geojson: '{"value": 1}', + geopoint: "40.0,50.0", + integer: 1, + list: "1.0,2.0,3.0", + number: 1.1, + object: '{"value": 1}', + string: "string", + time: "00:00:00", + year: 2025, + yearmonth: "2025-01", + }, + ]) + }) + + it("should load package from database", async () => { + const adapter = createAdapter("sqlite") + const database = await adapter.connectDatabase(path) + + await database.schema.dropTable("table1").ifExists().execute() + await database.schema + .createTable("table1") + .ifNotExists() + .addColumn("id", "integer", column => column.notNull()) + .addColumn("name", "text") + .execute() + + await database.schema.dropTable("table2").ifExists().execute() + await database.schema + .createTable("table2") + .ifNotExists() + .addColumn("id", "integer", column => column.notNull()) + .addColumn("number", "numeric") + .addColumn("datetime", "datetime") + .execute() + + const datapackage = await loadPackageFromDatabase(path, { + format: "mysql", + includeTables: ["table1", "table2"], + }) + + expect(datapackage).toEqual({ + resources: [ + { + path, + name: "table1", + format: "mysql", + dialect: { table: "table1" }, + schema: { + fields: [ + { name: "id", type: "integer", constraints: { required: true } }, + { name: "name", type: "string" }, + ], + }, + }, + { + path, + name: "table2", + format: "mysql", + dialect: { table: "table2" }, + schema: { + fields: [ + { name: "id", type: "integer", constraints: { required: true } }, + { name: "number", type: "number" }, + { name: "datetime", type: "datetime" }, + ], + }, + }, + ], + }) + }) +}) diff --git a/database/adapters/mysql.ts b/database/adapters/mysql.ts new file mode 100644 index 00000000..7a68f058 --- /dev/null +++ b/database/adapters/mysql.ts @@ -0,0 +1,96 @@ +import type { FieldType } from "@dpkit/core" +import { MysqlDialect } from "kysely" +import { createPool } from "mysql2" +import type { DatabaseType } from "../field/index.ts" +import { BaseAdapter } from "./base.ts" + +// TODO: Support more native types + +export class MysqlAdapter extends BaseAdapter { + nativeTypes = [ + "boolean", + "datetime", + "integer", + "number", + "string", + "year", + ] satisfies FieldType[] + + createDialect(path: string) { + return new MysqlDialect({ + pool: createPool({ uri: path }), + }) + } + + normalizeType(databaseType: DatabaseType): FieldType { + switch (databaseType.toLowerCase()) { + case "tinyint": + case "smallint": + case "mediumint": + case "int": + case "integer": + case "bigint": + return "integer" + case "decimal": + case "numeric": + case "float": + case "double": + case "real": + return "number" + case "bit": + case "bool": + case "boolean": + return "boolean" + case "char": + case "varchar": + case "tinytext": + case "text": + case "mediumtext": + case "longtext": + case "enum": + case "set": + return "string" + case "date": + return "date" + case "time": + return "time" + case "datetime": + case "timestamp": + return "datetime" + case "year": + return "year" + case "json": + return "object" + case "geometry": + case "point": + case "linestring": + case "polygon": + case "multipoint": + case "multilinestring": + case "multipolygon": + case "geometrycollection": + return "geojson" + default: + return "string" + } + } + + denormalizeType(fieldType: FieldType): DatabaseType { + switch (fieldType) { + case "boolean": + return "boolean" + case "datetime": + return "datetime" + case "integer": + return "integer" + case "number": + return "double precision" + case "string": + return "text" + case "year": + return "integer" + default: + return "text" + } + } +} diff --git a/database/adapters/postgresql.spec.ts b/database/adapters/postgresql.spec.ts new file mode 100644 index 00000000..fe26a58f --- /dev/null +++ b/database/adapters/postgresql.spec.ts @@ -0,0 +1,188 @@ +import { useRecording } from "@dpkit/test" +import { DataFrame, DataType, Series } from "nodejs-polars" +import { describe, expect, it } from "vitest" +import { loadPackageFromDatabase } from "../package/index.ts" +import { inferDatabaseSchema } from "../schema/index.ts" +import { loadDatabaseTable, saveDatabaseTable } from "../table/index.ts" +import { createAdapter } from "./create.ts" + +useRecording() + +const path = process.env.DPKIT_POSTGRESQL_URL + +// Vitest runs in-file tests sequentially so we can use the same table +const dialect = { table: "dpkit" } +const record1 = { id: 1, name: "english" } +const record2 = { id: 2, name: "中文" } + +describe.skipIf(!path)("PostgresqlAdapter", () => { + if (!path) return + + it("should infer schema", async () => { + const source = DataFrame([ + Series("string", ["string"], DataType.Utf8), + Series("integer", [1], DataType.Int32), + Series("number", [1.1], DataType.Float64), + ]).lazy() + + await saveDatabaseTable(source, { + path, + dialect, + format: "postgresql", + overwrite: true, + }) + + const schema = await inferDatabaseSchema({ + path, + dialect, + format: "postgresql", + }) + + expect(schema).toEqual({ + fields: [ + { name: "string", type: "string" }, + { name: "integer", type: "integer" }, + { name: "number", type: "number" }, + ], + }) + }) + + it("should save/load table", async () => { + const source = DataFrame([record1, record2]).lazy() + + await saveDatabaseTable(source, { + path, + dialect, + format: "postgresql", + overwrite: true, + }) + + const target = await loadDatabaseTable({ + path, + dialect, + format: "postgresql", + }) + + expect((await target.collect()).toRecords()).toEqual([record1, record2]) + }) + + it("should save/load table with various data types", async () => { + const source = DataFrame([ + Series("array", ["[1, 2, 3]"], DataType.String), + Series("boolean", [true], DataType.Bool), + Series("date", [new Date(Date.UTC(2025, 0, 1))], DataType.Date), + Series("datetime", [new Date(Date.UTC(2025, 0, 1))], DataType.Datetime), + Series("duration", ["P23DT23H"], DataType.String), + Series("geojson", ['{"value": 1}'], DataType.String), + Series("geopoint", [[40.0, 50.0]], DataType.List(DataType.Float32)), + Series("integer", [1], DataType.Int32), + Series("list", [[1.0, 2.0, 3.0]], DataType.List(DataType.Float32)), + Series("number", [1.1], DataType.Float64), + Series("object", ['{"value": 1}']), + Series("string", ["string"], DataType.String), + Series("time", [new Date(Date.UTC(2025, 0, 1))], DataType.Time), + Series("year", [2025], DataType.Int32), + Series("yearmonth", [[2025, 1]], DataType.List(DataType.Int16)), + ]).lazy() + + await saveDatabaseTable(source, { + path, + dialect, + format: "postgresql", + overwrite: true, + fieldTypes: { + geojson: "geojson", + geopoint: "geopoint", + list: "list", + object: "object", + // TODO: Remove time after: + // https://github.com/pola-rs/nodejs-polars/issues/364 + time: "time", + year: "year", + yearmonth: "yearmonth", + }, + }) + + const target = await loadDatabaseTable( + { path, dialect, format: "postgresql" }, + { denormalized: true }, + ) + + expect((await target.collect()).toRecords()).toEqual([ + { + array: "[1, 2, 3]", + boolean: true, + date: "2025-01-01", + datetime: new Date(Date.UTC(2025, 0, 1)), + duration: "P23DT23H", + geojson: '{"value": 1}', + geopoint: "40.0,50.0", + integer: 1, + list: "1.0,2.0,3.0", + number: 1.1, + object: '{"value": 1}', + string: "string", + time: "00:00:00", + year: 2025, + yearmonth: "2025-01", + }, + ]) + }) + + it("should load package from database", async () => { + const adapter = createAdapter("sqlite") + const database = await adapter.connectDatabase(path) + + await database.schema.dropTable("table1").ifExists().execute() + await database.schema + .createTable("table1") + .ifNotExists() + .addColumn("id", "integer", column => column.notNull()) + .addColumn("name", "text") + .execute() + + await database.schema.dropTable("table2").ifExists().execute() + await database.schema + .createTable("table2") + .ifNotExists() + .addColumn("id", "integer", column => column.notNull()) + .addColumn("number", "numeric") + .addColumn("datetime", "timestamp") + .execute() + + const datapackage = await loadPackageFromDatabase(path, { + format: "postgresql", + includeTables: ["table1", "table2"], + }) + + expect(datapackage).toEqual({ + resources: [ + { + path, + name: "table1", + format: "postgresql", + dialect: { table: "table1" }, + schema: { + fields: [ + { name: "id", type: "integer", constraints: { required: true } }, + { name: "name", type: "string" }, + ], + }, + }, + { + path, + name: "table2", + format: "postgresql", + dialect: { table: "table2" }, + schema: { + fields: [ + { name: "id", type: "integer", constraints: { required: true } }, + { name: "number", type: "number" }, + { name: "datetime", type: "datetime" }, + ], + }, + }, + ], + }) + }) +}) diff --git a/database/adapters/postgresql.ts b/database/adapters/postgresql.ts new file mode 100644 index 00000000..5b3b4d92 --- /dev/null +++ b/database/adapters/postgresql.ts @@ -0,0 +1,106 @@ +import type { FieldType } from "@dpkit/core" +import { PostgresDialect } from "kysely" +import { Pool } from "pg" +import type { DatabaseType } from "../field/index.ts" +import { BaseAdapter } from "./base.ts" + +// TODO: Support more native types + +export class PostgresqlAdapter extends BaseAdapter { + nativeTypes = [ + "boolean", + "datetime", + "integer", + "number", + "string", + "year", + ] satisfies FieldType[] + + createDialect(path: string) { + return new PostgresDialect({ + pool: new Pool({ connectionString: path }), + }) + } + + normalizeType(databaseType: DatabaseType): FieldType { + switch (databaseType.toLowerCase()) { + case "smallint": + case "integer": + case "int": + case "int2": + case "int4": + case "int8": + case "bigint": + case "smallserial": + case "serial": + case "bigserial": + return "integer" + case "decimal": + case "numeric": + case "real": + case "float4": + case "double precision": + case "float8": + return "number" + case "boolean": + case "bool": + return "boolean" + case "char": + case "character": + case "varchar": + case "character varying": + case "text": + case "citext": + case "uuid": + return "string" + case "date": + return "date" + case "time": + case "time without time zone": + case "time with time zone": + case "timetz": + return "time" + case "timestamp": + case "timestamp without time zone": + case "timestamp with time zone": + case "timestamptz": + return "datetime" + case "interval": + return "duration" + case "json": + case "jsonb": + return "object" + case "point": + case "line": + case "lseg": + case "box": + case "path": + case "polygon": + case "circle": + case "geometry": + case "geography": + return "geojson" + default: + return "string" + } + } + + denormalizeType(fieldType: FieldType): DatabaseType { + switch (fieldType) { + case "boolean": + return "boolean" + case "datetime": + return "timestamp" + case "integer": + return "integer" + case "number": + return "double precision" + case "string": + return "text" + case "year": + return "integer" + default: + return "text" + } + } +} diff --git a/database/adapters/sqlite.spec.ts b/database/adapters/sqlite.spec.ts new file mode 100644 index 00000000..a250fef9 --- /dev/null +++ b/database/adapters/sqlite.spec.ts @@ -0,0 +1,180 @@ +import { getTempFilePath } from "@dpkit/file" +import { useRecording } from "@dpkit/test" +import { DataFrame, DataType, Series } from "nodejs-polars" +import { describe, expect, it } from "vitest" +import { loadPackageFromDatabase } from "../package/index.ts" +import { inferDatabaseSchema } from "../schema/index.ts" +import { loadDatabaseTable, saveDatabaseTable } from "../table/index.ts" +import { createAdapter } from "./create.ts" + +useRecording() + +const dialect = { table: "dpkit" } +const record1 = { id: 1, name: "english" } +const record2 = { id: 2, name: "中文" } + +// TODO: Enable after fixing problem: +// https://github.com/pnpm/pnpm/issues/9073 +describe.skipIf(process.env.CI)("SqliteAdapter", () => { + it("should infer schema", async () => { + const path = getTempFilePath() + + const source = DataFrame([ + Series("string", ["string"], DataType.Utf8), + Series("integer", [1], DataType.Int32), + Series("number", [1.1], DataType.Float64), + ]).lazy() + + await saveDatabaseTable(source, { path, dialect, format: "sqlite" }) + const schema = await inferDatabaseSchema({ + path, + dialect, + format: "sqlite", + }) + + expect(schema).toEqual({ + fields: [ + { name: "string", type: "string" }, + { name: "integer", type: "integer" }, + { name: "number", type: "number" }, + ], + }) + }) + + it("should save/load table", async () => { + const path = getTempFilePath() + + const source = DataFrame([record1, record2]).lazy() + await saveDatabaseTable(source, { path, dialect, format: "sqlite" }) + const target = await loadDatabaseTable({ path, dialect, format: "sqlite" }) + + expect((await target.collect()).toRecords()).toEqual([record1, record2]) + }) + + it("should save/load table with protocol", async () => { + const path = `sqlite://${getTempFilePath()}` + + const source = DataFrame([record1, record2]).lazy() + await saveDatabaseTable(source, { path, dialect, format: "sqlite" }) + const target = await loadDatabaseTable({ path, dialect, format: "sqlite" }) + + expect((await target.collect()).toRecords()).toEqual([record1, record2]) + }) + + it("should save/load table with various data types", async () => { + const path = `sqlite://${getTempFilePath()}` + + const source = DataFrame([ + Series("array", ["[1, 2, 3]"], DataType.String), + Series("boolean", [true], DataType.Bool), + Series("date", [new Date(Date.UTC(2025, 0, 1))], DataType.Date), + Series("datetime", [new Date(Date.UTC(2025, 0, 1))], DataType.Datetime), + Series("duration", ["P23DT23H"], DataType.String), + Series("geojson", ['{"value": 1}'], DataType.String), + Series("geopoint", [[40.0, 50.0]], DataType.List(DataType.Float32)), + Series("integer", [1], DataType.Int32), + Series("list", [[1.0, 2.0, 3.0]], DataType.List(DataType.Float32)), + Series("number", [1.1], DataType.Float64), + Series("object", ['{"value": 1}']), + Series("string", ["string"], DataType.String), + Series("time", [new Date(Date.UTC(2025, 0, 1))], DataType.Time), + Series("year", [2025], DataType.Int32), + Series("yearmonth", [[2025, 1]], DataType.List(DataType.Int16)), + ]).lazy() + + await saveDatabaseTable(source, { + path, + dialect, + format: "sqlite", + fieldTypes: { + geojson: "geojson", + geopoint: "geopoint", + list: "list", + object: "object", + // TODO: Remove time after: + // https://github.com/pola-rs/nodejs-polars/issues/364 + time: "time", + year: "year", + yearmonth: "yearmonth", + }, + }) + + const target = await loadDatabaseTable( + { path, dialect, format: "sqlite" }, + { denormalized: true }, + ) + + expect((await target.collect()).toRecords()).toEqual([ + { + array: "[1, 2, 3]", + boolean: "true", + date: "2025-01-01", + datetime: "2025-01-01T00:00:00", + duration: "P23DT23H", + geojson: '{"value": 1}', + geopoint: "40.0,50.0", + integer: 1, + list: "1.0,2.0,3.0", + number: 1.1, + object: '{"value": 1}', + string: "string", + time: "00:00:00", + year: 2025, + yearmonth: "2025-01", + }, + ]) + }) + + it("should load package from database", async () => { + const path = getTempFilePath() + const adapter = createAdapter("sqlite") + const database = await adapter.connectDatabase(path) + + await database.schema + .createTable("table1") + .addColumn("id", "integer", column => column.notNull()) + .addColumn("name", "text") + .execute() + + await database.schema + .createTable("table2") + .addColumn("id", "integer", column => column.notNull()) + .addColumn("number", "numeric") + .addColumn("boolean", "boolean") + .execute() + + const datapackage = await loadPackageFromDatabase(path, { + format: "sqlite", + }) + + expect(datapackage).toEqual({ + resources: [ + { + path, + name: "table1", + format: "sqlite", + dialect: { table: "table1" }, + schema: { + fields: [ + { name: "id", type: "integer", constraints: { required: true } }, + { name: "name", type: "string" }, + ], + }, + }, + { + path, + name: "table2", + format: "sqlite", + dialect: { table: "table2" }, + schema: { + fields: [ + { name: "id", type: "integer", constraints: { required: true } }, + { name: "number", type: "number" }, + { name: "boolean", type: "string" }, + ], + }, + }, + ], + }) + }) +}) diff --git a/database/adapters/sqlite.ts b/database/adapters/sqlite.ts new file mode 100644 index 00000000..dce89175 --- /dev/null +++ b/database/adapters/sqlite.ts @@ -0,0 +1,48 @@ +import type { FieldType } from "@dpkit/core" +import Database from "better-sqlite3" +import { SqliteDialect } from "kysely" +import type { DatabaseType } from "../field/index.ts" +import { BaseAdapter } from "./base.ts" + +export class SqliteAdapter extends BaseAdapter { + nativeTypes = ["integer", "number", "string", "year"] satisfies FieldType[] + + createDialect(path: string) { + return new SqliteDialect({ + database: new Database(path.replace(/^sqlite:\/\//, "")), + }) + } + + normalizeType(databaseType: DatabaseType): FieldType { + switch (databaseType.toLowerCase()) { + case "blob": + return "string" + case "text": + return "string" + case "integer": + return "integer" + case "numeric": + case "real": + return "number" + default: + return "string" + } + } + + denormalizeType(fieldType: FieldType): DatabaseType { + switch (fieldType) { + case "boolean": + return "integer" + case "integer": + return "integer" + case "number": + return "real" + case "string": + return "text" + case "year": + return "integer" + default: + return "text" + } + } +} diff --git a/database/field/Field.ts b/database/field/Field.ts new file mode 100644 index 00000000..a6d0b4b9 --- /dev/null +++ b/database/field/Field.ts @@ -0,0 +1,3 @@ +import type { ColumnMetadata } from "kysely" + +export interface DatabaseField extends ColumnMetadata {} diff --git a/database/field/Type.ts b/database/field/Type.ts new file mode 100644 index 00000000..a1b74258 --- /dev/null +++ b/database/field/Type.ts @@ -0,0 +1,3 @@ +import type { DatabaseField } from "./Field.ts" + +export type DatabaseType = DatabaseField["dataType"] diff --git a/database/field/index.ts b/database/field/index.ts new file mode 100644 index 00000000..b6ff85ad --- /dev/null +++ b/database/field/index.ts @@ -0,0 +1,2 @@ +export type { DatabaseField } from "./Field.ts" +export type { DatabaseType } from "./Type.ts" diff --git a/database/index.ts b/database/index.ts new file mode 100644 index 00000000..e5beef8e --- /dev/null +++ b/database/index.ts @@ -0,0 +1,4 @@ +export * from "./schema/index.ts" +export * from "./package/index.ts" +export * from "./table/index.ts" +export * from "./plugin.ts" diff --git a/db/package.json b/database/package.json similarity index 52% rename from db/package.json rename to database/package.json index 8be59e81..7c844570 100644 --- a/db/package.json +++ b/database/package.json @@ -1,5 +1,5 @@ { - "name": "@dpkit/db", + "name": "@dpkit/database", "exports": "./build/index.js", "type": "module", "version": "0.3.0", @@ -17,9 +17,25 @@ "validation", "quality", "fair", - "db" + "database" ], "scripts": { "build": "tsc" + }, + "dependencies": { + "@dpkit/core": "workspace:*", + "@dpkit/table": "workspace:*", + "better-sqlite3": "^12.2.0", + "kysely": "^0.28.5", + "lru-cache": "^11.2.1", + "mysql2": "^3.14.4", + "nodejs-polars": "^0.21.1", + "pg": "^8.16.3" + }, + "devDependencies": { + "@dpkit/file": "workspace:*", + "@dpkit/test": "workspace:*", + "@types/better-sqlite3": "^7.6.13", + "@types/pg": "^8.15.5" } } diff --git a/database/package/index.ts b/database/package/index.ts new file mode 100644 index 00000000..270af671 --- /dev/null +++ b/database/package/index.ts @@ -0,0 +1 @@ +export { loadPackageFromDatabase } from "./load.ts" diff --git a/database/package/load.ts b/database/package/load.ts new file mode 100644 index 00000000..a38c9238 --- /dev/null +++ b/database/package/load.ts @@ -0,0 +1,47 @@ +import type { Package } from "@dpkit/core" +import { createAdapter } from "../adapters/create.ts" +import type { DatabaseFormat } from "../resource/index.ts" + +export async function loadPackageFromDatabase( + connectionString: string, + options: { + format: DatabaseFormat + includeTables?: string[] + excludeTables?: string[] + }, +) { + const { includeTables, excludeTables } = options + + const adapter = createAdapter(options.format) + const database = await adapter.connectDatabase(connectionString) + const databaseSchemas = await database.introspection.getTables() + + const dataPackage: Package = { + resources: [], + } + + for (const databaseSchema of databaseSchemas) { + const name = databaseSchema.name + + if (includeTables && !includeTables.includes(name)) { + continue + } + + if (excludeTables?.includes(name)) { + continue + } + + const schema = adapter.normalizeSchema(databaseSchema) + const dialect = { table: name } + + dataPackage.resources.push({ + name, + path: connectionString, + format: options.format, + dialect, + schema, + }) + } + + return dataPackage +} diff --git a/database/plugin.ts b/database/plugin.ts new file mode 100644 index 00000000..184213f2 --- /dev/null +++ b/database/plugin.ts @@ -0,0 +1,52 @@ +import type { Resource } from "@dpkit/core" +import { inferResourceFormat } from "@dpkit/core" +import type { TablePlugin } from "@dpkit/table" +import type { SaveTableOptions, Table } from "@dpkit/table" +import { loadPackageFromDatabase } from "./package/index.ts" +import { inferDatabaseSchema } from "./schema/index.ts" +import { loadDatabaseTable } from "./table/index.ts" +import { saveDatabaseTable } from "./table/index.ts" + +export class DatabasePlugin implements TablePlugin { + async loadPackage(source: string) { + const databaseFormat = getDatabaseFormat({ path: source }) + if (!databaseFormat) return undefined + + return await loadPackageFromDatabase(source, { + format: databaseFormat, + }) + } + + async inferSchema(resource: Partial) { + const databaseFormat = getDatabaseFormat(resource) + if (!databaseFormat) return undefined + + return await inferDatabaseSchema({ ...resource, format: databaseFormat }) + } + + async loadTable(resource: Partial) { + const databaseFormat = getDatabaseFormat(resource) + if (!databaseFormat) return undefined + + return await loadDatabaseTable({ ...resource, format: databaseFormat }) + } + + async saveTable(table: Table, options: SaveTableOptions) { + const { path, format } = options + + const databaseFormat = getDatabaseFormat({ path, format }) + if (!databaseFormat) return undefined + + return await saveDatabaseTable(table, { + ...options, + format: databaseFormat, + }) + } +} + +function getDatabaseFormat(resource: Partial) { + const format = inferResourceFormat(resource) + return format === "postgresql" || format === "mysql" || format === "sqlite" + ? format + : undefined +} diff --git a/database/resource/Format.ts b/database/resource/Format.ts new file mode 100644 index 00000000..da7e240d --- /dev/null +++ b/database/resource/Format.ts @@ -0,0 +1 @@ +export type DatabaseFormat = "sqlite" | "postgresql" | "mysql" diff --git a/database/resource/index.ts b/database/resource/index.ts new file mode 100644 index 00000000..dea20589 --- /dev/null +++ b/database/resource/index.ts @@ -0,0 +1 @@ +export type { DatabaseFormat } from "./Format.ts" diff --git a/database/schema/Schema.ts b/database/schema/Schema.ts new file mode 100644 index 00000000..7aa0fdc1 --- /dev/null +++ b/database/schema/Schema.ts @@ -0,0 +1,5 @@ +import type { TableMetadata } from "kysely" + +export interface DatabaseSchema extends TableMetadata { + primaryKey?: string[] +} diff --git a/database/schema/index.ts b/database/schema/index.ts new file mode 100644 index 00000000..694d37bb --- /dev/null +++ b/database/schema/index.ts @@ -0,0 +1,2 @@ +export { inferDatabaseSchema } from "./infer.ts" +export type { DatabaseSchema } from "./Schema.ts" diff --git a/database/schema/infer.spec.ts b/database/schema/infer.spec.ts new file mode 100644 index 00000000..9e3fc4d4 --- /dev/null +++ b/database/schema/infer.spec.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from "vitest" +import { inferDatabaseSchema } from "./infer.ts" + +describe("inferDatabaseSchema", () => { + it("throws error when resource path is not defined", async () => { + await expect( + inferDatabaseSchema({ + format: "sqlite", + dialect: { table: "dpkit" }, + }), + ).rejects.toThrow("Resource path is not defined") + }) + + it("throws error when table name is not defined in dialect", async () => { + await expect( + inferDatabaseSchema({ + path: "path", + format: "sqlite", + }), + ).rejects.toThrow("Table name is not defined in dialect") + }) + + it("throws error when format is not supported", async () => { + await expect( + inferDatabaseSchema({ + path: "path", + format: "unsupported" as any, + dialect: { table: "dpkit" }, + }), + ).rejects.toThrow('Unsupported database format: "unsupported"') + }) +}) diff --git a/database/schema/infer.ts b/database/schema/infer.ts new file mode 100644 index 00000000..7105b5b2 --- /dev/null +++ b/database/schema/infer.ts @@ -0,0 +1,32 @@ +import type { Resource } from "@dpkit/core" +import { loadResourceDialect } from "@dpkit/core" +import { createAdapter } from "../adapters/create.ts" + +export async function inferDatabaseSchema( + resource: Partial & { format: "postgresql" | "mysql" | "sqlite" }, +) { + const adapter = createAdapter(resource.format) + if (!adapter) { + throw new Error("Supported database format is not defined") + } + + const dialect = await loadResourceDialect(resource.dialect) + if (!dialect?.table) { + throw new Error("Table name is not defined in dialect") + } + + const path = typeof resource.path === "string" ? resource.path : undefined + if (!path) { + throw new Error("Resource path is not defined") + } + + const database = await adapter.connectDatabase(path) + const databaseSchemas = await database.introspection.getTables() + + const databaseSchema = databaseSchemas.find(s => s.name === dialect.table) + if (!databaseSchema) { + throw new Error(`Table is not found in database: ${dialect.table}`) + } + + return adapter.normalizeSchema(databaseSchema) +} diff --git a/database/table/index.ts b/database/table/index.ts new file mode 100644 index 00000000..8c8238f0 --- /dev/null +++ b/database/table/index.ts @@ -0,0 +1,2 @@ +export { loadDatabaseTable } from "./load.ts" +export { saveDatabaseTable } from "./save.ts" diff --git a/database/table/load.spec.ts b/database/table/load.spec.ts new file mode 100644 index 00000000..36801850 --- /dev/null +++ b/database/table/load.spec.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from "vitest" +import { loadDatabaseTable } from "./load.ts" + +describe("loadDatabaseTable", () => { + it("throws error when resource path is not defined", async () => { + await expect( + loadDatabaseTable({ + format: "sqlite", + dialect: { table: "dpkit" }, + }), + ).rejects.toThrow("Resource path is not defined") + }) + + it("throws error when table name is not defined in dialect", async () => { + await expect( + loadDatabaseTable({ + path: "path", + format: "sqlite", + }), + ).rejects.toThrow("Table name is not defined in dialect") + }) + + it("throws error when format is not supported", async () => { + await expect( + loadDatabaseTable({ + path: "path", + format: "unsupported" as any, + dialect: { table: "dpkit" }, + }), + ).rejects.toThrow('Unsupported database format: "unsupported"') + }) +}) diff --git a/database/table/load.ts b/database/table/load.ts new file mode 100644 index 00000000..fdfc36c9 --- /dev/null +++ b/database/table/load.ts @@ -0,0 +1,39 @@ +import { loadResourceDialect, loadResourceSchema } from "@dpkit/core" +import type { Resource } from "@dpkit/core" +import { normalizeTable } from "@dpkit/table" +import type { LoadTableOptions } from "@dpkit/table" +import { DataFrame } from "nodejs-polars" +import { createAdapter } from "../adapters/create.ts" +import { inferDatabaseSchema } from "../schema/index.ts" + +// Currently, we use slow non-rust implementation as in the future +// polars-rust might be able to provide a faster native implementation + +export async function loadDatabaseTable( + resource: Partial & { format: "postgresql" | "mysql" | "sqlite" }, + options?: LoadTableOptions, +) { + const dialect = await loadResourceDialect(resource.dialect) + if (!dialect?.table) { + throw new Error("Table name is not defined in dialect") + } + + const path = typeof resource.path === "string" ? resource.path : undefined + if (!path) { + throw new Error("Resource path is not defined") + } + + const adapter = createAdapter(resource.format) + const database = await adapter.connectDatabase(path) + const records = await database.selectFrom(dialect.table).selectAll().execute() + + let table = DataFrame(records).lazy() + + if (!options?.denormalized) { + let schema = await loadResourceSchema(resource.schema) + if (!schema) schema = await inferDatabaseSchema(resource) + table = await normalizeTable(table, schema) + } + + return table +} diff --git a/database/table/save.spec.ts b/database/table/save.spec.ts new file mode 100644 index 00000000..74bac46b --- /dev/null +++ b/database/table/save.spec.ts @@ -0,0 +1,26 @@ +import { DataFrame } from "nodejs-polars" +import { describe, expect, it } from "vitest" +import { saveDatabaseTable } from "./save.js" + +describe("saveDatabaseTable", () => { + const mockTable = DataFrame({ col1: [1, 2, 3] }).lazy() + + it("throws error when table name is not defined in dialect", async () => { + await expect( + saveDatabaseTable(mockTable, { + path: "test.db", + format: "sqlite", + }), + ).rejects.toThrow("Table name is not defined in dialect") + }) + + it("throws error when format is not supported", async () => { + await expect( + saveDatabaseTable(mockTable, { + path: "test.db", + format: "unsupported" as any, + dialect: { table: "test_table" }, + }), + ).rejects.toThrow('Unsupported database format: "unsupported"') + }) +}) diff --git a/database/table/save.ts b/database/table/save.ts new file mode 100644 index 00000000..d6216151 --- /dev/null +++ b/database/table/save.ts @@ -0,0 +1,90 @@ +import type { SaveTableOptions, Table } from "@dpkit/table" +import { denormalizeTable, inferSchemaFromTable } from "@dpkit/table" +import type { Kysely } from "kysely" +import { createAdapter } from "../adapters/create.ts" +import type { DatabaseSchema } from "../schema/index.ts" + +// Currently, we use slow non-rust implementation as in the future +// polars-rust might be able to provide a faster native implementation +// (if not supported we can use COPY in PostgreSQL/MySQL) + +export async function saveDatabaseTable( + table: Table, + options: SaveTableOptions & { format: "postgresql" | "mysql" | "sqlite" }, +) { + const { path, format, dialect, overwrite } = options + + const tableName = dialect?.table + if (!tableName) { + throw new Error("Table name is not defined in dialect") + } + + const schema = await inferSchemaFromTable(table, { + ...options, + keepStrings: true, + }) + + const adapter = createAdapter(format) + table = await denormalizeTable(table, schema, { + nativeTypes: adapter.nativeTypes, + }) + + const database = await adapter.connectDatabase(path) + const databaseSchema = adapter.denormalizeSchema(schema, tableName) + + await defineTable(database, databaseSchema, { overwrite }) + await populateTable(database, tableName, table) + + return path +} + +async function defineTable( + database: Kysely, + databaseSchema: DatabaseSchema, + options: { + overwrite?: boolean + }, +) { + if (options.overwrite) { + await database.schema.dropTable(databaseSchema.name).ifExists().execute() + } + + let query = database.schema.createTable(databaseSchema.name) + + for (const field of databaseSchema.columns) { + // @ts-ignore + query = query.addColumn(field.name, field.dataType) + } + + if (databaseSchema.primaryKey) { + query = query.addPrimaryKeyConstraint( + `${databaseSchema.name}_pkey`, + // @ts-ignore + databaseSchema.primaryKey, + ) + } + + await query.execute() +} + +async function populateTable( + database: Kysely, + tableName: string, + table: Table, +) { + let offset = 0 + const df = await table.collect({ streaming: true }) + while (true) { + const buffer = df.slice(offset, offset + BUFFER_SIZE) + offset += BUFFER_SIZE + + const records = buffer.toRecords() + if (!records.length) { + break + } + + await database.insertInto(tableName).values(records).execute() + } +} + +const BUFFER_SIZE = 10_000 diff --git a/db/tsconfig.json b/database/tsconfig.json similarity index 100% rename from db/tsconfig.json rename to database/tsconfig.json diff --git a/database/typedoc.json b/database/typedoc.json new file mode 100644 index 00000000..f8e49f3a --- /dev/null +++ b/database/typedoc.json @@ -0,0 +1,4 @@ +{ + "entryPoints": ["index.ts"], + "skipErrorChecking": true +} diff --git a/datahub/README.md b/datahub/README.md index 378c12ba..4605982c 100644 --- a/datahub/README.md +++ b/datahub/README.md @@ -1,3 +1,3 @@ # @dpkit/datahub -dpkit is a fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). +dpkit is a fast data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/datahub/package/load.spec.ts b/datahub/package/load.spec.ts index 433566ec..8cc75902 100644 --- a/datahub/package/load.spec.ts +++ b/datahub/package/load.spec.ts @@ -2,9 +2,9 @@ import { useRecording } from "@dpkit/test" import { describe, expect, it } from "vitest" import { loadPackageFromDatahub } from "./load.ts" -describe.skip("loadPackageFromDatahub", () => { - useRecording() +useRecording() +describe.skip("loadPackageFromDatahub", () => { it("should load a package", async () => { const dataPackage = await loadPackageFromDatahub( "https://datahub.io/core/eu-emissions-trading-system#readme", diff --git a/db/README.md b/db/README.md deleted file mode 100644 index b7492b0b..00000000 --- a/db/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# @dpkit/db - -dpkit is a fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/db/index.ts b/db/index.ts deleted file mode 100644 index e69de29b..00000000 diff --git a/docs/astro.config.ts b/docs/astro.config.ts index 393a173c..88e96c66 100644 --- a/docs/astro.config.ts +++ b/docs/astro.config.ts @@ -13,6 +13,7 @@ const PACKAGES = { "@dpkit/ckan": "../ckan", "@dpkit/core": "../core", "@dpkit/csv": "../csv", + "@dpkit/database": "../database", "@dpkit/datahub": "../datahub", "@dpkit/file": "../file", "@dpkit/github": "../github", diff --git a/docs/content/docs/guides/arrow.md b/docs/content/docs/guides/arrow.md index 87a6ce5d..950a3260 100644 --- a/docs/content/docs/guides/arrow.md +++ b/docs/content/docs/guides/arrow.md @@ -15,6 +15,10 @@ npm install @dpkit/arrow ## Basic Usage +:::tip +You can use `loadTable` and `saveTable` from `@dpkit/all` instead of `@dpkit/arrow` to load and save ARROW files if the framework can infer that files are in the `arrow/feather` format. +::: + ### Loading Data ```typescript diff --git a/docs/content/docs/guides/csv.md b/docs/content/docs/guides/csv.md index 8c855c6e..d86a814e 100644 --- a/docs/content/docs/guides/csv.md +++ b/docs/content/docs/guides/csv.md @@ -8,22 +8,22 @@ Comprehensive CSV and TSV file handling with automatic format detection, advance ## Introduction +:::tip +You can use `loadTable` and `saveTable` from `@dpkit/all` instead of `@dpkit/csv` to load and save CSV files if the framework can infer that files are in the `csv/tsv` format. +::: + The CSV plugin is a part of the [dpkit](https://github.com/datisthq/dpkit) ecosystem providing these capabilities: - `loadCsvTable` - `saveCsvTable` - `inferCsvDialect` -These functions are low-level and handles only CSV files on the IO and dialect level. So, for example, `loadCsvTable` will always return all the fields having a string type. - -For having both loading and processing of CSV files, the [dpkit](https://github.com/datisthq/dpkit) ecosystem provides the `readTable` function which is a high-level function that handles both loading and processing of CSV files. - -The CSV plugin automatically handles `.csv` and `.tsv` files when using dpkit: +For example: ```typescript -import { readTable } from "@dpkit/all" +import { loadCsvTable } from "@dpkit/csv" -const table = await readTable({path: "table.csv"}) +const table = await loadCsvTable({path: "table.csv"}) // the field types will be automatically inferred // or you can provide a Table Schema ``` diff --git a/docs/content/docs/guides/database.md b/docs/content/docs/guides/database.md new file mode 100644 index 00000000..8fa84cd2 --- /dev/null +++ b/docs/content/docs/guides/database.md @@ -0,0 +1,108 @@ +--- +title: Working with Database +sidebar: + label: Database + order: 5 +--- +Database connectivity and operations with support for SQLite, PostgreSQL, and MySQL through Kysely query builder integration. + +## Introduction + +:::tip +You can use `loadTable` and `saveTable` from `@dpkit/all` instead of `@dpkit/database` to load and save database tables if the framework can infer the database connection format. +::: + +The Database plugin is a part of the [dpkit](https://github.com/datisthq/dpkit) ecosystem providing these capabilities: + +- `loadDatabaseTable` +- `saveDatabaseTable` +- `inferDatabaseSchema` +- `loadPackageFromDatabase` + +For example: + +```typescript +import { loadDatabaseTable } from "@dpkit/database" + +const table = await loadDatabaseTable({ + path: "sqlite://database.db", + dialect: { table: "users" } +}) +// field types will be automatically inferred from database schema +// or you can provide a Table Schema +``` + +:::tip +The ouput of `loadDatabaseTable` is a Polars LazyDataFrame, allowing you to use all of the power of Polars for data processing. +::: + +## Supported Databases + +The plugin supports three database types: + +- **SQLite** - File-based database (`sqlite://path/to/file.db`) +- **PostgreSQL** - Network database (`postgresql://user:pass@host:port/db`) +- **MySQL** - Network database (`mysql://user:pass@host:port/db`) + +## Connection Formats + +Database connections are specified using standard connection strings: + +```typescript +// SQLite +const sqliteTable = await loadDatabaseTable({ + path: "sqlite://data.db", + dialect: { table: "products" } +}) + +// PostgreSQL +const pgTable = await loadDatabaseTable({ + path: "postgresql://user:password@localhost:5432/mydb", + dialect: { table: "orders" } +}) + +// MySQL +const mysqlTable = await loadDatabaseTable({ + path: "mysql://user:password@localhost:3306/mydb", + dialect: { table: "customers" } +}) +``` + +## Schema Inference + +The database adapter automatically infers Table Schema from database table definitions: + +```typescript +import { inferDatabaseSchema } from "@dpkit/database" + +const schema = await inferDatabaseSchema({ + path: "sqlite://shop.db", + dialect: { table: "products" } +}) +// Returns a Table Schema with field types matching database columns +``` + +## Package Loading + +Load entire Data Packages from databases: + +```typescript +import { loadPackageFromDatabase } from "@dpkit/database" + +const package = await loadPackageFromDatabase("sqlite://catalog.db") +// Loads all tables as package resources +``` + +## Database Adapters + +The plugin uses database-specific adapters built on Kysely: + +- **BaseAdapter** - Common functionality for all databases +- **SqliteAdapter** - SQLite-specific operations using better-sqlite3 +- **PostgreSQLAdapter** - PostgreSQL operations using pg driver +- **MySQLAdapter** - MySQL operations using mysql2 driver + +Each adapter handles: +- Type normalization between database and Table Schema types +- Connection management with LRU caching +- Database-specific SQL dialect handling diff --git a/docs/content/docs/guides/json.md b/docs/content/docs/guides/json.md index 62b1aa8b..624d9a19 100644 --- a/docs/content/docs/guides/json.md +++ b/docs/content/docs/guides/json.md @@ -17,6 +17,10 @@ npm install @dpkit/json ## Basic Usage +:::tip +You can use `loadTable` and `saveTable` from `@dpkit/all` instead of `@dpkit/json` to load and save JSON files if the framework can infer that files are in the `json/jsonl` format. +::: + ### Loading JSON Data ```typescript @@ -39,22 +43,22 @@ const table = await loadJsonTable({ ### Loading JSONL Data ```typescript -import { loadJsonlTable } from "@dpkit/json" +import { loadJsonTable } from "@dpkit/json" // Load JSONL (JSON Lines) format -const table = await loadJsonlTable({ path: "data.jsonl" }) +const table = await loadJsonTable({ path: "data.jsonl", format: 'jsonl' }) ``` ### Saving Data ```typescript -import { saveJsonTable, saveJsonlTable } from "@dpkit/json" +import { saveJsonTable } from "@dpkit/json" // Save as JSON await saveJsonTable(table, { path: "output.json" }) // Save as JSONL -await saveJsonlTable(table, { path: "output.jsonl" }) +await saveJsonTable(table, { path: "output.jsonl", format: 'jsonl' }) ``` ## Data Formats diff --git a/docs/content/docs/guides/ods.md b/docs/content/docs/guides/ods.md index 52b4a745..2fa194cb 100644 --- a/docs/content/docs/guides/ods.md +++ b/docs/content/docs/guides/ods.md @@ -8,25 +8,25 @@ Comprehensive OpenDocument Spreadsheet (ODS) file handling with sheet selection, ## Introduction +:::tip +You can use `loadTable` and `saveTable` from `@dpkit/all` instead of `@dpkit/ods` to load and save ODS files if the framework can infer that files are in the `ods` format. +::: + The ODS plugin is a part of the [dpkit](https://github.com/datisthq/dpkit) ecosystem providing these capabilities: - `loadOdsTable` - `saveOdsTable` -These functions handle ODS files at the IO and dialect level, supporting LibreOffice Calc and OpenOffice Calc formats. - -For complete loading and processing of ODS files, the [dpkit](https://github.com/datisthq/dpkit) ecosystem provides the `readTable` function which is a high-level function that handles both loading and processing of ODS files, and `saveTable` for saving ODS files. - -The ODS plugin automatically handles `.ods` files when using dpkit: +For example: ```typescript -import { readTable, saveTable } from "@dpkit/all" +import { loadOdsTable, saveOdsTable } from "@dpkit/ods" -const table = await readTable({path: "table.ods"}) +const table = await loadOdsTable({path: "table.ods"}) // the field types will be automatically inferred // or you can provide a Table Schema -await saveTable(table, {path: "output.ods"}) +await saveOdsTable(table, {path: "output.ods"}) ``` ## Basic Usage @@ -34,17 +34,17 @@ await saveTable(table, {path: "output.ods"}) ### Reading ODS Files :::tip -The ouput of `readTable` is a Polars LazyDataFrame, allowing you to use all of the power of Polars for data processing. +The ouput of `loadOdsTable` is a Polars LazyDataFrame, allowing you to use all of the power of Polars for data processing. ::: ```typescript -import { readTable } from "@dpkit/all" +import { loadOdsTable } from "@dpkit/ods" // Load a simple ODS file -const table = await readTable({ path: "data.ods" }) +const table = await loadOdsTable({ path: "data.ods" }) // Load with custom dialect (specify sheet) -const table = await readTable({ +const table = await loadOdsTable({ path: "data.ods", dialect: { sheetName: "Sheet2", @@ -53,7 +53,7 @@ const table = await readTable({ }) // Load multiple ODS files (concatenated) -const table = await readTable({ +const table = await loadOdsTable({ path: ["part1.ods", "part2.ods", "part3.ods"] }) @@ -65,13 +65,13 @@ df.describe() ### Saving ODS Files ```typescript -import { saveTable } from "@dpkit/all" +import { saveOdsTable } from "@dpkit/ods" // Save with default options -await saveTable(table, { path: "output.ods" }) +await saveOdsTable(table, { path: "output.ods" }) // Save with custom sheet name -await saveTable(table, { +await saveOdsTable(table, { path: "output.ods", dialect: { sheetName: "Data" @@ -84,10 +84,10 @@ await saveTable(table, { ### Sheet Selection ```typescript -import { readTable } from "@dpkit/all" +import { loadOdsTable } from "@dpkit/ods" // Select by sheet number (1-indexed) -const table = await readTable({ +const table = await loadOdsTable({ path: "workbook.ods", dialect: { sheetNumber: 2 // Load second sheet @@ -95,7 +95,7 @@ const table = await readTable({ }) // Select by sheet name -const table = await readTable({ +const table = await loadOdsTable({ path: "workbook.ods", dialect: { sheetName: "Sales Data" @@ -106,14 +106,14 @@ const table = await readTable({ ### Multi-Header Row Processing ```typescript -import { readTable } from "@dpkit/all" +import { loadOdsTable } from "@dpkit/ods" // ODS with multiple header rows: // Year | 2023 | 2023 | 2024 | 2024 // Quarter | Q1 | Q2 | Q1 | Q2 // Revenue | 100 | 120 | 110 | 130 -const table = await readTable({ +const table = await loadOdsTable({ path: "multi-header.ods", dialect: { headerRows: [1, 2], @@ -126,10 +126,10 @@ const table = await readTable({ ### Comment Row Handling ```typescript -import { readTable } from "@dpkit/all" +import { loadOdsTable } from "@dpkit/ods" // ODS with comment rows -const table = await readTable({ +const table = await loadOdsTable({ path: "with-comments.ods", dialect: { commentRows: [1, 2], // Skip first two rows @@ -138,7 +138,7 @@ const table = await readTable({ }) // Skip rows with comment character -const table = await readTable({ +const table = await loadOdsTable({ path: "data.ods", dialect: { commentChar: "#" // Skip rows starting with # @@ -149,15 +149,15 @@ const table = await readTable({ ### Remote File Loading ```typescript -import { readTable } from "@dpkit/all" +import { loadOdsTable } from "@dpkit/ods" // Load from URL -const table = await readTable({ +const table = await loadOdsTable({ path: "https://example.com/data.ods" }) // Load multiple remote files -const table = await readTable({ +const table = await loadOdsTable({ path: [ "https://api.example.com/data-2023.ods", "https://api.example.com/data-2024.ods" @@ -168,10 +168,10 @@ const table = await readTable({ ### Header Options ```typescript -import { readTable } from "@dpkit/all" +import { loadOdsTable } from "@dpkit/ods" // No header row (use generated column names) -const table = await readTable({ +const table = await loadOdsTable({ path: "data.ods", dialect: { header: false @@ -180,7 +180,7 @@ const table = await readTable({ // Columns will be: column_1, column_2, column_3, etc. // Custom header row offset -const table = await readTable({ +const table = await loadOdsTable({ path: "data.ods", dialect: { headerRows: [3] // Use third row as header diff --git a/docs/content/docs/guides/table.md b/docs/content/docs/guides/table.md index af9f83e2..93652e4e 100644 --- a/docs/content/docs/guides/table.md +++ b/docs/content/docs/guides/table.md @@ -9,11 +9,11 @@ The `@dpkit/table` package provides high-performance data validation and process ## Examples -### Basic Table Inspection +### Basic Table Validation ```typescript import { DataFrame } from "nodejs-polars" -import { inspectTable } from "@dpkit/table" +import { validateTable } from "@dpkit/table" import type { Schema } from "@dpkit/core" // Create a table from data @@ -32,8 +32,8 @@ const schema: Schema = { ] } -// Inspect the table -const errors = await inspectTable(table, { schema }) +// validate the table +const errors = await validateTable(table, { schema }) console.log(errors) // Array of validation errors ``` @@ -51,7 +51,7 @@ const table = DataFrame({ }).lazy() const inferredSchema = await inferSchema(table, { - sampleSize: 100, // Sample size for inference + sampleRows: 100, // Sample size for inference confidence: 0.9, // Confidence threshold monthFirst: false, // Date format preference commaDecimal: false // Decimal separator preference @@ -85,7 +85,7 @@ const equalSchema: Schema = { ### Table Processing ```typescript -import { processTable } from "@dpkit/table" +import { normalizeTable } from "@dpkit/table" // Process table with schema (converts string columns to proper types) const table = DataFrame({ @@ -104,7 +104,7 @@ const schema: Schema = { ] } -const processedTable = await processTable(table, { schema }) +const processedTable = await normalizeTable(table, { schema }) const result = await processedTable.collect() // Result will have properly typed columns: @@ -116,7 +116,7 @@ const result = await processedTable.collect() ### Error Handling ```typescript -const result = await inspectTable(table, { schema }) +const result = await validateTable(table, { schema }) result.errors.forEach(error => { switch (error.type) { diff --git a/docs/content/docs/guides/xlsx.md b/docs/content/docs/guides/xlsx.md index 6efd4393..164c671c 100644 --- a/docs/content/docs/guides/xlsx.md +++ b/docs/content/docs/guides/xlsx.md @@ -8,25 +8,25 @@ Comprehensive Excel (.xlsx) file handling with sheet selection, advanced header ## Introduction +:::tip +You can use `loadTable` and `saveTable` from `@dpkit/all` instead of `@dpkit/xlsx` to load and save XLSX files if the framework can infer that files are in the `xlsx` format. +::: + The XLSX plugin is a part of the [dpkit](https://github.com/datisthq/dpkit) ecosystem providing these capabilities: - `loadXlsxTable` - `saveXlsxTable` -These functions handle XLSX files at the IO and dialect level, supporting Microsoft Excel formats. - -For complete loading and processing of XLSX files, the [dpkit](https://github.com/datisthq/dpkit) ecosystem provides the `readTable` function which is a high-level function that handles both loading and processing of XLSX files, and `saveTable` for saving XLSX files. - -The XLSX plugin automatically handles `.xlsx` files when using dpkit: +For example: ```typescript -import { readTable, saveTable } from "@dpkit/all" +import { loadXlsxTable, saveXlsxTable } from "@dpkit/xlsx" -const table = await readTable({path: "table.xlsx"}) +const table = await loadXlsxTable({path: "table.xlsx"}) // the field types will be automatically inferred // or you can provide a Table Schema -await saveTable(table, {path: "output.xlsx"}) +await saveXlsxTable(table, {path: "output.xlsx"}) ``` ## Basic Usage @@ -34,17 +34,17 @@ await saveTable(table, {path: "output.xlsx"}) ### Reading XLSX Files :::tip -The ouput of `readTable` is a Polars LazyDataFrame, allowing you to use all of the power of Polars for data processing. +The ouput of `loadXlsxTable` is a Polars LazyDataFrame, allowing you to use all of the power of Polars for data processing. ::: ```typescript -import { readTable } from "@dpkit/all" +import { loadXlsxTable } from "@dpkit/xlsx" // Load a simple XLSX file -const table = await readTable({ path: "data.xlsx" }) +const table = await loadXlsxTable({ path: "data.xlsx" }) // Load with custom dialect (specify sheet) -const table = await readTable({ +const table = await loadXlsxTable({ path: "data.xlsx", dialect: { sheetName: "Sheet2", @@ -53,7 +53,7 @@ const table = await readTable({ }) // Load multiple XLSX files (concatenated) -const table = await readTable({ +const table = await loadXlsxTable({ path: ["part1.xlsx", "part2.xlsx", "part3.xlsx"] }) @@ -65,13 +65,13 @@ df.describe() ### Saving XLSX Files ```typescript -import { saveTable } from "@dpkit/all" +import { saveXlsxTable } from "@dpkit/xlsx" // Save with default options -await saveTable(table, { path: "output.xlsx" }) +await saveXlsxTable(table, { path: "output.xlsx" }) // Save with custom sheet name -await saveTable(table, { +await saveXlsxTable(table, { path: "output.xlsx", dialect: { sheetName: "Data" @@ -84,10 +84,10 @@ await saveTable(table, { ### Sheet Selection ```typescript -import { readTable } from "@dpkit/all" +import { loadXlsxTable } from "@dpkit/xlsx" // Select by sheet number (1-indexed) -const table = await readTable({ +const table = await loadXlsxTable({ path: "workbook.xlsx", dialect: { sheetNumber: 2 // Load second sheet @@ -95,7 +95,7 @@ const table = await readTable({ }) // Select by sheet name -const table = await readTable({ +const table = await loadXlsxTable({ path: "workbook.xlsx", dialect: { sheetName: "Sales Data" @@ -106,14 +106,14 @@ const table = await readTable({ ### Multi-Header Row Processing ```typescript -import { readTable } from "@dpkit/all" +import { loadXlsxTable } from "@dpkit/xlsx" // XLSX with multiple header rows: // Year | 2023 | 2023 | 2024 | 2024 // Quarter | Q1 | Q2 | Q1 | Q2 // Revenue | 100 | 120 | 110 | 130 -const table = await readTable({ +const table = await loadXlsxTable({ path: "multi-header.xlsx", dialect: { headerRows: [1, 2], @@ -126,10 +126,10 @@ const table = await readTable({ ### Comment Row Handling ```typescript -import { readTable } from "@dpkit/all" +import { loadXlsxTable } from "@dpkit/xlsx" // XLSX with comment rows -const table = await readTable({ +const table = await loadXlsxTable({ path: "with-comments.xlsx", dialect: { commentRows: [1, 2], // Skip first two rows @@ -138,7 +138,7 @@ const table = await readTable({ }) // Skip rows with comment character -const table = await readTable({ +const table = await loadXlsxTable({ path: "data.xlsx", dialect: { commentChar: "#" // Skip rows starting with # @@ -149,15 +149,15 @@ const table = await readTable({ ### Remote File Loading ```typescript -import { readTable } from "@dpkit/all" +import { loadXlsxTable } from "@dpkit/xlsx" // Load from URL -const table = await readTable({ +const table = await loadXlsxTable({ path: "https://example.com/data.xlsx" }) // Load multiple remote files -const table = await readTable({ +const table = await loadXlsxTable({ path: [ "https://api.example.com/data-2023.xlsx", "https://api.example.com/data-2024.xlsx" @@ -168,10 +168,10 @@ const table = await readTable({ ### Header Options ```typescript -import { readTable } from "@dpkit/all" +import { loadXlsxTable } from "@dpkit/xlsx" // No header row (use generated column names) -const table = await readTable({ +const table = await loadXlsxTable({ path: "data.xlsx", dialect: { header: false @@ -180,7 +180,7 @@ const table = await readTable({ // Columns will be: column_1, column_2, column_3, etc. // Custom header row offset -const table = await readTable({ +const table = await loadXlsxTable({ path: "data.xlsx", dialect: { headerRows: [3] // Use third row as header diff --git a/docs/package.json b/docs/package.json index 21b35b20..33133250 100644 --- a/docs/package.json +++ b/docs/package.json @@ -11,7 +11,7 @@ "@astrojs/starlight": "0.35.2", "@dpkit/all": "workspace:*", "astro": "5.12.9", - "nodejs-polars": "0.21.0", + "nodejs-polars": "0.21.1", "sharp": "0.34.2", "starlight-changelogs": "0.1.1", "starlight-scroll-to-top": "0.3.1", diff --git a/file/README.md b/file/README.md index 9fa4059e..a5720f71 100644 --- a/file/README.md +++ b/file/README.md @@ -1,3 +1,3 @@ # @dpkit/data -dpkit is a fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). +dpkit is a fast data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/file/file/save.ts b/file/file/save.ts index e96e1003..eac4e0ea 100644 --- a/file/file/save.ts +++ b/file/file/save.ts @@ -1,6 +1,12 @@ import { Readable } from "node:stream" import { saveFileStream } from "../stream/index.ts" -export async function saveFile(path: string, buffer: Buffer) { - await saveFileStream(Readable.from(buffer), { path }) +export async function saveFile( + path: string, + buffer: Buffer, + options?: { overwrite?: boolean }, +) { + const { overwrite } = options ?? {} + + await saveFileStream(Readable.from(buffer), { path, overwrite }) } diff --git a/file/stream/save.ts b/file/stream/save.ts index c29e54dc..5c2ab1b0 100644 --- a/file/stream/save.ts +++ b/file/stream/save.ts @@ -8,11 +8,17 @@ export async function saveFileStream( stream: Readable, options: { path: string + overwrite?: boolean }, ) { - // It is an equivalent to ensureDir function - await mkdir(dirname(options.path), { recursive: true }) + const { path, overwrite } = options - // The "wx" flag ensures that the file won't overwrite an existing file - await pipeline(stream, createWriteStream(options.path, { flags: "wx" })) + // It is an equivalent to ensureDir function that won't overwrite an existing directory + await mkdir(dirname(path), { recursive: true }) + + await pipeline( + stream, + // The "wx" flag ensures that the file won't overwrite an existing file + createWriteStream(path, { flags: overwrite ? "w" : "wx" }), + ) } diff --git a/folder/README.md b/folder/README.md index 3e865578..12b9c7cf 100644 --- a/folder/README.md +++ b/folder/README.md @@ -1,3 +1,3 @@ # @dpkit/folder -dpkit is a fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). +dpkit is a fast data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/github/README.md b/github/README.md index bc961eab..a30daa19 100644 --- a/github/README.md +++ b/github/README.md @@ -1,3 +1,3 @@ # @dpkit/github -dpkit is a fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). +dpkit is a fast data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/github/package/process/denormalize.ts b/github/package/denormalize.ts similarity index 95% rename from github/package/process/denormalize.ts rename to github/package/denormalize.ts index caae857a..5924ddcf 100644 --- a/github/package/process/denormalize.ts +++ b/github/package/denormalize.ts @@ -1,5 +1,5 @@ import type { Package } from "@dpkit/core" -import type { GithubPackage } from "../Package.ts" +import type { GithubPackage } from "./Package.ts" /** * Denormalizes a Frictionless Data Package to Github repository metadata format diff --git a/github/package/load.spec.ts b/github/package/load.spec.ts index f3cba35b..74cecdd1 100644 --- a/github/package/load.spec.ts +++ b/github/package/load.spec.ts @@ -2,9 +2,9 @@ import { useRecording } from "@dpkit/test" import { describe, expect, it } from "vitest" import { loadPackageFromGithub } from "./load.ts" -describe("loadPackageFromGithub", () => { - useRecording() +useRecording() +describe("loadPackageFromGithub", () => { it("should load a package", async () => { const datapackage = await loadPackageFromGithub( "https://github.com/roll/data", diff --git a/github/package/load.ts b/github/package/load.ts index 98a708b3..50f26977 100644 --- a/github/package/load.ts +++ b/github/package/load.ts @@ -2,7 +2,7 @@ import { mergePackages } from "@dpkit/core" import { makeGithubApiRequest } from "../github/index.ts" import type { GithubResource } from "../resource/index.ts" import type { GithubPackage } from "./Package.ts" -import { normalizeGithubPackage } from "./process/normalize.ts" +import { normalizeGithubPackage } from "./normalize.ts" /** * Load a package from a Github repository diff --git a/github/package/process/normalize.ts b/github/package/normalize.ts similarity index 94% rename from github/package/process/normalize.ts rename to github/package/normalize.ts index 06aab98b..fdbbf87a 100644 --- a/github/package/process/normalize.ts +++ b/github/package/normalize.ts @@ -1,6 +1,6 @@ import type { Contributor, License, Package } from "@dpkit/core" -import { normalizeGithubResource } from "../../resource/process/normalize.ts" -import type { GithubPackage } from "../Package.ts" +import { normalizeGithubResource } from "../resource/index.ts" +import type { GithubPackage } from "./Package.ts" /** * Normalizes a Github repository to Frictionless Data package format diff --git a/github/package/save.spec.ts b/github/package/save.spec.ts index a644dfc6..b2f9e6e6 100644 --- a/github/package/save.spec.ts +++ b/github/package/save.spec.ts @@ -3,9 +3,9 @@ import { loadPackageDescriptor } from "@dpkit/core" import { describe, expect, it } from "vitest" import { savePackageToGithub } from "./save.ts" -describe("savePackageToGithub", () => { - //useRecording() +//useRecording() +describe("savePackageToGithub", () => { it.skip("should save a package", async () => { const dataPackage = await loadPackageDescriptor( "core/package/fixtures/package.json", diff --git a/github/resource/process/denormalize.ts b/github/resource/denormalize.ts similarity index 91% rename from github/resource/process/denormalize.ts rename to github/resource/denormalize.ts index 863fd5fc..40e0099a 100644 --- a/github/resource/process/denormalize.ts +++ b/github/resource/denormalize.ts @@ -1,5 +1,5 @@ import type { Resource } from "@dpkit/core" -import type { GithubResource } from "../Resource.ts" +import type { GithubResource } from "./Resource.ts" /** * Denormalizes a Frictionless Data resource to Github file format diff --git a/github/resource/index.ts b/github/resource/index.ts index b9685df7..abcb93f2 100644 --- a/github/resource/index.ts +++ b/github/resource/index.ts @@ -1,3 +1,3 @@ export type { GithubResource } from "./Resource.ts" -export { normalizeGithubResource } from "./process/normalize.ts" -export { denormalizeGithubResource } from "./process/denormalize.ts" +export { normalizeGithubResource } from "./normalize.ts" +export { denormalizeGithubResource } from "./denormalize.ts" diff --git a/github/resource/process/normalize.ts b/github/resource/normalize.ts similarity index 95% rename from github/resource/process/normalize.ts rename to github/resource/normalize.ts index 9a724356..a91ab07c 100644 --- a/github/resource/process/normalize.ts +++ b/github/resource/normalize.ts @@ -1,6 +1,6 @@ import type { Resource } from "@dpkit/core" import { getFilename, getFormat, getName } from "@dpkit/core" -import type { GithubResource } from "../Resource.ts" +import type { GithubResource } from "./Resource.ts" /** * Normalizes a Github file to Frictionless Data resource format diff --git a/inline/README.md b/inline/README.md index af18cf09..9b7f1f4a 100644 --- a/inline/README.md +++ b/inline/README.md @@ -1,3 +1,3 @@ # @dpkit/inline -dpkit is a fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). +dpkit is a fast data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/inline/package.json b/inline/package.json index fada8601..190557d1 100644 --- a/inline/package.json +++ b/inline/package.json @@ -23,7 +23,7 @@ "build": "tsc" }, "dependencies": { - "nodejs-polars": "^0.21.0", + "nodejs-polars": "^0.21.1", "@dpkit/core": "workspace:*", "@dpkit/table": "workspace:*" } diff --git a/inline/plugin.ts b/inline/plugin.ts index f06a395c..3f96c9a5 100644 --- a/inline/plugin.ts +++ b/inline/plugin.ts @@ -1,13 +1,14 @@ import type { Resource } from "@dpkit/core" +import type { LoadTableOptions } from "@dpkit/table" import type { TablePlugin } from "@dpkit/table" import { loadInlineTable } from "./table/index.ts" export class InlinePlugin implements TablePlugin { - async loadTable(resource: Resource) { + async loadTable(resource: Resource, options?: LoadTableOptions) { const isInline = getIsInline(resource) if (!isInline) return undefined - return await loadInlineTable(resource) + return await loadInlineTable(resource, options) } } diff --git a/inline/table/load.spec.ts b/inline/table/load.spec.ts index 746c36ad..6012c31e 100644 --- a/inline/table/load.spec.ts +++ b/inline/table/load.spec.ts @@ -2,34 +2,20 @@ import { describe, expect, it } from "vitest" import { loadInlineTable } from "./load.ts" describe("loadInlineTable", () => { - it("should handle no data", async () => { - const resource = { - name: "test", - type: "table", - data: undefined, - schema: undefined, - } - - // @ts-ignore - const table = await loadInlineTable(resource) - const df = await table.collect() + it("should throw on no data", async () => { + const resource = { name: "test" } - expect([]).toEqual(df.toRecords()) + await expect(loadInlineTable(resource)).rejects.toThrow( + "Resource data is not defined or tabular", + ) }) - it("should handle bad data", async () => { - const resource = { - name: "test", - type: "table", - data: "bad", - schema: undefined, - } - - // @ts-ignore - const table = await loadInlineTable(resource) - const df = await table.collect() + it("should throw on bad data", async () => { + const resource = { name: "test", data: "bad" } - expect([]).toEqual(df.toRecords()) + await expect(loadInlineTable(resource)).rejects.toThrow( + "Resource data is not defined or tabular", + ) }) it("should read arrays", async () => { diff --git a/inline/table/load.ts b/inline/table/load.ts index 1306ad9d..4cb75937 100644 --- a/inline/table/load.ts +++ b/inline/table/load.ts @@ -1,19 +1,31 @@ import type { Resource } from "@dpkit/core" import { loadResourceDialect } from "@dpkit/core" +import { loadResourceSchema } from "@dpkit/core" import { getRecordsFromRows } from "@dpkit/table" +import type { LoadTableOptions } from "@dpkit/table" +import { inferSchemaFromTable, normalizeTable } from "@dpkit/table" import { DataFrame } from "nodejs-polars" -export async function loadInlineTable(resource: Partial) { - const dialect = await loadResourceDialect(resource.dialect) +export async function loadInlineTable( + resource: Partial, + options?: LoadTableOptions, +) { const data = resource.data - if (!Array.isArray(data)) { - return DataFrame().lazy() + throw new Error("Resource data is not defined or tabular") } + const dialect = await loadResourceDialect(resource.dialect) const isRows = data.every(row => Array.isArray(row)) + const records = isRows ? getRecordsFromRows(data, dialect) : data + let table = DataFrame(records).lazy() + + if (!options?.denormalized) { + let schema = await loadResourceSchema(resource.schema) + if (!schema) schema = await inferSchemaFromTable(table, options) + table = await normalizeTable(table, schema) + } - const table = DataFrame(records).lazy() return table } diff --git a/json/README.md b/json/README.md index dae27861..a2aaaa5b 100644 --- a/json/README.md +++ b/json/README.md @@ -1,3 +1,3 @@ # @dpkit/json -dpkit is a fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). +dpkit is a fast data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/json/package.json b/json/package.json index e306403f..a880ab16 100644 --- a/json/package.json +++ b/json/package.json @@ -27,7 +27,7 @@ "@dpkit/file": "workspace:*", "@dpkit/table": "workspace:*", "csv-sniffer": "^0.1.1", - "nodejs-polars": "^0.21.0" + "nodejs-polars": "^0.21.1" }, "devDependencies": { "@dpkit/test": "workspace:*" diff --git a/json/plugin.ts b/json/plugin.ts index d57a9ead..24843cd8 100644 --- a/json/plugin.ts +++ b/json/plugin.ts @@ -1,43 +1,32 @@ import type { Resource } from "@dpkit/core" import { inferResourceFormat } from "@dpkit/core" +import type { LoadTableOptions } from "@dpkit/table" import type { TablePlugin } from "@dpkit/table" import type { SaveTableOptions, Table } from "@dpkit/table" -import { loadJsonTable, loadJsonlTable } from "./table/index.ts" -import { saveJsonTable, saveJsonlTable } from "./table/index.ts" +import { loadJsonTable } from "./table/index.ts" +import { saveJsonTable } from "./table/index.ts" export class JsonPlugin implements TablePlugin { - async loadTable(resource: Partial) { - const formatInfo = getFormatInfo(resource) + async loadTable(resource: Partial, options?: LoadTableOptions) { + const jsonFormat = getJsonFormat(resource) + if (!jsonFormat) return undefined - if (formatInfo.isJson) { - return await loadJsonTable(resource) - } - - if (formatInfo.isJsonl) { - return await loadJsonlTable(resource) - } - - return undefined + return await loadJsonTable({ ...resource, format: jsonFormat }, options) } async saveTable(table: Table, options: SaveTableOptions) { - const formatInfo = getFormatInfo(options) - - if (formatInfo.isJson) { - return await saveJsonTable(table, options) - } + const { path, format } = options - if (formatInfo.isJsonl) { - return await saveJsonlTable(table, options) - } + const jsonFormat = getJsonFormat({ path, format }) + if (!jsonFormat) return undefined - return undefined + return await saveJsonTable(table, { ...options, format: jsonFormat }) } } -function getFormatInfo(resource: Partial) { +function getJsonFormat(resource: Partial) { const format = inferResourceFormat(resource) - const isJson = format === "json" - const isJsonl = format === "jsonl" || format === "ndjson" - return { isJson, isJsonl } + return format === "json" || format === "jsonl" || format === "ndjson" + ? format + : undefined } diff --git a/json/table/fixtures/generated/loadJsonTable-format-json-file-variations-should-load-remote-file-multipart_4065232820/recording.har b/json/table/fixtures/generated/loadJsonTable-format-json-file-variations-should-load-remote-file-multipart_4065232820/recording.har new file mode 100644 index 00000000..84a0c834 --- /dev/null +++ b/json/table/fixtures/generated/loadJsonTable-format-json-file-variations-should-load-remote-file-multipart_4065232820/recording.har @@ -0,0 +1,298 @@ +{ + "log": { + "_recordingName": "loadJsonTable (format=json)-file variations-should load remote file (multipart)", + "creator": { + "comment": "persister:fs", + "name": "Polly.JS", + "version": "6.0.6" + }, + "entries": [ + { + "_id": "9c1d4218bc0b2124f4ba05daec949590", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [], + "headersSize": 123, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://raw.githubusercontent.com/frictionlessdata/frictionless-py/refs/heads/main/data/table.keyed.json" + }, + "response": { + "bodySize": 78, + "content": { + "mimeType": "text/plain; charset=utf-8", + "size": 78, + "text": "[\n {\n \"id\": 1,\n \"name\": \"english\"\n },\n {\n \"id\": 2,\n \"name\": \"中国人\"\n }\n]\n" + }, + "cookies": [], + "headers": [ + { + "name": "accept-ranges", + "value": "bytes" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "cache-control", + "value": "max-age=300" + }, + { + "name": "connection", + "value": "keep-alive" + }, + { + "name": "content-encoding", + "value": "gzip" + }, + { + "name": "content-length", + "value": "78" + }, + { + "name": "content-security-policy", + "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" + }, + { + "name": "content-type", + "value": "text/plain; charset=utf-8" + }, + { + "name": "cross-origin-resource-policy", + "value": "cross-origin" + }, + { + "name": "date", + "value": "Sat, 06 Sep 2025 09:19:02 GMT" + }, + { + "name": "etag", + "value": "W/\"6f3db326388b97300ef06688ba9a05ac48b9af407f4cea02dee6a3c83568b3cb\"" + }, + { + "name": "expires", + "value": "Sat, 06 Sep 2025 09:24:02 GMT" + }, + { + "name": "source-age", + "value": "0" + }, + { + "name": "strict-transport-security", + "value": "max-age=31536000" + }, + { + "name": "vary", + "value": "Authorization,Accept-Encoding" + }, + { + "name": "via", + "value": "1.1 varnish" + }, + { + "name": "x-cache", + "value": "HIT" + }, + { + "name": "x-cache-hits", + "value": "2" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "x-fastly-request-id", + "value": "5066a6f614c16ec6ab0f45f7283892db15363cfa" + }, + { + "name": "x-frame-options", + "value": "deny" + }, + { + "name": "x-github-request-id", + "value": "F8AB:245F89:4A8552:5C58D3:68BBFC7B" + }, + { + "name": "x-served-by", + "value": "cache-lis1490038-LIS" + }, + { + "name": "x-timer", + "value": "S1757150342.223758,VS0,VE0" + }, + { + "name": "x-xss-protection", + "value": "1; mode=block" + } + ], + "headersSize": 898, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-09-06T09:19:02.199Z", + "time": 44, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 44 + } + }, + { + "_id": "9c1d4218bc0b2124f4ba05daec949590", + "_order": 1, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [], + "headersSize": 123, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://raw.githubusercontent.com/frictionlessdata/frictionless-py/refs/heads/main/data/table.keyed.json" + }, + "response": { + "bodySize": 78, + "content": { + "mimeType": "text/plain; charset=utf-8", + "size": 78, + "text": "[\n {\n \"id\": 1,\n \"name\": \"english\"\n },\n {\n \"id\": 2,\n \"name\": \"中国人\"\n }\n]\n" + }, + "cookies": [], + "headers": [ + { + "name": "accept-ranges", + "value": "bytes" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "cache-control", + "value": "max-age=300" + }, + { + "name": "connection", + "value": "keep-alive" + }, + { + "name": "content-encoding", + "value": "gzip" + }, + { + "name": "content-length", + "value": "78" + }, + { + "name": "content-security-policy", + "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" + }, + { + "name": "content-type", + "value": "text/plain; charset=utf-8" + }, + { + "name": "cross-origin-resource-policy", + "value": "cross-origin" + }, + { + "name": "date", + "value": "Sat, 06 Sep 2025 09:19:02 GMT" + }, + { + "name": "etag", + "value": "W/\"6f3db326388b97300ef06688ba9a05ac48b9af407f4cea02dee6a3c83568b3cb\"" + }, + { + "name": "expires", + "value": "Sat, 06 Sep 2025 09:24:02 GMT" + }, + { + "name": "source-age", + "value": "0" + }, + { + "name": "strict-transport-security", + "value": "max-age=31536000" + }, + { + "name": "vary", + "value": "Authorization,Accept-Encoding" + }, + { + "name": "via", + "value": "1.1 varnish" + }, + { + "name": "x-cache", + "value": "HIT" + }, + { + "name": "x-cache-hits", + "value": "1" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "x-fastly-request-id", + "value": "5180a060a2f78ed02e811c2b496022ccfc4aafc3" + }, + { + "name": "x-frame-options", + "value": "deny" + }, + { + "name": "x-github-request-id", + "value": "F8AB:245F89:4A8552:5C58D3:68BBFC7B" + }, + { + "name": "x-served-by", + "value": "cache-lis1490046-LIS" + }, + { + "name": "x-timer", + "value": "S1757150342.310871,VS0,VE1" + }, + { + "name": "x-xss-protection", + "value": "1; mode=block" + } + ], + "headersSize": 898, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-09-06T09:19:02.199Z", + "time": 124, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 124 + } + } + ], + "pages": [], + "version": "1.2" + } +} diff --git a/json/table/fixtures/generated/loadJsonTable-format-json-file-variations-should-load-remote-file_1579387671/recording.har b/json/table/fixtures/generated/loadJsonTable-format-json-file-variations-should-load-remote-file_1579387671/recording.har new file mode 100644 index 00000000..a7403787 --- /dev/null +++ b/json/table/fixtures/generated/loadJsonTable-format-json-file-variations-should-load-remote-file_1579387671/recording.har @@ -0,0 +1,156 @@ +{ + "log": { + "_recordingName": "loadJsonTable (format=json)-file variations-should load remote file", + "creator": { + "comment": "persister:fs", + "name": "Polly.JS", + "version": "6.0.6" + }, + "entries": [ + { + "_id": "9c1d4218bc0b2124f4ba05daec949590", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [], + "headersSize": 123, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://raw.githubusercontent.com/frictionlessdata/frictionless-py/refs/heads/main/data/table.keyed.json" + }, + "response": { + "bodySize": 78, + "content": { + "mimeType": "text/plain; charset=utf-8", + "size": 78, + "text": "[\n {\n \"id\": 1,\n \"name\": \"english\"\n },\n {\n \"id\": 2,\n \"name\": \"中国人\"\n }\n]\n" + }, + "cookies": [], + "headers": [ + { + "name": "accept-ranges", + "value": "bytes" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "cache-control", + "value": "max-age=300" + }, + { + "name": "connection", + "value": "keep-alive" + }, + { + "name": "content-encoding", + "value": "gzip" + }, + { + "name": "content-length", + "value": "78" + }, + { + "name": "content-security-policy", + "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" + }, + { + "name": "content-type", + "value": "text/plain; charset=utf-8" + }, + { + "name": "cross-origin-resource-policy", + "value": "cross-origin" + }, + { + "name": "date", + "value": "Sat, 06 Sep 2025 09:19:02 GMT" + }, + { + "name": "etag", + "value": "W/\"6f3db326388b97300ef06688ba9a05ac48b9af407f4cea02dee6a3c83568b3cb\"" + }, + { + "name": "expires", + "value": "Sat, 06 Sep 2025 09:24:02 GMT" + }, + { + "name": "source-age", + "value": "0" + }, + { + "name": "strict-transport-security", + "value": "max-age=31536000" + }, + { + "name": "vary", + "value": "Authorization,Accept-Encoding" + }, + { + "name": "via", + "value": "1.1 varnish" + }, + { + "name": "x-cache", + "value": "HIT" + }, + { + "name": "x-cache-hits", + "value": "1" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "x-fastly-request-id", + "value": "d8d344a9e2e5dca685df48c067b9524f557bbef7" + }, + { + "name": "x-frame-options", + "value": "deny" + }, + { + "name": "x-github-request-id", + "value": "F8AB:245F89:4A8552:5C58D3:68BBFC7B" + }, + { + "name": "x-served-by", + "value": "cache-lis1490038-LIS" + }, + { + "name": "x-timer", + "value": "S1757150342.967417,VS0,VE174" + }, + { + "name": "x-xss-protection", + "value": "1; mode=block" + } + ], + "headersSize": 900, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-09-06T09:19:01.782Z", + "time": 378, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 378 + } + } + ], + "pages": [], + "version": "1.2" + } +} diff --git a/json/table/fixtures/generated/loadJsonTable-format-jsonl-file-variations-should-load-remote-file-multipart_278600870/recording.har b/json/table/fixtures/generated/loadJsonTable-format-jsonl-file-variations-should-load-remote-file-multipart_278600870/recording.har new file mode 100644 index 00000000..2d9e5fd6 --- /dev/null +++ b/json/table/fixtures/generated/loadJsonTable-format-jsonl-file-variations-should-load-remote-file-multipart_278600870/recording.har @@ -0,0 +1,298 @@ +{ + "log": { + "_recordingName": "loadJsonTable (format=jsonl)-file variations-should load remote file (multipart)", + "creator": { + "comment": "persister:fs", + "name": "Polly.JS", + "version": "6.0.6" + }, + "entries": [ + { + "_id": "d30ef9a2f7ea66a64aee2d7670e18212", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [], + "headersSize": 118, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://raw.githubusercontent.com/frictionlessdata/frictionless-py/refs/heads/main/data/table.jsonl" + }, + "response": { + "bodySize": 63, + "content": { + "mimeType": "text/plain; charset=utf-8", + "size": 63, + "text": "{\"id\":1,\"name\":\"english\"}\n{\"id\":2,\"name\":\"中国人\"}\n" + }, + "cookies": [], + "headers": [ + { + "name": "accept-ranges", + "value": "bytes" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "cache-control", + "value": "max-age=300" + }, + { + "name": "connection", + "value": "keep-alive" + }, + { + "name": "content-encoding", + "value": "gzip" + }, + { + "name": "content-length", + "value": "63" + }, + { + "name": "content-security-policy", + "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" + }, + { + "name": "content-type", + "value": "text/plain; charset=utf-8" + }, + { + "name": "cross-origin-resource-policy", + "value": "cross-origin" + }, + { + "name": "date", + "value": "Sat, 06 Sep 2025 09:48:56 GMT" + }, + { + "name": "etag", + "value": "W/\"645c6d84ada374adbb879fdf78d3dedcd90e40bc81e8eeb882d1022ceff52f8f\"" + }, + { + "name": "expires", + "value": "Sat, 06 Sep 2025 09:53:56 GMT" + }, + { + "name": "source-age", + "value": "10" + }, + { + "name": "strict-transport-security", + "value": "max-age=31536000" + }, + { + "name": "vary", + "value": "Authorization,Accept-Encoding" + }, + { + "name": "via", + "value": "1.1 varnish" + }, + { + "name": "x-cache", + "value": "HIT" + }, + { + "name": "x-cache-hits", + "value": "2" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "x-fastly-request-id", + "value": "0a521df75e4284c45e7153ab664c87b4ba4f8d7f" + }, + { + "name": "x-frame-options", + "value": "deny" + }, + { + "name": "x-github-request-id", + "value": "70CB:1E7364:48DBF2:5ABE7A:68BBFD13" + }, + { + "name": "x-served-by", + "value": "cache-lis1490042-LIS" + }, + { + "name": "x-timer", + "value": "S1757152137.989277,VS0,VE0" + }, + { + "name": "x-xss-protection", + "value": "1; mode=block" + } + ], + "headersSize": 899, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-09-06T09:48:56.978Z", + "time": 55, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 55 + } + }, + { + "_id": "d30ef9a2f7ea66a64aee2d7670e18212", + "_order": 1, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [], + "headersSize": 118, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://raw.githubusercontent.com/frictionlessdata/frictionless-py/refs/heads/main/data/table.jsonl" + }, + "response": { + "bodySize": 63, + "content": { + "mimeType": "text/plain; charset=utf-8", + "size": 63, + "text": "{\"id\":1,\"name\":\"english\"}\n{\"id\":2,\"name\":\"中国人\"}\n" + }, + "cookies": [], + "headers": [ + { + "name": "accept-ranges", + "value": "bytes" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "cache-control", + "value": "max-age=300" + }, + { + "name": "connection", + "value": "keep-alive" + }, + { + "name": "content-encoding", + "value": "gzip" + }, + { + "name": "content-length", + "value": "63" + }, + { + "name": "content-security-policy", + "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" + }, + { + "name": "content-type", + "value": "text/plain; charset=utf-8" + }, + { + "name": "cross-origin-resource-policy", + "value": "cross-origin" + }, + { + "name": "date", + "value": "Sat, 06 Sep 2025 09:48:57 GMT" + }, + { + "name": "etag", + "value": "W/\"645c6d84ada374adbb879fdf78d3dedcd90e40bc81e8eeb882d1022ceff52f8f\"" + }, + { + "name": "expires", + "value": "Sat, 06 Sep 2025 09:53:57 GMT" + }, + { + "name": "source-age", + "value": "10" + }, + { + "name": "strict-transport-security", + "value": "max-age=31536000" + }, + { + "name": "vary", + "value": "Authorization,Accept-Encoding" + }, + { + "name": "via", + "value": "1.1 varnish" + }, + { + "name": "x-cache", + "value": "HIT" + }, + { + "name": "x-cache-hits", + "value": "1" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "x-fastly-request-id", + "value": "4ea9a06527a8d016c7d17ae0e64153075617d7d8" + }, + { + "name": "x-frame-options", + "value": "deny" + }, + { + "name": "x-github-request-id", + "value": "70CB:1E7364:48DBF2:5ABE7A:68BBFD13" + }, + { + "name": "x-served-by", + "value": "cache-lis1490039-LIS" + }, + { + "name": "x-timer", + "value": "S1757152137.085754,VS0,VE1" + }, + { + "name": "x-xss-protection", + "value": "1; mode=block" + } + ], + "headersSize": 899, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-09-06T09:48:56.978Z", + "time": 153, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 153 + } + } + ], + "pages": [], + "version": "1.2" + } +} diff --git a/json/table/fixtures/generated/loadJsonTable-format-jsonl-file-variations-should-load-remote-file-multipart_469959651/recording.har b/json/table/fixtures/generated/loadJsonTable-format-jsonl-file-variations-should-load-remote-file-multipart_469959651/recording.har new file mode 100644 index 00000000..12b4276c --- /dev/null +++ b/json/table/fixtures/generated/loadJsonTable-format-jsonl-file-variations-should-load-remote-file-multipart_469959651/recording.har @@ -0,0 +1,298 @@ +{ + "log": { + "_recordingName": "loadJsonTable (format:jsonl)-file variations-should load remote file (multipart)", + "creator": { + "comment": "persister:fs", + "name": "Polly.JS", + "version": "6.0.6" + }, + "entries": [ + { + "_id": "d30ef9a2f7ea66a64aee2d7670e18212", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [], + "headersSize": 118, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://raw.githubusercontent.com/frictionlessdata/frictionless-py/refs/heads/main/data/table.jsonl" + }, + "response": { + "bodySize": 63, + "content": { + "mimeType": "text/plain; charset=utf-8", + "size": 63, + "text": "{\"id\":1,\"name\":\"english\"}\n{\"id\":2,\"name\":\"中国人\"}\n" + }, + "cookies": [], + "headers": [ + { + "name": "accept-ranges", + "value": "bytes" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "cache-control", + "value": "max-age=300" + }, + { + "name": "connection", + "value": "keep-alive" + }, + { + "name": "content-encoding", + "value": "gzip" + }, + { + "name": "content-length", + "value": "63" + }, + { + "name": "content-security-policy", + "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" + }, + { + "name": "content-type", + "value": "text/plain; charset=utf-8" + }, + { + "name": "cross-origin-resource-policy", + "value": "cross-origin" + }, + { + "name": "date", + "value": "Sat, 06 Sep 2025 09:48:47 GMT" + }, + { + "name": "etag", + "value": "W/\"645c6d84ada374adbb879fdf78d3dedcd90e40bc81e8eeb882d1022ceff52f8f\"" + }, + { + "name": "expires", + "value": "Sat, 06 Sep 2025 09:53:47 GMT" + }, + { + "name": "source-age", + "value": "0" + }, + { + "name": "strict-transport-security", + "value": "max-age=31536000" + }, + { + "name": "vary", + "value": "Authorization,Accept-Encoding" + }, + { + "name": "via", + "value": "1.1 varnish" + }, + { + "name": "x-cache", + "value": "HIT" + }, + { + "name": "x-cache-hits", + "value": "1" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "x-fastly-request-id", + "value": "53bba32dbbbc9e146d96ddd45fa097833b6cb138" + }, + { + "name": "x-frame-options", + "value": "deny" + }, + { + "name": "x-github-request-id", + "value": "70CB:1E7364:48DBF2:5ABE7A:68BBFD13" + }, + { + "name": "x-served-by", + "value": "cache-lis1490044-LIS" + }, + { + "name": "x-timer", + "value": "S1757152128.733345,VS0,VE1" + }, + { + "name": "x-xss-protection", + "value": "1; mode=block" + } + ], + "headersSize": 898, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-09-06T09:48:47.713Z", + "time": 65, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 65 + } + }, + { + "_id": "d30ef9a2f7ea66a64aee2d7670e18212", + "_order": 1, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [], + "headersSize": 118, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://raw.githubusercontent.com/frictionlessdata/frictionless-py/refs/heads/main/data/table.jsonl" + }, + "response": { + "bodySize": 63, + "content": { + "mimeType": "text/plain; charset=utf-8", + "size": 63, + "text": "{\"id\":1,\"name\":\"english\"}\n{\"id\":2,\"name\":\"中国人\"}\n" + }, + "cookies": [], + "headers": [ + { + "name": "accept-ranges", + "value": "bytes" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "cache-control", + "value": "max-age=300" + }, + { + "name": "connection", + "value": "keep-alive" + }, + { + "name": "content-encoding", + "value": "gzip" + }, + { + "name": "content-length", + "value": "63" + }, + { + "name": "content-security-policy", + "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" + }, + { + "name": "content-type", + "value": "text/plain; charset=utf-8" + }, + { + "name": "cross-origin-resource-policy", + "value": "cross-origin" + }, + { + "name": "date", + "value": "Sat, 06 Sep 2025 09:48:48 GMT" + }, + { + "name": "etag", + "value": "W/\"645c6d84ada374adbb879fdf78d3dedcd90e40bc81e8eeb882d1022ceff52f8f\"" + }, + { + "name": "expires", + "value": "Sat, 06 Sep 2025 09:53:48 GMT" + }, + { + "name": "source-age", + "value": "1" + }, + { + "name": "strict-transport-security", + "value": "max-age=31536000" + }, + { + "name": "vary", + "value": "Authorization,Accept-Encoding" + }, + { + "name": "via", + "value": "1.1 varnish" + }, + { + "name": "x-cache", + "value": "HIT" + }, + { + "name": "x-cache-hits", + "value": "1" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "x-fastly-request-id", + "value": "885c5115d17a55a20313298f031bffe054dc20b7" + }, + { + "name": "x-frame-options", + "value": "deny" + }, + { + "name": "x-github-request-id", + "value": "70CB:1E7364:48DBF2:5ABE7A:68BBFD13" + }, + { + "name": "x-served-by", + "value": "cache-lis1490037-LIS" + }, + { + "name": "x-timer", + "value": "S1757152128.075366,VS0,VE1" + }, + { + "name": "x-xss-protection", + "value": "1; mode=block" + } + ], + "headersSize": 898, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-09-06T09:48:47.713Z", + "time": 489, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 489 + } + } + ], + "pages": [], + "version": "1.2" + } +} diff --git a/json/table/fixtures/generated/loadJsonTable-format-jsonl-file-variations-should-load-remote-file-multipart_958872155/recording.har b/json/table/fixtures/generated/loadJsonTable-format-jsonl-file-variations-should-load-remote-file-multipart_958872155/recording.har new file mode 100644 index 00000000..980f1ac5 --- /dev/null +++ b/json/table/fixtures/generated/loadJsonTable-format-jsonl-file-variations-should-load-remote-file-multipart_958872155/recording.har @@ -0,0 +1,298 @@ +{ + "log": { + "_recordingName": "loadJsonTable (format: jsonl)-file variations-should load remote file (multipart)", + "creator": { + "comment": "persister:fs", + "name": "Polly.JS", + "version": "6.0.6" + }, + "entries": [ + { + "_id": "d30ef9a2f7ea66a64aee2d7670e18212", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [], + "headersSize": 118, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://raw.githubusercontent.com/frictionlessdata/frictionless-py/refs/heads/main/data/table.jsonl" + }, + "response": { + "bodySize": 63, + "content": { + "mimeType": "text/plain; charset=utf-8", + "size": 63, + "text": "{\"id\":1,\"name\":\"english\"}\n{\"id\":2,\"name\":\"中国人\"}\n" + }, + "cookies": [], + "headers": [ + { + "name": "accept-ranges", + "value": "bytes" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "cache-control", + "value": "max-age=300" + }, + { + "name": "connection", + "value": "keep-alive" + }, + { + "name": "content-encoding", + "value": "gzip" + }, + { + "name": "content-length", + "value": "63" + }, + { + "name": "content-security-policy", + "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" + }, + { + "name": "content-type", + "value": "text/plain; charset=utf-8" + }, + { + "name": "cross-origin-resource-policy", + "value": "cross-origin" + }, + { + "name": "date", + "value": "Sat, 06 Sep 2025 09:48:49 GMT" + }, + { + "name": "etag", + "value": "W/\"645c6d84ada374adbb879fdf78d3dedcd90e40bc81e8eeb882d1022ceff52f8f\"" + }, + { + "name": "expires", + "value": "Sat, 06 Sep 2025 09:53:49 GMT" + }, + { + "name": "source-age", + "value": "2" + }, + { + "name": "strict-transport-security", + "value": "max-age=31536000" + }, + { + "name": "vary", + "value": "Authorization,Accept-Encoding" + }, + { + "name": "via", + "value": "1.1 varnish" + }, + { + "name": "x-cache", + "value": "HIT" + }, + { + "name": "x-cache-hits", + "value": "2" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "x-fastly-request-id", + "value": "597150a0e7525a655a942aad579e124a20e56aa5" + }, + { + "name": "x-frame-options", + "value": "deny" + }, + { + "name": "x-github-request-id", + "value": "70CB:1E7364:48DBF2:5ABE7A:68BBFD13" + }, + { + "name": "x-served-by", + "value": "cache-lis1490045-LIS" + }, + { + "name": "x-timer", + "value": "S1757152130.775330,VS0,VE0" + }, + { + "name": "x-xss-protection", + "value": "1; mode=block" + } + ], + "headersSize": 898, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-09-06T09:48:49.763Z", + "time": 176, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 176 + } + }, + { + "_id": "d30ef9a2f7ea66a64aee2d7670e18212", + "_order": 1, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [], + "headersSize": 118, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://raw.githubusercontent.com/frictionlessdata/frictionless-py/refs/heads/main/data/table.jsonl" + }, + "response": { + "bodySize": 63, + "content": { + "mimeType": "text/plain; charset=utf-8", + "size": 63, + "text": "{\"id\":1,\"name\":\"english\"}\n{\"id\":2,\"name\":\"中国人\"}\n" + }, + "cookies": [], + "headers": [ + { + "name": "accept-ranges", + "value": "bytes" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "cache-control", + "value": "max-age=300" + }, + { + "name": "connection", + "value": "keep-alive" + }, + { + "name": "content-encoding", + "value": "gzip" + }, + { + "name": "content-length", + "value": "63" + }, + { + "name": "content-security-policy", + "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" + }, + { + "name": "content-type", + "value": "text/plain; charset=utf-8" + }, + { + "name": "cross-origin-resource-policy", + "value": "cross-origin" + }, + { + "name": "date", + "value": "Sat, 06 Sep 2025 09:48:50 GMT" + }, + { + "name": "etag", + "value": "W/\"645c6d84ada374adbb879fdf78d3dedcd90e40bc81e8eeb882d1022ceff52f8f\"" + }, + { + "name": "expires", + "value": "Sat, 06 Sep 2025 09:53:50 GMT" + }, + { + "name": "source-age", + "value": "3" + }, + { + "name": "strict-transport-security", + "value": "max-age=31536000" + }, + { + "name": "vary", + "value": "Authorization,Accept-Encoding" + }, + { + "name": "via", + "value": "1.1 varnish" + }, + { + "name": "x-cache", + "value": "HIT" + }, + { + "name": "x-cache-hits", + "value": "2" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "x-fastly-request-id", + "value": "2582fac5fd67d140d3efed493e00744839e0adfa" + }, + { + "name": "x-frame-options", + "value": "deny" + }, + { + "name": "x-github-request-id", + "value": "70CB:1E7364:48DBF2:5ABE7A:68BBFD13" + }, + { + "name": "x-served-by", + "value": "cache-lis1490037-LIS" + }, + { + "name": "x-timer", + "value": "S1757152130.005758,VS0,VE0" + }, + { + "name": "x-xss-protection", + "value": "1; mode=block" + } + ], + "headersSize": 898, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-09-06T09:48:49.764Z", + "time": 291, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 291 + } + } + ], + "pages": [], + "version": "1.2" + } +} diff --git a/json/table/fixtures/generated/loadJsonTable-format-jsonl-file-variations-should-load-remote-file_1232694372/recording.har b/json/table/fixtures/generated/loadJsonTable-format-jsonl-file-variations-should-load-remote-file_1232694372/recording.har new file mode 100644 index 00000000..937f301f --- /dev/null +++ b/json/table/fixtures/generated/loadJsonTable-format-jsonl-file-variations-should-load-remote-file_1232694372/recording.har @@ -0,0 +1,156 @@ +{ + "log": { + "_recordingName": "loadJsonTable (format: jsonl)-file variations-should load remote file", + "creator": { + "comment": "persister:fs", + "name": "Polly.JS", + "version": "6.0.6" + }, + "entries": [ + { + "_id": "d30ef9a2f7ea66a64aee2d7670e18212", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [], + "headersSize": 118, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://raw.githubusercontent.com/frictionlessdata/frictionless-py/refs/heads/main/data/table.jsonl" + }, + "response": { + "bodySize": 63, + "content": { + "mimeType": "text/plain; charset=utf-8", + "size": 63, + "text": "{\"id\":1,\"name\":\"english\"}\n{\"id\":2,\"name\":\"中国人\"}\n" + }, + "cookies": [], + "headers": [ + { + "name": "accept-ranges", + "value": "bytes" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "cache-control", + "value": "max-age=300" + }, + { + "name": "connection", + "value": "keep-alive" + }, + { + "name": "content-encoding", + "value": "gzip" + }, + { + "name": "content-length", + "value": "63" + }, + { + "name": "content-security-policy", + "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" + }, + { + "name": "content-type", + "value": "text/plain; charset=utf-8" + }, + { + "name": "cross-origin-resource-policy", + "value": "cross-origin" + }, + { + "name": "date", + "value": "Sat, 06 Sep 2025 09:48:49 GMT" + }, + { + "name": "etag", + "value": "W/\"645c6d84ada374adbb879fdf78d3dedcd90e40bc81e8eeb882d1022ceff52f8f\"" + }, + { + "name": "expires", + "value": "Sat, 06 Sep 2025 09:53:49 GMT" + }, + { + "name": "source-age", + "value": "2" + }, + { + "name": "strict-transport-security", + "value": "max-age=31536000" + }, + { + "name": "vary", + "value": "Authorization,Accept-Encoding" + }, + { + "name": "via", + "value": "1.1 varnish" + }, + { + "name": "x-cache", + "value": "HIT" + }, + { + "name": "x-cache-hits", + "value": "1" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "x-fastly-request-id", + "value": "7f21a6318294e6ac27b5e46b74b0fa0c689ea32c" + }, + { + "name": "x-frame-options", + "value": "deny" + }, + { + "name": "x-github-request-id", + "value": "70CB:1E7364:48DBF2:5ABE7A:68BBFD13" + }, + { + "name": "x-served-by", + "value": "cache-lis1490045-LIS" + }, + { + "name": "x-timer", + "value": "S1757152130.675646,VS0,VE1" + }, + { + "name": "x-xss-protection", + "value": "1; mode=block" + } + ], + "headersSize": 898, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-09-06T09:48:49.438Z", + "time": 289, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 289 + } + } + ], + "pages": [], + "version": "1.2" + } +} diff --git a/json/table/fixtures/generated/loadJsonTable-format-jsonl-file-variations-should-load-remote-file_1248351709/recording.har b/json/table/fixtures/generated/loadJsonTable-format-jsonl-file-variations-should-load-remote-file_1248351709/recording.har new file mode 100644 index 00000000..6f7c676a --- /dev/null +++ b/json/table/fixtures/generated/loadJsonTable-format-jsonl-file-variations-should-load-remote-file_1248351709/recording.har @@ -0,0 +1,156 @@ +{ + "log": { + "_recordingName": "loadJsonTable (format=jsonl)-file variations-should load remote file", + "creator": { + "comment": "persister:fs", + "name": "Polly.JS", + "version": "6.0.6" + }, + "entries": [ + { + "_id": "d30ef9a2f7ea66a64aee2d7670e18212", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [], + "headersSize": 118, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://raw.githubusercontent.com/frictionlessdata/frictionless-py/refs/heads/main/data/table.jsonl" + }, + "response": { + "bodySize": 63, + "content": { + "mimeType": "text/plain; charset=utf-8", + "size": 63, + "text": "{\"id\":1,\"name\":\"english\"}\n{\"id\":2,\"name\":\"中国人\"}\n" + }, + "cookies": [], + "headers": [ + { + "name": "accept-ranges", + "value": "bytes" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "cache-control", + "value": "max-age=300" + }, + { + "name": "connection", + "value": "keep-alive" + }, + { + "name": "content-encoding", + "value": "gzip" + }, + { + "name": "content-length", + "value": "63" + }, + { + "name": "content-security-policy", + "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" + }, + { + "name": "content-type", + "value": "text/plain; charset=utf-8" + }, + { + "name": "cross-origin-resource-policy", + "value": "cross-origin" + }, + { + "name": "date", + "value": "Sat, 06 Sep 2025 09:48:56 GMT" + }, + { + "name": "etag", + "value": "W/\"645c6d84ada374adbb879fdf78d3dedcd90e40bc81e8eeb882d1022ceff52f8f\"" + }, + { + "name": "expires", + "value": "Sat, 06 Sep 2025 09:53:56 GMT" + }, + { + "name": "source-age", + "value": "10" + }, + { + "name": "strict-transport-security", + "value": "max-age=31536000" + }, + { + "name": "vary", + "value": "Authorization,Accept-Encoding" + }, + { + "name": "via", + "value": "1.1 varnish" + }, + { + "name": "x-cache", + "value": "HIT" + }, + { + "name": "x-cache-hits", + "value": "1" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "x-fastly-request-id", + "value": "a80b0c364196e682dbea6ccd62521f21226f26e5" + }, + { + "name": "x-frame-options", + "value": "deny" + }, + { + "name": "x-github-request-id", + "value": "70CB:1E7364:48DBF2:5ABE7A:68BBFD13" + }, + { + "name": "x-served-by", + "value": "cache-lis1490042-LIS" + }, + { + "name": "x-timer", + "value": "S1757152137.880736,VS0,VE1" + }, + { + "name": "x-xss-protection", + "value": "1; mode=block" + } + ], + "headersSize": 899, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-09-06T09:48:56.611Z", + "time": 320, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 320 + } + } + ], + "pages": [], + "version": "1.2" + } +} diff --git a/json/table/fixtures/generated/loadJsonTable-format-jsonl-file-variations-should-load-remote-file_928976444/recording.har b/json/table/fixtures/generated/loadJsonTable-format-jsonl-file-variations-should-load-remote-file_928976444/recording.har new file mode 100644 index 00000000..898e13b9 --- /dev/null +++ b/json/table/fixtures/generated/loadJsonTable-format-jsonl-file-variations-should-load-remote-file_928976444/recording.har @@ -0,0 +1,156 @@ +{ + "log": { + "_recordingName": "loadJsonTable (format:jsonl)-file variations-should load remote file", + "creator": { + "comment": "persister:fs", + "name": "Polly.JS", + "version": "6.0.6" + }, + "entries": [ + { + "_id": "d30ef9a2f7ea66a64aee2d7670e18212", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [], + "headersSize": 118, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://raw.githubusercontent.com/frictionlessdata/frictionless-py/refs/heads/main/data/table.jsonl" + }, + "response": { + "bodySize": 63, + "content": { + "mimeType": "text/plain; charset=utf-8", + "size": 63, + "text": "{\"id\":1,\"name\":\"english\"}\n{\"id\":2,\"name\":\"中国人\"}\n" + }, + "cookies": [], + "headers": [ + { + "name": "accept-ranges", + "value": "bytes" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "cache-control", + "value": "max-age=300" + }, + { + "name": "connection", + "value": "keep-alive" + }, + { + "name": "content-encoding", + "value": "gzip" + }, + { + "name": "content-length", + "value": "63" + }, + { + "name": "content-security-policy", + "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" + }, + { + "name": "content-type", + "value": "text/plain; charset=utf-8" + }, + { + "name": "cross-origin-resource-policy", + "value": "cross-origin" + }, + { + "name": "date", + "value": "Sat, 06 Sep 2025 09:48:47 GMT" + }, + { + "name": "etag", + "value": "W/\"645c6d84ada374adbb879fdf78d3dedcd90e40bc81e8eeb882d1022ceff52f8f\"" + }, + { + "name": "expires", + "value": "Sat, 06 Sep 2025 09:53:47 GMT" + }, + { + "name": "source-age", + "value": "0" + }, + { + "name": "strict-transport-security", + "value": "max-age=31536000" + }, + { + "name": "vary", + "value": "Authorization,Accept-Encoding" + }, + { + "name": "via", + "value": "1.1 varnish" + }, + { + "name": "x-cache", + "value": "HIT" + }, + { + "name": "x-cache-hits", + "value": "0" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "x-fastly-request-id", + "value": "f04fd549afb0463dd3c9156f080da7e9afbe20f3" + }, + { + "name": "x-frame-options", + "value": "deny" + }, + { + "name": "x-github-request-id", + "value": "70CB:1E7364:48DBF2:5ABE7A:68BBFD13" + }, + { + "name": "x-served-by", + "value": "cache-lis1490044-LIS" + }, + { + "name": "x-timer", + "value": "S1757152127.156714,VS0,VE156" + }, + { + "name": "x-xss-protection", + "value": "1; mode=block" + } + ], + "headersSize": 900, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-09-06T09:48:46.866Z", + "time": 801, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 801 + } + } + ], + "pages": [], + "version": "1.2" + } +} diff --git a/json/table/fixtures/generated/loadJsonTable-json-file-variations-should-load-remote-file-multipart_1453173580/recording.har b/json/table/fixtures/generated/loadJsonTable-json-file-variations-should-load-remote-file-multipart_1453173580/recording.har new file mode 100644 index 00000000..ee81f014 --- /dev/null +++ b/json/table/fixtures/generated/loadJsonTable-json-file-variations-should-load-remote-file-multipart_1453173580/recording.har @@ -0,0 +1,298 @@ +{ + "log": { + "_recordingName": "loadJsonTable (json)-file variations-should load remote file (multipart)", + "creator": { + "comment": "persister:fs", + "name": "Polly.JS", + "version": "6.0.6" + }, + "entries": [ + { + "_id": "9c1d4218bc0b2124f4ba05daec949590", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [], + "headersSize": 123, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://raw.githubusercontent.com/frictionlessdata/frictionless-py/refs/heads/main/data/table.keyed.json" + }, + "response": { + "bodySize": 78, + "content": { + "mimeType": "text/plain; charset=utf-8", + "size": 78, + "text": "[\n {\n \"id\": 1,\n \"name\": \"english\"\n },\n {\n \"id\": 2,\n \"name\": \"中国人\"\n }\n]\n" + }, + "cookies": [], + "headers": [ + { + "name": "accept-ranges", + "value": "bytes" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "cache-control", + "value": "max-age=300" + }, + { + "name": "connection", + "value": "keep-alive" + }, + { + "name": "content-encoding", + "value": "gzip" + }, + { + "name": "content-length", + "value": "78" + }, + { + "name": "content-security-policy", + "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" + }, + { + "name": "content-type", + "value": "text/plain; charset=utf-8" + }, + { + "name": "cross-origin-resource-policy", + "value": "cross-origin" + }, + { + "name": "date", + "value": "Sat, 06 Sep 2025 09:20:45 GMT" + }, + { + "name": "etag", + "value": "W/\"6f3db326388b97300ef06688ba9a05ac48b9af407f4cea02dee6a3c83568b3cb\"" + }, + { + "name": "expires", + "value": "Sat, 06 Sep 2025 09:25:45 GMT" + }, + { + "name": "source-age", + "value": "104" + }, + { + "name": "strict-transport-security", + "value": "max-age=31536000" + }, + { + "name": "vary", + "value": "Authorization,Accept-Encoding" + }, + { + "name": "via", + "value": "1.1 varnish" + }, + { + "name": "x-cache", + "value": "HIT" + }, + { + "name": "x-cache-hits", + "value": "1" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "x-fastly-request-id", + "value": "83ee989a33056a098f9dfb4091b78717e2e56b68" + }, + { + "name": "x-frame-options", + "value": "deny" + }, + { + "name": "x-github-request-id", + "value": "F8AB:245F89:4A8552:5C58D3:68BBFC7B" + }, + { + "name": "x-served-by", + "value": "cache-lis1490058-LIS" + }, + { + "name": "x-timer", + "value": "S1757150446.813180,VS0,VE1" + }, + { + "name": "x-xss-protection", + "value": "1; mode=block" + } + ], + "headersSize": 900, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-09-06T09:20:45.793Z", + "time": 39, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 39 + } + }, + { + "_id": "9c1d4218bc0b2124f4ba05daec949590", + "_order": 1, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [], + "headersSize": 123, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://raw.githubusercontent.com/frictionlessdata/frictionless-py/refs/heads/main/data/table.keyed.json" + }, + "response": { + "bodySize": 78, + "content": { + "mimeType": "text/plain; charset=utf-8", + "size": 78, + "text": "[\n {\n \"id\": 1,\n \"name\": \"english\"\n },\n {\n \"id\": 2,\n \"name\": \"中国人\"\n }\n]\n" + }, + "cookies": [], + "headers": [ + { + "name": "accept-ranges", + "value": "bytes" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "cache-control", + "value": "max-age=300" + }, + { + "name": "connection", + "value": "keep-alive" + }, + { + "name": "content-encoding", + "value": "gzip" + }, + { + "name": "content-length", + "value": "78" + }, + { + "name": "content-security-policy", + "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" + }, + { + "name": "content-type", + "value": "text/plain; charset=utf-8" + }, + { + "name": "cross-origin-resource-policy", + "value": "cross-origin" + }, + { + "name": "date", + "value": "Sat, 06 Sep 2025 09:20:45 GMT" + }, + { + "name": "etag", + "value": "W/\"6f3db326388b97300ef06688ba9a05ac48b9af407f4cea02dee6a3c83568b3cb\"" + }, + { + "name": "expires", + "value": "Sat, 06 Sep 2025 09:25:45 GMT" + }, + { + "name": "source-age", + "value": "104" + }, + { + "name": "strict-transport-security", + "value": "max-age=31536000" + }, + { + "name": "vary", + "value": "Authorization,Accept-Encoding" + }, + { + "name": "via", + "value": "1.1 varnish" + }, + { + "name": "x-cache", + "value": "HIT" + }, + { + "name": "x-cache-hits", + "value": "1" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "x-fastly-request-id", + "value": "f180e2b07c1f1a1588a78fe97405d267bf9f86f9" + }, + { + "name": "x-frame-options", + "value": "deny" + }, + { + "name": "x-github-request-id", + "value": "F8AB:245F89:4A8552:5C58D3:68BBFC7B" + }, + { + "name": "x-served-by", + "value": "cache-lis1490041-LIS" + }, + { + "name": "x-timer", + "value": "S1757150446.901348,VS0,VE1" + }, + { + "name": "x-xss-protection", + "value": "1; mode=block" + } + ], + "headersSize": 900, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-09-06T09:20:45.793Z", + "time": 134, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 134 + } + } + ], + "pages": [], + "version": "1.2" + } +} diff --git a/json/table/fixtures/generated/loadJsonTable-json-file-variations-should-load-remote-file_4010456287/recording.har b/json/table/fixtures/generated/loadJsonTable-json-file-variations-should-load-remote-file_4010456287/recording.har new file mode 100644 index 00000000..7573b973 --- /dev/null +++ b/json/table/fixtures/generated/loadJsonTable-json-file-variations-should-load-remote-file_4010456287/recording.har @@ -0,0 +1,156 @@ +{ + "log": { + "_recordingName": "loadJsonTable (json)-file variations-should load remote file", + "creator": { + "comment": "persister:fs", + "name": "Polly.JS", + "version": "6.0.6" + }, + "entries": [ + { + "_id": "9c1d4218bc0b2124f4ba05daec949590", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [], + "headersSize": 123, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://raw.githubusercontent.com/frictionlessdata/frictionless-py/refs/heads/main/data/table.keyed.json" + }, + "response": { + "bodySize": 78, + "content": { + "mimeType": "text/plain; charset=utf-8", + "size": 78, + "text": "[\n {\n \"id\": 1,\n \"name\": \"english\"\n },\n {\n \"id\": 2,\n \"name\": \"中国人\"\n }\n]\n" + }, + "cookies": [], + "headers": [ + { + "name": "accept-ranges", + "value": "bytes" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "cache-control", + "value": "max-age=300" + }, + { + "name": "connection", + "value": "keep-alive" + }, + { + "name": "content-encoding", + "value": "gzip" + }, + { + "name": "content-length", + "value": "78" + }, + { + "name": "content-security-policy", + "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" + }, + { + "name": "content-type", + "value": "text/plain; charset=utf-8" + }, + { + "name": "cross-origin-resource-policy", + "value": "cross-origin" + }, + { + "name": "date", + "value": "Sat, 06 Sep 2025 09:20:45 GMT" + }, + { + "name": "etag", + "value": "W/\"6f3db326388b97300ef06688ba9a05ac48b9af407f4cea02dee6a3c83568b3cb\"" + }, + { + "name": "expires", + "value": "Sat, 06 Sep 2025 09:25:45 GMT" + }, + { + "name": "source-age", + "value": "104" + }, + { + "name": "strict-transport-security", + "value": "max-age=31536000" + }, + { + "name": "vary", + "value": "Authorization,Accept-Encoding" + }, + { + "name": "via", + "value": "1.1 varnish" + }, + { + "name": "x-cache", + "value": "HIT" + }, + { + "name": "x-cache-hits", + "value": "0" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "x-fastly-request-id", + "value": "513c911a368bd2333b3bda6c5b701356a18be44d" + }, + { + "name": "x-frame-options", + "value": "deny" + }, + { + "name": "x-github-request-id", + "value": "F8AB:245F89:4A8552:5C58D3:68BBFC7B" + }, + { + "name": "x-served-by", + "value": "cache-lis1490058-LIS" + }, + { + "name": "x-timer", + "value": "S1757150446.723354,VS0,VE1" + }, + { + "name": "x-xss-protection", + "value": "1; mode=block" + } + ], + "headersSize": 900, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-09-06T09:20:45.535Z", + "time": 215, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 215 + } + } + ], + "pages": [], + "version": "1.2" + } +} diff --git a/json/table/fixtures/generated/loadJsonTable-jsonl-file-variations-should-load-remote-file-multipart_1746504766/recording.har b/json/table/fixtures/generated/loadJsonTable-jsonl-file-variations-should-load-remote-file-multipart_1746504766/recording.har new file mode 100644 index 00000000..c013b295 --- /dev/null +++ b/json/table/fixtures/generated/loadJsonTable-jsonl-file-variations-should-load-remote-file-multipart_1746504766/recording.har @@ -0,0 +1,298 @@ +{ + "log": { + "_recordingName": "loadJsonTable (jsonl)-file variations-should load remote file (multipart)", + "creator": { + "comment": "persister:fs", + "name": "Polly.JS", + "version": "6.0.6" + }, + "entries": [ + { + "_id": "d30ef9a2f7ea66a64aee2d7670e18212", + "_order": 1, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [], + "headersSize": 118, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://raw.githubusercontent.com/frictionlessdata/frictionless-py/refs/heads/main/data/table.jsonl" + }, + "response": { + "bodySize": 63, + "content": { + "mimeType": "text/plain; charset=utf-8", + "size": 63, + "text": "{\"id\":1,\"name\":\"english\"}\n{\"id\":2,\"name\":\"中国人\"}\n" + }, + "cookies": [], + "headers": [ + { + "name": "accept-ranges", + "value": "bytes" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "cache-control", + "value": "max-age=300" + }, + { + "name": "connection", + "value": "keep-alive" + }, + { + "name": "content-encoding", + "value": "gzip" + }, + { + "name": "content-length", + "value": "63" + }, + { + "name": "content-security-policy", + "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" + }, + { + "name": "content-type", + "value": "text/plain; charset=utf-8" + }, + { + "name": "cross-origin-resource-policy", + "value": "cross-origin" + }, + { + "name": "date", + "value": "Sat, 06 Sep 2025 09:21:32 GMT" + }, + { + "name": "etag", + "value": "W/\"645c6d84ada374adbb879fdf78d3dedcd90e40bc81e8eeb882d1022ceff52f8f\"" + }, + { + "name": "expires", + "value": "Sat, 06 Sep 2025 09:26:32 GMT" + }, + { + "name": "source-age", + "value": "8" + }, + { + "name": "strict-transport-security", + "value": "max-age=31536000" + }, + { + "name": "vary", + "value": "Authorization,Accept-Encoding" + }, + { + "name": "via", + "value": "1.1 varnish" + }, + { + "name": "x-cache", + "value": "HIT" + }, + { + "name": "x-cache-hits", + "value": "1" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "x-fastly-request-id", + "value": "78c538745057a51cdc3335c18ce22d5671cbea32" + }, + { + "name": "x-frame-options", + "value": "deny" + }, + { + "name": "x-github-request-id", + "value": "70CB:1E7364:48DBF2:5ABE7A:68BBFD13" + }, + { + "name": "x-served-by", + "value": "cache-lis1490047-LIS" + }, + { + "name": "x-timer", + "value": "S1757150493.594105,VS0,VE1" + }, + { + "name": "x-xss-protection", + "value": "1; mode=block" + } + ], + "headersSize": 898, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-09-06T09:21:32.455Z", + "time": 163, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 163 + } + }, + { + "_id": "d30ef9a2f7ea66a64aee2d7670e18212", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [], + "headersSize": 118, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://raw.githubusercontent.com/frictionlessdata/frictionless-py/refs/heads/main/data/table.jsonl" + }, + "response": { + "bodySize": 63, + "content": { + "mimeType": "text/plain; charset=utf-8", + "size": 63, + "text": "{\"id\":1,\"name\":\"english\"}\n{\"id\":2,\"name\":\"中国人\"}\n" + }, + "cookies": [], + "headers": [ + { + "name": "accept-ranges", + "value": "bytes" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "cache-control", + "value": "max-age=300" + }, + { + "name": "connection", + "value": "keep-alive" + }, + { + "name": "content-encoding", + "value": "gzip" + }, + { + "name": "content-length", + "value": "63" + }, + { + "name": "content-security-policy", + "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" + }, + { + "name": "content-type", + "value": "text/plain; charset=utf-8" + }, + { + "name": "cross-origin-resource-policy", + "value": "cross-origin" + }, + { + "name": "date", + "value": "Sat, 06 Sep 2025 09:21:32 GMT" + }, + { + "name": "etag", + "value": "W/\"645c6d84ada374adbb879fdf78d3dedcd90e40bc81e8eeb882d1022ceff52f8f\"" + }, + { + "name": "expires", + "value": "Sat, 06 Sep 2025 09:26:32 GMT" + }, + { + "name": "source-age", + "value": "8" + }, + { + "name": "strict-transport-security", + "value": "max-age=31536000" + }, + { + "name": "vary", + "value": "Authorization,Accept-Encoding" + }, + { + "name": "via", + "value": "1.1 varnish" + }, + { + "name": "x-cache", + "value": "HIT" + }, + { + "name": "x-cache-hits", + "value": "1" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "x-fastly-request-id", + "value": "84ed45f55f55a58015153e227c7ac8ecfe2d6c7d" + }, + { + "name": "x-frame-options", + "value": "deny" + }, + { + "name": "x-github-request-id", + "value": "70CB:1E7364:48DBF2:5ABE7A:68BBFD13" + }, + { + "name": "x-served-by", + "value": "cache-lis1490044-LIS" + }, + { + "name": "x-timer", + "value": "S1757150493.595364,VS0,VE1" + }, + { + "name": "x-xss-protection", + "value": "1; mode=block" + } + ], + "headersSize": 898, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-09-06T09:21:32.455Z", + "time": 164, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 164 + } + } + ], + "pages": [], + "version": "1.2" + } +} diff --git a/csv/table/fixtures/generated/loadCsvTable-file-variations-should-load-remote-file_4234893388/recording.har b/json/table/fixtures/generated/loadJsonTable-jsonl-file-variations-should-load-remote-file_3190404165/recording.har similarity index 76% rename from csv/table/fixtures/generated/loadCsvTable-file-variations-should-load-remote-file_4234893388/recording.har rename to json/table/fixtures/generated/loadJsonTable-jsonl-file-variations-should-load-remote-file_3190404165/recording.har index 1897e489..e7019f82 100644 --- a/csv/table/fixtures/generated/loadCsvTable-file-variations-should-load-remote-file_4234893388/recording.har +++ b/json/table/fixtures/generated/loadJsonTable-jsonl-file-variations-should-load-remote-file_3190404165/recording.har @@ -1,6 +1,6 @@ { "log": { - "_recordingName": "loadCsvTable-file variations-should load remote file", + "_recordingName": "loadJsonTable (jsonl)-file variations-should load remote file", "creator": { "comment": "persister:fs", "name": "Polly.JS", @@ -8,25 +8,25 @@ }, "entries": [ { - "_id": "c00cb7e4f4b92fa8d3f3fc16f382ce35", + "_id": "d30ef9a2f7ea66a64aee2d7670e18212", "_order": 0, "cache": {}, "request": { "bodySize": 0, "cookies": [], "headers": [], - "headersSize": 115, + "headersSize": 118, "httpVersion": "HTTP/1.1", "method": "GET", "queryString": [], - "url": "https://raw.githubusercontent.com/datisthq/dpkit/refs/heads/main/core/package/fixtures/table.csv" + "url": "https://raw.githubusercontent.com/frictionlessdata/frictionless-py/refs/heads/main/data/table.jsonl" }, "response": { - "bodySize": 51, + "bodySize": 63, "content": { "mimeType": "text/plain; charset=utf-8", - "size": 51, - "text": "id,name\n1,english\n2,中国人\n" + "size": 63, + "text": "{\"id\":1,\"name\":\"english\"}\n{\"id\":2,\"name\":\"中国人\"}\n" }, "cookies": [], "headers": [ @@ -52,7 +52,7 @@ }, { "name": "content-length", - "value": "51" + "value": "63" }, { "name": "content-security-policy", @@ -68,15 +68,15 @@ }, { "name": "date", - "value": "Thu, 07 Aug 2025 07:49:21 GMT" + "value": "Sat, 06 Sep 2025 09:21:24 GMT" }, { "name": "etag", - "value": "W/\"0172a2fd99319bed82fe7cccbd7a44b27a77f7200caf0d04b7f23cbb6b81026d\"" + "value": "W/\"645c6d84ada374adbb879fdf78d3dedcd90e40bc81e8eeb882d1022ceff52f8f\"" }, { "name": "expires", - "value": "Thu, 07 Aug 2025 07:54:21 GMT" + "value": "Sat, 06 Sep 2025 09:26:24 GMT" }, { "name": "source-age", @@ -108,7 +108,7 @@ }, { "name": "x-fastly-request-id", - "value": "3c3523724b8cd2b888233b8accaa910abd099969" + "value": "2df082b1b7d9f6324fd8e2b9bed15180b71e7151" }, { "name": "x-frame-options", @@ -116,29 +116,29 @@ }, { "name": "x-github-request-id", - "value": "D0E2:1C49BB:154BB4B:18AC644:68945A7D" + "value": "70CB:1E7364:48DBF2:5ABE7A:68BBFD13" }, { "name": "x-served-by", - "value": "cache-lis1490032-LIS" + "value": "cache-lis1490058-LIS" }, { "name": "x-timer", - "value": "S1754552962.771787,VS0,VE182" + "value": "S1757150485.674322,VS0,VE171" }, { "name": "x-xss-protection", "value": "1; mode=block" } ], - "headersSize": 903, + "headersSize": 901, "httpVersion": "HTTP/1.1", "redirectURL": "", "status": 200, "statusText": "OK" }, - "startedDateTime": "2025-08-07T07:49:21.578Z", - "time": 499, + "startedDateTime": "2025-09-06T09:21:24.546Z", + "time": 320, "timings": { "blocked": -1, "connect": -1, @@ -146,7 +146,7 @@ "receive": 0, "send": 0, "ssl": -1, - "wait": 499 + "wait": 320 } } ], diff --git a/json/table/index.ts b/json/table/index.ts index 357f58c2..0b10c6ac 100644 --- a/json/table/index.ts +++ b/json/table/index.ts @@ -1,2 +1,2 @@ -export { loadJsonTable, loadJsonlTable } from "./load.ts" -export { saveJsonTable, saveJsonlTable } from "./save.ts" +export { loadJsonTable } from "./load.ts" +export { saveJsonTable } from "./save.ts" diff --git a/json/table/load.spec.ts b/json/table/load.spec.ts index 8251c6ea..c542c0c3 100644 --- a/json/table/load.spec.ts +++ b/json/table/load.spec.ts @@ -1,7 +1,7 @@ import { writeTempFile } from "@dpkit/file" import { useRecording } from "@dpkit/test" import { describe, expect, it } from "vitest" -import { loadJsonTable, loadJsonlTable } from "./load.ts" +import { loadJsonTable } from "./load.ts" useRecording() @@ -12,6 +12,7 @@ describe("loadJsonTable", () => { const path = await writeTempFile(body) const table = await loadJsonTable({ path }) + expect((await table.collect()).toRecords()).toEqual([ { id: 1, name: "english" }, { id: 2, name: "中文" }, @@ -23,7 +24,10 @@ describe("loadJsonTable", () => { const path1 = await writeTempFile(body) const path2 = await writeTempFile(body) - const table = await loadJsonTable({ path: [path1, path2] }) + const table = await loadJsonTable({ + path: [path1, path2], + }) + expect((await table.collect()).toRecords()).toEqual([ { id: 1, name: "english" }, { id: 2, name: "中文" }, @@ -123,13 +127,13 @@ describe("loadJsonTable", () => { }) }) -describe("loadJsonlTable", () => { +describe("loadJsonTable (format=jsonl)", () => { describe("file variations", () => { it("should load local file", async () => { const body = '{"id":1,"name":"english"}\n{"id":2,"name":"中文"}' const path = await writeTempFile(body) - const table = await loadJsonlTable({ path }) + const table = await loadJsonTable({ path, format: "jsonl" }) expect((await table.collect()).toRecords()).toEqual([ { id: 1, name: "english" }, { id: 2, name: "中文" }, @@ -141,7 +145,11 @@ describe("loadJsonlTable", () => { const path1 = await writeTempFile(body) const path2 = await writeTempFile(body) - const table = await loadJsonlTable({ path: [path1, path2] }) + const table = await loadJsonTable({ + path: [path1, path2], + format: "jsonl", + }) + expect((await table.collect()).toRecords()).toEqual([ { id: 1, name: "english" }, { id: 2, name: "中文" }, @@ -151,8 +159,9 @@ describe("loadJsonlTable", () => { }) it("should load remote file", async () => { - const table = await loadJsonlTable({ + const table = await loadJsonTable({ path: "https://raw.githubusercontent.com/frictionlessdata/frictionless-py/refs/heads/main/data/table.jsonl", + format: "jsonl", }) expect((await table.collect()).toRecords()).toEqual([ @@ -162,11 +171,12 @@ describe("loadJsonlTable", () => { }) it("should load remote file (multipart)", async () => { - const table = await loadJsonlTable({ + const table = await loadJsonTable({ path: [ "https://raw.githubusercontent.com/frictionlessdata/frictionless-py/refs/heads/main/data/table.jsonl", "https://raw.githubusercontent.com/frictionlessdata/frictionless-py/refs/heads/main/data/table.jsonl", ], + format: "jsonl", }) expect((await table.collect()).toRecords()).toEqual([ @@ -183,8 +193,9 @@ describe("loadJsonlTable", () => { const body = '{"id":1,"name":"english"}\n{"id":2,"name":"中文"}' const path = await writeTempFile(body) - const table = await loadJsonlTable({ + const table = await loadJsonTable({ path, + format: "jsonl", dialect: { itemKeys: ["name"] }, }) @@ -198,8 +209,9 @@ describe("loadJsonlTable", () => { const body = '["id","name"]\n[1,"english"]\n[2,"中文"]' const path = await writeTempFile(body) - const table = await loadJsonlTable({ + const table = await loadJsonTable({ path, + format: "jsonl", dialect: { itemType: "array" }, }) @@ -213,8 +225,9 @@ describe("loadJsonlTable", () => { const body = '{"id":1,"name":"english"}\n{"id":2,"name":"中文"}' const path = await writeTempFile(body) - const table = await loadJsonlTable({ + const table = await loadJsonTable({ path, + format: "jsonl", dialect: { itemType: "object" }, }) diff --git a/json/table/load.ts b/json/table/load.ts index 599b7b46..03b7a9c3 100644 --- a/json/table/load.ts +++ b/json/table/load.ts @@ -1,28 +1,23 @@ import type { Dialect, Resource } from "@dpkit/core" import { loadResourceDialect } from "@dpkit/core" +import { loadResourceSchema } from "@dpkit/core" import { loadFile, prefetchFiles } from "@dpkit/file" +import type { LoadTableOptions } from "@dpkit/table" +import { inferSchemaFromTable, normalizeTable } from "@dpkit/table" import type { Table } from "@dpkit/table" import { concat } from "nodejs-polars" import { DataFrame, scanJson } from "nodejs-polars" import { decodeJsonBuffer } from "../buffer/index.ts" -export async function loadJsonTable(resource: Partial) { - return await loadTable(resource, { isLines: false }) -} - -export async function loadJsonlTable(resource: Partial) { - return await loadTable(resource, { isLines: true }) -} - -async function loadTable( - resource: Partial, - options: { isLines: boolean }, +export async function loadJsonTable( + resource: Partial & { format?: "json" | "jsonl" | "ndjson" }, + options?: LoadTableOptions, ) { - const { isLines } = options + const isLines = resource.format === "jsonl" || resource.format === "ndjson" const paths = await prefetchFiles(resource.path) if (!paths.length) { - return DataFrame().lazy() + throw new Error("Resource path is not defined") } const dialect = await loadResourceDialect(resource.dialect) @@ -45,7 +40,14 @@ async function loadTable( tables.push(table) } - const table = concat(tables) + let table = concat(tables) + + if (!options?.denormalized) { + let schema = await loadResourceSchema(resource.schema) + if (!schema) schema = await inferSchemaFromTable(table, options) + table = await normalizeTable(table, schema) + } + return table } diff --git a/json/table/save.spec.ts b/json/table/save.spec.ts index b3ce57a7..366bd7e8 100644 --- a/json/table/save.spec.ts +++ b/json/table/save.spec.ts @@ -1,8 +1,10 @@ import { readFile } from "node:fs/promises" import { getTempFilePath } from "@dpkit/file" +import { DataFrame, DataType, Series } from "nodejs-polars" import { readRecords } from "nodejs-polars" import { describe, expect, it } from "vitest" -import { saveJsonTable, saveJsonlTable } from "./save.ts" +import { loadJsonTable } from "./load.ts" +import { saveJsonTable } from "./save.ts" const row1 = { id: 1, name: "english" } const row2 = { id: 2, name: "中文" } @@ -19,7 +21,11 @@ describe("saveJsonTable", () => { it("should handle property", async () => { const path = getTempFilePath() - await saveJsonTable(table, { path, dialect: { property: "key" } }) + + await saveJsonTable(table, { + path, + dialect: { property: "key" }, + }) const content = await readFile(path, "utf-8") expect(content).toEqual(JSON.stringify({ key: [row1, row2] }, null, 2)) @@ -27,7 +33,11 @@ describe("saveJsonTable", () => { it("should handle item keys", async () => { const path = getTempFilePath() - await saveJsonTable(table, { path, dialect: { itemKeys: ["name"] } }) + + await saveJsonTable(table, { + path, + dialect: { itemKeys: ["name"] }, + }) const content = await readFile(path, "utf-8") expect(content).toEqual( @@ -37,7 +47,11 @@ describe("saveJsonTable", () => { it("should handle item type (array)", async () => { const path = getTempFilePath() - await saveJsonTable(table, { path, dialect: { itemType: "array" } }) + + await saveJsonTable(table, { + path, + dialect: { itemType: "array" }, + }) const content = await readFile(path, "utf-8") expect(content).toEqual( @@ -48,12 +62,72 @@ describe("saveJsonTable", () => { ), ) }) + + it("should save and load various data types", async () => { + const path = getTempFilePath() + + const source = DataFrame([ + Series("array", ["[1, 2, 3]"], DataType.String), + Series("boolean", [true], DataType.Bool), + Series("date", [new Date(Date.UTC(2025, 0, 1))], DataType.Date), + Series("datetime", [new Date(Date.UTC(2025, 0, 1))], DataType.Datetime), + Series("duration", ["P23DT23H"], DataType.String), + Series("geojson", ['{"value": 1}'], DataType.String), + Series("geopoint", [[40.0, 50.0]], DataType.List(DataType.Float32)), + Series("integer", [1], DataType.Int32), + Series("list", [[1.0, 2.0, 3.0]], DataType.List(DataType.Float32)), + Series("number", [1.1], DataType.Float64), + Series("object", ['{"value": 1}']), + Series("string", ["string"], DataType.String), + Series("time", [new Date(Date.UTC(2025, 0, 1))], DataType.Time), + Series("year", [2025], DataType.Int32), + Series("yearmonth", [[2025, 1]], DataType.List(DataType.Int16)), + ]).lazy() + + await saveJsonTable(source, { + path, + fieldTypes: { + array: "array", + geojson: "geojson", + geopoint: "geopoint", + list: "list", + object: "object", + // TODO: Remove time after: + // https://github.com/pola-rs/nodejs-polars/issues/364 + time: "time", + year: "year", + yearmonth: "yearmonth", + }, + }) + + const target = await loadJsonTable({ path }, { denormalized: true }) + expect((await target.collect()).toRecords()).toEqual([ + { + array: "[1, 2, 3]", + boolean: true, + date: "2025-01-01", + datetime: "2025-01-01T00:00:00", + duration: "P23DT23H", + geojson: '{"value": 1}', + geopoint: "40.0,50.0", + integer: 1, + list: [1.0, 2.0, 3.0], + number: 1.1, + object: '{"value": 1}', + string: "string", + time: "00:00:00", + year: 2025, + yearmonth: "2025-01", + }, + ]) + }) }) -describe("saveJsonlTable", () => { +describe("saveJsonTable (format=jsonl)", () => { it("should save table to file", async () => { const path = getTempFilePath() - await saveJsonlTable(table, { path }) + + await saveJsonTable(table, { path, format: "jsonl" }) const content = await readFile(path, "utf-8") expect(content).toEqual( @@ -63,7 +137,11 @@ describe("saveJsonlTable", () => { it("should handle item keys", async () => { const path = getTempFilePath() - await saveJsonlTable(table, { path, dialect: { itemKeys: ["name"] } }) + await saveJsonTable(table, { + path, + format: "jsonl", + dialect: { itemKeys: ["name"] }, + }) const content = await readFile(path, "utf-8") expect(content).toEqual( @@ -76,7 +154,11 @@ describe("saveJsonlTable", () => { it("should handle item type (array)", async () => { const path = getTempFilePath() - await saveJsonlTable(table, { path, dialect: { itemType: "array" } }) + await saveJsonTable(table, { + path, + format: "jsonl", + dialect: { itemType: "array" }, + }) const content = await readFile(path, "utf-8") expect(content).toEqual( @@ -90,7 +172,11 @@ describe("saveJsonlTable", () => { it("should handle item type (object)", async () => { const path = getTempFilePath() - await saveJsonlTable(table, { path, dialect: { itemType: "object" } }) + await saveJsonTable(table, { + path, + format: "jsonl", + dialect: { itemType: "object" }, + }) const content = await readFile(path, "utf-8") expect(content).toEqual( diff --git a/json/table/save.ts b/json/table/save.ts index 4298d6e2..f73e2c52 100644 --- a/json/table/save.ts +++ b/json/table/save.ts @@ -1,28 +1,31 @@ import type { Dialect } from "@dpkit/core" import { saveFile } from "@dpkit/file" +import { denormalizeTable, inferSchemaFromTable } from "@dpkit/table" import type { SaveTableOptions, Table } from "@dpkit/table" import { decodeJsonBuffer, encodeJsonBuffer } from "../buffer/index.ts" // TODO: rebase on sinkJSON when it is available // https://github.com/pola-rs/nodejs-polars/issues/353 -export async function saveJsonTable(table: Table, options: SaveTableOptions) { - return await saveTable(table, { ...options, isLines: false }) -} - -export async function saveJsonlTable(table: Table, options: SaveTableOptions) { - return await saveTable(table, { ...options, isLines: true }) -} - -async function saveTable( +export async function saveJsonTable( table: Table, - options: SaveTableOptions & { isLines: boolean }, + options: SaveTableOptions & { format?: "json" | "jsonl" | "ndjson" }, ) { - const { path, dialect, isLines } = options - const df = await table.collect() + const { path, dialect, overwrite, format } = options + const isLines = format === "jsonl" || format === "ndjson" + + const schema = await inferSchemaFromTable(table, { + ...options, + keepStrings: true, + }) + + table = await denormalizeTable(table, schema, { + nativeTypes: ["boolean", "integer", "list", "number", "string", "year"], + }) // We use polars to serialize the data // But encode it manually to support dialects/formatting + const df = await table.collect() let buffer = df.writeJSON({ format: isLines ? "lines" : "json" }) let data = decodeJsonBuffer(buffer, { isLines }) @@ -31,7 +34,7 @@ async function saveTable( } buffer = encodeJsonBuffer(data, { isLines }) - await saveFile(path, buffer) + await saveFile(path, buffer, { overwrite }) return path } diff --git a/ods/README.md b/ods/README.md index 64a82639..9adbee21 100644 --- a/ods/README.md +++ b/ods/README.md @@ -1,3 +1,3 @@ # @dpkit/ods -dpkit is a fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). +dpkit is a fast data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/ods/package.json b/ods/package.json index 1ee7342b..dba8fccc 100644 --- a/ods/package.json +++ b/ods/package.json @@ -26,7 +26,7 @@ "@dpkit/core": "workspace:*", "@dpkit/file": "workspace:*", "@dpkit/table": "workspace:*", - "nodejs-polars": "^0.21.0", + "nodejs-polars": "^0.21.1", "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz" }, "devDependencies": { diff --git a/ods/plugin.ts b/ods/plugin.ts index fcc0c481..4c43f5b8 100644 --- a/ods/plugin.ts +++ b/ods/plugin.ts @@ -1,19 +1,22 @@ import type { Resource } from "@dpkit/core" import { inferResourceFormat } from "@dpkit/core" +import type { LoadTableOptions } from "@dpkit/table" import type { TablePlugin } from "@dpkit/table" import type { SaveTableOptions, Table } from "@dpkit/table" import { loadOdsTable, saveOdsTable } from "./table/index.ts" export class OdsPlugin implements TablePlugin { - async loadTable(resource: Partial) { + async loadTable(resource: Partial, options?: LoadTableOptions) { const isOds = getIsOds(resource) if (!isOds) return undefined - return await loadOdsTable(resource) + return await loadOdsTable(resource, options) } async saveTable(table: Table, options: SaveTableOptions) { - const isOds = getIsOds(options) + const { path, format } = options + + const isOds = getIsOds({ path, format }) if (!isOds) return undefined return await saveOdsTable(table, options) diff --git a/ods/table/load.ts b/ods/table/load.ts index 7309ec8d..38ef5a04 100644 --- a/ods/table/load.ts +++ b/ods/table/load.ts @@ -1,15 +1,21 @@ import { loadResourceDialect } from "@dpkit/core" import type { Resource } from "@dpkit/core" +import { loadResourceSchema } from "@dpkit/core" import { loadFile, prefetchFiles } from "@dpkit/file" +import type { LoadTableOptions } from "@dpkit/table" +import { inferSchemaFromTable, normalizeTable } from "@dpkit/table" import type { DataRow, Table } from "@dpkit/table" import { getRecordsFromRows } from "@dpkit/table" import { DataFrame, concat } from "nodejs-polars" import { read, utils } from "xlsx" -export async function loadOdsTable(resource: Partial) { +export async function loadOdsTable( + resource: Partial, + options?: LoadTableOptions, +) { const paths = await prefetchFiles(resource.path) if (!paths.length) { - return DataFrame().lazy() + throw new Error("Resource path is not defined") } const dialect = await loadResourceDialect(resource.dialect) @@ -36,6 +42,13 @@ export async function loadOdsTable(resource: Partial) { } } - const table = concat(tables) + let table = concat(tables) + + if (!options?.denormalized) { + let schema = await loadResourceSchema(resource.schema) + if (!schema) schema = await inferSchemaFromTable(table, options) + table = await normalizeTable(table, schema) + } + return table } diff --git a/ods/table/save.spec.ts b/ods/table/save.spec.ts index d99afb6f..56b5b28c 100644 --- a/ods/table/save.spec.ts +++ b/ods/table/save.spec.ts @@ -1,6 +1,8 @@ import { getTempFilePath } from "@dpkit/file" +import { DataFrame, DataType, Series } from "nodejs-polars" import { readRecords } from "nodejs-polars" import { describe, expect, it } from "vitest" +import { loadOdsTable } from "./load.ts" import { saveOdsTable } from "./save.ts" import { readTestData } from "./test.ts" @@ -16,4 +18,63 @@ describe("saveOdsTable", () => { const data = await readTestData(path) expect(data).toEqual([row1, row2]) }) + + it("should save and load various data types", async () => { + const path = getTempFilePath() + + const source = DataFrame([ + Series("array", ["[1, 2, 3]"], DataType.String), + Series("boolean", [true], DataType.Bool), + Series("date", [new Date(Date.UTC(2025, 0, 1))], DataType.Date), + Series("datetime", [new Date(Date.UTC(2025, 0, 1))], DataType.Datetime), + Series("duration", ["P23DT23H"], DataType.String), + Series("geojson", ['{"value": 1}'], DataType.String), + Series("geopoint", [[40.0, 50.0]], DataType.List(DataType.Float32)), + Series("integer", [1], DataType.Int32), + Series("list", [[1.0, 2.0, 3.0]], DataType.List(DataType.Float32)), + Series("number", [1.1], DataType.Float64), + Series("object", ['{"value": 1}']), + Series("string", ["string"], DataType.String), + Series("time", [new Date(Date.UTC(2025, 0, 1))], DataType.Time), + Series("year", [2025], DataType.Int32), + Series("yearmonth", [[2025, 1]], DataType.List(DataType.Int16)), + ]).lazy() + + await saveOdsTable(source, { + path, + fieldTypes: { + array: "array", + geojson: "geojson", + geopoint: "geopoint", + list: "list", + object: "object", + // TODO: Remove time after: + // https://github.com/pola-rs/nodejs-polars/issues/364 + time: "time", + year: "year", + yearmonth: "yearmonth", + }, + }) + + const target = await loadOdsTable({ path }, { denormalized: true }) + expect((await target.collect()).toRecords()).toEqual([ + { + array: "[1, 2, 3]", + boolean: true, + date: "2025-01-01", + datetime: "2025-01-01T00:00:00", + duration: "P23DT23H", + geojson: '{"value": 1}', + geopoint: "40.0,50.0", + integer: 1, + list: "1.0,2.0,3.0", + number: 1.1, + object: '{"value": 1}', + string: "string", + time: "00:00:00", + year: 2025, + yearmonth: "2025-01", + }, + ]) + }) }) diff --git a/ods/table/save.ts b/ods/table/save.ts index 9bcd3faf..ccfa65b2 100644 --- a/ods/table/save.ts +++ b/ods/table/save.ts @@ -1,10 +1,20 @@ import { loadResourceDialect } from "@dpkit/core" import { saveFile } from "@dpkit/file" +import { denormalizeTable, inferSchemaFromTable } from "@dpkit/table" import type { SaveTableOptions, Table } from "@dpkit/table" import { utils, write } from "xlsx" export async function saveOdsTable(table: Table, options: SaveTableOptions) { - const { path } = options + const { path, overwrite } = options + + const schema = await inferSchemaFromTable(table, { + ...options, + keepStrings: true, + }) + + table = await denormalizeTable(table, schema, { + nativeTypes: ["boolean", "integer", "number", "string", "year"], + }) const df = await table.collect() const dialect = await loadResourceDialect(options.dialect) @@ -15,7 +25,7 @@ export async function saveOdsTable(table: Table, options: SaveTableOptions) { utils.book_append_sheet(book, sheet, sheetName) const buffer = write(book, { type: "buffer", bookType: "ods" }) - await saveFile(path, buffer) + await saveFile(path, buffer, { overwrite }) return path } diff --git a/parquet/README.md b/parquet/README.md index 710b083a..32048c15 100644 --- a/parquet/README.md +++ b/parquet/README.md @@ -1,3 +1,3 @@ # @dpkit/parquet -dpkit is a fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). +dpkit is a fast data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/parquet/package.json b/parquet/package.json index 26c0e96d..4004fc19 100644 --- a/parquet/package.json +++ b/parquet/package.json @@ -27,7 +27,7 @@ "@dpkit/file": "workspace:*", "@dpkit/table": "workspace:*", "csv-sniffer": "^0.1.1", - "nodejs-polars": "^0.21.0" + "nodejs-polars": "^0.21.1" }, "devDependencies": { "@dpkit/test": "workspace:*" diff --git a/parquet/plugin.ts b/parquet/plugin.ts index d978d341..a6f26f22 100644 --- a/parquet/plugin.ts +++ b/parquet/plugin.ts @@ -1,19 +1,22 @@ import type { Resource } from "@dpkit/core" import { inferResourceFormat } from "@dpkit/core" +import type { LoadTableOptions } from "@dpkit/table" import type { TablePlugin } from "@dpkit/table" import type { SaveTableOptions, Table } from "@dpkit/table" import { loadParquetTable, saveParquetTable } from "./table/index.ts" export class ParquetPlugin implements TablePlugin { - async loadTable(resource: Partial) { + async loadTable(resource: Partial, options?: LoadTableOptions) { const isParquet = getIsParquet(resource) if (!isParquet) return undefined - return await loadParquetTable(resource) + return await loadParquetTable(resource, options) } async saveTable(table: Table, options: SaveTableOptions) { - const isParquet = getIsParquet(options) + const { path, format } = options + + const isParquet = getIsParquet({ path, format }) if (!isParquet) return undefined return await saveParquetTable(table, options) diff --git a/parquet/table/load.spec.ts b/parquet/table/load.spec.ts index 36a7cbf9..13965eb0 100644 --- a/parquet/table/load.spec.ts +++ b/parquet/table/load.spec.ts @@ -4,9 +4,9 @@ import { DataFrame } from "nodejs-polars" import { describe, expect, it } from "vitest" import { loadParquetTable } from "./load.ts" -describe("loadParquetTable", () => { - useRecording() +useRecording() +describe("loadParquetTable", () => { describe("file variations", () => { it("should load local file", async () => { const path = getTempFilePath() diff --git a/parquet/table/load.ts b/parquet/table/load.ts index 6622669a..3c8e0df8 100644 --- a/parquet/table/load.ts +++ b/parquet/table/load.ts @@ -1,13 +1,18 @@ import type { Resource } from "@dpkit/core" +import { loadResourceSchema } from "@dpkit/core" import { prefetchFiles } from "@dpkit/file" +import type { LoadTableOptions } from "@dpkit/table" +import { inferSchemaFromTable, normalizeTable } from "@dpkit/table" import { concat } from "nodejs-polars" -import { DataFrame } from "nodejs-polars" import { scanParquet } from "nodejs-polars" -export async function loadParquetTable(resource: Partial) { +export async function loadParquetTable( + resource: Partial, + options?: LoadTableOptions, +) { const [firstPath, ...restPaths] = await prefetchFiles(resource.path) if (!firstPath) { - return DataFrame().lazy() + throw new Error("Resource path is not defined") } let table = scanParquet(firstPath) @@ -15,5 +20,11 @@ export async function loadParquetTable(resource: Partial) { table = concat([table, ...restPaths.map(path => scanParquet(path))]) } + if (!options?.denormalized) { + let schema = await loadResourceSchema(resource.schema) + if (!schema) schema = await inferSchemaFromTable(table, options) + table = await normalizeTable(table, schema) + } + return table } diff --git a/parquet/table/save.spec.ts b/parquet/table/save.spec.ts index ae0c1abc..615dde4a 100644 --- a/parquet/table/save.spec.ts +++ b/parquet/table/save.spec.ts @@ -1,5 +1,5 @@ import { getTempFilePath } from "@dpkit/file" -import { DataFrame } from "nodejs-polars" +import { DataFrame, DataType, Series } from "nodejs-polars" import { describe, expect, it } from "vitest" import { loadParquetTable } from "./load.ts" import { saveParquetTable } from "./save.ts" @@ -14,11 +14,70 @@ describe("saveParquetTable", () => { await saveParquetTable(source, { path }) - const target = await loadParquetTable({ path }) - expect((await target.collect()).toRecords()).toEqual([ + const table = await loadParquetTable({ path }) + expect((await table.collect()).toRecords()).toEqual([ { id: 1.0, name: "Alice" }, { id: 2.0, name: "Bob" }, { id: 3.0, name: "Charlie" }, ]) }) + + it("should save and load various data types", async () => { + const path = getTempFilePath() + + const source = DataFrame([ + Series("array", ["[1, 2, 3]"], DataType.String), + Series("boolean", [true], DataType.Bool), + Series("date", [new Date(Date.UTC(2025, 0, 1))], DataType.Date), + Series("datetime", [new Date(Date.UTC(2025, 0, 1))], DataType.Datetime), + Series("duration", ["P23DT23H"], DataType.String), + Series("geojson", ['{"value": 1}'], DataType.String), + Series("geopoint", [[40.0, 50.0]], DataType.List(DataType.Float32)), + Series("integer", [1], DataType.Int32), + Series("list", [[1.0, 2.0, 3.0]], DataType.List(DataType.Float32)), + Series("number", [1.1], DataType.Float64), + Series("object", ['{"value": 1}']), + Series("string", ["string"], DataType.String), + Series("time", [new Date(Date.UTC(2025, 0, 1))], DataType.Time), + Series("year", [2025], DataType.Int32), + Series("yearmonth", [[2025, 1]], DataType.List(DataType.Int16)), + ]).lazy() + + await saveParquetTable(source, { + path, + fieldTypes: { + array: "array", + geojson: "geojson", + geopoint: "geopoint", + list: "list", + object: "object", + // TODO: Remove time after: + // https://github.com/pola-rs/nodejs-polars/issues/364 + time: "time", + year: "year", + yearmonth: "yearmonth", + }, + }) + + const target = await loadParquetTable({ path }, { denormalized: true }) + expect((await target.collect()).toRecords()).toEqual([ + { + array: "[1, 2, 3]", + boolean: true, + date: "2025-01-01", + datetime: new Date(Date.UTC(2025, 0, 1)), + duration: "P23DT23H", + geojson: '{"value": 1}', + geopoint: "40.0,50.0", + integer: 1, + list: [1.0, 2.0, 3.0], + number: 1.1, + object: '{"value": 1}', + string: "string", + time: "00:00:00", + year: 2025, + yearmonth: "2025-01", + }, + ]) + }) }) diff --git a/parquet/table/save.ts b/parquet/table/save.ts index 4fae1f09..a8b84596 100644 --- a/parquet/table/save.ts +++ b/parquet/table/save.ts @@ -1,10 +1,33 @@ +import { assertLocalPathVacant } from "@dpkit/file" +import { denormalizeTable, inferSchemaFromTable } from "@dpkit/table" import type { SaveTableOptions, Table } from "@dpkit/table" export async function saveParquetTable( table: Table, options: SaveTableOptions, ) { - const { path } = options + const { path, overwrite } = options + + if (!overwrite) { + await assertLocalPathVacant(path) + } + + const schema = await inferSchemaFromTable(table, { + ...options, + keepStrings: true, + }) + + table = await denormalizeTable(table, schema, { + nativeTypes: [ + "boolean", + "datetime", + "integer", + "list", + "number", + "string", + "year", + ], + }) await table .sinkParquet(path, { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ce4e3f8b..f03124b5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -62,6 +62,9 @@ importers: '@dpkit/csv': specifier: workspace:* version: link:../csv + '@dpkit/database': + specifier: workspace:* + version: link:../database '@dpkit/datahub': specifier: workspace:* version: link:../datahub @@ -114,8 +117,8 @@ importers: specifier: ^0.1.1 version: 0.1.1 nodejs-polars: - specifier: ^0.21.0 - version: 0.21.0 + specifier: ^0.21.1 + version: 0.21.1 devDependencies: '@dpkit/test': specifier: workspace:* @@ -170,24 +173,62 @@ importers: specifier: ^0.1.1 version: 0.1.1 nodejs-polars: - specifier: ^0.21.0 - version: 0.21.0 + specifier: ^0.21.1 + version: 0.21.1 devDependencies: '@dpkit/test': specifier: workspace:* version: link:../test - datahub: + database: dependencies: '@dpkit/core': specifier: workspace:* version: link:../core + '@dpkit/table': + specifier: workspace:* + version: link:../table + better-sqlite3: + specifier: ^12.2.0 + version: 12.2.0 + kysely: + specifier: ^0.28.5 + version: 0.28.5 + lru-cache: + specifier: ^11.2.1 + version: 11.2.1 + mysql2: + specifier: ^3.14.4 + version: 3.14.4 + nodejs-polars: + specifier: ^0.21.1 + version: 0.21.1 + pg: + specifier: ^8.16.3 + version: 8.16.3 devDependencies: + '@dpkit/file': + specifier: workspace:* + version: link:../file '@dpkit/test': specifier: workspace:* version: link:../test + '@types/better-sqlite3': + specifier: ^7.6.13 + version: 7.6.13 + '@types/pg': + specifier: ^8.15.5 + version: 8.15.5 - db: {} + datahub: + dependencies: + '@dpkit/core': + specifier: workspace:* + version: link:../core + devDependencies: + '@dpkit/test': + specifier: workspace:* + version: link:../test docs: devDependencies: @@ -201,8 +242,8 @@ importers: specifier: 5.12.9 version: 5.12.9(@types/node@24.2.0)(rollup@4.46.2)(tsx@4.20.3)(typescript@5.9.2)(yaml@2.8.1) nodejs-polars: - specifier: 0.21.0 - version: 0.21.0 + specifier: 0.21.1 + version: 0.21.1 sharp: specifier: 0.34.2 version: 0.34.2 @@ -283,8 +324,8 @@ importers: specifier: workspace:* version: link:../table nodejs-polars: - specifier: ^0.21.0 - version: 0.21.0 + specifier: ^0.21.1 + version: 0.21.1 json: dependencies: @@ -301,8 +342,8 @@ importers: specifier: ^0.1.1 version: 0.1.1 nodejs-polars: - specifier: ^0.21.0 - version: 0.21.0 + specifier: ^0.21.1 + version: 0.21.1 devDependencies: '@dpkit/test': specifier: workspace:* @@ -320,8 +361,8 @@ importers: specifier: workspace:* version: link:../table nodejs-polars: - specifier: ^0.21.0 - version: 0.21.0 + specifier: ^0.21.1 + version: 0.21.1 xlsx: specifier: https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz version: https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz @@ -345,8 +386,8 @@ importers: specifier: ^0.1.1 version: 0.1.1 nodejs-polars: - specifier: ^0.21.0 - version: 0.21.0 + specifier: ^0.21.1 + version: 0.21.1 devDependencies: '@dpkit/test': specifier: workspace:* @@ -358,8 +399,8 @@ importers: specifier: workspace:* version: link:../core nodejs-polars: - specifier: ^0.21.0 - version: 0.21.0 + specifier: ^0.21.1 + version: 0.21.1 test: dependencies: @@ -385,8 +426,8 @@ importers: specifier: workspace:* version: link:../table nodejs-polars: - specifier: ^0.21.0 - version: 0.21.0 + specifier: ^0.21.1 + version: 0.21.1 xlsx: specifier: https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz version: https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz @@ -1358,6 +1399,9 @@ packages: '@tybys/wasm-util@0.10.0': resolution: {integrity: sha512-VyyPYFlOMNylG45GoAe0xDoLwWuowvf92F9kySqzYh8vmYm7D2u4iUJKa1tOUpS70Ku13ASrOkS4ScXFsTaCNQ==} + '@types/better-sqlite3@7.6.13': + resolution: {integrity: sha512-NMv9ASNARoKksWtsq/SHakpYAYnhBrQgGD8zkLYk/jaK8jUGn08CfEdTRgYhMypUQAfzSP8W6gNLe0q19/t4VA==} + '@types/debug@4.1.12': resolution: {integrity: sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==} @@ -1397,6 +1441,9 @@ packages: '@types/node@24.2.0': resolution: {integrity: sha512-3xyG3pMCq3oYCNg7/ZP+E1ooTaGB4cG8JWRsqqOYQdbWNY4zbaV0Ennrd7stjiJEFZCaybcIgpTjJWHRfBSIDw==} + '@types/pg@8.15.5': + resolution: {integrity: sha512-LF7lF6zWEKxuT3/OR8wAZGzkg4ENGXFNyiV/JeOt9z5B+0ZVwbql9McqX5c/WStFq1GaGso7H1AzP/qSzmlCKQ==} + '@types/sax@1.2.7': resolution: {integrity: sha512-rO73L89PJxeYM3s3pPPjiPgVVcymqU490g0YO5n5By0k2Erzj6tay/4lr1CHAAU4JyOWd1rpQ8bCf6cZfHU96A==} @@ -1549,6 +1596,10 @@ packages: engines: {node: 18.20.8 || ^20.3.0 || >=22.0.0, npm: '>=9.6.5', pnpm: '>=7.1.0'} hasBin: true + aws-ssl-profiles@1.1.2: + resolution: {integrity: sha512-NZKeq9AfyQvEeNlN0zSYAaWrmBffJh3IELMZfRpJVWgrpEbtEpnjvzqBPf+mxoI287JohRDoa+/nsfqqiZmF6g==} + engines: {node: '>= 6.0.0'} + axobject-query@4.1.0: resolution: {integrity: sha512-qIj0G9wZbMGNLjLmg1PT6v2mE9AH2zlnADJD/2tC6E00hgmhUOfEB6greHPAfLRSufHqROIUTkw6E+M3lH0PTQ==} engines: {node: '>= 0.4'} @@ -1579,6 +1630,16 @@ packages: resolution: {integrity: sha512-pbnl5XzGBdrFU/wT4jqmJVPn2B6UHPBOhzMQkY/SPUPB6QtUXtmBHBIwCbXJol93mOpGMnQyP/+BB19q04xj7g==} engines: {node: '>=4'} + better-sqlite3@12.2.0: + resolution: {integrity: sha512-eGbYq2CT+tos1fBwLQ/tkBt9J5M3JEHjku4hbvQUePCckkvVf14xWj+1m7dGoK81M/fOjFT7yM9UMeKT/+vFLQ==} + engines: {node: 20.x || 22.x || 23.x || 24.x} + + bindings@1.5.0: + resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==} + + bl@4.1.0: + resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} + blob-to-buffer@1.2.9: resolution: {integrity: sha512-BF033y5fN6OCofD3vgHmNtwZWRcq9NLyyxyILx9hfMy1sXYy4ojFl765hJ2lP0YaN2fuxPaLO2Vzzoxy0FLFFA==} @@ -1613,6 +1674,9 @@ packages: resolution: {integrity: sha512-Db1SbgBS/fg/392AblrMJk97KggmvYhr4pB5ZIMTWtaivCPMWLkmb7m21cJvpvgK+J3nsU2CmmixNBZx4vFj/w==} engines: {node: '>=8.0.0'} + buffer@5.7.1: + resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==} + bytes@3.1.2: resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} engines: {node: '>= 0.8'} @@ -1670,6 +1734,9 @@ packages: resolution: {integrity: sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==} engines: {node: '>= 14.16.0'} + chownr@1.1.4: + resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==} + ci-info@3.9.0: resolution: {integrity: sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ==} engines: {node: '>=8'} @@ -1788,10 +1855,18 @@ packages: decode-named-character-reference@1.2.0: resolution: {integrity: sha512-c6fcElNV6ShtZXmsgNgFFV5tVX2PaV4g+MOAkb8eXHvn6sryJBrZa9r0zV6+dtTyoCKxtDy5tyQ5ZwQuidtd+Q==} + decompress-response@6.0.0: + resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==} + engines: {node: '>=10'} + deep-eql@5.0.2: resolution: {integrity: sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==} engines: {node: '>=6'} + deep-extend@0.6.0: + resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==} + engines: {node: '>=4.0.0'} + define-data-property@1.1.4: resolution: {integrity: sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==} engines: {node: '>= 0.4'} @@ -1803,6 +1878,10 @@ packages: defu@6.1.4: resolution: {integrity: sha512-mEQCMmwJu317oSz8CwdIOdwf3xMif1ttiM8LTufzc3g6kR+9Pe236twL8j3IYT1F7GfRgGcW6MWxzZjLIkuHIg==} + denque@2.1.0: + resolution: {integrity: sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==} + engines: {node: '>=0.10'} + depd@2.0.0: resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} engines: {node: '>= 0.8'} @@ -1885,6 +1964,9 @@ packages: resolution: {integrity: sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==} engines: {node: '>= 0.8'} + end-of-stream@1.4.5: + resolution: {integrity: sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==} + enquirer@2.4.1: resolution: {integrity: sha512-rRqJg/6gd538VHvR3PSrdRBb/1Vy2YfzHqzvbhGIQpDRKIa4FgV/54b5Q1xYSxOOwKvjXweS26E0Q+nAMwp2pQ==} engines: {node: '>=8.6'} @@ -1978,6 +2060,10 @@ packages: resolution: {integrity: sha512-Fqs7ChZm72y40wKjOFXBKg7nJZvQJmewP5/7LtePDdnah/+FH9Hp5sgMujSCMPXlxOAW2//1jrW9pnsY7o20vQ==} engines: {node: '>=18'} + expand-template@2.0.3: + resolution: {integrity: sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==} + engines: {node: '>=6'} + expect-type@1.2.2: resolution: {integrity: sha512-JhFGDVJ7tmDJItKhYgJCGLOWjuK9vPxiXoUFLwLDc99NlmklilbiQJwoctZtt13+xMw91MCk/REan6MWHqDjyA==} engines: {node: '>=12.0.0'} @@ -2030,6 +2116,9 @@ packages: resolution: {integrity: sha512-d+l3qxjSesT4V7v2fh+QnmFnUWv9lSpjarhShNTgBOfA0ttejbQUAlHLitbjkoRiDulW0OPoQPYIGhIC8ohejg==} engines: {node: '>=18'} + file-uri-to-path@1.0.0: + resolution: {integrity: sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==} + fill-range@7.1.1: resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} engines: {node: '>=8'} @@ -2067,6 +2156,9 @@ packages: resolution: {integrity: sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==} engines: {node: '>= 0.6'} + fs-constants@1.0.0: + resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==} + fs-extra@10.1.0: resolution: {integrity: sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==} engines: {node: '>=12'} @@ -2087,6 +2179,9 @@ packages: function-bind@1.1.2: resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} + generate-function@2.3.1: + resolution: {integrity: sha512-eeB5GfMNeevm/GRYq20ShmsaGcmI81kIX2K9XQx5miC8KdHaC6Jm0qQ8ZNeGOi7wYB8OsdxKs+Y2oVuTFuVwKQ==} + get-east-asian-width@1.3.0: resolution: {integrity: sha512-vpeMIQKxczTD/0s2CdEWHcb0eeJe6TFjxb+J5xgX7hScxqrGuyjmv4c1D4A/gelKfyox0gJJwIHF+fLjeaM8kQ==} engines: {node: '>=18'} @@ -2106,6 +2201,9 @@ packages: get-tsconfig@4.10.1: resolution: {integrity: sha512-auHyJ4AgMz7vgS8Hp3N6HXSmlMdUyhSUrfBF16w153rxtLIEOE+HGqaBppczZvnHLqQJfiHotCYpNhl0lUROFQ==} + github-from-package@0.0.0: + resolution: {integrity: sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==} + github-slugger@2.0.0: resolution: {integrity: sha512-IaOQ9puYtjrkq7Y0Ygl9KDZnrf/aiUJYUpVf89y8kyaxbRG7Y1SrX/jaumrv81vc61+kiMempujsM3Yw7w5qcw==} @@ -2257,6 +2355,13 @@ packages: resolution: {integrity: sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==} engines: {node: '>=0.10.0'} + iconv-lite@0.7.0: + resolution: {integrity: sha512-cf6L2Ds3h57VVmkZe+Pn+5APsT7FpqJtEhhieDCvrE2MK5Qk9MyffgQyuxQTm6BChfeZNtcOLHp9IcWRVcIcBQ==} + engines: {node: '>=0.10.0'} + + ieee754@1.2.1: + resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} + ignore@5.3.2: resolution: {integrity: sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==} engines: {node: '>= 4'} @@ -2267,6 +2372,9 @@ packages: inherits@2.0.4: resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} + ini@1.3.8: + resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==} + inline-style-parser@0.2.4: resolution: {integrity: sha512-0aO8FkhNZlj/ZIbNi7Lxxr12obT7cL1moPfE4tg1LkX7LlLfC6DeX4l2ZEud1ukP9jNQyNnfzQVqwbwmAATY4Q==} @@ -2330,6 +2438,9 @@ packages: resolution: {integrity: sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==} engines: {node: '>=12'} + is-property@1.0.2: + resolution: {integrity: sha512-Ks/IoX00TtClbGQr4TWXemAnktAQvYB7HzcCxDGqEZU6oCmb2INHuOoKxbtR+HFkmYWBKv/dOZtGRiAjDhj92g==} + is-stream@3.0.0: resolution: {integrity: sha512-LnQR4bZ9IADDRSkvpqMGvt/tEJWclzklNgSw48V5EAaAeDd6qGvN8ei6k5p0tvxSR171VmGyHuTiAOfxAbr8kA==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} @@ -2409,6 +2520,10 @@ packages: resolution: {integrity: sha512-dhG34DXATL5hSxJbIexCft8FChFXtmskoZYnoPWjXQuebWYCNkVeV3KkGegCK9CP1oswI/vQibS2GY7Em/sJJA==} engines: {node: '>= 8'} + kysely@0.28.5: + resolution: {integrity: sha512-rlB0I/c6FBDWPcQoDtkxi9zIvpmnV5xoIalfCMSMCa7nuA6VGA3F54TW9mEgX4DVf10sXAWCF5fDbamI/5ZpKA==} + engines: {node: '>=20.0.0'} + linkify-it@5.0.0: resolution: {integrity: sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ==} @@ -2429,6 +2544,9 @@ packages: resolution: {integrity: sha512-HgMmCqIJSAKqo68l0rS2AanEWfkxaZ5wNiEFb5ggm08lDs9Xl2KxBlX3PTcaD2chBM1gXAYf491/M2Rv8Jwayg==} engines: {node: '>= 0.6.0'} + long@5.3.2: + resolution: {integrity: sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==} + longest-streak@3.1.0: resolution: {integrity: sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g==} @@ -2438,6 +2556,18 @@ packages: lru-cache@10.4.3: resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} + lru-cache@11.2.1: + resolution: {integrity: sha512-r8LA6i4LP4EeWOhqBaZZjDWwehd1xUJPCJd9Sv300H0ZmcUER4+JPh7bqqZeqs1o5pgtgvXm+d9UGrB5zZGDiQ==} + engines: {node: 20 || >=22} + + lru-cache@7.18.3: + resolution: {integrity: sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==} + engines: {node: '>=12'} + + lru.min@1.1.2: + resolution: {integrity: sha512-Nv9KddBcQSlQopmBHXSsZVY5xsdlZkdH/Iey0BlcBYggMd4two7cZnKOK9vmy3nY0O5RGH99z1PCeTpPqszUYg==} + engines: {bun: '>=1.0.0', deno: '>=1.30.0', node: '>=8.0.0'} + lunr@2.3.9: resolution: {integrity: sha512-zTU3DaZaF3Rt9rhN3uBMGQD3dD2/vFQqnvZCDv4dl5iOzq2IZQqTxu90r4E5J+nP70J3ilqVCrbho2eWaeW8Ow==} @@ -2666,14 +2796,24 @@ packages: engines: {node: '>=4'} hasBin: true + mimic-response@3.1.0: + resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} + engines: {node: '>=10'} + minimatch@9.0.5: resolution: {integrity: sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==} engines: {node: '>=16 || 14 >=14.17'} + minimist@1.2.8: + resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} + minipass@7.1.2: resolution: {integrity: sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==} engines: {node: '>=16 || 14 >=14.17'} + mkdirp-classic@0.5.3: + resolution: {integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==} + morgan@1.10.1: resolution: {integrity: sha512-223dMRJtI/l25dJKWpgij2cMtywuG/WiUKXdvwfbhGKBhy1puASqXwFzmWZ7+K73vUPoR7SS2Qz2cI/g9MKw0A==} engines: {node: '>= 0.8.0'} @@ -2692,11 +2832,22 @@ packages: ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + mysql2@3.14.4: + resolution: {integrity: sha512-Cs/jx3WZPNrYHVz+Iunp9ziahaG5uFMvD2R8Zlmc194AqXNxt9HBNu7ZsPYrUtmJsF0egETCWIdMIYAwOGjL1w==} + engines: {node: '>= 8.0'} + + named-placeholders@1.1.3: + resolution: {integrity: sha512-eLoBxg6wE/rZkJPhU/xRX1WTpkFEwDJEN96oxFrTsqBdbT5ec295Q+CoHrL9IT0DipqKhmGcaZmwOt8OON5x1w==} + engines: {node: '>=12.0.0'} + nanoid@3.3.11: resolution: {integrity: sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true + napi-build-utils@2.0.0: + resolution: {integrity: sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==} + negotiator@0.6.3: resolution: {integrity: sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==} engines: {node: '>= 0.6'} @@ -2712,6 +2863,10 @@ packages: resolution: {integrity: sha512-WDD0bdg9mbq6F4mRxEYcPWwfA1vxd0mrvKOyxI7Xj/atfRHVeutzuWByG//jfm4uPzp0y4Kj051EORCBSQMycw==} engines: {node: '>=12.0.0'} + node-abi@3.75.0: + resolution: {integrity: sha512-OhYaY5sDsIka7H7AtijtI9jwGYLyl29eQn/W623DiN/MIv5sUqc4g7BIDThX+gb7di9f6xK02nkp8sdfFWZLTg==} + engines: {node: '>=10'} + node-fetch-native@1.6.7: resolution: {integrity: sha512-g9yhqoedzIUm0nTnTqAQvueMPVOuIY16bqgAJJC8XOOubYFNwz6IER9qs0Gq2Xd0+CecCKFjtdDTMA4u4xG06Q==} @@ -2727,57 +2882,57 @@ packages: node-mock-http@1.0.2: resolution: {integrity: sha512-zWaamgDUdo9SSLw47we78+zYw/bDr5gH8pH7oRRs8V3KmBtu8GLgGIbV2p/gRPd3LWpEOpjQj7X1FOU3VFMJ8g==} - nodejs-polars-android-arm64@0.21.0: - resolution: {integrity: sha512-X2Y6iulWGJttGxOIqcZjSJ256TW4FJH/otkBdA6qnmS8jLTD6oDmrxvoMXEOroIBUOTWLC4wqFCeVIDHFMyUDQ==} + nodejs-polars-android-arm64@0.21.1: + resolution: {integrity: sha512-g/alCp4Hra/ZtVdEwhAb/yDwpdVmLzzK54Z4pL8LwHw/+FK8D1lULk2gDEoaTGy2WDUEa0lOR1PCHwvoXj9XGQ==} engines: {node: '>= 16'} cpu: [arm64] os: [android] - nodejs-polars-darwin-arm64@0.21.0: - resolution: {integrity: sha512-0QUeH7VW4hnojHBm5fUMfoWdWHfpt313cspK9sAjkhWtKtXEHs9Ade2Xqg7a2T2XTRpvcakhdQ4SkG6Wq9mJRg==} + nodejs-polars-darwin-arm64@0.21.1: + resolution: {integrity: sha512-55GQ0vRlUewTJLTdIl0RrpVl+LuxX9EFrMcMa4G/YlCSiBmzhEliE9pfX0NxKG5NPIQA187K9fLN2ZqwZCMKcA==} engines: {node: '>= 16'} cpu: [arm64] os: [darwin] - nodejs-polars-darwin-x64@0.21.0: - resolution: {integrity: sha512-xJh3JGWn3ZMAKwpFpv0EA8Z8IoiMgXs2WfE6ibTfLxVIHGJMZoI0eEGTnrQaPIkX+bXfiqEWtCcQrgQXfu4sew==} + nodejs-polars-darwin-x64@0.21.1: + resolution: {integrity: sha512-3MuSKiHTIlfsvo/LBIgG7kBsxRd/dafGF2mmooCoyIHYETUWamCzSbM3cta5jNOTamDdIWhayuVVk4bQevfRrg==} engines: {node: '>= 16'} cpu: [x64] os: [darwin] - nodejs-polars-linux-arm64-gnu@0.21.0: - resolution: {integrity: sha512-UUVfc7Kjiwpdsh9YoIP/jTUSMWpEmbJk7/LWCEZAZsOiQKbHriPKNdzoG+4zhaNsu/mPDgOKZyXjY2OI5JEUYg==} + nodejs-polars-linux-arm64-gnu@0.21.1: + resolution: {integrity: sha512-woztH1q88m+dO2mR2lLTxd4Bi88GmsLQAgmDVskxwPUiv8VTOOKtItTzf+Vp2Kg4WIRFAhFiQiTyCqfK9SAhcg==} engines: {node: '>= 16'} cpu: [arm64] os: [linux] - nodejs-polars-linux-arm64-musl@0.21.0: - resolution: {integrity: sha512-L26Xs1PAThiAcz9Om9KtEdyhreaFPhgdH0SI0MXATS63bLsIYK12CKC3DkEn+dwqqt/UBT8ApHnqQjeiluWJ2g==} + nodejs-polars-linux-arm64-musl@0.21.1: + resolution: {integrity: sha512-7X82Jwutwu6MjhdxHfSX5a9Qgnkrmw11VsSHt755a+/zHym25bcSBFTafZ/wDKIG8gGH7GUKjLTR9Wn0fzHkoQ==} engines: {node: '>= 16'} cpu: [arm64] os: [linux] - nodejs-polars-linux-x64-gnu@0.21.0: - resolution: {integrity: sha512-zvYOo8GeQBN+U9ZyXEEDSwjzmQyZIvGydmD5IYUX5iR6SwFrJ6EiYx6Lr8Pv4QPdYH9DQXxIb44xgst8FCjIWA==} + nodejs-polars-linux-x64-gnu@0.21.1: + resolution: {integrity: sha512-vsf15a0LWBjD3ce7lF10mhI/Ks2SUyVVbxur8r7wYfSEkKPfVmPprWvhTZJ7jjTnmxIvZ9Q+P5McA7mzzq/dAQ==} engines: {node: '>= 16'} cpu: [x64] os: [linux] - nodejs-polars-linux-x64-musl@0.21.0: - resolution: {integrity: sha512-FfvZjx4AuH+zCLNoM9OhlrQyeAEsh0gzGE6UDIvc+/FqsV2ISkn6NLSvwhWj9lvDK+8mZZWCTlxIj/aLqWq2Pw==} + nodejs-polars-linux-x64-musl@0.21.1: + resolution: {integrity: sha512-6bcdDXRyqvDg5JKXOlrnzuqK6HhBggR10a+G/xbmU4tl8dBdm6RvhHQBdMczQF6XD7dRWk8fy6lcW+LjxGrWgQ==} engines: {node: '>= 16'} cpu: [x64] os: [linux] - nodejs-polars-win32-x64-msvc@0.21.0: - resolution: {integrity: sha512-+KAyDvfeEV4naz2XdGGCZlGIv5Jd4dxm7vSvlL63bUeSk0yrPdbCRUCIwsFLV3dpsJDmBCiwBwJpQe7sT4atUA==} + nodejs-polars-win32-x64-msvc@0.21.1: + resolution: {integrity: sha512-Mvgu6fi1iuvKzlyCHMUtDh7z8pESMpic3ejQ17dmcuyJqOX8oxgIoupF8awWqFjWeNWTvEdt2bS+RgAIONHNOQ==} engines: {node: '>= 16'} cpu: [x64] os: [win32] - nodejs-polars@0.21.0: - resolution: {integrity: sha512-EBPp1YlRWcghkFMYZ7eXoGm0UhbA9AHUuBRaseZrtww/F10yphyXdgjN+0riJJjL+XFs5AFXc3k5IQjoobs0BQ==} - engines: {node: '>= 18'} + nodejs-polars@0.21.1: + resolution: {integrity: sha512-tdKUZuWwZaqDEC4ovVqb/K15uEbV2DizHN59Wl2lJBp3GA1pHAN+7S/7GPDs+xlo3eHDqaslcS2iJDqslZmuWw==} + engines: {node: '>= 20'} normalize-path@3.0.0: resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==} @@ -2825,6 +2980,9 @@ packages: resolution: {integrity: sha512-737ZY3yNnXy37FHkQxPzt4UZ2UWPWiCZWLvFZ4fu5cueciegX0zGPnrlY6bwRg4FdQOe9YU8MkmJwGhoMybl8A==} engines: {node: '>= 0.8'} + once@1.4.0: + resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} + oniguruma-parser@0.12.1: resolution: {integrity: sha512-8Unqkvk1RYc6yq2WBYRj4hdnsAxVze8i7iPfQr8e4uSP3tRv0rpZcbGUDvxfQQcdwHt/e9PrMvGCsa8OqG9X3w==} @@ -2933,6 +3091,40 @@ packages: resolution: {integrity: sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==} engines: {node: '>= 14.16'} + pg-cloudflare@1.2.7: + resolution: {integrity: sha512-YgCtzMH0ptvZJslLM1ffsY4EuGaU0cx4XSdXLRFae8bPP4dS5xL1tNB3k2o/N64cHJpwU7dxKli/nZ2lUa5fLg==} + + pg-connection-string@2.9.1: + resolution: {integrity: sha512-nkc6NpDcvPVpZXxrreI/FOtX3XemeLl8E0qFr6F2Lrm/I8WOnaWNhIPK2Z7OHpw7gh5XJThi6j6ppgNoaT1w4w==} + + pg-int8@1.0.1: + resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==} + engines: {node: '>=4.0.0'} + + pg-pool@3.10.1: + resolution: {integrity: sha512-Tu8jMlcX+9d8+QVzKIvM/uJtp07PKr82IUOYEphaWcoBhIYkoHpLXN3qO59nAI11ripznDsEzEv8nUxBVWajGg==} + peerDependencies: + pg: '>=8.0' + + pg-protocol@1.10.3: + resolution: {integrity: sha512-6DIBgBQaTKDJyxnXaLiLR8wBpQQcGWuAESkRBX/t6OwA8YsqP+iVSiond2EDy6Y/dsGk8rh/jtax3js5NeV7JQ==} + + pg-types@2.2.0: + resolution: {integrity: sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==} + engines: {node: '>=4'} + + pg@8.16.3: + resolution: {integrity: sha512-enxc1h0jA/aq5oSDMvqyW3q89ra6XIIDZgCX9vkMrnz5DFTw/Ny3Li2lFQ+pt3L6MCgm/5o2o8HW9hiJji+xvw==} + engines: {node: '>= 16.0.0'} + peerDependencies: + pg-native: '>=3.0.1' + peerDependenciesMeta: + pg-native: + optional: true + + pgpass@1.0.5: + resolution: {integrity: sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==} + picocolors@1.1.1: resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} @@ -2962,6 +3154,27 @@ packages: resolution: {integrity: sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg==} engines: {node: ^10 || ^12 || >=14} + postgres-array@2.0.0: + resolution: {integrity: sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==} + engines: {node: '>=4'} + + postgres-bytea@1.0.0: + resolution: {integrity: sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==} + engines: {node: '>=0.10.0'} + + postgres-date@1.0.7: + resolution: {integrity: sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==} + engines: {node: '>=0.10.0'} + + postgres-interval@1.2.0: + resolution: {integrity: sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==} + engines: {node: '>=0.10.0'} + + prebuild-install@7.1.3: + resolution: {integrity: sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==} + engines: {node: '>=10'} + hasBin: true + prettier@2.8.8: resolution: {integrity: sha512-tdN8qQGvNjw4CHbY+XXk0JgCXn9QiF21a55rBe5LJAU+kDyC4WQn4+awm2Xfk2lQMk5fKup9XgzTZtGkjBdP9Q==} engines: {node: '>=10.13.0'} @@ -2989,6 +3202,9 @@ packages: resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} engines: {node: '>= 0.10'} + pump@3.0.3: + resolution: {integrity: sha512-todwxLMY7/heScKmntwQG8CXVkWUOdYxIvY2s0VWAAMh/nd8SoYiRaKjlr7+iCs984f2P8zvrfWcDDYVb73NfA==} + punycode.js@2.3.1: resolution: {integrity: sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==} engines: {node: '>=6'} @@ -3025,10 +3241,18 @@ packages: resolution: {integrity: sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==} engines: {node: '>= 0.8'} + rc@1.2.8: + resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==} + hasBin: true + read-yaml-file@1.1.0: resolution: {integrity: sha512-VIMnQi/Z4HT2Fxuwg5KrY174U1VdUIASQVWXXyqtNRtxSr9IYkn1rsI6Tb6HsrHCmB7gVpNwX6JxPTHcH6IoTA==} engines: {node: '>=6'} + readable-stream@3.6.2: + resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==} + engines: {node: '>= 6'} + readdirp@4.1.2: resolution: {integrity: sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==} engines: {node: '>= 14.18.0'} @@ -3164,6 +3388,9 @@ packages: resolution: {integrity: sha512-dW41u5VfLXu8SJh5bwRmyYUbAoSB3c9uQh6L8h/KtsFREPWpbX1lrljJo186Jc4nmci/sGUZ9a0a0J2zgfq2hw==} engines: {node: '>= 0.8.0'} + seq-queue@0.0.5: + resolution: {integrity: sha512-hr3Wtp/GZIc/6DAGPDcV4/9WoZhjrkXsi5B/07QgX8tsdc6ilr7BFM6PM6rbdAX1kFSDYeZGLipIZZKyQP0O5Q==} + serve-static@1.16.2: resolution: {integrity: sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw==} engines: {node: '>= 0.8.0'} @@ -3216,6 +3443,12 @@ packages: resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} engines: {node: '>=14'} + simple-concat@1.0.1: + resolution: {integrity: sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==} + + simple-get@4.0.1: + resolution: {integrity: sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==} + simple-invariant@2.0.1: resolution: {integrity: sha512-1sbhsxqI+I2tqlmjbz99GXNmZtr6tKIyEgGGnJw/MKGblalqk/XoOYYFJlBzTKZCxx8kLaD3FD5s9BEEjx5Pyg==} engines: {node: '>=10'} @@ -3261,9 +3494,17 @@ packages: spawndamnit@3.0.1: resolution: {integrity: sha512-MmnduQUuHCoFckZoWnXsTg7JaiLBJrKFj9UI2MbRPGaJeVpsLcVBu6P/IGZovziM/YBsellCmsprgNA+w0CzVg==} + split2@4.2.0: + resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} + engines: {node: '>= 10.x'} + sprintf-js@1.0.3: resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} + sqlstring@2.3.3: + resolution: {integrity: sha512-qC9iz2FlN7DQl3+wjwn3802RTyjCx7sDvfQEXchwa6CWOx07/WVfh91gBmQ9fahw8snwGEWU3xGzOt4tFyHLxg==} + engines: {node: '>= 0.6'} + stackback@0.0.2: resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} @@ -3309,6 +3550,9 @@ packages: resolution: {integrity: sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==} engines: {node: '>=18'} + string_decoder@1.3.0: + resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} + stringify-entities@4.0.4: resolution: {integrity: sha512-IwfBptatlO+QCJUo19AqvrPNqlVMpW9YEL2LIVY+Rpv2qsjCGxaDLNRgeGsQWJhfItebuJhsGSLjaBbNSQ+ieg==} @@ -3328,6 +3572,10 @@ packages: resolution: {integrity: sha512-aulFJcD6YK8V1G7iRB5tigAP4TsHBZZrOV8pjV++zdUwmeV8uzbY7yn6h9MswN62adStNZFuCIx4haBnRuMDaw==} engines: {node: '>=18'} + strip-json-comments@2.0.1: + resolution: {integrity: sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==} + engines: {node: '>=0.10.0'} + style-to-js@1.1.17: resolution: {integrity: sha512-xQcBGDxJb6jjFCTzvQtfiPn6YvvP2O8U1MDIPNfJQlWMYfktPy+iGsHE7cssjs7y84d9fQaK4UF3RIJaAHSoYA==} @@ -3338,6 +3586,13 @@ packages: resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} engines: {node: '>=8'} + tar-fs@2.1.3: + resolution: {integrity: sha512-090nwYJDmlhwFwEW3QQl+vaNnxsO2yVsd45eTKRBzSzu+hlb1w2K9inVq5b0ngXuLVqQ4ApvsUHHnu/zQNkWAg==} + + tar-stream@2.2.0: + resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==} + engines: {node: '>=6'} + temp-dir@3.0.0: resolution: {integrity: sha512-nHc6S/bwIilKHNRgK/3jlhDoIHcp45YgyiwcAk46Tr0LfEqGBVpmiAyuiuxeVE44m3mXnEeVhaipLOEWmH+Njw==} engines: {node: '>=14.16'} @@ -3428,6 +3683,9 @@ packages: engines: {node: '>=18.0.0'} hasBin: true + tunnel-agent@0.6.0: + resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==} + type-fest@1.4.0: resolution: {integrity: sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==} engines: {node: '>=10'} @@ -3744,12 +4002,19 @@ packages: resolution: {integrity: sha512-G8ura3S+3Z2G+mkgNRq8dqaFZAuxfsxpBB8OCTGRTCtp+l/v9nbFNmCUP1BZMts3G1142MsZfn6eeUKrr4PD1Q==} engines: {node: '>=18'} + wrappy@1.0.2: + resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} + xlsx@https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz: resolution: {tarball: https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz} version: 0.20.3 engines: {node: '>=0.8'} hasBin: true + xtend@4.0.2: + resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} + engines: {node: '>=0.4'} + xxhash-wasm@1.1.0: resolution: {integrity: sha512-147y/6YNh+tlp6nd/2pWq38i9h6mz/EuQ6njIrmW8D1BS5nCqs0P6DG+m6zTGnNz5I+uhZ0SHxBs9BsPrwcKDA==} @@ -4753,6 +5018,10 @@ snapshots: tslib: 2.8.1 optional: true + '@types/better-sqlite3@7.6.13': + dependencies: + '@types/node': 24.2.0 + '@types/debug@4.1.12': dependencies: '@types/ms': 2.1.0 @@ -4793,6 +5062,12 @@ snapshots: dependencies: undici-types: 7.10.0 + '@types/pg@8.15.5': + dependencies: + '@types/node': 24.2.0 + pg-protocol: 1.10.3 + pg-types: 2.2.0 + '@types/sax@1.2.7': dependencies: '@types/node': 24.2.0 @@ -5053,6 +5328,8 @@ snapshots: - uploadthing - yaml + aws-ssl-profiles@1.1.2: {} + axobject-query@4.1.0: {} bail@2.0.2: {} @@ -5079,6 +5356,21 @@ snapshots: dependencies: is-windows: 1.0.2 + better-sqlite3@12.2.0: + dependencies: + bindings: 1.5.0 + prebuild-install: 7.1.3 + + bindings@1.5.0: + dependencies: + file-uri-to-path: 1.0.0 + + bl@4.1.0: + dependencies: + buffer: 5.7.1 + inherits: 2.0.4 + readable-stream: 3.6.2 + blob-to-buffer@1.2.9: {} blueimp-md5@2.19.0: {} @@ -5129,6 +5421,11 @@ snapshots: buffer-crc32@1.0.0: {} + buffer@5.7.1: + dependencies: + base64-js: 1.5.1 + ieee754: 1.2.1 + bytes@3.1.2: {} cac@6.7.14: {} @@ -5175,6 +5472,8 @@ snapshots: dependencies: readdirp: 4.1.2 + chownr@1.1.4: {} + ci-info@3.9.0: {} ci-info@4.3.0: {} @@ -5269,8 +5568,14 @@ snapshots: dependencies: character-entities: 2.0.2 + decompress-response@6.0.0: + dependencies: + mimic-response: 3.1.0 + deep-eql@5.0.2: {} + deep-extend@0.6.0: {} + define-data-property@1.1.4: dependencies: es-define-property: 1.0.1 @@ -5285,6 +5590,8 @@ snapshots: defu@6.1.4: {} + denque@2.1.0: {} + depd@2.0.0: {} dequal@2.0.3: {} @@ -5341,6 +5648,10 @@ snapshots: encodeurl@2.0.0: {} + end-of-stream@1.4.5: + dependencies: + once: 1.4.0 + enquirer@2.4.1: dependencies: ansi-colors: 4.1.3 @@ -5467,6 +5778,8 @@ snapshots: exit-hook@4.0.0: {} + expand-template@2.0.3: {} + expect-type@1.2.2: {} express@4.21.2: @@ -5550,6 +5863,8 @@ snapshots: dependencies: is-unicode-supported: 2.1.0 + file-uri-to-path@1.0.0: {} + fill-range@7.1.1: dependencies: to-regex-range: 5.0.1 @@ -5601,6 +5916,8 @@ snapshots: fresh@0.5.2: {} + fs-constants@1.0.0: {} + fs-extra@10.1.0: dependencies: graceful-fs: 4.2.11 @@ -5624,6 +5941,10 @@ snapshots: function-bind@1.1.2: {} + generate-function@2.3.1: + dependencies: + is-property: 1.0.2 + get-east-asian-width@1.3.0: {} get-intrinsic@1.3.0: @@ -5654,6 +5975,8 @@ snapshots: resolve-pkg-maps: 1.0.0 optional: true + github-from-package@0.0.0: {} + github-slugger@2.0.0: {} glob-parent@5.1.2: @@ -5943,12 +6266,20 @@ snapshots: dependencies: safer-buffer: 2.1.2 + iconv-lite@0.7.0: + dependencies: + safer-buffer: 2.1.2 + + ieee754@1.2.1: {} + ignore@5.3.2: {} import-meta-resolve@4.1.0: {} inherits@2.0.4: {} + ini@1.3.8: {} + inline-style-parser@0.2.4: {} ipaddr.js@1.9.1: {} @@ -5992,6 +6323,8 @@ snapshots: is-plain-obj@4.1.0: {} + is-property@1.0.2: {} + is-stream@3.0.0: {} is-stream@4.0.1: {} @@ -6066,6 +6399,8 @@ snapshots: klona@2.0.6: {} + kysely@0.28.5: {} + linkify-it@5.0.0: dependencies: uc.micro: 2.1.0 @@ -6082,12 +6417,20 @@ snapshots: loglevel@1.9.2: {} + long@5.3.2: {} + longest-streak@3.1.0: {} loupe@3.2.0: {} lru-cache@10.4.3: {} + lru-cache@11.2.1: {} + + lru-cache@7.18.3: {} + + lru.min@1.1.2: {} + lunr@2.3.9: {} magic-string@0.30.17: @@ -6601,12 +6944,18 @@ snapshots: mime@1.6.0: {} + mimic-response@3.1.0: {} + minimatch@9.0.5: dependencies: brace-expansion: 2.0.2 + minimist@1.2.8: {} + minipass@7.1.2: {} + mkdirp-classic@0.5.3: {} + morgan@1.10.1: dependencies: basic-auth: 2.0.1 @@ -6625,8 +6974,26 @@ snapshots: ms@2.1.3: {} + mysql2@3.14.4: + dependencies: + aws-ssl-profiles: 1.1.2 + denque: 2.1.0 + generate-function: 2.3.1 + iconv-lite: 0.7.0 + long: 5.3.2 + lru.min: 1.1.2 + named-placeholders: 1.1.3 + seq-queue: 0.0.5 + sqlstring: 2.3.3 + + named-placeholders@1.1.3: + dependencies: + lru-cache: 7.18.3 + nanoid@3.3.11: {} + napi-build-utils@2.0.0: {} + negotiator@0.6.3: {} neotraverse@0.6.18: {} @@ -6637,6 +7004,10 @@ snapshots: nocache@3.0.4: {} + node-abi@3.75.0: + dependencies: + semver: 7.7.2 + node-fetch-native@1.6.7: {} node-fetch@2.7.0: @@ -6645,40 +7016,40 @@ snapshots: node-mock-http@1.0.2: {} - nodejs-polars-android-arm64@0.21.0: + nodejs-polars-android-arm64@0.21.1: optional: true - nodejs-polars-darwin-arm64@0.21.0: + nodejs-polars-darwin-arm64@0.21.1: optional: true - nodejs-polars-darwin-x64@0.21.0: + nodejs-polars-darwin-x64@0.21.1: optional: true - nodejs-polars-linux-arm64-gnu@0.21.0: + nodejs-polars-linux-arm64-gnu@0.21.1: optional: true - nodejs-polars-linux-arm64-musl@0.21.0: + nodejs-polars-linux-arm64-musl@0.21.1: optional: true - nodejs-polars-linux-x64-gnu@0.21.0: + nodejs-polars-linux-x64-gnu@0.21.1: optional: true - nodejs-polars-linux-x64-musl@0.21.0: + nodejs-polars-linux-x64-musl@0.21.1: optional: true - nodejs-polars-win32-x64-msvc@0.21.0: + nodejs-polars-win32-x64-msvc@0.21.1: optional: true - nodejs-polars@0.21.0: + nodejs-polars@0.21.1: optionalDependencies: - nodejs-polars-android-arm64: 0.21.0 - nodejs-polars-darwin-arm64: 0.21.0 - nodejs-polars-darwin-x64: 0.21.0 - nodejs-polars-linux-arm64-gnu: 0.21.0 - nodejs-polars-linux-arm64-musl: 0.21.0 - nodejs-polars-linux-x64-gnu: 0.21.0 - nodejs-polars-linux-x64-musl: 0.21.0 - nodejs-polars-win32-x64-msvc: 0.21.0 + nodejs-polars-android-arm64: 0.21.1 + nodejs-polars-darwin-arm64: 0.21.1 + nodejs-polars-darwin-x64: 0.21.1 + nodejs-polars-linux-arm64-gnu: 0.21.1 + nodejs-polars-linux-arm64-musl: 0.21.1 + nodejs-polars-linux-x64-gnu: 0.21.1 + nodejs-polars-linux-x64-musl: 0.21.1 + nodejs-polars-win32-x64-msvc: 0.21.1 normalize-path@3.0.0: {} @@ -6717,6 +7088,10 @@ snapshots: on-headers@1.1.0: {} + once@1.4.0: + dependencies: + wrappy: 1.0.2 + oniguruma-parser@0.12.1: {} oniguruma-to-es@4.3.3: @@ -6820,6 +7195,41 @@ snapshots: pathval@2.0.1: {} + pg-cloudflare@1.2.7: + optional: true + + pg-connection-string@2.9.1: {} + + pg-int8@1.0.1: {} + + pg-pool@3.10.1(pg@8.16.3): + dependencies: + pg: 8.16.3 + + pg-protocol@1.10.3: {} + + pg-types@2.2.0: + dependencies: + pg-int8: 1.0.1 + postgres-array: 2.0.0 + postgres-bytea: 1.0.0 + postgres-date: 1.0.7 + postgres-interval: 1.2.0 + + pg@8.16.3: + dependencies: + pg-connection-string: 2.9.1 + pg-pool: 3.10.1(pg@8.16.3) + pg-protocol: 1.10.3 + pg-types: 2.2.0 + pgpass: 1.0.5 + optionalDependencies: + pg-cloudflare: 1.2.7 + + pgpass@1.0.5: + dependencies: + split2: 4.2.0 + picocolors@1.1.1: {} picomatch@2.3.1: {} @@ -6844,6 +7254,31 @@ snapshots: picocolors: 1.1.1 source-map-js: 1.2.1 + postgres-array@2.0.0: {} + + postgres-bytea@1.0.0: {} + + postgres-date@1.0.7: {} + + postgres-interval@1.2.0: + dependencies: + xtend: 4.0.2 + + prebuild-install@7.1.3: + dependencies: + detect-libc: 2.0.4 + expand-template: 2.0.3 + github-from-package: 0.0.0 + minimist: 1.2.8 + mkdirp-classic: 0.5.3 + napi-build-utils: 2.0.0 + node-abi: 3.75.0 + pump: 3.0.3 + rc: 1.2.8 + simple-get: 4.0.1 + tar-fs: 2.1.3 + tunnel-agent: 0.6.0 + prettier@2.8.8: {} pretty-ms@9.2.0: @@ -6866,6 +7301,11 @@ snapshots: forwarded: 0.2.0 ipaddr.js: 1.9.1 + pump@3.0.3: + dependencies: + end-of-stream: 1.4.5 + once: 1.4.0 + punycode.js@2.3.1: {} qs@6.13.0: @@ -6895,6 +7335,13 @@ snapshots: iconv-lite: 0.4.24 unpipe: 1.0.0 + rc@1.2.8: + dependencies: + deep-extend: 0.6.0 + ini: 1.3.8 + minimist: 1.2.8 + strip-json-comments: 2.0.1 + read-yaml-file@1.1.0: dependencies: graceful-fs: 4.2.11 @@ -6902,6 +7349,12 @@ snapshots: pify: 4.0.1 strip-bom: 3.0.0 + readable-stream@3.6.2: + dependencies: + inherits: 2.0.4 + string_decoder: 1.3.0 + util-deprecate: 1.0.2 + readdirp@4.1.2: {} recma-build-jsx@1.0.0: @@ -7140,6 +7593,8 @@ snapshots: transitivePeerDependencies: - supports-color + seq-queue@0.0.5: {} + serve-static@1.16.2: dependencies: encodeurl: 2.0.0 @@ -7257,6 +7712,14 @@ snapshots: signal-exit@4.1.0: {} + simple-concat@1.0.1: {} + + simple-get@4.0.1: + dependencies: + decompress-response: 6.0.0 + once: 1.4.0 + simple-concat: 1.0.1 + simple-invariant@2.0.1: {} simple-swizzle@0.2.2: @@ -7295,8 +7758,12 @@ snapshots: cross-spawn: 7.0.6 signal-exit: 4.1.0 + split2@4.2.0: {} + sprintf-js@1.0.3: {} + sqlstring@2.3.3: {} + stackback@0.0.2: {} starlight-changelogs@0.1.1(@astrojs/starlight@0.35.2(astro@5.12.9(@types/node@24.2.0)(rollup@4.46.2)(tsx@4.20.3)(typescript@5.9.2)(yaml@2.8.1)))(astro@5.12.9(@types/node@24.2.0)(rollup@4.46.2)(tsx@4.20.3)(typescript@5.9.2)(yaml@2.8.1)): @@ -7347,6 +7814,10 @@ snapshots: get-east-asian-width: 1.3.0 strip-ansi: 7.1.0 + string_decoder@1.3.0: + dependencies: + safe-buffer: 5.2.1 + stringify-entities@4.0.4: dependencies: character-entities-html4: 2.1.0 @@ -7364,6 +7835,8 @@ snapshots: strip-final-newline@4.0.0: {} + strip-json-comments@2.0.1: {} + style-to-js@1.1.17: dependencies: style-to-object: 1.0.9 @@ -7376,6 +7849,21 @@ snapshots: dependencies: has-flag: 4.0.0 + tar-fs@2.1.3: + dependencies: + chownr: 1.1.4 + mkdirp-classic: 0.5.3 + pump: 3.0.3 + tar-stream: 2.2.0 + + tar-stream@2.2.0: + dependencies: + bl: 4.1.0 + end-of-stream: 1.4.5 + fs-constants: 1.0.0 + inherits: 2.0.4 + readable-stream: 3.6.2 + temp-dir@3.0.0: {} tempy@3.1.0: @@ -7446,6 +7934,10 @@ snapshots: fsevents: 2.3.3 optional: true + tunnel-agent@0.6.0: + dependencies: + safe-buffer: 5.2.1 + type-fest@1.4.0: {} type-fest@2.19.0: {} @@ -7727,8 +8219,12 @@ snapshots: string-width: 7.2.0 strip-ansi: 7.1.0 + wrappy@1.0.2: {} + xlsx@https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz: {} + xtend@4.0.2: {} + xxhash-wasm@1.1.0: {} yaml@2.8.1: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 73f80eb2..60a5161c 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -7,7 +7,7 @@ packages: - core - csv - datahub - - db + - database - docs - file - folder diff --git a/table/OVERVIEW.md b/table/OVERVIEW.md deleted file mode 100644 index 9ac88dd8..00000000 --- a/table/OVERVIEW.md +++ /dev/null @@ -1,222 +0,0 @@ -# @dpkit/table - -The `@dpkit/table` package provides high-performance data validation and processing capabilities for tabular data. Built on top of **nodejs-polars** (a Rust-based DataFrame library), it offers robust schema validation, type inference, and error handling for CSV, Excel, and other tabular data formats. - -## Examples - -### Basic Table Inspection - -```typescript -import { DataFrame } from "nodejs-polars" -import { inspectTable } from "@dpkit/table" -import type { Schema } from "@dpkit/core" - -// Create a table from data -const table = DataFrame({ - id: [1, 2, 3], - name: ["John", "Jane", "Bob"], - email: ["john@example.com", "jane@example.com", "bob@example.com"] -}).lazy() - -// Define schema with constraints -const schema: Schema = { - fields: [ - { name: "id", type: "integer", constraints: { required: true, unique: true } }, - { name: "name", type: "string", constraints: { required: true } }, - { name: "email", type: "string", constraints: { pattern: "^[^@]+@[^@]+\\.[^@]+$" } } - ] -} - -// Inspect the table -const errors = await inspectTable(table, { schema }) -console.log(errors) // Array of validation errors -``` - -### Schema Inference - -```typescript -import { inferSchema } from "@dpkit/table" - -// Automatically infer schema from data patterns -const table = DataFrame({ - id: ["1", "2", "3"], - price: ["10.50", "25.00", "15.75"], - date: ["2023-01-15", "2023-02-20", "2023-03-25"], - active: ["true", "false", "true"] -}).lazy() - -const inferredSchema = await inferSchema(table, { - sampleSize: 100, // Sample size for inference - confidence: 0.9, // Confidence threshold - monthFirst: false, // Date format preference - commaDecimal: false // Decimal separator preference -}) - -// Result: automatically detected integer, number, date, and boolean types -``` - -### Field Matching Strategies - -```typescript -// Subset matching - data can have extra fields -const schema: Schema = { - fieldsMatch: "subset", - fields: [ - { name: "id", type: "number" }, - { name: "name", type: "string" } - ] -} - -// Equal matching - field names must match regardless of order -const equalSchema: Schema = { - fieldsMatch: "equal", - fields: [ - { name: "id", type: "number" }, - { name: "name", type: "string" } - ] -} -``` - -### Table Processing - -```typescript -import { processTable } from "@dpkit/table" - -// Process table with schema (converts string columns to proper types) -const table = DataFrame({ - id: ["1", "2", "3"], // String data - price: ["10.50", "25.00", "15.75"], - active: ["true", "false", "true"], - date: ["2023-01-15", "2023-02-20", "2023-03-25"] -}).lazy() - -const schema: Schema = { - fields: [ - { name: "id", type: "integer" }, - { name: "price", type: "number" }, - { name: "active", type: "boolean" }, - { name: "date", type: "date" } - ] -} - -const processedTable = await processTable(table, { schema }) -const result = await processedTable.collect() - -// Result will have properly typed columns: -// { id: 1, price: 10.50, active: true, date: Date('2023-01-15') } -// { id: 2, price: 25.00, active: false, date: Date('2023-02-20') } -// { id: 3, price: 15.75, active: true, date: Date('2023-03-25') } -``` - -### Error Handling - -```typescript -const result = await inspectTable(table, { schema }) - -result.errors.forEach(error => { - switch (error.type) { - case "cell/required": - console.log(`Required field missing in row ${error.rowNumber}: '${error.fieldName}'`) - break - case "cell/unique": - console.log(`Duplicate value in row ${error.rowNumber}: '${error.cell}'`) - break - case "cell/pattern": - console.log(`Pattern mismatch: '${error.cell}' doesn't match ${error.constraint}`) - break - } -}) -``` - -## Core Architecture - -### Table Type -The package uses `LazyDataFrame` from nodejs-polars as its core table representation, enabling lazy evaluation and efficient processing of large datasets through vectorized operations. - -### Schema Integration -Integrates seamlessly with `@dpkit/core` schemas, bridging Data Package field definitions with Polars data types for comprehensive validation workflows. - -## Key Features - -### 1. Multi-Level Validation System - -**Field-Level Validation:** -- **Type Validation**: Converts and validates data types (string → integer, etc.) -- **Name Validation**: Ensures field names match schema requirements -- **Constraint Validation**: Enforces required, unique, enum, pattern, min/max values, and length constraints - -**Table-Level Validation:** -- **Field Presence**: Validates missing/extra fields based on flexible matching strategies -- **Schema Compatibility**: Ensures data structure aligns with schema definitions - -**Row-Level Validation:** -- **Primary Key Uniqueness**: Validates unique identifiers -- **Composite Keys**: Supports multi-column unique constraints - -### 2. Comprehensive Field Types - -**Primitive Types:** -- `string`, `integer`, `number`, `boolean` - -**Temporal Types:** -- `date`, `datetime`, `time`, `year`, `yearmonth`, `duration` - -**Spatial Types:** -- `geopoint`, `geojson` - -**Complex Types:** -- `array`, `list`, `object` - -### 3. Smart Schema Inference - -Automatically infers field types and formats from data using: -- Regex pattern matching with configurable confidence thresholds -- Locale-specific format detection (comma decimals, date formats) -- Complex type recognition (objects, arrays, temporal data) - -### 4. Flexible Field Matching Strategies - -- **exact**: Fields must match exactly in order and count -- **equal**: Same fields in any order -- **subset**: Data must contain all schema fields (extras allowed) -- **superset**: Schema must contain all data fields -- **partial**: At least one field must match - -### 5. Advanced Data Processing - -**Format-Aware Parsing:** -- Handles missing values at schema and field levels -- Supports group/decimal character customization -- Processes currency symbols and whitespace -- Parses complex formats (ISO dates, scientific notation) - -**Performance Optimizations:** -- Sample-based validation for large datasets -- Lazy evaluation for memory efficiency -- Vectorized constraint checking -- Configurable error limits and batch processing - -## Error Handling - -### Comprehensive Error Taxonomy - -**Field Errors:** -- Name mismatches between schema and data -- Type conversion failures -- Missing or extra field violations - -**Cell Errors:** -- Type validation failures with specific conversion details -- Constraint violations (required, unique, enum, pattern, range, length) -- Format parsing errors with problematic values - -**Row Errors:** -- Unique key constraint violations -- Composite constraint failures - -### Error Details -Each error includes: -- Precise row and column locations -- Actual vs expected values -- Specific error type classification -- Actionable error messages for debugging diff --git a/table/README.md b/table/README.md index b79dedba..dd056ff5 100644 --- a/table/README.md +++ b/table/README.md @@ -1,3 +1,3 @@ # @dpkit/table -dpkit is a fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). +dpkit is a fast data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/table/error/Field.ts b/table/error/Field.ts index b421f4e6..936196d2 100644 --- a/table/error/Field.ts +++ b/table/error/Field.ts @@ -1,4 +1,4 @@ -import type { Field } from "@dpkit/core" +import type { FieldType } from "@dpkit/core" import type { BaseTableError } from "./Base.ts" export interface BaseFieldError extends BaseTableError { @@ -12,6 +12,6 @@ export interface FieldNameError extends BaseFieldError { export interface FieldTypeError extends BaseFieldError { type: "field/type" - fieldType: Field["type"] - actualFieldType: Field["type"] + fieldType: FieldType + actualFieldType: FieldType } diff --git a/table/field/checks/enum.spec.ts b/table/field/checks/enum.spec.ts index 5d6100a8..973d167e 100644 --- a/table/field/checks/enum.spec.ts +++ b/table/field/checks/enum.spec.ts @@ -1,9 +1,9 @@ import type { Schema } from "@dpkit/core" import { DataFrame } from "nodejs-polars" import { describe, expect, it } from "vitest" -import { inspectTable } from "../../table/index.ts" +import { validateTable } from "../../table/index.ts" -describe("inspectTable (cell/enum)", () => { +describe("validateTable (cell/enum)", () => { it("should not report errors for string values that are in the enum", async () => { const table = DataFrame({ status: ["pending", "approved", "rejected", "pending"], @@ -21,7 +21,7 @@ describe("inspectTable (cell/enum)", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toHaveLength(0) }) @@ -42,7 +42,7 @@ describe("inspectTable (cell/enum)", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors.filter(e => e.type === "cell/enum")).toHaveLength(2) expect(errors).toContainEqual({ type: "cell/enum", @@ -75,7 +75,7 @@ describe("inspectTable (cell/enum)", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors.filter(e => e.type === "cell/enum")).toHaveLength(0) }) @@ -96,7 +96,7 @@ describe("inspectTable (cell/enum)", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors.filter(e => e.type === "cell/enum")).toHaveLength(2) expect(errors).toContainEqual({ type: "cell/enum", diff --git a/table/field/checks/maxLength.spec.ts b/table/field/checks/maxLength.spec.ts index c4e90d19..95424693 100644 --- a/table/field/checks/maxLength.spec.ts +++ b/table/field/checks/maxLength.spec.ts @@ -1,9 +1,9 @@ import type { Schema } from "@dpkit/core" import { DataFrame } from "nodejs-polars" import { describe, expect, it } from "vitest" -import { inspectTable } from "../../table/index.ts" +import { validateTable } from "../../table/index.ts" -describe("inspectTable (cell/maxLength)", () => { +describe("validateTable (cell/maxLength)", () => { it("should not report errors for string values that meet the maxLength constraint", async () => { const table = DataFrame({ code: ["A123", "B456", "C789"], @@ -19,7 +19,7 @@ describe("inspectTable (cell/maxLength)", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toHaveLength(0) }) @@ -38,7 +38,7 @@ describe("inspectTable (cell/maxLength)", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors.filter(e => e.type === "cell/maxLength")).toHaveLength(1) expect(errors).toContainEqual({ type: "cell/maxLength", diff --git a/table/field/checks/maximum.spec.ts b/table/field/checks/maximum.spec.ts index 6c3acb81..0e2f07fb 100644 --- a/table/field/checks/maximum.spec.ts +++ b/table/field/checks/maximum.spec.ts @@ -1,9 +1,9 @@ import type { Schema } from "@dpkit/core" import { DataFrame } from "nodejs-polars" import { describe, expect, it } from "vitest" -import { inspectTable } from "../../table/index.ts" +import { validateTable } from "../../table/index.ts" -describe("inspectTable (cell/maximum)", () => { +describe("validateTable (cell/maximum)", () => { it("should not report errors for valid values", async () => { const table = DataFrame({ price: [10.5, 20.75, 30.0], @@ -19,7 +19,7 @@ describe("inspectTable (cell/maximum)", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toHaveLength(0) }) @@ -38,7 +38,7 @@ describe("inspectTable (cell/maximum)", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors.filter(e => e.type === "cell/maximum")).toHaveLength(1) expect(errors).toContainEqual({ type: "cell/maximum", @@ -63,7 +63,7 @@ describe("inspectTable (cell/maximum)", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors.filter(e => e.type === "cell/exclusiveMaximum")).toHaveLength( 2, ) diff --git a/table/field/checks/minLength.spec.ts b/table/field/checks/minLength.spec.ts index c13aa025..293fd960 100644 --- a/table/field/checks/minLength.spec.ts +++ b/table/field/checks/minLength.spec.ts @@ -1,9 +1,9 @@ import type { Schema } from "@dpkit/core" import { DataFrame } from "nodejs-polars" import { describe, expect, it } from "vitest" -import { inspectTable } from "../../table/index.ts" +import { validateTable } from "../../table/index.ts" -describe("inspectTable (cell/minLength)", () => { +describe("validateTable (cell/minLength)", () => { it("should not report errors for string values that meet the minLength constraint", async () => { const table = DataFrame({ code: ["A123", "B456", "C789"], @@ -19,7 +19,7 @@ describe("inspectTable (cell/minLength)", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toHaveLength(0) }) @@ -38,7 +38,7 @@ describe("inspectTable (cell/minLength)", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors.filter(e => e.type === "cell/minLength")).toHaveLength(2) expect(errors).toContainEqual({ type: "cell/minLength", diff --git a/table/field/checks/minimum.spec.ts b/table/field/checks/minimum.spec.ts index 8e1a307f..0b77ba38 100644 --- a/table/field/checks/minimum.spec.ts +++ b/table/field/checks/minimum.spec.ts @@ -1,9 +1,9 @@ import type { Schema } from "@dpkit/core" import { DataFrame } from "nodejs-polars" import { describe, expect, it } from "vitest" -import { inspectTable } from "../../table/index.ts" +import { validateTable } from "../../table/index.ts" -describe("inspectTable (cell/minimum)", () => { +describe("validateTable (cell/minimum)", () => { it("should not report errors for valid values", async () => { const table = DataFrame({ price: [10.5, 20.75, 30.0], @@ -19,7 +19,7 @@ describe("inspectTable (cell/minimum)", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toHaveLength(0) }) @@ -38,7 +38,7 @@ describe("inspectTable (cell/minimum)", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors.filter(e => e.type === "cell/minimum")).toHaveLength(1) expect(errors).toContainEqual({ type: "cell/minimum", @@ -63,7 +63,7 @@ describe("inspectTable (cell/minimum)", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors.filter(e => e.type === "cell/exclusiveMinimum")).toHaveLength( 2, ) diff --git a/table/field/checks/pattern.spec.ts b/table/field/checks/pattern.spec.ts index 29e60d60..34ad57ae 100644 --- a/table/field/checks/pattern.spec.ts +++ b/table/field/checks/pattern.spec.ts @@ -1,9 +1,9 @@ import type { Schema } from "@dpkit/core" import { DataFrame } from "nodejs-polars" import { describe, expect, it } from "vitest" -import { inspectTable } from "../../table/index.ts" +import { validateTable } from "../../table/index.ts" -describe("inspectTable (cell/pattern)", () => { +describe("validateTable (cell/pattern)", () => { it("should not report errors for string values that match the pattern", async () => { const table = DataFrame({ email: ["john@example.com", "alice@domain.org", "test@test.io"], @@ -21,7 +21,7 @@ describe("inspectTable (cell/pattern)", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toHaveLength(0) }) @@ -42,7 +42,7 @@ describe("inspectTable (cell/pattern)", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors.filter(e => e.type === "cell/pattern")).toHaveLength(2) expect(errors).toContainEqual({ type: "cell/pattern", diff --git a/table/field/checks/required.spec.ts b/table/field/checks/required.spec.ts index 1468ebd5..3f1f9e0e 100644 --- a/table/field/checks/required.spec.ts +++ b/table/field/checks/required.spec.ts @@ -1,9 +1,9 @@ import type { Schema } from "@dpkit/core" import { DataFrame } from "nodejs-polars" import { describe, expect, it } from "vitest" -import { inspectTable } from "../../table/index.ts" +import { validateTable } from "../../table/index.ts" -describe("inspectTable (cell/required)", () => { +describe("validateTable (cell/required)", () => { it("should report a cell/required error", async () => { const table = DataFrame({ id: [1, null, 3], @@ -13,7 +13,7 @@ describe("inspectTable (cell/required)", () => { fields: [{ name: "id", type: "number", constraints: { required: true } }], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toHaveLength(1) expect(errors).toContainEqual({ diff --git a/table/field/checks/type.spec.ts b/table/field/checks/type.spec.ts index 027a5215..04aa7844 100644 --- a/table/field/checks/type.spec.ts +++ b/table/field/checks/type.spec.ts @@ -1,10 +1,10 @@ import type { Schema } from "@dpkit/core" import { DataFrame } from "nodejs-polars" import { describe, expect, it } from "vitest" -import { inspectTable } from "../../table/index.ts" +import { validateTable } from "../../table/index.ts" -describe("inspectTable", () => { - it("should inspect string to integer conversion errors", async () => { +describe("validateTable", () => { + it("should validate string to integer conversion errors", async () => { const table = DataFrame({ id: ["1", "bad", "3", "4x"], }).lazy() @@ -13,7 +13,7 @@ describe("inspectTable", () => { fields: [{ name: "id", type: "integer" }], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toHaveLength(2) expect(errors).toContainEqual({ @@ -30,7 +30,7 @@ describe("inspectTable", () => { }) }) - it("should inspect string to number conversion errors", async () => { + it("should validate string to number conversion errors", async () => { const table = DataFrame({ price: ["10.5", "twenty", "30.75", "$40"], }).lazy() @@ -39,7 +39,7 @@ describe("inspectTable", () => { fields: [{ name: "price", type: "number" }], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toHaveLength(2) expect(errors).toContainEqual({ @@ -56,7 +56,7 @@ describe("inspectTable", () => { }) }) - it("should inspect string to boolean conversion errors", async () => { + it("should validate string to boolean conversion errors", async () => { const table = DataFrame({ active: ["true", "yes", "false", "0", "1"], }).lazy() @@ -65,7 +65,7 @@ describe("inspectTable", () => { fields: [{ name: "active", type: "boolean" }], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toHaveLength(1) expect(errors).toContainEqual({ @@ -76,7 +76,7 @@ describe("inspectTable", () => { }) }) - it("should inspect string to date conversion errors", async () => { + it("should validate string to date conversion errors", async () => { const table = DataFrame({ created: ["2023-01-15", "Jan 15, 2023", "20230115", "not-a-date"], }).lazy() @@ -85,7 +85,7 @@ describe("inspectTable", () => { fields: [{ name: "created", type: "date" }], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toHaveLength(3) expect(errors).toContainEqual({ @@ -108,7 +108,7 @@ describe("inspectTable", () => { }) }) - it("should inspect string to time conversion errors", async () => { + it("should validate string to time conversion errors", async () => { const table = DataFrame({ time: ["14:30:00", "2:30pm", "invalid", "14h30"], }).lazy() @@ -117,7 +117,7 @@ describe("inspectTable", () => { fields: [{ name: "time", type: "time" }], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toHaveLength(3) expect(errors).toContainEqual({ @@ -140,7 +140,7 @@ describe("inspectTable", () => { }) }) - it("should inspect string to year conversion errors", async () => { + it("should validate string to year conversion errors", async () => { const table = DataFrame({ year: ["2023", "23", "MMXXIII", "two-thousand-twenty-three"], }).lazy() @@ -149,7 +149,7 @@ describe("inspectTable", () => { fields: [{ name: "year", type: "year" }], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toHaveLength(3) expect(errors).toContainEqual({ @@ -172,7 +172,7 @@ describe("inspectTable", () => { }) }) - it("should inspect string to datetime conversion errors", async () => { + it("should validate string to datetime conversion errors", async () => { const table = DataFrame({ timestamp: [ "2023-01-15T14:30:00", @@ -186,7 +186,7 @@ describe("inspectTable", () => { fields: [{ name: "timestamp", type: "datetime" }], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) // Adjust the expectations to match actual behavior expect(errors.length).toBeGreaterThan(0) @@ -216,12 +216,12 @@ describe("inspectTable", () => { fields: [{ name: "id", type: "integer" }], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toHaveLength(0) }) - it("should inspect with non-string source data", async () => { + it("should validate with non-string source data", async () => { const table = DataFrame({ is_active: [true, false, 1, 0], }).lazy() @@ -230,9 +230,9 @@ describe("inspectTable", () => { fields: [{ name: "is_active", type: "boolean" }], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) - // Since the column isn't string type, inspectField will not process it + // Since the column isn't string type, validateField will not normalize it expect(errors).toHaveLength(0) }) }) diff --git a/table/field/checks/unique.spec.ts b/table/field/checks/unique.spec.ts index c53727c8..dc3d6b29 100644 --- a/table/field/checks/unique.spec.ts +++ b/table/field/checks/unique.spec.ts @@ -1,10 +1,10 @@ import type { Schema } from "@dpkit/core" import { DataFrame } from "nodejs-polars" import { describe, expect, it } from "vitest" -import { inspectTable } from "../../table/index.ts" +import { validateTable } from "../../table/index.ts" // TODO: recover -describe("inspectTable (cell/unique)", () => { +describe("validateTable (cell/unique)", () => { it("should not report errors when all values are unique", async () => { const table = DataFrame({ id: [1, 2, 3, 4, 5], @@ -20,7 +20,7 @@ describe("inspectTable (cell/unique)", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toHaveLength(0) }) @@ -39,7 +39,7 @@ describe("inspectTable (cell/unique)", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors.filter(e => e.type === "cell/unique")).toHaveLength(1) expect(errors).toContainEqual({ @@ -65,7 +65,7 @@ describe("inspectTable (cell/unique)", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors.filter(e => e.type === "cell/unique")).toHaveLength(2) expect(errors).toContainEqual({ type: "cell/unique", @@ -96,7 +96,7 @@ describe("inspectTable (cell/unique)", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toHaveLength(0) }) }) diff --git a/table/field/index.ts b/table/field/index.ts index a9480dfe..c0ecec8e 100644 --- a/table/field/index.ts +++ b/table/field/index.ts @@ -1,4 +1,5 @@ export { parseField } from "./parse.ts" -export { inspectField } from "./inspect.ts" -export type { PolarsField } from "./Field.ts" +export { validateField } from "./validate.ts" export { matchField } from "./match.ts" +export { stringifyField } from "./stringify.ts" +export type { PolarsField } from "./Field.ts" diff --git a/table/field/parse.spec.ts b/table/field/parse.spec.ts index ef2e24ef..99d88511 100644 --- a/table/field/parse.spec.ts +++ b/table/field/parse.spec.ts @@ -1,7 +1,7 @@ import type { Schema } from "@dpkit/core" import { DataFrame } from "nodejs-polars" import { describe, expect, it } from "vitest" -import { processTable } from "../table/index.ts" +import { normalizeTable } from "../table/index.ts" describe("parseField", () => { describe("missing values", () => { @@ -29,7 +29,7 @@ describe("parseField", () => { fields: [{ name: "name", type: "string", missingValues: fieldLevel }], } - const ldf = await processTable(table, { schema }) + const ldf = await normalizeTable(table, schema) const df = await ldf.collect() expect(df.getColumn("name").get(0)).toEqual(value) diff --git a/table/field/parse.ts b/table/field/parse.ts index 6a9f8383..0ac6c831 100644 --- a/table/field/parse.ts +++ b/table/field/parse.ts @@ -1,4 +1,4 @@ -import type { Field, Schema } from "@dpkit/core" +import type { Field } from "@dpkit/core" import { col, lit, when } from "nodejs-polars" import type { Expr } from "nodejs-polars" import { parseArrayField } from "./types/array.ts" @@ -19,21 +19,13 @@ import { parseYearmonthField } from "./types/yearmonth.ts" const DEFAULT_MISSING_VALUES = [""] -export function parseField( - field: Field, - options?: { expr?: Expr; schema?: Schema }, -) { - let expr = options?.expr ?? col(field.name) +export function parseField(field: Field, expr?: Expr) { + expr = expr ?? col(field.name) - const missingValues = - field.missingValues ?? - options?.schema?.missingValues ?? + const flattenMissingValues = + field.missingValues?.map(it => (typeof it === "string" ? it : it.value)) ?? DEFAULT_MISSING_VALUES - const flattenMissingValues = missingValues.map(item => - typeof item === "string" ? item : item.value, - ) - if (flattenMissingValues.length) { expr = when(expr.isIn(flattenMissingValues)) .then(lit(null)) @@ -42,36 +34,36 @@ export function parseField( } switch (field.type) { - case "string": - return parseStringField(field, expr) - case "integer": - return parseIntegerField(field, expr) - case "number": - return parseNumberField(field, expr) + case "array": + return parseArrayField(field, expr) case "boolean": return parseBooleanField(field, expr) case "date": return parseDateField(field, expr) case "datetime": return parseDatetimeField(field, expr) + case "duration": + return parseDurationField(field, expr) + case "geojson": + return parseGeojsonField(field, expr) + case "geopoint": + return parseGeopointField(field, expr) + case "integer": + return parseIntegerField(field, expr) + case "list": + return parseListField(field, expr) + case "number": + return parseNumberField(field, expr) + case "object": + return parseObjectField(field, expr) + case "string": + return parseStringField(field, expr) case "time": return parseTimeField(field, expr) case "year": return parseYearField(field, expr) case "yearmonth": return parseYearmonthField(field, expr) - case "list": - return parseListField(field, expr) - case "array": - return parseArrayField(field, expr) - case "geopoint": - return parseGeopointField(field, expr) - case "object": - return parseObjectField(field, expr) - case "geojson": - return parseGeojsonField(field, expr) - case "duration": - return parseDurationField(field, expr) default: return expr } diff --git a/table/field/stringify.spec.ts b/table/field/stringify.spec.ts new file mode 100644 index 00000000..32db737c --- /dev/null +++ b/table/field/stringify.spec.ts @@ -0,0 +1,50 @@ +import type { Schema } from "@dpkit/core" +import { DataFrame } from "nodejs-polars" +import { describe, expect, it } from "vitest" +import { denormalizeTable } from "../table/index.ts" + +describe("stringifyField", () => { + describe("missing values", () => { + it.each([ + // Schema-level - null values should be converted to first missing value + [null, "", {}], + [null, "", { schemaLevel: [] }], // defaults to "" + [null, "-", { schemaLevel: ["-"] }], + [null, "x", { schemaLevel: ["x"] }], + + // Regular values should remain unchanged + ["hello", "hello", {}], + ["world", "world", { schemaLevel: ["-"] }], + + // Field-level missing values take precedence + [null, "", {}], // default field-level missing value + [null, "-", { fieldLevel: ["-"] }], + [null, "n/a", { fieldLevel: ["n/a"] }], + + // Regular values with field-level settings + ["test", "test", { fieldLevel: ["-"] }], + ["value", "value", { fieldLevel: ["n/a"] }], + + // Field-level overrides schema-level + [null, "-", { schemaLevel: ["x"], fieldLevel: ["-"] }], + [null, "x", { schemaLevel: ["-"], fieldLevel: ["x"] }], + + // Multiple missing values - should use first one + [null, "-", { fieldLevel: ["-", "n/a", "null"] }], + [null, "n/a", { schemaLevel: ["n/a", "NULL", ""] }], + + // @ts-ignore + ])("%s -> %s %s", async (value, expected, { fieldLevel, schemaLevel }) => { + const table = DataFrame({ name: [value] }).lazy() + const schema: Schema = { + missingValues: schemaLevel, + fields: [{ name: "name", type: "string", missingValues: fieldLevel }], + } + + const ldf = await denormalizeTable(table, schema) + const df = await ldf.collect() + + expect(df.toRecords()[0]?.name).toEqual(expected) + }) + }) +}) diff --git a/table/field/stringify.ts b/table/field/stringify.ts new file mode 100644 index 00000000..6b62ad0b --- /dev/null +++ b/table/field/stringify.ts @@ -0,0 +1,84 @@ +import type { Field } from "@dpkit/core" +import { col, lit, when } from "nodejs-polars" +import type { Expr } from "nodejs-polars" +import { stringifyArrayField } from "./types/array.ts" +import { stringifyBooleanField } from "./types/boolean.ts" +import { stringifyDateField } from "./types/date.ts" +import { stringifyDatetimeField } from "./types/datetime.ts" +import { stringifyDurationField } from "./types/duration.ts" +import { stringifyGeojsonField } from "./types/geojson.ts" +import { stringifyGeopointField } from "./types/geopoint.ts" +import { stringifyIntegerField } from "./types/integer.ts" +import { stringifyListField } from "./types/list.ts" +import { stringifyNumberField } from "./types/number.ts" +import { stringifyObjectField } from "./types/object.ts" +import { stringifyStringField } from "./types/string.ts" +import { stringifyTimeField } from "./types/time.ts" +import { stringifyYearField } from "./types/year.ts" +import { stringifyYearmonthField } from "./types/yearmonth.ts" + +const DEFAULT_MISSING_VALUE = "" + +export function stringifyField(field: Field, expr?: Expr) { + expr = expr ?? col(field.name) + + switch (field.type) { + case "array": + expr = stringifyArrayField(field, expr) + break + case "boolean": + expr = stringifyBooleanField(field, expr) + break + case "date": + expr = stringifyDateField(field, expr) + break + case "datetime": + expr = stringifyDatetimeField(field, expr) + break + case "duration": + expr = stringifyDurationField(field, expr) + break + case "geojson": + expr = stringifyGeojsonField(field, expr) + break + case "geopoint": + expr = stringifyGeopointField(field, expr) + break + case "integer": + expr = stringifyIntegerField(field, expr) + break + case "list": + expr = stringifyListField(field, expr) + break + case "number": + expr = stringifyNumberField(field, expr) + break + case "object": + expr = stringifyObjectField(field, expr) + break + case "string": + expr = stringifyStringField(field, expr) + break + case "time": + expr = stringifyTimeField(field, expr) + break + case "year": + expr = stringifyYearField(field, expr) + break + case "yearmonth": + expr = stringifyYearmonthField(field, expr) + break + } + + const flattenMissingValues = field.missingValues?.map(it => + typeof it === "string" ? it : it.value, + ) + + const missingValue = flattenMissingValues?.[0] ?? DEFAULT_MISSING_VALUE + expr = when(expr.isNull()) + .then(lit(missingValue)) + .otherwise(expr) + .alias(field.name) + + return expr +} diff --git a/table/field/types/array.spec.ts b/table/field/types/array.spec.ts index d881f1b8..01d2ec1f 100644 --- a/table/field/types/array.spec.ts +++ b/table/field/types/array.spec.ts @@ -1,6 +1,6 @@ -import { DataFrame } from "nodejs-polars" +import { DataFrame, DataType, Series } from "nodejs-polars" import { describe, expect, it } from "vitest" -import { processTable } from "../../table/index.ts" +import { denormalizeTable, normalizeTable } from "../../table/index.ts" describe("parseArrayField", () => { it.each([ @@ -25,15 +25,47 @@ describe("parseArrayField", () => { ["null", null], ["undefined", null], ])("%s -> %s", async (cell, value) => { - const table = DataFrame({ name: [cell] }).lazy() + const table = DataFrame([Series("name", [cell], DataType.String)]).lazy() + const schema = { fields: [{ name: "name", type: "array" as const }], } - const ldf = await processTable(table, { schema }) + const ldf = await normalizeTable(table, schema) const df = await ldf.collect() - const res = df.getColumn("name").get(0) expect(res ? JSON.parse(res) : res).toEqual(value) }) }) + +describe("stringifyArrayField", () => { + it.each([ + // JSON array strings should be returned as-is + ["[1,2,3]", "[1,2,3]"], + ['["a","b","c"]', '["a","b","c"]'], + ['[{"name":"John"},{"name":"Jane"}]', '[{"name":"John"},{"name":"Jane"}]'], + ["[]", "[]"], + + // Complex nested arrays + [ + '[{"users":[{"id":1,"name":"Alice"}],"count":2}]', + '[{"users":[{"id":1,"name":"Alice"}],"count":2}]', + ], + ["[[1,2],[3,4]]", "[[1,2],[3,4]]"], + ['[true,false,null,"text",42,3.14]', '[true,false,null,"text",42,3.14]'], + + // Null handling + [null, ""], + ])("%s -> %s", async (value, expected) => { + const table = DataFrame([Series("name", [value], DataType.String)]).lazy() + + const schema = { + fields: [{ name: "name", type: "array" as const }], + } + + const ldf = await denormalizeTable(table, schema) + const df = await ldf.collect() + + expect(df.toRecords()[0]?.name).toEqual(expected) + }) +}) diff --git a/table/field/types/array.ts b/table/field/types/array.ts index f3692d60..7c21ec25 100644 --- a/table/field/types/array.ts +++ b/table/field/types/array.ts @@ -10,3 +10,10 @@ export function parseArrayField(field: ArrayField, expr?: Expr) { return when(expr.str.contains("^\\[")).then(expr).otherwise(lit(null)) } + +export function stringifyArrayField(field: ArrayField, expr?: Expr) { + expr = expr ?? col(field.name) + + // TODO: implement + return expr +} diff --git a/table/field/types/boolean.spec.ts b/table/field/types/boolean.spec.ts index 3618e98b..e1517a31 100644 --- a/table/field/types/boolean.spec.ts +++ b/table/field/types/boolean.spec.ts @@ -1,6 +1,7 @@ -import { DataFrame } from "nodejs-polars" +import { DataFrame, DataType, Series } from "nodejs-polars" import { describe, expect, it } from "vitest" -import { processTable } from "../../table/index.ts" +import { normalizeTable } from "../../table/index.ts" +import { denormalizeTable } from "../../table/index.ts" describe("parseBooleanField", () => { it.each([ @@ -45,14 +46,46 @@ describe("parseBooleanField", () => { ["non", false, { trueValues: ["oui", "si"], falseValues: ["non", "no"] }], ["no", false, { trueValues: ["oui", "si"], falseValues: ["non", "no"] }], ])("%s -> %s %o", async (cell, value, options) => { - const table = DataFrame({ name: [cell] }).lazy() + const table = DataFrame([Series("name", [cell], DataType.String)]).lazy() + const schema = { fields: [{ name: "name", type: "boolean" as const, ...options }], } - const ldf = await processTable(table, { schema }) + const ldf = await normalizeTable(table, schema) const df = await ldf.collect() expect(df.toRecords()[0]?.name).toEqual(value) }) }) + +describe("stringifyBooleanField", () => { + it.each([ + // Default values + [true, "true", {}], + [false, "false", {}], + + // Custom true values + [true, "Y", { trueValues: ["Y", "y", "yes"] }], + [false, "false", { trueValues: ["Y", "y", "yes"] }], + + // Custom false values + [true, "true", { falseValues: ["N", "n", "no"] }], + [false, "N", { falseValues: ["N", "n", "no"] }], + + // Custom true and false values + [true, "oui", { trueValues: ["oui", "si"], falseValues: ["non", "no"] }], + [false, "non", { trueValues: ["oui", "si"], falseValues: ["non", "no"] }], + ])("%s -> %s %o", async (value, expected, options) => { + const table = DataFrame([Series("name", [value], DataType.Bool)]).lazy() + + const schema = { + fields: [{ name: "name", type: "boolean" as const, ...options }], + } + + const ldf = await denormalizeTable(table, schema) + const df = await ldf.collect() + + expect(df.toRecords()[0]?.name).toEqual(expected) + }) +}) diff --git a/table/field/types/boolean.ts b/table/field/types/boolean.ts index 7bca8739..054cae60 100644 --- a/table/field/types/boolean.ts +++ b/table/field/types/boolean.ts @@ -9,8 +9,8 @@ const DEFAULT_FALSE_VALUES = ["false", "False", "FALSE", "0"] export function parseBooleanField(field: BooleanField, expr?: Expr) { expr = expr ?? col(field.name) - const trueValues = field.trueValues || DEFAULT_TRUE_VALUES - const falseValues = field.falseValues || DEFAULT_FALSE_VALUES + const trueValues = field.trueValues ?? DEFAULT_TRUE_VALUES + const falseValues = field.falseValues ?? DEFAULT_FALSE_VALUES for (const value of trueValues) expr = expr.replace(value, "1") for (const value of falseValues) expr = expr.replace(value, "0") @@ -24,3 +24,18 @@ export function parseBooleanField(field: BooleanField, expr?: Expr) { .otherwise(lit(null)) .alias(field.name) } + +const DEFAULT_TRUE_VALUE = "true" +const DEFAULT_FALSE_VALUE = "false" + +export function stringifyBooleanField(field: BooleanField, expr?: Expr) { + expr = expr ?? col(field.name) + + const trueValue = field.trueValues?.[0] ?? DEFAULT_TRUE_VALUE + const falseValue = field.falseValues?.[0] ?? DEFAULT_FALSE_VALUE + + return when(expr.eq(lit(true))) + .then(lit(trueValue)) + .otherwise(lit(falseValue)) + .alias(field.name) +} diff --git a/table/field/types/date.spec.ts b/table/field/types/date.spec.ts index 5cf5beb5..75b3c2bd 100644 --- a/table/field/types/date.spec.ts +++ b/table/field/types/date.spec.ts @@ -1,6 +1,7 @@ -import { DataFrame } from "nodejs-polars" +import { DataFrame, DataType, Series } from "nodejs-polars" import { describe, expect, it } from "vitest" -import { processTable } from "../../table/index.ts" +import { normalizeTable } from "../../table/index.ts" +import { denormalizeTable } from "../../table/index.ts" describe("parseDateField", () => { it.each([ @@ -20,12 +21,36 @@ describe("parseDateField", () => { // Invalid format ["21/11/06", null, { format: "invalid" }], ])("%s -> %s %o", async (cell, expected, options) => { - const table = DataFrame({ name: [cell] }).lazy() + const table = DataFrame([Series("name", [cell], DataType.String)]).lazy() + + const schema = { + fields: [{ name: "name", type: "date" as const, ...options }], + } + + const ldf = await normalizeTable(table, schema) + const df = await ldf.collect() + + expect(df.toRecords()[0]?.name).toEqual(expected) + }) +}) + +describe("stringifyDateField", () => { + it.each([ + // Default format + [new Date(Date.UTC(2019, 0, 1)), "2019-01-01", {}], + [new Date(Date.UTC(2006, 10, 21)), "2006-11-21", {}], + + // Custom format + [new Date(Date.UTC(2006, 10, 21)), "21/11/2006", { format: "%d/%m/%Y" }], + [new Date(Date.UTC(2006, 10, 21)), "2006/11/21", { format: "%Y/%m/%d" }], + ])("%s -> %s %o", async (value, expected, options) => { + const table = DataFrame([Series("name", [value], DataType.Date)]).lazy() + const schema = { fields: [{ name: "name", type: "date" as const, ...options }], } - const ldf = await processTable(table, { schema }) + const ldf = await denormalizeTable(table, schema) const df = await ldf.collect() expect(df.toRecords()[0]?.name).toEqual(expected) diff --git a/table/field/types/date.ts b/table/field/types/date.ts index 36ef1a3f..3199a413 100644 --- a/table/field/types/date.ts +++ b/table/field/types/date.ts @@ -15,3 +15,11 @@ export function parseDateField(field: DateField, expr?: Expr) { return expr.str.strptime(DataType.Date, format) } + +export function stringifyDateField(field: DateField, expr?: Expr) { + expr = expr ?? col(field.name) + + const format = field.format ?? DEFAULT_FORMAT + + return expr.date.strftime(format) +} diff --git a/table/field/types/datetime.spec.ts b/table/field/types/datetime.spec.ts index c1abbd7a..89297edc 100644 --- a/table/field/types/datetime.spec.ts +++ b/table/field/types/datetime.spec.ts @@ -1,14 +1,14 @@ -import { DataFrame } from "nodejs-polars" +import { DataFrame, DataType, Series } from "nodejs-polars" import { describe, expect, it } from "vitest" -import { processTable } from "../../table/index.ts" +import { normalizeTable } from "../../table/index.ts" +import { denormalizeTable } from "../../table/index.ts" -// TODO: Enable this test suite -// Currently, it seems to have a weird datetime translation bug on the Polars side -// as resutls within the dataframe are correct, but exported ones are not +// TODO: Enable this test suite after this issue is fixed: +// https://github.com/pola-rs/nodejs-polars/issues/365 describe.skip("parseDatetimeField", () => { it.each([ // Default format - ["2014-01-01T06:00:00", new Date(2014, 0, 1, 6, 0, 0), {}], + ["2014-01-01T06:00:00", new Date(Date.UTC(2014, 0, 1, 6, 0, 0)), {}], ["2014-01-01T06:00:00Z", new Date(Date.UTC(2014, 0, 1, 6, 0, 0)), {}], ["Mon 1st Jan 2014 9 am", null, {}], ["invalid", null, {}], @@ -27,12 +27,44 @@ describe.skip("parseDatetimeField", () => { // Invalid format ["21/11/06 16:30", null, { format: "invalid" }], ])("%s -> %s %o", async (cell, expected, options) => { - const table = DataFrame({ name: [cell] }).lazy() + const table = DataFrame([Series("name", [cell], DataType.String)]).lazy() + + const schema = { + fields: [{ name: "name", type: "datetime" as const, ...options }], + } + + const ldf = await normalizeTable(table, schema) + const df = await ldf.collect() + + expect(df.toRecords()[0]?.name).toEqual(expected) + }) +}) + +describe("stringifyDatetimeField", () => { + it.each([ + // Default format + [new Date(Date.UTC(2014, 0, 1, 6, 0, 0)), "2014-01-01T06:00:00", {}], + [new Date(Date.UTC(2006, 10, 21, 16, 30, 0)), "2006-11-21T16:30:00", {}], + + // Custom format + [ + new Date(Date.UTC(2006, 10, 21, 16, 30, 0)), + "21/11/2006 16:30", + { format: "%d/%m/%Y %H:%M" }, + ], + [ + new Date(Date.UTC(2014, 0, 1, 6, 0, 0)), + "2014/01/01T06:00:00", + { format: "%Y/%m/%dT%H:%M:%S" }, + ], + ])("%s -> %s %o", async (value, expected, options) => { + const table = DataFrame([Series("name", [value], DataType.Datetime)]).lazy() + const schema = { fields: [{ name: "name", type: "datetime" as const, ...options }], } - const ldf = await processTable(table, { schema }) + const ldf = await denormalizeTable(table, schema) const df = await ldf.collect() expect(df.toRecords()[0]?.name).toEqual(expected) diff --git a/table/field/types/datetime.ts b/table/field/types/datetime.ts index 2951a6fd..c12ab8a4 100644 --- a/table/field/types/datetime.ts +++ b/table/field/types/datetime.ts @@ -16,3 +16,11 @@ export function parseDatetimeField(field: DatetimeField, expr?: Expr) { return expr.str.strptime(DataType.Datetime, format) } + +export function stringifyDatetimeField(field: DatetimeField, expr?: Expr) { + expr = expr ?? col(field.name) + + const format = field.format ?? DEFAULT_FORMAT + + return expr.date.strftime(format) +} diff --git a/table/field/types/duration.spec.ts b/table/field/types/duration.spec.ts index 48a441e5..742022f0 100644 --- a/table/field/types/duration.spec.ts +++ b/table/field/types/duration.spec.ts @@ -1,20 +1,47 @@ -import { DataFrame } from "nodejs-polars" +import { DataFrame, DataType, Series } from "nodejs-polars" import { describe, expect, it } from "vitest" -import { processTable } from "../../table/index.ts" +import { denormalizeTable, normalizeTable } from "../../table/index.ts" describe("parseDurationField", () => { it.each([["P23DT23H", "P23DT23H", {}]])( "$0 -> $1 $2", async (cell, value, options) => { - const table = DataFrame({ name: [cell] }).lazy() + const table = DataFrame([Series("name", [cell], DataType.String)]).lazy() const schema = { fields: [{ name: "name", type: "duration" as const, ...options }], } - const ldf = await processTable(table, { schema }) + const ldf = await normalizeTable(table, schema) const df = await ldf.collect() expect(df.getColumn("name").get(0)).toEqual(value) }, ) }) + +describe("stringifyDurationField", () => { + it.each([ + // ISO 8601 duration strings should be returned as-is + ["P23DT23H", "P23DT23H"], + ["P1Y2M3DT4H5M6S", "P1Y2M3DT4H5M6S"], + ["PT30M", "PT30M"], + ["P1D", "P1D"], + ["PT1H", "PT1H"], + ["P1W", "P1W"], + ["PT0S", "PT0S"], + + // Null handling + [null, ""], + ])("%s -> %s", async (value, expected) => { + const table = DataFrame([Series("name", [value], DataType.String)]).lazy() + + const schema = { + fields: [{ name: "name", type: "duration" as const }], + } + + const ldf = await denormalizeTable(table, schema) + const df = await ldf.collect() + + expect(df.toRecords()[0]?.name).toEqual(expected) + }) +}) diff --git a/table/field/types/duration.ts b/table/field/types/duration.ts index 3d79c042..cca2d21e 100644 --- a/table/field/types/duration.ts +++ b/table/field/types/duration.ts @@ -9,3 +9,9 @@ export function parseDurationField(field: DurationField, expr?: Expr) { return expr } + +export function stringifyDurationField(field: DurationField, expr?: Expr) { + expr = expr ?? col(field.name) + + return expr +} diff --git a/table/field/types/geojson.spec.ts b/table/field/types/geojson.spec.ts index 5b09578b..93da3adb 100644 --- a/table/field/types/geojson.spec.ts +++ b/table/field/types/geojson.spec.ts @@ -1,6 +1,6 @@ -import { DataFrame } from "nodejs-polars" +import { DataFrame, DataType, Series } from "nodejs-polars" import { describe, expect, it } from "vitest" -import { processTable } from "../../table/index.ts" +import { denormalizeTable, normalizeTable } from "../../table/index.ts" describe("parseGeojsonField", () => { it.each([ @@ -25,15 +25,67 @@ describe("parseGeojsonField", () => { ["null", null], ["undefined", null], ])("%s -> %s", async (cell, value) => { - const table = DataFrame({ name: [cell] }).lazy() + const table = DataFrame([Series("name", [cell], DataType.String)]).lazy() + const schema = { fields: [{ name: "name", type: "geojson" as const }], } - const ldf = await processTable(table, { schema }) + const ldf = await normalizeTable(table, schema) const df = await ldf.collect() const res = df.getColumn("name").get(0) expect(res ? JSON.parse(res) : res).toEqual(value) }) }) + +describe("stringifyGeojsonField", () => { + it.each([ + // GeoJSON Point + [ + '{"type":"Point","coordinates":[125.6,10.1]}', + '{"type":"Point","coordinates":[125.6,10.1]}', + ], + + // GeoJSON LineString + [ + '{"type":"LineString","coordinates":[[125.6,10.1],[125.7,10.2]]}', + '{"type":"LineString","coordinates":[[125.6,10.1],[125.7,10.2]]}', + ], + + // GeoJSON Polygon + [ + '{"type":"Polygon","coordinates":[[[125.6,10.1],[125.7,10.1],[125.7,10.2],[125.6,10.2],[125.6,10.1]]]}', + '{"type":"Polygon","coordinates":[[[125.6,10.1],[125.7,10.1],[125.7,10.2],[125.6,10.2],[125.6,10.1]]]}', + ], + + // GeoJSON Feature + [ + '{"type":"Feature","geometry":{"type":"Point","coordinates":[125.6,10.1]},"properties":{"name":"Sample Point"}}', + '{"type":"Feature","geometry":{"type":"Point","coordinates":[125.6,10.1]},"properties":{"name":"Sample Point"}}', + ], + + // GeoJSON FeatureCollection + [ + '{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[125.6,10.1]},"properties":{"name":"Point 1"}}]}', + '{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[125.6,10.1]},"properties":{"name":"Point 1"}}]}', + ], + + // Empty GeoJSON object + ["{}", "{}"], + + // Null handling + [null, ""], + ])("%s -> %s", async (value, expected) => { + const table = DataFrame([Series("name", [value], DataType.String)]).lazy() + + const schema = { + fields: [{ name: "name", type: "geojson" as const }], + } + + const ldf = await denormalizeTable(table, schema) + const df = await ldf.collect() + + expect(df.toRecords()[0]?.name).toEqual(expected) + }) +}) diff --git a/table/field/types/geojson.ts b/table/field/types/geojson.ts index 98881550..d4986074 100644 --- a/table/field/types/geojson.ts +++ b/table/field/types/geojson.ts @@ -10,3 +10,9 @@ export function parseGeojsonField(field: GeojsonField, expr?: Expr) { return when(expr.str.contains("^\\{")).then(expr).otherwise(lit(null)) } + +export function stringifyGeojsonField(field: GeojsonField, expr?: Expr) { + expr = expr ?? col(field.name) + + return expr +} diff --git a/table/field/types/geopoint.spec.ts b/table/field/types/geopoint.spec.ts index fff80217..71c87dcb 100644 --- a/table/field/types/geopoint.spec.ts +++ b/table/field/types/geopoint.spec.ts @@ -1,6 +1,6 @@ -import { DataFrame } from "nodejs-polars" +import { DataFrame, DataType, Series } from "nodejs-polars" import { describe, expect, it } from "vitest" -import { processTable } from "../../table/index.ts" +import { denormalizeTable, normalizeTable } from "../../table/index.ts" describe("parseGeopointField", () => { describe("default format", () => { @@ -23,19 +23,20 @@ describe("parseGeopointField", () => { //["lon,45.50", null], //["90.50,45.50,0", null], ])("%s -> %s", async (cell, value) => { - const table = DataFrame({ name: [cell] }).lazy() + const table = DataFrame([Series("name", [cell], DataType.String)]).lazy() + const schema = { fields: [{ name: "name", type: "geopoint" as const }], } - const ldf = await processTable(table, { schema }) + const ldf = await normalizeTable(table, schema) const df = await ldf.collect() expect(df.toRecords()[0]?.name).toEqual(value) }) }) - describe.skip("array format", () => { + describe("array format", () => { it.each([ // Valid geopoints in array format ["[90.50, 45.50]", [90.5, 45.5]], @@ -48,27 +49,29 @@ describe("parseGeopointField", () => { [" [90.50, 45.50] ", [90.5, 45.5]], // Invalid formats - ["not a geopoint", null], - ["", null], - ["[90.50]", null], - ["[90.50, 45.50, 0]", null], - ["['lon', 'lat']", null], + // TODO: fix this + //["not a geopoint", null], + //["", null], + //["[90.50]", null], + //["[90.50, 45.50, 0]", null], + //["['lon', 'lat']", null], ])("%s -> %s", async (cell, value) => { - const table = DataFrame({ name: [cell] }).lazy() + const table = DataFrame([Series("name", [cell], DataType.String)]).lazy() + const schema = { fields: [ { name: "name", type: "geopoint" as const, format: "array" as const }, ], } - const ldf = await processTable(table, { schema }) + const ldf = await normalizeTable(table, schema) const df = await ldf.collect() - expect(df.getColumn("name").get(0)).toEqual(value) + expect(df.toRecords()[0]?.name).toEqual(value) }) }) - describe.skip("object format", () => { + describe("object format", () => { it.each([ // Valid geopoints in object format ['{"lon": 90.50, "lat": 45.50}', [90.5, 45.5]], @@ -80,14 +83,110 @@ describe("parseGeopointField", () => { // With whitespace [' {"lon": 90.50, "lat": 45.50} ', [90.5, 45.5]], + // TODO: fix this // Invalid formats - ["not a geopoint", null], - ["", null], - ['{"longitude": 90.50, "latitude": 45.50}', null], - ['{"lon": 90.50}', null], - ['{"lat": 45.50}', null], + //["not a geopoint", null], + //["", null], + //['{"longitude": 90.50, "latitude": 45.50}', null], + //['{"lon": 90.50}', null], + //['{"lat": 45.50}', null], ])("%s -> %s", async (cell, value) => { - const table = DataFrame({ name: [cell] }).lazy() + const table = DataFrame([Series("name", [cell], DataType.String)]).lazy() + + const schema = { + fields: [ + { + name: "name", + type: "geopoint" as const, + format: "object" as const, + }, + ], + } + + const ldf = await normalizeTable(table, schema) + const df = await ldf.collect() + + expect(df.toRecords()[0]?.name).toEqual(value) + }) + }) +}) + +describe("stringifyGeopointField", () => { + describe("default format", () => { + it.each([ + // Coordinate arrays to default format (lon,lat) + [[90.5, 45.5], "90.5,45.5"], + [[0, 0], "0.0,0.0"], + [[-122.4, 37.78], "-122.4,37.78"], + [[-180.0, -90.0], "-180.0,-90.0"], + [[180.0, 90.0], "180.0,90.0"], + + // With precise decimals + [[125.6789, 10.1234], "125.6789,10.1234"], + + // Null handling + //[null, null], + ])("%s -> %s", async (value, expected) => { + const table = DataFrame([ + Series("name", [value], DataType.List(DataType.Float64)), + ]).lazy() + + const schema = { + fields: [{ name: "name", type: "geopoint" as const }], + } + + const ldf = await denormalizeTable(table, schema) + const df = await ldf.collect() + + expect(df.toRecords()[0]?.name).toEqual(expected) + }) + }) + + describe("array format", () => { + it.each([ + // Coordinate arrays to array format string + [[90.5, 45.5], "[90.5,45.5]"], + [[0, 0], "[0.0,0.0]"], + [[-122.4, 37.78], "[-122.4,37.78]"], + [[-180.0, -90.0], "[-180.0,-90.0]"], + [[180.0, 90.0], "[180.0,90.0]"], + + // Null handling + //[null, null], + ])("%s -> %s", async (value, expected) => { + const table = DataFrame([ + Series("name", [value], DataType.List(DataType.Float64)), + ]).lazy() + + const schema = { + fields: [ + { name: "name", type: "geopoint" as const, format: "array" as const }, + ], + } + + const ldf = await denormalizeTable(table, schema) + const df = await ldf.collect() + + expect(df.toRecords()[0]?.name).toEqual(expected) + }) + }) + + describe("object format", () => { + it.each([ + // Coordinate arrays to object format string + [[90.5, 45.5], '{"lon":90.5,"lat":45.5}'], + [[0, 0], '{"lon":0.0,"lat":0.0}'], + [[-122.4, 37.78], '{"lon":-122.4,"lat":37.78}'], + [[-180.0, -90.0], '{"lon":-180.0,"lat":-90.0}'], + [[180.0, 90.0], '{"lon":180.0,"lat":90.0}'], + + // Null handling + //[null, null], + ])("%s -> %s", async (value, expected) => { + const table = DataFrame([ + Series("name", [value], DataType.List(DataType.Float64)), + ]).lazy() + const schema = { fields: [ { @@ -98,10 +197,10 @@ describe("parseGeopointField", () => { ], } - const ldf = await processTable(table, { schema }) + const ldf = await denormalizeTable(table, schema) const df = await ldf.collect() - expect(df.getColumn("name").get(0)).toEqual(value) + expect(df.toRecords()[0]?.name).toEqual(expected) }) }) }) diff --git a/table/field/types/geopoint.ts b/table/field/types/geopoint.ts index c9077d95..601bd5d7 100644 --- a/table/field/types/geopoint.ts +++ b/table/field/types/geopoint.ts @@ -1,30 +1,74 @@ import type { GeopointField } from "@dpkit/core" -import { DataType, col, lit } from "nodejs-polars" +import { DataType, col, concatList, concatString, lit } from "nodejs-polars" import type { Expr } from "nodejs-polars" +// TODO: +// Add more validation: +// - Check the length of the list is 2 (no list.lenghts in polars currently) +// - Check the values are within -180..180 and -90..90 +// - Return null instead of list if any of the values are out of range + export function parseGeopointField(field: GeopointField, expr?: Expr) { expr = expr ?? col(field.name) // Default format is "lon,lat" string - const format = field.format || "default" + const format = field.format ?? "default" if (format === "default") { expr = expr.str.split(",").cast(DataType.List(DataType.Float64)) - // TODO: - // Add more validation: - // - Check the length of the list is 2 (no list.lenghts in polars currently) - // - Check the values are within -180..180 and -90..90 - // - Return null instead of list if any of the values are out of range + } + + if (format === "array") { + expr = expr.str + .replaceAll("[\\[\\]\\s]", "") + .str.split(",") + .cast(DataType.List(DataType.Float64)) } if (format === "object") { - // TODO: implement - expr = lit(null).alias(field.name) + expr = concatList([ + expr.str.jsonPathMatch("$.lon").cast(DataType.Float64), + expr.str.jsonPathMatch("$.lat").cast(DataType.Float64), + ]).alias(field.name) + } + + return expr +} + +export function stringifyGeopointField(field: GeopointField, expr?: Expr) { + expr = expr ?? col(field.name) + + // Default format is "lon,lat" string + const format = field.format ?? "default" + + if (format === "default") { + return expr.cast(DataType.List(DataType.String)).lst.join(",") } if (format === "array") { - // TODO: implement - expr = lit(null).alias(field.name) + return concatString( + [ + lit("["), + expr.lst.get(0).cast(DataType.String), + lit(","), + expr.lst.get(1).cast(DataType.String), + lit("]"), + ], + "", + ).alias(field.name) as Expr + } + + if (format === "object") { + return concatString( + [ + lit('{"lon":'), + expr.lst.get(0).cast(DataType.String), + lit(',"lat":'), + expr.lst.get(1).cast(DataType.String), + lit("}"), + ], + "", + ).alias(field.name) as Expr } return expr diff --git a/table/field/types/integer.spec.ts b/table/field/types/integer.spec.ts index 02fa74c4..c26e9509 100644 --- a/table/field/types/integer.spec.ts +++ b/table/field/types/integer.spec.ts @@ -1,6 +1,6 @@ -import { DataFrame } from "nodejs-polars" +import { DataFrame, DataType, Series } from "nodejs-polars" import { describe, expect, it } from "vitest" -import { processTable } from "../../table/index.ts" +import { denormalizeTable, normalizeTable } from "../../table/index.ts" describe("parseIntegerField", () => { it.each([ @@ -51,15 +51,71 @@ describe("parseIntegerField", () => { //[" -1,000 ", -1000, { groupChar: "," }], ["000,001", 1, { groupChar: "," }], ])("$0 -> $1 $2", async (cell, value, options) => { - const table = DataFrame({ name: [cell] }).lazy() + const table = DataFrame([Series("name", [cell], DataType.String)]).lazy() + const schema = { fields: [{ name: "name", type: "integer" as const, ...options }], } - const ldf = await processTable(table, { schema }) + const ldf = await normalizeTable(table, schema) const df = await ldf.collect() expect(df.getColumn("name").get(0)).toEqual(value) expect(df.getColumn("name").get(0)).toEqual(value) }) + + describe("categories", () => { + it.each([ + // Flat categories + ["1", 1, { categories: [1, 2] }], + ["2", 2, { categories: [1, 2] }], + ["3", null, { categories: [1, 2] }], + + // Object categories + ["1", 1, { categories: [{ value: 1, label: "One" }] }], + ["2", null, { categories: [{ value: 1, label: "One" }] }], + ])("$0 -> $1 $2", async (cell, value, options) => { + const table = DataFrame([Series("name", [cell], DataType.String)]).lazy() + + const schema = { + fields: [{ name: "name", type: "integer" as const, ...options }], + } + + const ldf = await normalizeTable(table, schema) + const df = await ldf.collect() + + expect(df.toRecords()[0]?.name).toEqual(value) + }) + }) +}) + +describe("stringifyIntegerField", () => { + it.each([ + // Basic integer to string conversion + [1, "1"], + [2, "2"], + [1000, "1000"], + [42, "42"], + [-1, "-1"], + [-100, "-100"], + [0, "0"], + + // Large integers + [1234567890, "1234567890"], + [-1234567890, "-1234567890"], + + // Null handling + [null, ""], + ])("%s -> %s", async (value, expected) => { + const table = DataFrame([Series("name", [value], DataType.Int64)]).lazy() + + const schema = { + fields: [{ name: "name", type: "integer" as const }], + } + + const ldf = await denormalizeTable(table, schema) + const df = await ldf.collect() + + expect(df.toRecords()[0]?.name).toEqual(expected) + }) }) diff --git a/table/field/types/integer.ts b/table/field/types/integer.ts index f975fe78..0765a912 100644 --- a/table/field/types/integer.ts +++ b/table/field/types/integer.ts @@ -1,6 +1,5 @@ import type { IntegerField } from "@dpkit/core" -import { DataType } from "nodejs-polars" -import { col } from "nodejs-polars" +import { DataType, col, lit, when } from "nodejs-polars" import type { Expr } from "nodejs-polars" // TODO: support categories @@ -8,24 +7,51 @@ import type { Expr } from "nodejs-polars" export function parseIntegerField(field: IntegerField, expr?: Expr) { expr = expr ?? col(field.name) + const groupChar = field.groupChar + const bareNumber = field.bareNumber + const flattenCategories = field.categories?.map(it => + typeof it === "number" ? it : it.value, + ) + // Handle non-bare numbers (with currency symbols, percent signs, etc.) - if (field.bareNumber === false) { + if (bareNumber === false) { // Preserve the minus sign when removing leading characters expr = expr.str.replaceAll("^[^\\d\\-]+", "") expr = expr.str.replaceAll("[^\\d\\-]+$", "") } // Handle group character (thousands separator) - if (field.groupChar) { + if (groupChar) { // Escape special characters for regex - const escapedGroupChar = field.groupChar.replace( - /[.*+?^${}()|[\]\\]/g, - "\\$&", - ) + const escapedGroupChar = groupChar.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") expr = expr.str.replaceAll(escapedGroupChar, "") } // Cast to int64 (will handle values up to 2^63-1) expr = expr.cast(DataType.Int64) + + // Currently, only string categories are supported + if (flattenCategories) { + return when(expr.isIn(flattenCategories)) + .then(expr) + .otherwise(lit(null)) + .alias(field.name) + } + + return expr +} + +export function stringifyIntegerField(field: IntegerField, expr?: Expr) { + expr = expr ?? col(field.name) + + // Convert to string + expr = expr.cast(DataType.String) + + //const groupChar = field.groupChar + //const bareNumber = field.bareNumber + + // TODO: Add group character formatting (thousands separator) when needed + // TODO: Add non-bare number formatting (currency symbols, etc.) when needed + return expr } diff --git a/table/field/types/list.spec.ts b/table/field/types/list.spec.ts index 606452df..1e66c258 100644 --- a/table/field/types/list.spec.ts +++ b/table/field/types/list.spec.ts @@ -1,6 +1,6 @@ -import { DataFrame } from "nodejs-polars" +import { DataFrame, DataType, Series } from "nodejs-polars" import { describe, expect, it } from "vitest" -import { processTable } from "../../table/index.ts" +import { denormalizeTable, normalizeTable } from "../../table/index.ts" describe("parseListField", () => { describe("default settings (string items, comma delimiter)", () => { @@ -28,12 +28,13 @@ describe("parseListField", () => { // Null handling //[null, null], ])("%s -> %s", async (cell, value) => { - const table = DataFrame({ name: [cell] }).lazy() + const table = DataFrame([Series("name", [cell], DataType.String)]).lazy() + const schema = { fields: [{ name: "name", type: "list" as const }], } - const ldf = await processTable(table, { schema }) + const ldf = await normalizeTable(table, schema) const df = await ldf.collect() expect(df.toRecords()[0]?.name).toEqual(value) @@ -65,14 +66,15 @@ describe("parseListField", () => { ["1,a,3", [1, null, 3]], ["1.5,2,3", [null, 2, 3]], ])("%s -> %s", async (cell, value) => { - const table = DataFrame({ name: [cell] }).lazy() + const table = DataFrame([Series("name", [cell], DataType.String)]).lazy() + const schema = { fields: [ { name: "name", type: "list" as const, itemType: "integer" as const }, ], } - const ldf = await processTable(table, { schema }) + const ldf = await normalizeTable(table, schema) const df = await ldf.collect() expect(df.toRecords()[0]?.name).toEqual(value) @@ -103,14 +105,15 @@ describe("parseListField", () => { // Invalid numbers become null ["1.1,a,3.3", [1.1, null, 3.3]], ])("%s -> %s", async (cell, value) => { - const table = DataFrame({ name: [cell] }).lazy() + const table = DataFrame([Series("name", [cell], DataType.String)]).lazy() + const schema = { fields: [ { name: "name", type: "list" as const, itemType: "number" as const }, ], } - const ldf = await processTable(table, { schema }) + const ldf = await normalizeTable(table, schema) const df = await ldf.collect() expect(df.toRecords()[0]?.name).toEqual(value) @@ -135,15 +138,157 @@ describe("parseListField", () => { // Empty items in list ["a;;c", ["a", "", "c"]], ])("%s -> %s", async (cell, value) => { - const table = DataFrame({ name: [cell] }).lazy() + const table = DataFrame([Series("name", [cell], DataType.String)]).lazy() + const schema = { fields: [{ name: "name", type: "list" as const, delimiter: ";" }], } - const ldf = await processTable(table, { schema }) + const ldf = await normalizeTable(table, schema) const df = await ldf.collect() expect(df.toRecords()[0]?.name).toEqual(value) }) }) }) + +describe("stringifyListField", () => { + describe("default settings (string items, comma delimiter)", () => { + it.each([ + // Basic list stringifying + [["a", "b", "c"], "a,b,c"], + [["foo", "bar", "baz"], "foo,bar,baz"], + [["1", "2", "3"], "1,2,3"], + + // Single item + [["single"], "single"], + + // Empty items in list + [["a", "", "c"], "a,,c"], + [["", "b", ""], ",b,"], + [["", "", "", ""], ",,,"], + + // Null handling + [[null, "b", null], "b"], + [["a", null, "c"], "a,c"], + + // Empty array + [[], ""], + ])("%s -> %s", async (value, expected) => { + const table = DataFrame([ + Series("name", [value], DataType.List(DataType.String)), + ]).lazy() + + const schema = { + fields: [{ name: "name", type: "list" as const }], + } + + const ldf = await denormalizeTable(table, schema) + const df = await ldf.collect() + + expect(df.toRecords()[0]?.name).toEqual(expected) + }) + }) + + // TODO: Remove Int16Array once this issue is fixed: + // https://github.com/pola-rs/nodejs-polars/issues/362 + describe("integer item type", () => { + it.each([ + // Integer lists to string + [new Int16Array([1, 2, 3]), "1,2,3"], + [new Int16Array([0, -1, 42]), "0,-1,42"], + [new Int16Array([-10, 0, 10]), "-10,0,10"], + + // Single item + [new Int16Array([42]), "42"], + + // With nulls + //[[1, null, 3], "1,,3"], + //[[null, 2, null], ",2,"], + + // Empty array + [[], ""], + ])("%s -> %s", async (value, expected) => { + const table = DataFrame([ + Series("name", [value], DataType.List(DataType.Int16)), + ]).lazy() + + const schema = { + fields: [ + { name: "name", type: "list" as const, itemType: "integer" as const }, + ], + } + + const ldf = await denormalizeTable(table, schema) + const df = await ldf.collect() + + expect(df.toRecords()[0]?.name).toEqual(expected) + }) + }) + + describe("number item type", () => { + it.each([ + // Number lists to string + [[1.5, 2.1, 3.7], "1.5,2.1,3.7"], + [[0, -1.1, 42], "0.0,-1.1,42.0"], + [[-10.5, 0, 10], "-10.5,0.0,10.0"], + + // Single item + [[3.14], "3.14"], + + // With nulls + [[1.1, null, 3.3], "1.1,3.3"], + [[null, 2.2, null], "2.2"], + + // Empty array + [[], ""], + ])("%s -> %s", async (value, expected) => { + const table = DataFrame([ + Series("name", [value], DataType.List(DataType.Float64)), + ]).lazy() + + const schema = { + fields: [ + { name: "name", type: "list" as const, itemType: "number" as const }, + ], + } + + const ldf = await denormalizeTable(table, schema) + const df = await ldf.collect() + + expect(df.toRecords()[0]?.name).toEqual(expected) + }) + }) + + describe("custom delimiter", () => { + it.each([ + // Semicolon delimiter + [["a", "b", "c"], "a;b;c"], + [["1", "2", "3"], "1;2;3"], + + // Single item + [["single"], "single"], + + // Empty items in list + [["a", "", "c"], "a;;c"], + [["", "b", ""], ";b;"], + + // Numeric items + [[1.0, 2.0, 3.0], "1.0;2.0;3.0"], + + // Empty array + [[], ""], + ])("%s -> %s", async (value, expected) => { + const table = DataFrame([Series("name", [value], DataType.String)]).lazy() + + const schema = { + fields: [{ name: "name", type: "list" as const, delimiter: ";" }], + } + + const ldf = await denormalizeTable(table, schema) + const df = await ldf.collect() + + expect(df.toRecords()[0]?.name).toEqual(expected) + }) + }) +}) diff --git a/table/field/types/list.ts b/table/field/types/list.ts index 458956a4..09cad5c3 100644 --- a/table/field/types/list.ts +++ b/table/field/types/list.ts @@ -9,16 +9,27 @@ export function parseListField(field: ListField, expr?: Expr) { expr = expr ?? col(field.name) const delimiter = field.delimiter ?? "," + const itemType = field.itemType let dtype: any = DataType.String - if (field.itemType === "integer") dtype = DataType.Int64 - if (field.itemType === "number") dtype = DataType.Float64 - if (field.itemType === "boolean") dtype = DataType.Bool - if (field.itemType === "datetime") dtype = DataType.Datetime - if (field.itemType === "date") dtype = DataType.Date - if (field.itemType === "time") dtype = DataType.Time + if (itemType === "integer") dtype = DataType.Int64 + if (itemType === "number") dtype = DataType.Float64 + if (itemType === "boolean") dtype = DataType.Bool + if (itemType === "datetime") dtype = DataType.Datetime + if (itemType === "date") dtype = DataType.Date + if (itemType === "time") dtype = DataType.Time expr = expr.str.split(delimiter).cast(DataType.List(dtype)) return expr } + +export function stringifyListField(field: ListField, expr?: Expr) { + expr = expr ?? col(field.name) + + const delimiter = field.delimiter ?? "," + + return expr + .cast(DataType.List(DataType.String)) + .lst.join({ separator: delimiter, ignoreNulls: true }) +} diff --git a/table/field/types/number.spec.ts b/table/field/types/number.spec.ts index bb375915..cb9c4314 100644 --- a/table/field/types/number.spec.ts +++ b/table/field/types/number.spec.ts @@ -1,6 +1,6 @@ -import { DataFrame } from "nodejs-polars" +import { DataFrame, DataType, Series } from "nodejs-polars" import { describe, expect, it } from "vitest" -import { processTable } from "../../table/index.ts" +import { denormalizeTable, normalizeTable } from "../../table/index.ts" describe("parseNumberField", () => { it.each([ @@ -63,14 +63,55 @@ describe("parseNumberField", () => { { bareNumber: false, groupChar: ".", decimalChar: "," }, ], ])("$0 -> $1 $2", async (cell, value, options) => { - const table = DataFrame({ name: [cell] }).lazy() + const table = DataFrame([Series("name", [cell], DataType.String)]).lazy() + const schema = { fields: [{ name: "name", type: "number" as const, ...options }], } - const ldf = await processTable(table, { schema }) + const ldf = await normalizeTable(table, schema) const df = await ldf.collect() expect(df.getColumn("name").get(0)).toEqual(value) }) }) + +describe("stringifyNumberField", () => { + it.each([ + // Basic number to string conversion + [1.0, "1.0"], + [2.0, "2.0"], + [1000.0, "1000.0"], + [3.14, "3.14"], + [42.5, "42.5"], + [-1.0, "-1.0"], + [-100.5, "-100.5"], + [0.0, "0.0"], + + // Numbers with many decimal places + //[3.141592653589793, "3.141592653589793"], + [-123.456789, "-123.456789"], + + // Large numbers + [1234567890.123, "1234567890.123"], + [-9876543210.987, "-9876543210.987"], + + // Small numbers + [0.001, "0.001"], + [-0.0001, "-0.0001"], + + // Null handling + [null, ""], + ])("%s -> %s", async (value, expected) => { + const table = DataFrame([Series("name", [value], DataType.Float64)]).lazy() + + const schema = { + fields: [{ name: "name", type: "number" as const }], + } + + const ldf = await denormalizeTable(table, schema) + const df = await ldf.collect() + + expect(df.toRecords()[0]?.name).toEqual(expected) + }) +}) diff --git a/table/field/types/number.ts b/table/field/types/number.ts index 10c0f062..c5791117 100644 --- a/table/field/types/number.ts +++ b/table/field/types/number.ts @@ -7,11 +7,12 @@ export function parseNumberField(field: NumberField, expr?: Expr) { expr = expr ?? col(field.name) // Extract the decimal and group characters - const decimalChar = field.decimalChar || "." - const groupChar = field.groupChar || "" + const decimalChar = field.decimalChar ?? "." + const groupChar = field.groupChar ?? "" + const bareNumber = field.bareNumber ?? true // Handle non-bare numbers (with currency symbols, percent signs, etc.) - if (field.bareNumber === false) { + if (bareNumber === false) { // Remove leading non-digit characters (except minus sign and allowed decimal points) const allowedDecimalChars = decimalChar === "." ? "\\." : `\\.${decimalChar}` @@ -46,3 +47,19 @@ export function parseNumberField(field: NumberField, expr?: Expr) { expr = expr.cast(DataType.Float64) return expr } + +export function stringifyNumberField(field: NumberField, expr?: Expr) { + expr = expr ?? col(field.name) + + // Convert to string + expr = expr.cast(DataType.String) + + //const decimalChar = field.decimalChar ?? "." + //const groupChar = field.groupChar ?? "" + + // TODO: Add decimal character formatting when needed + // TODO: Add group character formatting (thousands separator) when needed + // TODO: Add non-bare number formatting (currency symbols, etc.) when needed + + return expr +} diff --git a/table/field/types/object.spec.ts b/table/field/types/object.spec.ts index 64259d9a..36cdbf30 100644 --- a/table/field/types/object.spec.ts +++ b/table/field/types/object.spec.ts @@ -1,6 +1,6 @@ -import { DataFrame } from "nodejs-polars" +import { DataFrame, DataType, Series } from "nodejs-polars" import { describe, expect, it } from "vitest" -import { processTable } from "../../table/index.ts" +import { denormalizeTable, normalizeTable } from "../../table/index.ts" describe("parseObjectField", () => { it.each([ @@ -25,15 +25,50 @@ describe("parseObjectField", () => { ["null", null], ["undefined", null], ])("%s -> %s", async (cell, value) => { - const table = DataFrame({ name: [cell] }).lazy() + const table = DataFrame([Series("name", [cell], DataType.String)]).lazy() + const schema = { fields: [{ name: "name", type: "object" as const }], } - const ldf = await processTable(table, { schema }) + const ldf = await normalizeTable(table, schema) const df = await ldf.collect() const res = df.getColumn("name").get(0) expect(res ? JSON.parse(res) : res).toEqual(value) }) }) + +describe("stringifyObjectField", () => { + it.each([ + // JSON strings should be returned as-is + ['{"name":"John","age":30}', '{"name":"John","age":30}'], + ['{"numbers":[1,2,3]}', '{"numbers":[1,2,3]}'], + ['{"nested":{"prop":"value"}}', '{"nested":{"prop":"value"}}'], + ["{}", "{}"], + + // Complex nested objects as JSON strings + [ + '{"users":[{"id":1,"name":"Alice"},{"id":2,"name":"Bob"}]}', + '{"users":[{"id":1,"name":"Alice"},{"id":2,"name":"Bob"}]}', + ], + [ + '{"config":{"debug":true,"timeout":5000}}', + '{"config":{"debug":true,"timeout":5000}}', + ], + + // Null handling + [null, ""], + ])("%s -> %s", async (value, expected) => { + const table = DataFrame([Series("name", [value], DataType.String)]).lazy() + + const schema = { + fields: [{ name: "name", type: "object" as const }], + } + + const ldf = await denormalizeTable(table, schema) + const df = await ldf.collect() + + expect(df.toRecords()[0]?.name).toEqual(expected) + }) +}) diff --git a/table/field/types/object.ts b/table/field/types/object.ts index c61c41e4..164d7f94 100644 --- a/table/field/types/object.ts +++ b/table/field/types/object.ts @@ -10,3 +10,9 @@ export function parseObjectField(field: ObjectField, expr?: Expr) { return when(expr.str.contains("^\\{")).then(expr).otherwise(lit(null)) } + +export function stringifyObjectField(field: ObjectField, expr?: Expr) { + expr = expr ?? col(field.name) + + return expr +} diff --git a/table/field/types/string.spec.ts b/table/field/types/string.spec.ts index 3c5e6131..e62ba159 100644 --- a/table/field/types/string.spec.ts +++ b/table/field/types/string.spec.ts @@ -1,24 +1,195 @@ -import { DataFrame } from "nodejs-polars" +import { DataFrame, DataType, Series } from "nodejs-polars" import { describe, expect, it } from "vitest" -import { processTable } from "../../table/index.ts" +import { normalizeTable } from "../../table/index.ts" // TODO: Implement proper tests // TODO: Currently, it fails on to JS conversion from Polars -describe.skip("parseStringField", () => { - describe("categorical field", () => { - it.each([["apple", "apple", { categories: ["apple", "banana"] }]])( - "$0 -> $1 $2", - async (cell, value, options) => { - const table = DataFrame({ name: [cell] }).lazy() - const schema = { - fields: [{ name: "name", type: "string" as const, ...options }], - } - - const ldf = await processTable(table, { schema }) - const df = await ldf.collect() - - expect(df.getColumn("name").get(0)).toEqual(value) - }, - ) +describe("parseStringField", () => { + it.each([ + // Simplr string + ["string", "string"], + + // Null handling + ["", null], + ])("$0 -> $1", async (cell, value) => { + const table = DataFrame([Series("name", [cell], DataType.String)]).lazy() + + const schema = { + fields: [{ name: "name", type: "string" as const }], + } + + const ldf = await normalizeTable(table, schema) + const df = await ldf.collect() + + expect(df.getColumn("name").dtype).toEqual(DataType.String) + expect(df.toRecords()[0]?.name).toEqual(value) + }) + + describe("email format", () => { + it.each([ + // Valid emails + ["user@example.com", "user@example.com"], + ["test.email@domain.co.uk", "test.email@domain.co.uk"], + ["user+tag@example.org", "user+tag@example.org"], + ["first.last@subdomain.example.com", "first.last@subdomain.example.com"], + ["user123@test-domain.com", "user123@test-domain.com"], + + // Invalid emails + ["invalid-email", null], + ["@example.com", null], + ["user@", null], + ["user@@example.com", null], + ["user@example", null], + ["user name@example.com", null], + + // Null handling + ["", null], + ])("$0 -> $1", async (cell, value) => { + const table = DataFrame([Series("name", [cell], DataType.String)]).lazy() + + const schema = { + fields: [ + { name: "name", type: "string" as const, format: "email" as const }, + ], + } + + const ldf = await normalizeTable(table, schema) + const df = await ldf.collect() + + expect(df.getColumn("name").dtype).toEqual(DataType.String) + expect(df.toRecords()[0]?.name).toEqual(value) + }) + }) + + describe("uri format", () => { + it.each([ + // Valid URIs + ["https://example.com", "https://example.com"], + [ + "http://www.google.com/search?q=test", + "http://www.google.com/search?q=test", + ], + ["ftp://files.example.org/file.txt", "ftp://files.example.org/file.txt"], + ["mailto:user@example.com", "mailto:user@example.com"], + ["file:///path/to/file", "file:///path/to/file"], + ["ssh://user@host:22/path", "ssh://user@host:22/path"], + + // Invalid URIs + ["not-a-uri", null], + ["://missing-scheme", null], + ["http://", null], + ["example.com", null], + ["http:// space in uri", null], + + // Null handling + ["", null], + ])("$0 -> $1", async (cell, value) => { + const table = DataFrame([Series("name", [cell], DataType.String)]).lazy() + + const schema = { + fields: [ + { name: "name", type: "string" as const, format: "uri" as const }, + ], + } + + const ldf = await normalizeTable(table, schema) + const df = await ldf.collect() + + expect(df.getColumn("name").dtype).toEqual(DataType.String) + expect(df.toRecords()[0]?.name).toEqual(value) + }) + }) + + describe("binary format", () => { + it.each([ + // Valid base64 strings + ["SGVsbG8gV29ybGQ=", "SGVsbG8gV29ybGQ="], + ["YWJjZGVmZw==", "YWJjZGVmZw=="], + ["VGVzdA==", "VGVzdA=="], + ["QQ==", "QQ=="], + ["Zg==", "Zg=="], + ["Zm8=", "Zm8="], + ["Zm9v", "Zm9v"], + + // Invalid base64 strings + ["Hello World!", null], + ["SGVsbG8gV29ybGQ===", null], + ["Invalid@#$", null], + ["SGVsb(8gV29ybGQ=", null], + + // Null handling + ["", null], + ])("$0 -> $1", async (cell, value) => { + const table = DataFrame([Series("name", [cell], DataType.String)]).lazy() + + const schema = { + fields: [ + { name: "name", type: "string" as const, format: "binary" as const }, + ], + } + + const ldf = await normalizeTable(table, schema) + const df = await ldf.collect() + + expect(df.getColumn("name").dtype).toEqual(DataType.String) + expect(df.toRecords()[0]?.name).toEqual(value) + }) + }) + + describe("uuid format", () => { + it.each([ + // Valid UUIDs + [ + "f47ac10b-58cc-4372-a567-0e02b2c3d479", + "f47ac10b-58cc-4372-a567-0e02b2c3d479", + ], + + // Invalid UUIDs + ["f47ac10b-58cc-4372-a567-0e02b2c3d47X", null], + ["f47ac10b", null], + ["X", null], + + // Null handling + ["", null], + ])("$0 -> $1 $2", async (cell, value) => { + const table = DataFrame([Series("name", [cell], DataType.String)]).lazy() + + const schema = { + fields: [ + { name: "name", type: "string" as const, format: "uuid" as const }, + ], + } + + const ldf = await normalizeTable(table, schema) + const df = await ldf.collect() + + expect(df.getColumn("name").dtype).toEqual(DataType.Categorical) + expect(df.toRecords()[0]?.name).toEqual(value) + }) + }) + + describe("categories", () => { + it.each([ + // Flat categories + ["apple", "apple", { categories: ["apple", "banana"] }], + ["banana", "banana", { categories: ["apple", "banana"] }], + ["orange", null, { categories: ["apple", "banana"] }], + + // Object categories + ["apple", "apple", { categories: [{ value: "apple", label: "Apple" }] }], + ["orange", null, { categories: [{ value: "apple", label: "Apple" }] }], + ])("$0 -> $1 $2", async (cell, value, options) => { + const table = DataFrame([Series("name", [cell], DataType.String)]).lazy() + + const schema = { + fields: [{ name: "name", type: "string" as const, ...options }], + } + + const ldf = await normalizeTable(table, schema) + const df = await ldf.collect() + + expect(df.getColumn("name").dtype).toEqual(DataType.Categorical) + expect(df.toRecords()[0]?.name).toEqual(value) + }) }) }) diff --git a/table/field/types/string.ts b/table/field/types/string.ts index 466f8eac..69bc6871 100644 --- a/table/field/types/string.ts +++ b/table/field/types/string.ts @@ -1,18 +1,44 @@ import type { StringField } from "@dpkit/core" -import { DataType, col } from "nodejs-polars" +import { DataType, col, lit, when } from "nodejs-polars" import type { Expr } from "nodejs-polars" -// TODO: -// Add more validation: -// - Validate regex if format provided and it can be performant (e.g. uuid) -// - Validate categories -// TODO: support categoriesOrder +const FORMAT_REGEX = { + email: + "^[a-zA-Z0-9.!#$%&'*+/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)+$", + uri: "^[a-zA-Z][a-zA-Z0-9+.-]*:(//([^\\s/]+[^\\s]*|/[^\\s]*)|[^\\s/][^\\s]*)$", + binary: "^[A-Za-z0-9+/]*={0,2}$", + uuid: "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", +} as const + +// TODO: support categoriesOrder? export function parseStringField(field: StringField, expr?: Expr) { expr = expr ?? col(field.name) - if (field.categories) { - expr = expr.cast(DataType.Categorical) + const format = field.format + const flattenCategories = field.categories?.map(it => + typeof it === "string" ? it : it.value, + ) + + if (flattenCategories) { + return when(expr.isIn(flattenCategories)) + .then(expr.cast(DataType.Categorical)) + .otherwise(lit(null)) + .alias(field.name) } + if (format) { + const regex = FORMAT_REGEX[format] + return when(expr.str.contains(regex)) + .then(expr) + .otherwise(lit(null)) + .alias(field.name) + } + + return expr +} + +export function stringifyStringField(field: StringField, expr?: Expr) { + expr = expr ?? col(field.name) + return expr } diff --git a/table/field/types/time.spec.ts b/table/field/types/time.spec.ts index 3de524c4..fa273f05 100644 --- a/table/field/types/time.spec.ts +++ b/table/field/types/time.spec.ts @@ -1,6 +1,7 @@ -import { DataFrame } from "nodejs-polars" +import { DataFrame, DataType, Series } from "nodejs-polars" import { describe, expect, it } from "vitest" -import { processTable } from "../../table/index.ts" +import { normalizeTable } from "../../table/index.ts" +import { denormalizeTable } from "../../table/index.ts" describe("parseTimeField", () => { it.each([ @@ -25,12 +26,36 @@ describe("parseTimeField", () => { // Invalid format //["06:00", null, { format: "invalid" }], ])("$0 -> $1 $2", async (cell, expected, options) => { - const table = DataFrame({ name: [cell] }).lazy() + const table = DataFrame([Series("name", [cell], DataType.String)]).lazy() + + const schema = { + fields: [{ name: "name", type: "time" as const, ...options }], + } + + const ldf = await normalizeTable(table, schema) + const df = await ldf.collect() + + expect(df.toRecords()[0]?.name).toEqual(expected) + }) +}) + +describe("stringifyTimeField", () => { + it.each([ + // Default format + [new Date(Date.UTC(2014, 0, 1, 6, 0, 0)), "06:00:00", {}], + [new Date(Date.UTC(2014, 0, 1, 16, 30, 0)), "16:30:00", {}], + + // Custom format + [new Date(Date.UTC(2014, 0, 1, 6, 0, 0)), "06:00", { format: "%H:%M" }], + [new Date(Date.UTC(2014, 0, 1, 16, 30, 0)), "16:30", { format: "%H:%M" }], + ])("%s -> %s %o", async (value, expected, options) => { + const table = DataFrame([Series("name", [value], DataType.Time)]).lazy() + const schema = { fields: [{ name: "name", type: "time" as const, ...options }], } - const ldf = await processTable(table, { schema }) + const ldf = await denormalizeTable(table, schema) const df = await ldf.collect() expect(df.toRecords()[0]?.name).toEqual(expected) diff --git a/table/field/types/time.ts b/table/field/types/time.ts index b8d1f067..13ebe1c8 100644 --- a/table/field/types/time.ts +++ b/table/field/types/time.ts @@ -18,3 +18,11 @@ export function parseTimeField(field: TimeField, expr?: Expr) { .cast(DataType.Time) .alias(field.name) } + +export function stringifyTimeField(field: TimeField, expr?: Expr) { + expr = expr ?? col(field.name) + + const format = field.format ?? DEFAULT_FORMAT + + return expr.date.strftime(format) +} diff --git a/table/field/types/year.spec.ts b/table/field/types/year.spec.ts index a47c05e5..5c6c1993 100644 --- a/table/field/types/year.spec.ts +++ b/table/field/types/year.spec.ts @@ -1,6 +1,6 @@ -import { DataFrame } from "nodejs-polars" +import { DataFrame, DataType, Series } from "nodejs-polars" import { describe, expect, it } from "vitest" -import { processTable } from "../../table/index.ts" +import { denormalizeTable, normalizeTable } from "../../table/index.ts" describe("parseYearField", () => { it.each([ @@ -20,14 +20,40 @@ describe("parseYearField", () => { ["12345", null], ["123", null], ])("%s -> %s", async (cell, value) => { - const table = DataFrame({ name: [cell] }).lazy() + const table = DataFrame([Series("name", [cell], DataType.String)]).lazy() + const schema = { fields: [{ name: "name", type: "year" as const }], } - const ldf = await processTable(table, { schema }) + const ldf = await normalizeTable(table, schema) const df = await ldf.collect() expect(df.getColumn("name").get(0)).toEqual(value) }) }) + +describe("stringifyYearField", () => { + it.each([ + // Basic integer years to string conversion + [2000, "2000"], + [2023, "2023"], + [1999, "1999"], + [0, "0000"], + [9999, "9999"], + + // Edge cases with null values + [null, ""], + ])("%s -> %s", async (value, expected) => { + const table = DataFrame([Series("name", [value], DataType.Int16)]).lazy() + + const schema = { + fields: [{ name: "name", type: "year" as const }], + } + + const ldf = await denormalizeTable(table, schema) + const df = await ldf.collect() + + expect(df.toRecords()[0]?.name).toEqual(expected) + }) +}) diff --git a/table/field/types/year.ts b/table/field/types/year.ts index c57023b2..380e0631 100644 --- a/table/field/types/year.ts +++ b/table/field/types/year.ts @@ -15,3 +15,9 @@ export function parseYearField(field: YearField, expr?: Expr) { .then(expr) .otherwise(lit(null)) } + +export function stringifyYearField(field: YearField, expr?: Expr) { + expr = expr ?? col(field.name) + + return expr.cast(DataType.String).str.zFill(4) +} diff --git a/table/field/types/yearmonth.spec.ts b/table/field/types/yearmonth.spec.ts index 95f72aa4..4e77b679 100644 --- a/table/field/types/yearmonth.spec.ts +++ b/table/field/types/yearmonth.spec.ts @@ -1,20 +1,43 @@ -import { DataFrame } from "nodejs-polars" +import { DataFrame, DataType, Series } from "nodejs-polars" import { describe, expect, it } from "vitest" -import { processTable } from "../../table/index.ts" +import { normalizeTable } from "../../table/index.ts" +import { denormalizeTable } from "../../table/index.ts" describe("parseYearmonthField", () => { it.each([ ["2000-01", [2000, 1]], ["0-0", [0, 0]], ])("%s -> %s", async (cell, value) => { - const table = DataFrame({ name: [cell] }).lazy() + const table = DataFrame([Series("name", [cell], DataType.String)]).lazy() + const schema = { fields: [{ name: "name", type: "yearmonth" as const }], } - const ldf = await processTable(table, { schema }) + const ldf = await normalizeTable(table, schema) const df = await ldf.collect() expect(df.toRecords()[0]?.name).toEqual(value) }) }) + +describe("stringifyYearmonthField", () => { + it.each([ + [[2000, 1], "2000-01"], + [[2023, 12], "2023-12"], + [[0, 0], "0000-00"], + ])("%s -> %s", async (value, expected) => { + const table = DataFrame([ + Series("name", [value], DataType.List(DataType.Int16)), + ]).lazy() + + const schema = { + fields: [{ name: "name", type: "yearmonth" as const }], + } + + const ldf = await denormalizeTable(table, schema) + const df = await ldf.collect() + + expect(df.toRecords()[0]?.name).toEqual(expected) + }) +}) diff --git a/table/field/types/yearmonth.ts b/table/field/types/yearmonth.ts index 0bbb1b14..b284b87c 100644 --- a/table/field/types/yearmonth.ts +++ b/table/field/types/yearmonth.ts @@ -1,5 +1,5 @@ import type { YearmonthField } from "@dpkit/core" -import { DataType, col } from "nodejs-polars" +import { DataType, col, concatString } from "nodejs-polars" import type { Expr } from "nodejs-polars" // TODO: @@ -7,6 +7,8 @@ import type { Expr } from "nodejs-polars" // - Check the length of the list is 2 (no list.lenghts in polars currently) // - Check the values are year and month limits // - Return null instead of list if any of the values are out of range +// - Rebase on Struct when lst.toStruct() is available? + export function parseYearmonthField(field: YearmonthField, expr?: Expr) { expr = expr ?? col(field.name) @@ -14,3 +16,17 @@ export function parseYearmonthField(field: YearmonthField, expr?: Expr) { return expr } + +export function stringifyYearmonthField(field: YearmonthField, expr?: Expr) { + expr = expr ?? col(field.name) + + // TODO: remove int casting when resolved: + // https://github.com/pola-rs/nodejs-polars/issues/362 + return concatString( + [ + expr.lst.get(0).cast(DataType.Int16).cast(DataType.String).str.zFill(4), + expr.lst.get(1).cast(DataType.Int16).cast(DataType.String).str.zFill(2), + ], + "-", + ).alias(field.name) as Expr +} diff --git a/table/field/inspect.spec.ts b/table/field/validate.spec.ts similarity index 84% rename from table/field/inspect.spec.ts rename to table/field/validate.spec.ts index 978dcd25..b9885d62 100644 --- a/table/field/inspect.spec.ts +++ b/table/field/validate.spec.ts @@ -1,9 +1,9 @@ import type { Schema } from "@dpkit/core" import { DataFrame } from "nodejs-polars" import { describe, expect, it } from "vitest" -import { inspectTable } from "../table/inspect.ts" +import { validateTable } from "../table/validate.ts" -describe("inspectField", () => { +describe("validateField", () => { describe("field name validation", () => { it("should report an error when field names don't match", async () => { const table = DataFrame({ @@ -19,7 +19,7 @@ describe("inspectField", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toContainEqual({ type: "field/name", @@ -42,7 +42,7 @@ describe("inspectField", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toHaveLength(0) }) @@ -60,7 +60,7 @@ describe("inspectField", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toHaveLength(1) expect(errors).toContainEqual({ @@ -86,7 +86,7 @@ describe("inspectField", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toHaveLength(1) expect(errors).toContainEqual({ @@ -111,13 +111,13 @@ describe("inspectField", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toHaveLength(0) }) }) describe("cell types validation", () => { - it("should inspect string to integer conversion errors", async () => { + it("should validate string to integer conversion errors", async () => { const table = DataFrame({ id: ["1", "bad", "3", "4x"], }).lazy() @@ -131,7 +131,7 @@ describe("inspectField", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toHaveLength(2) expect(errors).toContainEqual({ @@ -148,7 +148,7 @@ describe("inspectField", () => { }) }) - it("should inspect string to number conversion errors", async () => { + it("should validate string to number conversion errors", async () => { const table = DataFrame({ price: ["10.5", "twenty", "30.75", "$40"], }).lazy() @@ -162,7 +162,7 @@ describe("inspectField", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toHaveLength(2) expect(errors).toContainEqual({ @@ -179,7 +179,7 @@ describe("inspectField", () => { }) }) - it("should inspect string to boolean conversion errors", async () => { + it("should validate string to boolean conversion errors", async () => { const table = DataFrame({ active: ["true", "yes", "false", "0", "1"], }).lazy() @@ -193,7 +193,7 @@ describe("inspectField", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toHaveLength(1) expect(errors).toContainEqual({ @@ -204,7 +204,7 @@ describe("inspectField", () => { }) }) - it("should inspect string to date conversion errors", async () => { + it("should validate string to date conversion errors", async () => { const table = DataFrame({ created: ["2023-01-15", "Jan 15, 2023", "20230115", "not-a-date"], }).lazy() @@ -218,7 +218,7 @@ describe("inspectField", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toHaveLength(3) expect(errors).toContainEqual({ @@ -241,7 +241,7 @@ describe("inspectField", () => { }) }) - it("should inspect string to time conversion errors", async () => { + it("should validate string to time conversion errors", async () => { const table = DataFrame({ time: ["14:30:00", "2:30pm", "invalid", "14h30"], }).lazy() @@ -255,7 +255,7 @@ describe("inspectField", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toHaveLength(3) expect(errors).toContainEqual({ @@ -278,7 +278,7 @@ describe("inspectField", () => { }) }) - it("should inspect string to year conversion errors", async () => { + it("should validate string to year conversion errors", async () => { const table = DataFrame({ year: ["2023", "23", "MMXXIII", "two-thousand-twenty-three"], }).lazy() @@ -292,7 +292,7 @@ describe("inspectField", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toHaveLength(3) expect(errors).toContainEqual({ @@ -315,7 +315,7 @@ describe("inspectField", () => { }) }) - it("should inspect string to datetime conversion errors", async () => { + it("should validate string to datetime conversion errors", async () => { const table = DataFrame({ timestamp: [ "2023-01-15T14:30:00", @@ -334,7 +334,7 @@ describe("inspectField", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) // Adjust the expectations to match actual behavior expect(errors.length).toBeGreaterThan(0) @@ -369,12 +369,12 @@ describe("inspectField", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toHaveLength(0) }) - it("should inspect with non-string source data", async () => { + it("should validate with non-string source data", async () => { const table = DataFrame({ is_active: [true, false, true, false], }).lazy() @@ -388,7 +388,7 @@ describe("inspectField", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) // Since the column matches the expected type, validation passes expect(errors).toHaveLength(0) diff --git a/table/field/inspect.ts b/table/field/validate.ts similarity index 83% rename from table/field/inspect.ts rename to table/field/validate.ts index 095a368d..b254fe5e 100644 --- a/table/field/inspect.ts +++ b/table/field/validate.ts @@ -12,7 +12,7 @@ import { checkCellRequired } from "./checks/required.ts" import { checkCellType } from "./checks/type.ts" import { checkCellUnique } from "./checks/unique.ts" -export function inspectField( +export function validateField( field: Field, options: { errorTable: Table @@ -22,20 +22,20 @@ export function inspectField( const { polarsField } = options const errors: TableError[] = [] - const nameErrors = inspectName(field, polarsField) + const nameErrors = validateName(field, polarsField) errors.push(...nameErrors) - const typeErrors = inspectType(field, polarsField) + const typeErrors = validateType(field, polarsField) errors.push(...typeErrors) const errorTable = !typeErrors.length - ? inspectCells(field, options.errorTable) + ? validateCells(field, options.errorTable) : options.errorTable return { errors, errorTable } } -function inspectName(field: Field, polarsField: PolarsField) { +function validateName(field: Field, polarsField: PolarsField) { const errors: TableError[] = [] if (field.name !== polarsField.name) { @@ -49,7 +49,7 @@ function inspectName(field: Field, polarsField: PolarsField) { return errors } -function inspectType(field: Field, polarsField: PolarsField) { +function validateType(field: Field, polarsField: PolarsField) { const errors: TableError[] = [] const mapping: Record = { @@ -72,13 +72,13 @@ function inspectType(field: Field, polarsField: PolarsField) { Utf8: "string", } - const actualFieldType = mapping[polarsField.type.variant] + const actualFieldType = mapping[polarsField.type.variant] ?? "any" if (actualFieldType !== field.type && actualFieldType !== "string") { errors.push({ type: "field/type", fieldName: field.name, - fieldType: field.type, + fieldType: field.type ?? "any", actualFieldType, }) } @@ -86,7 +86,7 @@ function inspectType(field: Field, polarsField: PolarsField) { return errors } -function inspectCells(field: Field, errorTable: Table) { +function validateCells(field: Field, errorTable: Table) { errorTable = checkCellType(field, errorTable) errorTable = checkCellRequired(field, errorTable) errorTable = checkCellPattern(field, errorTable) diff --git a/table/package.json b/table/package.json index 8356c173..0807b496 100644 --- a/table/package.json +++ b/table/package.json @@ -23,7 +23,7 @@ "build": "tsc" }, "dependencies": { - "nodejs-polars": "^0.21.0", + "nodejs-polars": "^0.21.1", "@dpkit/core": "workspace:*" } } diff --git a/table/plugin.ts b/table/plugin.ts index b6b57735..83f03127 100644 --- a/table/plugin.ts +++ b/table/plugin.ts @@ -1,11 +1,21 @@ -import type { Dialect, Plugin, Resource } from "@dpkit/core" +import type { Dialect, Plugin, Resource, Schema } from "@dpkit/core" +import type { InferSchemaOptions, SchemaOptions } from "./schema/index.ts" import type { Table } from "./table/index.ts" -export type InferDialectOptions = { sampleBytes?: number } -export type SaveTableOptions = { +export type InferDialectOptions = { + sampleBytes?: number +} + +export type LoadTableOptions = InferDialectOptions & + InferSchemaOptions & { + denormalized?: boolean + } + +export type SaveTableOptions = SchemaOptions & { path: string format?: string dialect?: Dialect + overwrite?: boolean } export interface TablePlugin extends Plugin { @@ -14,7 +24,15 @@ export interface TablePlugin extends Plugin { options?: InferDialectOptions, ): Promise - loadTable?(resource: Partial): Promise
+ inferSchema?( + resource: Partial, + options?: InferSchemaOptions, + ): Promise + + loadTable?( + resource: Partial, + options?: LoadTableOptions, + ): Promise
saveTable?( table: Table, diff --git a/table/row/checks/unique.spec.ts b/table/row/checks/unique.spec.ts index 34c91c6b..2b8849ce 100644 --- a/table/row/checks/unique.spec.ts +++ b/table/row/checks/unique.spec.ts @@ -1,9 +1,9 @@ import type { Schema } from "@dpkit/core" import { DataFrame } from "nodejs-polars" import { describe, expect, it } from "vitest" -import { inspectTable } from "../../table/index.ts" +import { validateTable } from "../../table/index.ts" -describe("inspectTable (row/unique)", () => { +describe("validateTable (row/unique)", () => { it("should not report errors when all rows are unique for primary key", async () => { const table = DataFrame({ id: [1, 2, 3, 4, 5], @@ -18,7 +18,7 @@ describe("inspectTable (row/unique)", () => { primaryKey: ["id"], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toHaveLength(0) }) @@ -36,7 +36,7 @@ describe("inspectTable (row/unique)", () => { primaryKey: ["id"], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors.filter(e => e.type === "row/unique")).toHaveLength(1) expect(errors).toContainEqual({ type: "row/unique", @@ -65,7 +65,7 @@ describe("inspectTable (row/unique)", () => { uniqueKeys: [["email"]], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toHaveLength(0) }) @@ -89,7 +89,7 @@ describe("inspectTable (row/unique)", () => { uniqueKeys: [["email"]], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors.filter(e => e.type === "row/unique")).toHaveLength(2) expect(errors).toContainEqual({ type: "row/unique", @@ -119,7 +119,7 @@ describe("inspectTable (row/unique)", () => { uniqueKeys: [["category", "subcategory"]], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors.filter(e => e.type === "row/unique")).toHaveLength(1) expect(errors).toContainEqual({ type: "row/unique", @@ -149,7 +149,7 @@ describe("inspectTable (row/unique)", () => { uniqueKeys: [["email"]], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors.filter(e => e.type === "row/unique")).toHaveLength(2) expect(errors).toContainEqual({ type: "row/unique", @@ -177,7 +177,7 @@ describe("inspectTable (row/unique)", () => { uniqueKeys: [["id"], ["id", "name"]], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toHaveLength(2) expect(errors).toContainEqual({ diff --git a/table/row/index.ts b/table/row/index.ts index 7a4e2c8b..0628f872 100644 --- a/table/row/index.ts +++ b/table/row/index.ts @@ -1 +1 @@ -export { inspectRows } from "./inspect.ts" +export { validateRows } from "./validate.ts" diff --git a/table/row/inspect.ts b/table/row/validate.ts similarity index 82% rename from table/row/inspect.ts rename to table/row/validate.ts index 097c6c49..2703d848 100644 --- a/table/row/inspect.ts +++ b/table/row/validate.ts @@ -3,7 +3,7 @@ import type { TableError } from "../error/Table.ts" import type { Table } from "../table/Table.ts" import { checkRowUnique } from "./checks/unique.ts" -export function inspectRows(schema: Schema, errorTable: Table) { +export function validateRows(schema: Schema, errorTable: Table) { const errors: TableError[] = [] errorTable = checkRowUnique(schema, errorTable) diff --git a/table/schema/Options.ts b/table/schema/Options.ts new file mode 100644 index 00000000..08594c7c --- /dev/null +++ b/table/schema/Options.ts @@ -0,0 +1,23 @@ +import type { GeojsonField, GeopointField, ListField } from "@dpkit/core" +import type { StringField } from "@dpkit/core" +import type { FieldType } from "@dpkit/core" + +export interface SchemaOptions { + fieldNames?: string[] + fieldTypes?: Record + missingValues?: string[] + stringFormat?: StringField["format"] + decimalChar?: string + groupChar?: string + bareNumber?: boolean + trueValues?: string[] + falseValues?: string[] + datetimeFormat?: string + dateFormat?: string + timeFormat?: string + arrayType?: "array" | "list" + listDelimiter?: string + listItemType?: ListField["itemType"] + geopointFormat?: GeopointField["format"] + geojsonFormat?: GeojsonField["format"] +} diff --git a/table/schema/Schema.ts b/table/schema/Schema.ts index c4ae7f44..60d6e45c 100644 --- a/table/schema/Schema.ts +++ b/table/schema/Schema.ts @@ -1,15 +1,5 @@ -import type { DataType } from "nodejs-polars" import type { PolarsField } from "../field/index.ts" export interface PolarsSchema { fields: PolarsField[] } - -export function getPolarsSchema( - typeMapping: Record, -): PolarsSchema { - const entries = Object.entries(typeMapping) - const fields = entries.map(([name, type]) => ({ name, type })) - - return { fields } -} diff --git a/table/schema/helpers.ts b/table/schema/helpers.ts new file mode 100644 index 00000000..5b15a458 --- /dev/null +++ b/table/schema/helpers.ts @@ -0,0 +1,11 @@ +import type { DataType } from "nodejs-polars" +import type { PolarsSchema } from "./Schema.ts" + +export function getPolarsSchema( + typeMapping: Record, +): PolarsSchema { + const entries = Object.entries(typeMapping) + const fields = entries.map(([name, type]) => ({ name, type })) + + return { fields } +} diff --git a/table/schema/index.ts b/table/schema/index.ts index 9d6f0cd4..11e8fa74 100644 --- a/table/schema/index.ts +++ b/table/schema/index.ts @@ -1,2 +1,5 @@ -export { type InferSchemaOptions, inferSchema } from "./infer.ts" -export { type PolarsSchema, getPolarsSchema } from "./Schema.ts" +export type { PolarsSchema } from "./Schema.ts" +export { getPolarsSchema } from "./helpers.ts" +export type { SchemaOptions } from "./Options.ts" +export { inferSchemaFromTable } from "./infer.ts" +export type { InferSchemaOptions } from "./infer.ts" diff --git a/table/schema/infer.spec.ts b/table/schema/infer.spec.ts index a04ea61e..3f3ac067 100644 --- a/table/schema/infer.spec.ts +++ b/table/schema/infer.spec.ts @@ -1,9 +1,9 @@ import { DataFrame, Series } from "nodejs-polars" import { DataType } from "nodejs-polars" import { describe, expect, it } from "vitest" -import { inferSchema } from "./infer.ts" +import { inferSchemaFromTable } from "./infer.ts" -describe("inferSchema", () => { +describe("inferSchemaFromTable", () => { it("should infer from native types", async () => { const table = DataFrame({ integer: Series("integer", [1, 2], DataType.Int32), @@ -17,7 +17,7 @@ describe("inferSchema", () => { ], } - expect(await inferSchema(table)).toEqual(schema) + expect(await inferSchemaFromTable(table)).toEqual(schema) }) it("should infer numeric", async () => { @@ -37,7 +37,7 @@ describe("inferSchema", () => { ], } - expect(await inferSchema(table)).toEqual(schema) + expect(await inferSchemaFromTable(table)).toEqual(schema) }) it("should infer numeric (commaDecimal)", async () => { @@ -53,7 +53,9 @@ describe("inferSchema", () => { ], } - expect(await inferSchema(table, { commaDecimal: true })).toEqual(schema) + expect(await inferSchemaFromTable(table, { commaDecimal: true })).toEqual( + schema, + ) }) it("should infer booleans", async () => { @@ -69,7 +71,7 @@ describe("inferSchema", () => { ], } - expect(await inferSchema(table)).toEqual(schema) + expect(await inferSchemaFromTable(table)).toEqual(schema) }) it("should infer objects", async () => { @@ -85,7 +87,7 @@ describe("inferSchema", () => { ], } - expect(await inferSchema(table)).toEqual(schema) + expect(await inferSchemaFromTable(table)).toEqual(schema) }) it("should infer arrays", async () => { @@ -101,7 +103,7 @@ describe("inferSchema", () => { ], } - expect(await inferSchema(table)).toEqual(schema) + expect(await inferSchemaFromTable(table)).toEqual(schema) }) it("should infer dates with ISO format", async () => { @@ -113,7 +115,7 @@ describe("inferSchema", () => { fields: [{ name: "name1", type: "date" }], } - expect(await inferSchema(table)).toEqual(schema) + expect(await inferSchemaFromTable(table)).toEqual(schema) }) it("should infer dates with slash format", async () => { @@ -139,8 +141,8 @@ describe("inferSchema", () => { ], } - expect(await inferSchema(table)).toEqual(schemaDefault) - expect(await inferSchema(table, { monthFirst: true })).toEqual( + expect(await inferSchemaFromTable(table)).toEqual(schemaDefault) + expect(await inferSchemaFromTable(table, { monthFirst: true })).toEqual( schemaMonthFirst, ) }) @@ -158,8 +160,8 @@ describe("inferSchema", () => { fields: [{ name: "dayMonth", type: "date", format: "%m-%d-%Y" }], } - expect(await inferSchema(table)).toEqual(schemaDefault) - expect(await inferSchema(table, { monthFirst: true })).toEqual( + expect(await inferSchemaFromTable(table)).toEqual(schemaDefault) + expect(await inferSchemaFromTable(table, { monthFirst: true })).toEqual( schemaMonthFirst, ) }) @@ -177,7 +179,7 @@ describe("inferSchema", () => { ], } - expect(await inferSchema(table)).toEqual(schema) + expect(await inferSchemaFromTable(table)).toEqual(schema) }) it("should infer times with 12-hour format", async () => { @@ -193,7 +195,7 @@ describe("inferSchema", () => { ], } - expect(await inferSchema(table)).toEqual(schema) + expect(await inferSchemaFromTable(table)).toEqual(schema) }) it("should infer times with timezone offset", async () => { @@ -205,7 +207,7 @@ describe("inferSchema", () => { fields: [{ name: "name", type: "time" }], } - expect(await inferSchema(table)).toEqual(schema) + expect(await inferSchemaFromTable(table)).toEqual(schema) }) it("should infer datetimes with ISO format", async () => { @@ -241,7 +243,7 @@ describe("inferSchema", () => { ], } - expect(await inferSchema(table)).toEqual(schema) + expect(await inferSchemaFromTable(table)).toEqual(schema) }) it("should infer datetimes with custom formats", async () => { @@ -286,8 +288,8 @@ describe("inferSchema", () => { ], } - expect(await inferSchema(table)).toEqual(schemaDefault) - expect(await inferSchema(table, { monthFirst: true })).toEqual( + expect(await inferSchemaFromTable(table)).toEqual(schemaDefault) + expect(await inferSchemaFromTable(table, { monthFirst: true })).toEqual( schemaMonthFirst, ) }) diff --git a/table/schema/infer.ts b/table/schema/infer.ts index e7611819..c5b7b535 100644 --- a/table/schema/infer.ts +++ b/table/schema/infer.ts @@ -2,18 +2,30 @@ import type { Field, Schema } from "@dpkit/core" import { col } from "nodejs-polars" import { getPolarsSchema } from "../schema/index.ts" import type { Table } from "../table/index.ts" +import type { SchemaOptions } from "./Options.ts" +// TODO: Implement actual options usage for inferring // TODO: Review default values being {fields: []} vs undefined -export type InferSchemaOptions = { +export interface InferSchemaOptions extends SchemaOptions { sampleRows?: number confidence?: number commaDecimal?: boolean monthFirst?: boolean + keepStrings?: boolean } -export async function inferSchema(table: Table, options?: InferSchemaOptions) { - const { sampleRows = 100, confidence = 0.9 } = options ?? {} +export async function inferSchemaFromTable( + table: Table, + options?: InferSchemaOptions, +) { + const { + sampleRows = 100, + confidence = 0.9, + fieldTypes, + keepStrings, + } = options ?? {} + const schema: Schema = { fields: [], } @@ -23,17 +35,26 @@ export async function inferSchema(table: Table, options?: InferSchemaOptions) { const sample = await table.head(sampleRows).collect() const polarsSchema = getPolarsSchema(sample.schema) + const fieldNames = options?.fieldNames ?? polarsSchema.fields.map(f => f.name) const failureThreshold = sample.height - Math.floor(sample.height * confidence) || 1 - for (const polarsField of polarsSchema.fields) { - const name = polarsField.name - const type = typeMapping[polarsField.type.variant] ?? "any" + for (const name of fieldNames) { + const polarsField = polarsSchema.fields.find(f => f.name === name) + if (!polarsField) { + throw new Error(`Field "${name}" not found in the table`) + } - let field = { name, type } + let type = + fieldTypes?.[name] ?? typeMapping[polarsField.type.variant] ?? "any" - if (type === "string") { + if (type === "array" && options?.arrayType === "list") { + type = "list" + } + + let field = { name, type } + if (!keepStrings && type === "string") { for (const [regex, patch] of Object.entries(regexMapping)) { const failures = sample .filter(col(name).str.contains(regex).not()) @@ -45,9 +66,11 @@ export async function inferSchema(table: Table, options?: InferSchemaOptions) { } } + enhanceField(field, options) schema.fields.push(field) } + enhanceSchema(schema, options) return schema } @@ -81,10 +104,7 @@ function createTypeMapping() { return mapping } -function createRegexMapping(options?: { - commaDecimal?: boolean - monthFirst?: boolean -}) { +function createRegexMapping(options?: InferSchemaOptions) { const { commaDecimal, monthFirst } = options ?? {} const mapping: Record> = { @@ -150,3 +170,36 @@ function createRegexMapping(options?: { return mapping } + +function enhanceField(field: Field, options?: InferSchemaOptions) { + if (field.type === "string") { + field.format = options?.stringFormat ?? field.format + } else if (field.type === "integer") { + field.groupChar = options?.groupChar ?? field.groupChar + field.bareNumber = options?.bareNumber ?? field.bareNumber + } else if (field.type === "number") { + field.decimalChar = options?.decimalChar ?? field.decimalChar + field.groupChar = options?.groupChar ?? field.groupChar + field.bareNumber = options?.bareNumber ?? field.bareNumber + } else if (field.type === "boolean") { + field.trueValues = options?.trueValues ?? field.trueValues + field.falseValues = options?.falseValues ?? field.falseValues + } else if (field.type === "datetime") { + field.format = options?.datetimeFormat ?? field.format + } else if (field.type === "date") { + field.format = options?.dateFormat ?? field.format + } else if (field.type === "time") { + field.format = options?.timeFormat ?? field.format + } else if (field.type === "list") { + field.delimiter = options?.listDelimiter ?? field.delimiter + field.itemType = options?.listItemType ?? field.itemType + } else if (field.type === "geopoint") { + field.format = options?.geopointFormat ?? field.format + } else if (field.type === "geojson") { + field.format = options?.geojsonFormat ?? field.format + } +} + +function enhanceSchema(schema: Schema, options?: InferSchemaOptions) { + schema.missingValues = options?.missingValues ?? schema.missingValues +} diff --git a/table/table/denormalize.ts b/table/table/denormalize.ts new file mode 100644 index 00000000..2f724b5c --- /dev/null +++ b/table/table/denormalize.ts @@ -0,0 +1,54 @@ +import type { Field, Schema } from "@dpkit/core" +import { col, lit } from "nodejs-polars" +import type { Expr } from "nodejs-polars" +import { stringifyField } from "../field/index.ts" +import type { PolarsSchema } from "../schema/index.ts" +import { getPolarsSchema } from "../schema/index.ts" +import type { Table } from "./Table.ts" + +const HEAD_ROWS = 100 + +type DenormalizeTableOptions = { + nativeTypes?: Exclude[] +} + +export async function denormalizeTable( + table: Table, + schema: Schema, + options?: DenormalizeTableOptions, +) { + const head = await table.head(HEAD_ROWS).collect() + const polarsSchema = getPolarsSchema(head.schema) + + return table.select( + Object.values(denormalizeFields(schema, polarsSchema, options)), + ) +} + +export function denormalizeFields( + schema: Schema, + polarsSchema: PolarsSchema, + options?: DenormalizeTableOptions, +) { + const { nativeTypes } = options ?? {} + const exprs: Record = {} + + for (const field of schema.fields) { + const polarsField = polarsSchema.fields.find(f => f.name === field.name) + let expr = lit(null).alias(field.name) + + if (polarsField) { + expr = col(polarsField.name).alias(field.name) + + if (!nativeTypes?.includes(field.type ?? "any")) { + const missingValues = field.missingValues ?? schema.missingValues + const mergedField = { ...field, missingValues } + expr = stringifyField(mergedField, expr) + } + } + + exprs[field.name] = expr + } + + return exprs +} diff --git a/table/table/index.ts b/table/table/index.ts index 5e83a27b..34724c19 100644 --- a/table/table/index.ts +++ b/table/table/index.ts @@ -1,5 +1,6 @@ -export { processTable } from "./process.ts" -export { inspectTable } from "./inspect.ts" +export { normalizeTable } from "./normalize.ts" +export { denormalizeTable } from "./denormalize.ts" +export { validateTable } from "./validate.ts" export type { Table } from "./Table.ts" export { skipCommentRows } from "./helpers.ts" export { joinHeaderRows } from "./helpers.ts" diff --git a/table/table/process.spec.ts b/table/table/normalize.spec.ts similarity index 83% rename from table/table/process.spec.ts rename to table/table/normalize.spec.ts index ea4eb244..96f60dc3 100644 --- a/table/table/process.spec.ts +++ b/table/table/normalize.spec.ts @@ -1,25 +1,9 @@ import type { Schema } from "@dpkit/core" import { DataFrame } from "nodejs-polars" import { describe, expect, it } from "vitest" -import { processTable } from "./process.ts" - -describe("processTable", () => { - it("should work without schema", async () => { - const table = DataFrame({ - id: [1, 2], - name: ["english", "中文"], - }).lazy() - - const records = [ - { id: 1, name: "english" }, - { id: 2, name: "中文" }, - ] - - const ldf = await processTable(table) - const df = await ldf.collect() - expect(df.toRecords()).toEqual(records) - }) +import { normalizeTable } from "./normalize.ts" +describe("normalizeTable", () => { it("should work with schema", async () => { const table = DataFrame({ id: [1, 2], @@ -38,7 +22,7 @@ describe("processTable", () => { { id: 2, name: "中文" }, ] - const ldf = await processTable(table, { schema }) + const ldf = await normalizeTable(table, schema) const df = await ldf.collect() expect(df.toRecords()).toEqual(records) }) @@ -62,7 +46,7 @@ describe("processTable", () => { { id: 2, name: "中文", other: null }, ] - const ldf = await processTable(table, { schema }) + const ldf = await normalizeTable(table, schema) const df = await ldf.collect() expect(df.toRecords()).toEqual(records) }) @@ -86,7 +70,7 @@ describe("processTable", () => { { id: 2, name: "中文" }, ] - const ldf = await processTable(table, { schema }) + const ldf = await normalizeTable(table, schema) const df = await ldf.collect() expect(df.toRecords()).toEqual(records) }) @@ -109,7 +93,7 @@ describe("processTable", () => { { id: 2, name: "中文" }, ] - const ldf = await processTable(table, { schema }) + const ldf = await normalizeTable(table, schema) const df = await ldf.collect() expect(df.toRecords()).toEqual(records) }) @@ -133,7 +117,7 @@ describe("processTable", () => { { id: 2, name: "中文" }, ] - const ldf = await processTable(table, { schema }) + const ldf = await normalizeTable(table, schema) const df = await ldf.collect() expect(df.toRecords()).toEqual(records) }) @@ -157,7 +141,7 @@ describe("processTable", () => { { id: 2, name: "中文" }, ] - const ldf = await processTable(table, { schema }) + const ldf = await normalizeTable(table, schema) const df = await ldf.collect() expect(df.toRecords()).toEqual(records) }) @@ -181,7 +165,7 @@ describe("processTable", () => { { id: 2, name: "中文" }, ] - const ldf = await processTable(table, { schema }) + const ldf = await normalizeTable(table, schema) const df = await ldf.collect() expect(df.toRecords()).toEqual(records) }) @@ -205,7 +189,7 @@ describe("processTable", () => { { id: 2, name: "中文" }, ] - const ldf = await processTable(table, { schema }) + const ldf = await normalizeTable(table, schema) const df = await ldf.collect() expect(df.toRecords()).toEqual(records) }) @@ -228,7 +212,7 @@ describe("processTable", () => { { id: 2, name: "中文" }, ] - const ldf = await processTable(table, { schema }) + const ldf = await normalizeTable(table, schema) const df = await ldf.collect() expect(df.toRecords()).toEqual(records) }) @@ -251,7 +235,7 @@ describe("processTable", () => { { id: 2, name: null }, ] - const ldf = await processTable(table, { schema }) + const ldf = await normalizeTable(table, schema) const df = await ldf.collect() expect(df.toRecords()).toEqual(records) }) diff --git a/table/table/process.ts b/table/table/normalize.ts similarity index 55% rename from table/table/process.ts rename to table/table/normalize.ts index 2b085473..a5ead8e8 100644 --- a/table/table/process.ts +++ b/table/table/normalize.ts @@ -4,34 +4,37 @@ import { DataType } from "nodejs-polars" import { col, lit } from "nodejs-polars" import { matchField } from "../field/index.ts" import { parseField } from "../field/index.ts" -import type { PolarsSchema } from "../schema/index.ts" import { getPolarsSchema } from "../schema/index.ts" +import type { PolarsSchema } from "../schema/index.ts" import type { Table } from "./Table.ts" -export async function processTable( +const HEAD_ROWS = 100 + +export async function normalizeTable( table: Table, + schema: Schema, options?: { - schema?: Schema - sampleSize?: number + noParse?: boolean }, ) { - const { schema, sampleSize = 100 } = options ?? {} + const { noParse } = options ?? {} - if (!schema) { - return table - } - - const sample = await table.head(sampleSize).collect() - const polarsSchema = getPolarsSchema(sample.schema) + const head = await table.head(HEAD_ROWS).collect() + const polarsSchema = getPolarsSchema(head.schema) - return table.select(Object.values(processFields(schema, polarsSchema))) + return table.select( + Object.values(normalizeFields(schema, polarsSchema, { noParse })), + ) } -export function processFields( +export function normalizeFields( schema: Schema, polarsSchema: PolarsSchema, - options?: { dontParse?: boolean }, + options?: { + noParse?: boolean + }, ) { + const { noParse } = options ?? {} const exprs: Record = {} for (const [index, field] of schema.fields.entries()) { @@ -41,8 +44,10 @@ export function processFields( if (polarsField) { expr = col(polarsField.name).alias(field.name) - if (!options?.dontParse && polarsField.type.equals(DataType.String)) { - expr = parseField(field, { expr, schema }) + if (!noParse && polarsField.type.equals(DataType.String)) { + const missingValues = field.missingValues ?? schema.missingValues + const mergedField = { ...field, missingValues } + expr = parseField(mergedField, expr) } } diff --git a/table/table/inspect.spec.ts b/table/table/validate.spec.ts similarity index 87% rename from table/table/inspect.spec.ts rename to table/table/validate.spec.ts index 75888ff9..62edd1e9 100644 --- a/table/table/inspect.spec.ts +++ b/table/table/validate.spec.ts @@ -1,9 +1,9 @@ import type { Schema } from "@dpkit/core" import { DataFrame } from "nodejs-polars" import { describe, expect, it } from "vitest" -import { inspectTable } from "./inspect.ts" +import { validateTable } from "./validate.ts" -describe("inspectTable", () => { +describe("validateTable", () => { describe("fields validation with fieldsMatch='exact'", () => { it("should pass when fields exactly match", async () => { const table = DataFrame({ @@ -18,7 +18,7 @@ describe("inspectTable", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toEqual([]) }) @@ -37,7 +37,7 @@ describe("inspectTable", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toEqual([ { type: "field/name", @@ -62,7 +62,7 @@ describe("inspectTable", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toContainEqual({ type: "fields/extra", fieldNames: ["age"], @@ -81,7 +81,7 @@ describe("inspectTable", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toContainEqual({ type: "fields/missing", fieldNames: ["name"], @@ -103,7 +103,7 @@ describe("inspectTable", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toEqual([]) }) @@ -122,7 +122,7 @@ describe("inspectTable", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toContainEqual({ type: "fields/extra", fieldNames: ["age"], @@ -142,7 +142,7 @@ describe("inspectTable", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toContainEqual({ type: "fields/missing", fieldNames: ["name"], @@ -166,7 +166,7 @@ describe("inspectTable", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toEqual([]) }) @@ -184,7 +184,7 @@ describe("inspectTable", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toEqual([]) }) @@ -201,7 +201,7 @@ describe("inspectTable", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toContainEqual({ type: "fields/missing", fieldNames: ["name"], @@ -223,7 +223,7 @@ describe("inspectTable", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toEqual([]) }) @@ -241,7 +241,7 @@ describe("inspectTable", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toEqual([]) }) @@ -260,7 +260,7 @@ describe("inspectTable", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toContainEqual({ type: "fields/extra", fieldNames: ["age"], @@ -283,7 +283,7 @@ describe("inspectTable", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toEqual([]) }) @@ -301,7 +301,7 @@ describe("inspectTable", () => { ], } - const errors = await inspectTable(table, { schema }) + const errors = await validateTable(table, { schema }) expect(errors).toContainEqual({ type: "fields/missing", fieldNames: ["id", "name"], diff --git a/table/table/inspect.ts b/table/table/validate.ts similarity index 87% rename from table/table/inspect.ts rename to table/table/validate.ts index 1e95570f..bb6fec13 100644 --- a/table/table/inspect.ts +++ b/table/table/validate.ts @@ -2,14 +2,14 @@ import type { Schema } from "@dpkit/core" import { col, lit } from "nodejs-polars" import type { TableError } from "../error/index.ts" import { matchField } from "../field/index.ts" -import { inspectField } from "../field/index.ts" -import { inspectRows } from "../row/index.ts" +import { validateField } from "../field/index.ts" +import { validateRows } from "../row/index.ts" import { getPolarsSchema } from "../schema/index.ts" import type { PolarsSchema } from "../schema/index.ts" import type { Table } from "./Table.ts" -import { processFields } from "./process.ts" +import { normalizeFields } from "./normalize.ts" -export async function inspectTable( +export async function validateTable( table: Table, options?: { schema?: Schema @@ -24,10 +24,10 @@ export async function inspectTable( const sample = await table.head(sampleRows).collect() const polarsSchema = getPolarsSchema(sample.schema) - const matchErrors = inspectFieldsMatch({ schema, polarsSchema }) + const matchErrors = validateFieldsMatch({ schema, polarsSchema }) errors.push(...matchErrors) - const fieldErrors = await inspectFields( + const fieldErrors = await validateFields( table, schema, polarsSchema, @@ -39,7 +39,7 @@ export async function inspectTable( return errors } -function inspectFieldsMatch(props: { +function validateFieldsMatch(props: { schema: Schema polarsSchema: PolarsSchema }) { @@ -122,7 +122,7 @@ function inspectFieldsMatch(props: { return errors } -async function inspectFields( +async function validateFields( table: Table, schema: Schema, polarsSchema: PolarsSchema, @@ -132,13 +132,13 @@ async function inspectFields( const targetNames: string[] = [] const sources = Object.entries( - processFields(schema, polarsSchema, { dontParse: true }), + normalizeFields(schema, polarsSchema, { noParse: true }), ).map(([name, expr]) => { return expr.alias(`source:${name}`) }) const targets = Object.entries( - processFields(schema, polarsSchema, { dontParse: false }), + normalizeFields(schema, polarsSchema, { noParse: false }), ).map(([name, expr]) => { const targetName = `target:${name}` targetNames.push(targetName) @@ -157,13 +157,13 @@ async function inspectFields( for (const [index, field] of schema.fields.entries()) { const polarsField = matchField(index, field, schema, polarsSchema) if (polarsField) { - const fieldResult = inspectField(field, { errorTable, polarsField }) + const fieldResult = validateField(field, { errorTable, polarsField }) errorTable = fieldResult.errorTable errors.push(...fieldResult.errors) } } - const rowsResult = inspectRows(schema, errorTable) + const rowsResult = validateRows(schema, errorTable) errorTable = rowsResult.errorTable errors.push(...rowsResult.errors) diff --git a/test/README.md b/test/README.md index e87239fa..c634d58f 100644 --- a/test/README.md +++ b/test/README.md @@ -1,3 +1,3 @@ # @dpkit/test -dpkit is a fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). +dpkit is a fast data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/vitest.config.ts b/vitest.config.ts index 573d75ec..42638a42 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -1,9 +1,11 @@ import { basename, dirname, join } from "node:path" -import { - configDefaults, - coverageConfigDefaults, - defineConfig, -} from "vitest/config" +import { loadEnvFile } from "node:process" +import { configDefaults, defineConfig } from "vitest/config" +import { coverageConfigDefaults } from "vitest/config" + +try { + loadEnvFile(join(import.meta.dirname, ".env")) +} catch {} export default defineConfig({ test: { diff --git a/xlsx/README.md b/xlsx/README.md index 3361aeeb..ab9d5330 100644 --- a/xlsx/README.md +++ b/xlsx/README.md @@ -1,3 +1,3 @@ # @dpkit/xlsx -dpkit is a fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). +dpkit is a fast data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/xlsx/package.json b/xlsx/package.json index 994ce9f7..9bcddd84 100644 --- a/xlsx/package.json +++ b/xlsx/package.json @@ -26,7 +26,7 @@ "@dpkit/core": "workspace:*", "@dpkit/file": "workspace:*", "@dpkit/table": "workspace:*", - "nodejs-polars": "^0.21.0", + "nodejs-polars": "^0.21.1", "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz" }, "devDependencies": { diff --git a/xlsx/plugin.ts b/xlsx/plugin.ts index 7a555bba..266be920 100644 --- a/xlsx/plugin.ts +++ b/xlsx/plugin.ts @@ -1,19 +1,22 @@ import type { Resource } from "@dpkit/core" import { inferResourceFormat } from "@dpkit/core" +import type { LoadTableOptions } from "@dpkit/table" import type { TablePlugin } from "@dpkit/table" import type { SaveTableOptions, Table } from "@dpkit/table" import { loadXlsxTable, saveXlsxTable } from "./table/index.ts" export class XlsxPlugin implements TablePlugin { - async loadTable(resource: Partial) { + async loadTable(resource: Partial, options?: LoadTableOptions) { const isXlsx = getIsXlsx(resource) if (!isXlsx) return undefined - return await loadXlsxTable(resource) + return await loadXlsxTable(resource, options) } async saveTable(table: Table, options: SaveTableOptions) { - const isXlsx = getIsXlsx(options) + const { path, format } = options + + const isXlsx = getIsXlsx({ path, format }) if (!isXlsx) return undefined return await saveXlsxTable(table, options) diff --git a/xlsx/table/load.ts b/xlsx/table/load.ts index 54d847b3..786515c9 100644 --- a/xlsx/table/load.ts +++ b/xlsx/table/load.ts @@ -1,15 +1,24 @@ import { loadResourceDialect } from "@dpkit/core" import type { Resource } from "@dpkit/core" +import { loadResourceSchema } from "@dpkit/core" import { loadFile, prefetchFiles } from "@dpkit/file" +import type { LoadTableOptions } from "@dpkit/table" +import { inferSchemaFromTable, normalizeTable } from "@dpkit/table" import type { DataRow, Table } from "@dpkit/table" import { getRecordsFromRows } from "@dpkit/table" import { DataFrame, concat } from "nodejs-polars" import { read, utils } from "xlsx" -export async function loadXlsxTable(resource: Partial) { +// Currently, we use slow non-rust implementation as in the future +// polars-rust might be able to provide a faster native implementation + +export async function loadXlsxTable( + resource: Partial, + options?: LoadTableOptions, +) { const paths = await prefetchFiles(resource.path) if (!paths.length) { - return DataFrame().lazy() + throw new Error("Resource path is not defined") } const dialect = await loadResourceDialect(resource.dialect) @@ -36,6 +45,13 @@ export async function loadXlsxTable(resource: Partial) { } } - const table = concat(tables) + let table = concat(tables) + + if (!options?.denormalized) { + let schema = await loadResourceSchema(resource.schema) + if (!schema) schema = await inferSchemaFromTable(table, options) + table = await normalizeTable(table, schema) + } + return table } diff --git a/xlsx/table/save.spec.ts b/xlsx/table/save.spec.ts index f34bd63b..0ec415e7 100644 --- a/xlsx/table/save.spec.ts +++ b/xlsx/table/save.spec.ts @@ -1,6 +1,8 @@ import { getTempFilePath } from "@dpkit/file" +import { DataFrame, DataType, Series } from "nodejs-polars" import { readRecords } from "nodejs-polars" import { describe, expect, it } from "vitest" +import { loadXlsxTable } from "./load.ts" import { saveXlsxTable } from "./save.ts" import { readTestData } from "./test.ts" @@ -16,4 +18,63 @@ describe("saveXlsxTable", () => { const data = await readTestData(path) expect(data).toEqual([row1, row2]) }) + + it("should save and load various data types", async () => { + const path = getTempFilePath() + + const source = DataFrame([ + Series("array", ["[1, 2, 3]"], DataType.String), + Series("boolean", [true], DataType.Bool), + Series("date", [new Date(Date.UTC(2025, 0, 1))], DataType.Date), + Series("datetime", [new Date(Date.UTC(2025, 0, 1))], DataType.Datetime), + Series("duration", ["P23DT23H"], DataType.String), + Series("geojson", ['{"value": 1}'], DataType.String), + Series("geopoint", [[40.0, 50.0]], DataType.List(DataType.Float32)), + Series("integer", [1], DataType.Int32), + Series("list", [[1.0, 2.0, 3.0]], DataType.List(DataType.Float32)), + Series("number", [1.1], DataType.Float64), + Series("object", ['{"value": 1}']), + Series("string", ["string"], DataType.String), + Series("time", [new Date(Date.UTC(2025, 0, 1))], DataType.Time), + Series("year", [2025], DataType.Int32), + Series("yearmonth", [[2025, 1]], DataType.List(DataType.Int16)), + ]).lazy() + + await saveXlsxTable(source, { + path, + fieldTypes: { + array: "array", + geojson: "geojson", + geopoint: "geopoint", + list: "list", + object: "object", + // TODO: Remove time after: + // https://github.com/pola-rs/nodejs-polars/issues/364 + time: "time", + year: "year", + yearmonth: "yearmonth", + }, + }) + + const target = await loadXlsxTable({ path }, { denormalized: true }) + expect((await target.collect()).toRecords()).toEqual([ + { + array: "[1, 2, 3]", + boolean: true, + date: "2025-01-01", + datetime: "2025-01-01T00:00:00", + duration: "P23DT23H", + geojson: '{"value": 1}', + geopoint: "40.0,50.0", + integer: 1, + list: "1.0,2.0,3.0", + number: 1.1, + object: '{"value": 1}', + string: "string", + time: "00:00:00", + year: 2025, + yearmonth: "2025-01", + }, + ]) + }) }) diff --git a/xlsx/table/save.ts b/xlsx/table/save.ts index 8bc145ad..407e2e9f 100644 --- a/xlsx/table/save.ts +++ b/xlsx/table/save.ts @@ -1,10 +1,23 @@ import { loadResourceDialect } from "@dpkit/core" import { saveFile } from "@dpkit/file" +import { denormalizeTable, inferSchemaFromTable } from "@dpkit/table" import type { SaveTableOptions, Table } from "@dpkit/table" import { utils, write } from "xlsx" +// Currently, we use slow non-rust implementation as in the future +// polars-rust might be able to provide a faster native implementation + export async function saveXlsxTable(table: Table, options: SaveTableOptions) { - const { path } = options + const { path, overwrite } = options + + const schema = await inferSchemaFromTable(table, { + ...options, + keepStrings: true, + }) + + table = await denormalizeTable(table, schema, { + nativeTypes: ["boolean", "integer", "number", "string", "year"], + }) const df = await table.collect() const dialect = await loadResourceDialect(options.dialect) @@ -15,7 +28,7 @@ export async function saveXlsxTable(table: Table, options: SaveTableOptions) { utils.book_append_sheet(book, sheet, sheetName) const buffer = write(book, { type: "buffer", bookType: "xlsx" }) - await saveFile(path, buffer) + await saveFile(path, buffer, { overwrite }) return path } diff --git a/zenodo/README.md b/zenodo/README.md index 6ba90aa7..b8df6c8e 100644 --- a/zenodo/README.md +++ b/zenodo/README.md @@ -1,3 +1,3 @@ # @dpkit/zenodo -dpkit is a fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). +dpkit is a fast data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/zenodo/package/process/denormalize.ts b/zenodo/package/denormalize.ts similarity index 96% rename from zenodo/package/process/denormalize.ts rename to zenodo/package/denormalize.ts index c2770475..84341b07 100644 --- a/zenodo/package/process/denormalize.ts +++ b/zenodo/package/denormalize.ts @@ -1,6 +1,6 @@ import type { Package } from "@dpkit/core" -import type { ZenodoCreator } from "../Creator.ts" -import type { ZenodoPackage } from "../Package.ts" +import type { ZenodoCreator } from "./Creator.ts" +import type { ZenodoPackage } from "./Package.ts" /** * Denormalizes a Frictionless Data Package to Zenodo Deposit metadata format diff --git a/zenodo/package/load.spec.ts b/zenodo/package/load.spec.ts index 2227e463..399f9b34 100644 --- a/zenodo/package/load.spec.ts +++ b/zenodo/package/load.spec.ts @@ -2,9 +2,9 @@ import { useRecording } from "@dpkit/test" import { describe, expect, it } from "vitest" import { loadPackageFromZenodo } from "./load.ts" -describe("loadPackageFromZenodo", () => { - useRecording() +useRecording() +describe("loadPackageFromZenodo", () => { it("should load a package", async () => { const datapackage = await loadPackageFromZenodo( "https://zenodo.org/records/15525711", diff --git a/zenodo/package/load.ts b/zenodo/package/load.ts index be9d5e32..034e74c3 100644 --- a/zenodo/package/load.ts +++ b/zenodo/package/load.ts @@ -1,7 +1,7 @@ import { mergePackages } from "@dpkit/core" import { makeZenodoApiRequest } from "../zenodo/index.ts" import type { ZenodoPackage } from "./Package.ts" -import { normalizeZenodoPackage } from "./process/normalize.ts" +import { normalizeZenodoPackage } from "./normalize.ts" /** * Load a package from a Zenodo deposit diff --git a/zenodo/package/process/normalize.ts b/zenodo/package/normalize.ts similarity index 94% rename from zenodo/package/process/normalize.ts rename to zenodo/package/normalize.ts index 813ee28d..eae41a1f 100644 --- a/zenodo/package/process/normalize.ts +++ b/zenodo/package/normalize.ts @@ -1,6 +1,6 @@ import type { Contributor, License, Package } from "@dpkit/core" -import { normalizeZenodoResource } from "../../resource/index.ts" -import type { ZenodoPackage } from "../Package.ts" +import { normalizeZenodoResource } from "../resource/index.ts" +import type { ZenodoPackage } from "./Package.ts" /** * Normalizes a Zenodo deposit to Frictionless Data package format diff --git a/zenodo/package/save.ts b/zenodo/package/save.ts index 041c7e1f..70618b72 100644 --- a/zenodo/package/save.ts +++ b/zenodo/package/save.ts @@ -1,14 +1,11 @@ import { blob } from "node:stream/consumers" import type { Descriptor, Package } from "@dpkit/core" import { denormalizePackage, stringifyDescriptor } from "@dpkit/core" -import { - getPackageBasepath, - loadFileStream, - saveResourceFiles, -} from "@dpkit/file" +import { loadFileStream, saveResourceFiles } from "@dpkit/file" +import { getPackageBasepath } from "@dpkit/file" import { makeZenodoApiRequest } from "../zenodo/index.ts" import type { ZenodoPackage } from "./Package.ts" -import { denormalizeZenodoPackage } from "./process/denormalize.ts" +import { denormalizeZenodoPackage } from "./denormalize.ts" /** * Save a package to Zenodo diff --git a/zenodo/resource/process/denormalize.ts b/zenodo/resource/denormalize.ts similarity index 87% rename from zenodo/resource/process/denormalize.ts rename to zenodo/resource/denormalize.ts index ceac0d46..0de8f188 100644 --- a/zenodo/resource/process/denormalize.ts +++ b/zenodo/resource/denormalize.ts @@ -1,5 +1,5 @@ import type { Resource } from "@dpkit/core" -import type { ZenodoResource } from "../Resource.ts" +import type { ZenodoResource } from "./Resource.ts" export function denormalizeZenodoResource(resource: Resource) { const zenodoResource: Partial = { diff --git a/zenodo/resource/index.ts b/zenodo/resource/index.ts index 9c1a6f5e..9dde4fdb 100644 --- a/zenodo/resource/index.ts +++ b/zenodo/resource/index.ts @@ -1,3 +1,3 @@ export type { ZenodoResource } from "./Resource.ts" -export { normalizeZenodoResource } from "./process/normalize.ts" -export { denormalizeZenodoResource } from "./process/denormalize.ts" +export { normalizeZenodoResource } from "./normalize.ts" +export { denormalizeZenodoResource } from "./denormalize.ts" diff --git a/zenodo/resource/process/normalize.ts b/zenodo/resource/normalize.ts similarity index 93% rename from zenodo/resource/process/normalize.ts rename to zenodo/resource/normalize.ts index b5b57af6..9cb44e7f 100644 --- a/zenodo/resource/process/normalize.ts +++ b/zenodo/resource/normalize.ts @@ -1,5 +1,5 @@ import { getFormat, getName } from "@dpkit/core" -import type { ZenodoResource } from "../Resource.ts" +import type { ZenodoResource } from "./Resource.ts" /** * Normalizes a Zenodo file to Frictionless Data resource format diff --git a/zip/README.md b/zip/README.md index 73ca459a..2f48649a 100644 --- a/zip/README.md +++ b/zip/README.md @@ -1,3 +1,3 @@ # @dpkit/zip -dpkit is a fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). +dpkit is a fast data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io).