feat: Database support

.changeset/clear-colts-prove.md

@@ -0,0 +1,24 @@
+---
+"@dpkit/database": minor
+"@dpkit/camtrap": minor
+"@dpkit/datahub": minor
+"@dpkit/parquet": minor
+"@dpkit/folder": minor
+"@dpkit/github": minor
+"@dpkit/inline": minor
+"@dpkit/zenodo": minor
+"@dpkit/arrow": minor
+"@dpkit/table": minor
+"@dpkit/ckan": minor
+"@dpkit/core": minor
+"@dpkit/file": minor
+"@dpkit/json": minor
+"@dpkit/test": minor
+"@dpkit/xlsx": minor
+"@dpkit/all": minor
+"@dpkit/csv": minor
+"@dpkit/ods": minor
+"@dpkit/zip": minor
+---
+
+Added database support

.env.example

@@ -0,0 +1,2 @@
+DPKIT_MYSQL_URL='<url>'
+DPKIT_POSTGRESQL_URL='<url>'

.github/workflows/general.yaml

@@ -11,6 +11,26 @@ on:
 jobs:
   test:
     runs-on: ubuntu-latest
+
+    services:
+      postgres:
+        image: postgres:16
+        env:
+          POSTGRES_PASSWORD: test
+          POSTGRES_USER: test
+          POSTGRES_DB: test
+        ports:
+          - 5432:5432
+      mysql:
+        image: mysql:8
+        env:
+          MYSQL_ROOT_PASSWORD: test
+          MYSQL_DATABASE: test
+          MYSQL_USER: test
+          MYSQL_PASSWORD: test
+        ports:
+          - 3306:3306
+
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v4
@@ -27,6 +47,9 @@ jobs:
         run: pnpm build
       - name: Test Packages
         run: pnpm test
+        env:
+          DPKIT_MYSQL_URL: mysql://test:test@localhost:3306/test
+          DPKIT_POSTGRESQL_URL: postgresql://test:test@localhost:5432/test
       - name: Upload Coverage
         uses: codecov/codecov-action@v5
         with:

.gitignore

@@ -64,7 +64,8 @@ compile/
 dist/
 .astro/
 /docs/content/docs/reference/
-**/.claude/settings.local.json
-AGENTS.md
-CLAUDE.md
 .cursor/
+.claude/
+.serena/
+.mcp.json
+.env

all/README.md

@@ -1,3 +1,3 @@
 # @dpkit/all
 
-dpkit is a fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io).
+dpkit is a fast data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io).

all/index.ts

@@ -14,5 +14,6 @@ export * from "@dpkit/zip"
 export * from "./dialect/index.ts"
 export * from "./package/index.ts"
 export * from "./resource/index.ts"
+export * from "./schema/index.ts"
 export * from "./table/index.ts"
 export * from "./plugin.ts"

all/package.json

@@ -25,6 +25,7 @@
     "@dpkit/ckan": "workspace:*",
     "@dpkit/core": "workspace:*",
     "@dpkit/csv": "workspace:*",
+    "@dpkit/database": "workspace:*",
     "@dpkit/datahub": "workspace:*",
     "@dpkit/file": "workspace:*",
     "@dpkit/folder": "workspace:*",

all/plugin.ts

@@ -1,6 +1,7 @@
 import { ArrowPlugin } from "@dpkit/arrow"
 import { CkanPlugin } from "@dpkit/ckan"
 import { CsvPlugin } from "@dpkit/csv"
+import { DatabasePlugin } from "@dpkit/database"
 import { DatahubPlugin } from "@dpkit/datahub"
 import { FolderPlugin } from "@dpkit/folder"
 import { GithubPlugin } from "@dpkit/github"
@@ -39,3 +40,6 @@ dpkit.register(JsonPlugin)
 dpkit.register(OdsPlugin)
 dpkit.register(ParquetPlugin)
 dpkit.register(XlsxPlugin)
+
+// Mixed functions
+dpkit.register(DatabasePlugin)

all/resource/infer.ts

@@ -2,7 +2,8 @@ import type { Resource } from "@dpkit/core"
 import { inferResourceFormat, inferResourceName } from "@dpkit/core"
 import { prefetchFile } from "@dpkit/file"
 import { inferFileBytes, inferFileEncoding, inferFileHash } from "@dpkit/file"
-import { inferTable } from "../table/index.ts"
+import { inferDialect } from "../dialect/index.ts"
+import { inferSchema } from "../schema/index.ts"
 
 // TODO: Support multipart resources? (clarify on the specs level)
 
@@ -18,6 +19,7 @@ export async function inferResource(resource: Partial<Resource>) {
 
   if (typeof resource.path === "string") {
     const localPath = await prefetchFile(resource.path)
+    const localResource = { ...resource, path: localPath }
 
     if (!result.encoding) {
       const encoding = await inferFileEncoding(localPath)
@@ -34,13 +36,15 @@ export async function inferResource(resource: Partial<Resource>) {
       result.hash = await inferFileHash(localPath)
     }
 
-    if (!result.schema) {
+    if (!result.dialect) {
       try {
-        const localResource = { ...resource, path: localPath }
-        const { dialect, schema } = await inferTable(localResource)
+        result.dialect = await inferDialect(localResource)
+      } catch {}
+    }
 
-        result.dialect = dialect
-        result.schema = schema
+    if (!result.schema) {
+      try {
+        result.schema = await inferSchema(localResource)
       } catch {}
     }
   }

all/resource/validate.ts

@@ -1,15 +1,18 @@
 import type { Descriptor, Resource } from "@dpkit/core"
+import { loadResourceSchema } from "@dpkit/core"
 import { loadDescriptor, validateResourceDescriptor } from "@dpkit/core"
 import { validateFile } from "@dpkit/file"
-import { validateTable } from "../table/index.ts"
+import { validateTable } from "@dpkit/table"
+import { inferSchema } from "../schema/index.ts"
+import { loadTable } from "../table/index.ts"
 
 // TODO: Support multipart resources? (clarify on the specs level)
 
 export async function validateResource(
-  pathOrDescriptorOrResource: string | Descriptor | Partial<Resource>,
+  source: string | Descriptor | Partial<Resource>,
   options?: { basepath?: string },
 ) {
-  let descriptor = pathOrDescriptorOrResource
+  let descriptor = source
   let basepath = options?.basepath
 
   if (typeof descriptor === "string") {
@@ -39,7 +42,14 @@ export async function validateResource(
   try {
     // TODO: rebase on not-rasing?
     // It will raise if the resource is not a table
-    return await validateTable(resource)
+    let schema = await loadResourceSchema(resource.schema)
+    if (!schema) schema = await inferSchema(resource)
+
+    const table = await loadTable(resource, { denormalized: true })
+    const errors = await validateTable(table, { schema })
+
+    const valid = errors.length === 0
+    return { valid, errors }
   } catch {}
 
   return { valid: true, errors: [] }

all/schema/index.ts

@@ -0,0 +1 @@
+export { inferSchema } from "./infer.ts"

all/schema/infer.ts

@@ -0,0 +1,21 @@
+import type { Resource, Schema } from "@dpkit/core"
+import type { InferSchemaOptions } from "@dpkit/table"
+import { inferSchemaFromTable } from "@dpkit/table"
+import { dpkit } from "../plugin.ts"
+import { loadTable } from "../table/index.ts"
+
+export async function inferSchema(
+  resource: Partial<Resource>,
+  options?: InferSchemaOptions,
+): Promise<Schema> {
+  for (const plugin of dpkit.plugins) {
+    const schema = await plugin.inferSchema?.(resource, options)
+    if (schema) {
+      return schema
+    }
+  }
+
+  const table = await loadTable(resource, { denormalized: true })
+  const schema = await inferSchemaFromTable(table, options)
+  return schema
+}

all/table/index.ts

@@ -1,5 +1,2 @@
 export { loadTable } from "./load.ts"
-export { readTable } from "./read.ts"
 export { saveTable } from "./save.ts"
-export { inferTable } from "./infer.ts"
-export { validateTable } from "./validate.ts"

all/table/infer.ts

@@ -1,7 +1,7 @@
 import type { Dialect, Resource, Schema } from "@dpkit/core"
 import { loadResourceDialect, loadResourceSchema } from "@dpkit/core"
 import type { Table } from "@dpkit/table"
-import { inferSchema } from "@dpkit/table"
+import { inferSchemaFromTable } from "@dpkit/table"
 import { inferDialect } from "../dialect/index.ts"
 import { loadTable } from "./load.ts"
 
@@ -15,11 +15,14 @@ export async function inferTable(
     dialect = await inferDialect(resource)
   }
 
-  const table = await loadTable({ ...resource, dialect })
+  const table = await loadTable(
+    { ...resource, dialect },
+    { denormalized: true },
+  )
 
   let schema = await loadResourceSchema(resource.schema)
   if (!schema) {
-    schema = await inferSchema(table)
+    schema = await inferSchemaFromTable(table)
   }
 
   return { dialect, schema, table }

all/table/load.ts

@@ -1,10 +1,13 @@
 import type { Resource } from "@dpkit/core"
-import type { Table } from "@dpkit/table"
+import type { LoadTableOptions, Table } from "@dpkit/table"
 import { dpkit } from "../plugin.ts"
 
-export async function loadTable(resource: Partial<Resource>): Promise<Table> {
+export async function loadTable(
+  resource: Partial<Resource>,
+  options?: LoadTableOptions,
+): Promise<Table> {
   for (const plugin of dpkit.plugins) {
-    const table = await plugin.loadTable?.(resource)
+    const table = await plugin.loadTable?.(resource, options)
     if (table) {
       return table
     }

arrow/README.md

@@ -1,3 +1,3 @@
 # @dpkit/arrow
 
-dpkit is a fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io).
+dpkit is a fast data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io).

arrow/package.json

@@ -27,7 +27,7 @@
     "@dpkit/file": "workspace:*",
     "@dpkit/table": "workspace:*",
     "csv-sniffer": "^0.1.1",
-    "nodejs-polars": "^0.21.0"
+    "nodejs-polars": "^0.21.1"
   },
   "devDependencies": {
     "@dpkit/test": "workspace:*"

arrow/plugin.ts

@@ -1,19 +1,22 @@
 import type { Resource } from "@dpkit/core"
 import { inferResourceFormat } from "@dpkit/core"
+import type { LoadTableOptions } from "@dpkit/table"
 import type { TablePlugin } from "@dpkit/table"
 import type { SaveTableOptions, Table } from "@dpkit/table"
 import { loadArrowTable, saveArrowTable } from "./table/index.ts"
 
 export class ArrowPlugin implements TablePlugin {
-  async loadTable(resource: Partial<Resource>) {
+  async loadTable(resource: Partial<Resource>, options?: LoadTableOptions) {
     const isArrow = getIsArrow(resource)
     if (!isArrow) return undefined
 
-    return await loadArrowTable(resource)
+    return await loadArrowTable(resource, options)
   }
 
   async saveTable(table: Table, options: SaveTableOptions) {
-    const isArrow = getIsArrow(options)
+    const { path, format } = options
+
+    const isArrow = getIsArrow({ path, format })
     if (!isArrow) return undefined
 
     return await saveArrowTable(table, options)

arrow/table/load.spec.ts

@@ -4,9 +4,9 @@ import { DataFrame } from "nodejs-polars"
 import { describe, expect, it } from "vitest"
 import { loadArrowTable } from "./load.ts"
 
-describe("loadArrowTable", () => {
-  useRecording()
+useRecording()
 
+describe("loadArrowTable", () => {
   describe("file variations", () => {
     it("should load local file", async () => {
       const path = getTempFilePath()

arrow/table/load.ts

@@ -1,19 +1,30 @@
 import type { Resource } from "@dpkit/core"
+import { loadResourceSchema } from "@dpkit/core"
 import { prefetchFiles } from "@dpkit/file"
+import type { LoadTableOptions } from "@dpkit/table"
+import { inferSchemaFromTable, normalizeTable } from "@dpkit/table"
 import { concat } from "nodejs-polars"
-import { DataFrame } from "nodejs-polars"
 import { scanIPC } from "nodejs-polars"
 
-export async function loadArrowTable(resource: Partial<Resource>) {
+export async function loadArrowTable(
+  resource: Partial<Resource>,
+  options?: LoadTableOptions,
+) {
   const [firstPath, ...restPaths] = await prefetchFiles(resource.path)
   if (!firstPath) {
-    return DataFrame().lazy()
+    throw new Error("Resource path is not defined")
   }
 
   let table = scanIPC(firstPath)
   if (restPaths.length) {
     table = concat([table, ...restPaths.map(path => scanIPC(path))])
   }
 
+  if (!options?.denormalized) {
+    let schema = await loadResourceSchema(resource.schema)
+    if (!schema) schema = await inferSchemaFromTable(table, options)
+    table = await normalizeTable(table, schema)
+  }
+
   return table
 }