feat: Json/Jsonl/Arrow/Parquet support, bootstrapped CLI, and other improvements/fixes

.changeset/breezy-knives-greet.md

@@ -0,0 +1,27 @@
+---
+"@dpkit/camtrap": minor
+"@dpkit/datahub": minor
+"@dpkit/parquet": minor
+"@dpkit/extend": minor
+"@dpkit/folder": minor
+"@dpkit/github": minor
+"@dpkit/inline": minor
+"@dpkit/zenodo": minor
+"@dpkit/arrow": minor
+"dpkit": minor
+"@dpkit/excel": minor
+"@dpkit/table": minor
+"@dpkit/ckan": minor
+"@dpkit/core": minor
+"@dpkit/file": minor
+"@dpkit/json": minor
+"@dpkit/test": minor
+"@dpkit/cli": minor
+"@dpkit/csv": minor
+"@dpkit/ods": minor
+"@dpkit/zip": minor
+"@dpkit/db": minor
+"@dpkit/ui": minor
+---
+
+Added Json/Jsonl/Arrow/Parquet support; Bootstrapped CLI

.gitignore

@@ -60,9 +60,10 @@ pids
 .vscode/
 .wrangler/
 build/
+compile/
 dist/
 .astro/
-/docs/content/docs/packages/
+/docs/content/docs/reference/
 **/.claude/settings.local.json
 AGENTS.md
 CLAUDE.md

arrow/CHANGELOG.md

@@ -0,0 +1,2 @@
+# @dpkit/arrow
+

bin/README.md → arrow/README.md

@@ -1,3 +1,3 @@
-# @dpkit/bin
+# @dpkit/arrow
 
 dpkit is a fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io).

arrow/index.ts

@@ -0,0 +1,2 @@
+export * from "./table/index.ts"
+export * from "./plugin.ts"

avro/package.json → arrow/package.json

@@ -1,8 +1,8 @@
 {
-  "name": "@dpkit/avro",
+  "name": "@dpkit/arrow",
   "type": "module",
-  "main": "build/index.js",
-  "version": "0.2.0",
+  "version": "0.1.0",
+  "exports": "./build/index.js",
   "license": "MIT",
   "author": "Evgeny Karev",
   "repository": "https://github.com/datisthq/dpkit",
@@ -17,9 +17,19 @@
     "validation",
     "quality",
     "fair",
-    "avro"
+    "parquet"
   ],
   "scripts": {
-    "build": "tsc --build"
+    "build": "tsc"
+  },
+  "dependencies": {
+    "@dpkit/core": "workspace:*",
+    "@dpkit/file": "workspace:*",
+    "@dpkit/table": "workspace:*",
+    "csv-sniffer": "^0.1.1",
+    "nodejs-polars": "^0.21.0"
+  },
+  "devDependencies": {
+    "@dpkit/test": "workspace:*"
   }
 }

arrow/plugin.ts

@@ -0,0 +1,26 @@
+import type { Resource } from "@dpkit/core"
+import { inferFormat } from "@dpkit/core"
+import type { TablePlugin } from "@dpkit/table"
+import type { SaveTableOptions, Table } from "@dpkit/table"
+import { loadArrowTable, saveArrowTable } from "./table/index.ts"
+
+export class ArrowPlugin implements TablePlugin {
+  async loadTable(resource: Partial<Resource>) {
+    const isArrow = getIsArrow(resource)
+    if (!isArrow) return undefined
+
+    return await loadArrowTable(resource)
+  }
+
+  async saveTable(table: Table, options: SaveTableOptions) {
+    const isArrow = getIsArrow(options)
+    if (!isArrow) return undefined
+
+    return await saveArrowTable(table, options)
+  }
+}
+
+function getIsArrow(resource: Partial<Resource>) {
+  const format = inferFormat(resource)
+  return format === "arrow" || format === "feather"
+}

arrow/table/index.ts

@@ -0,0 +1,2 @@
+export { loadArrowTable } from "./load.ts"
+export { saveArrowTable } from "./save.ts"

arrow/table/load.spec.ts

@@ -0,0 +1,64 @@
+import { getTempFilePath } from "@dpkit/file"
+import { useRecording } from "@dpkit/test"
+import { DataFrame } from "nodejs-polars"
+import { describe, expect, it } from "vitest"
+import { loadArrowTable } from "./load.ts"
+
+describe("loadArrowTable", () => {
+  useRecording()
+
+  describe("file variations", () => {
+    it("should load local file", async () => {
+      const path = getTempFilePath()
+      DataFrame({ id: [1, 2], name: ["english", "中文"] }).writeIPC(path)
+
+      const table = await loadArrowTable({ path })
+      expect((await table.collect()).toRecords()).toEqual([
+        { id: 1, name: "english" },
+        { id: 2, name: "中文" },
+      ])
+    })
+
+    it("should load local file (multipart)", async () => {
+      const path1 = getTempFilePath()
+      const path2 = getTempFilePath()
+      DataFrame({ id: [1, 2], name: ["english", "中文"] }).writeIPC(path1)
+      DataFrame({ id: [1, 2], name: ["english", "中文"] }).writeIPC(path2)
+
+      const table = await loadArrowTable({ path: [path1, path2] })
+      expect((await table.collect()).toRecords()).toEqual([
+        { id: 1, name: "english" },
+        { id: 2, name: "中文" },
+        { id: 1, name: "english" },
+        { id: 2, name: "中文" },
+      ])
+    })
+
+    it.skip("should load remote file", async () => {
+      const table = await loadArrowTable({
+        path: "https://github.com/frictionlessdata/frictionless-py/raw/refs/heads/main/data/table.arrow",
+      })
+
+      expect((await table.collect()).toRecords()).toEqual([
+        { id: 1, name: "english" },
+        { id: 2, name: "中国人" },
+      ])
+    })
+
+    it.skip("should load remote file (multipart)", async () => {
+      const table = await loadArrowTable({
+        path: [
+          "https://github.com/frictionlessdata/frictionless-py/raw/refs/heads/main/data/table.arrow",
+          "https://github.com/frictionlessdata/frictionless-py/raw/refs/heads/main/data/table.arrow",
+        ],
+      })
+
+      expect((await table.collect()).toRecords()).toEqual([
+        { id: 1, name: "english" },
+        { id: 2, name: "中国人" },
+        { id: 1, name: "english" },
+        { id: 2, name: "中国人" },
+      ])
+    })
+  })
+})

arrow/table/load.ts

@@ -0,0 +1,19 @@
+import type { Resource } from "@dpkit/core"
+import { prefetchFiles } from "@dpkit/file"
+import { concat } from "nodejs-polars"
+import { DataFrame } from "nodejs-polars"
+import { scanIPC } from "nodejs-polars"
+
+export async function loadArrowTable(resource: Partial<Resource>) {
+  const [firstPath, ...restPaths] = await prefetchFiles(resource.path)
+  if (!firstPath) {
+    return DataFrame().lazy()
+  }
+
+  let table = scanIPC(firstPath)
+  if (restPaths.length) {
+    table = concat([table, ...restPaths.map(path => scanIPC(path))])
+  }
+
+  return table
+}

arrow/table/save.spec.ts

@@ -0,0 +1,24 @@
+import { getTempFilePath } from "@dpkit/file"
+import { DataFrame } from "nodejs-polars"
+import { describe, expect, it } from "vitest"
+import { loadArrowTable } from "./load.ts"
+import { saveArrowTable } from "./save.ts"
+
+describe("saveArrowTable", () => {
+  it("should save table to Arrow file", async () => {
+    const path = getTempFilePath()
+    const source = DataFrame({
+      id: [1.0, 2.0, 3.0],
+      name: ["Alice", "Bob", "Charlie"],
+    }).lazy()
+
+    await saveArrowTable(source, { path })
+
+    const target = await loadArrowTable({ path })
+    expect((await target.collect()).toRecords()).toEqual([
+      { id: 1.0, name: "Alice" },
+      { id: 2.0, name: "Bob" },
+      { id: 3.0, name: "Charlie" },
+    ])
+  })
+})

arrow/table/save.ts

@@ -0,0 +1,13 @@
+import type { SaveTableOptions, Table } from "@dpkit/table"
+
+// TODO: rebase on sinkIPC when it is available
+// https://github.com/pola-rs/nodejs-polars/issues/353
+
+export async function saveArrowTable(table: Table, options: SaveTableOptions) {
+  const { path } = options
+
+  const df = await table.collect()
+  df.writeIPC(path)
+
+  return path
+}

arrow/tsconfig.json

@@ -0,0 +1,3 @@
+{
+  "extends": "../tsconfig.json"
+}

arrow/typedoc.json

@@ -0,0 +1,4 @@
+{
+  "entryPoints": ["index.ts"],
+  "skipErrorChecking": true
+}

camtrap/index.ts

@@ -1 +1 @@
-export * from "./package/index.js"
+export * from "./package/index.ts"

camtrap/package.json

@@ -1,8 +1,8 @@
 {
   "name": "@dpkit/camtrap",
   "type": "module",
-  "main": "build/index.js",
   "version": "0.5.1",
+  "exports": "./build/index.js",
   "license": "MIT",
   "author": "Evgeny Karev",
   "repository": "https://github.com/datisthq/dpkit",
@@ -20,7 +20,7 @@
     "camtrap"
   ],
   "scripts": {
-    "build": "tsc --build"
+    "build": "tsc"
   },
   "dependencies": {
     "@dpkit/core": "workspace:*"

camtrap/package/assert.ts

@@ -1,6 +1,6 @@
 import { AssertionError, validatePackageDescriptor } from "@dpkit/core"
 import type { Descriptor, Package } from "@dpkit/core"
-import type { CamtrapPackage } from "./Package.js"
+import type { CamtrapPackage } from "./Package.ts"
 
 const SUPPORTED_PROFILES = [
   "https://raw.githubusercontent.com/tdwg/camtrap-dp/1.0/camtrap-dp-profile.json",

camtrap/package/index.ts

@@ -1,2 +1,2 @@
-export type { CamtrapPackage } from "./Package.js"
-export { assertCamtrapPackage } from "./assert.js"
+export type { CamtrapPackage } from "./Package.ts"
+export { assertCamtrapPackage } from "./assert.ts"

camtrap/tsconfig.json

@@ -1,10 +1,3 @@
 {
-  "extends": "../tsconfig.json",
-  "include": ["**/*.ts"],
-  "exclude": ["**/build/"],
-  "compilerOptions": {
-    "outDir": "build",
-    "noEmit": false,
-    "declaration": true
-  }
+  "extends": "../tsconfig.json"
 }

camtrap/typedoc.json

@@ -1,5 +1,4 @@
 {
   "entryPoints": ["index.ts"],
-  "projectDocuments": ["CHANGELOG.md", "OVERVIEW.md"],
   "skipErrorChecking": true
 }

ckan/ckan/index.ts

@@ -0,0 +1 @@
+export { makeCkanApiRequest } from "./request.ts"

ckan/index.ts

@@ -1,4 +1,4 @@
-export * from "./package/index.js"
-export * from "./resource/index.js"
-export * from "./schema/index.js"
-export * from "./plugin.js"
+export * from "./package/index.ts"
+export * from "./resource/index.ts"
+export * from "./schema/index.ts"
+export * from "./plugin.ts"

ckan/package.json

@@ -1,8 +1,8 @@
 {
   "name": "@dpkit/ckan",
   "type": "module",
-  "main": "build/index.js",
   "version": "0.6.0",
+  "exports": "./build/index.js",
   "license": "MIT",
   "author": "Evgeny Karev",
   "repository": "https://github.com/datisthq/dpkit",
@@ -20,7 +20,7 @@
     "ckan"
   ],
   "scripts": {
-    "build": "tsc --build"
+    "build": "tsc"
   },
   "dependencies": {
     "@dpkit/core": "workspace:*",

ckan/package/Package.ts

@@ -1,6 +1,6 @@
-import type { CkanResource } from "../resource/index.js"
-import type { CkanOrganization } from "./Organization.js"
-import type { CkanTag } from "./Tag.js"
+import type { CkanResource } from "../resource/index.ts"
+import type { CkanOrganization } from "./Organization.ts"
+import type { CkanTag } from "./Tag.ts"
 
 /**
  * CKAN Package interface