From d4f641fb40bfd5a8109f53694baf09fb341c8091 Mon Sep 17 00:00:00 2001 From: roll Date: Tue, 12 Aug 2025 15:47:51 +0100 Subject: [PATCH 01/61] Added package copy command --- ckan/package/save.ts | 10 +++++----- cli/commands/package/copy.ts | 31 +++++++++++++++++++++++++++++++ cli/options/index.ts | 1 + cli/options/package.ts | 5 +++++ cli/options/path.ts | 5 +++++ folder/package/save.ts | 8 ++++---- github/package/save.ts | 14 +++++++------- zenodo/package/save.ts | 10 +++++----- zip/package/save.ts | 8 ++++---- 9 files changed, 67 insertions(+), 25 deletions(-) create mode 100644 cli/commands/package/copy.ts create mode 100644 cli/options/package.ts diff --git a/ckan/package/save.ts b/ckan/package/save.ts index 72fd4b81..34bc3cbe 100644 --- a/ckan/package/save.ts +++ b/ckan/package/save.ts @@ -54,20 +54,20 @@ export async function savePackageToCkan( basepath, withRemote: true, withoutFolders: true, - saveFile: async props => { - const filename = getFilename(props.normalizedPath) + saveFile: async options => { + const filename = getFilename(options.normalizedPath) const ckanResource = denormalizeCkanResource(resource) const payload = { ...ckanResource, package_id: datasetName, - name: props.denormalizedPath, + name: options.denormalizedPath, format: getFormat(filename)?.toUpperCase(), } const upload = { - name: props.denormalizedPath, - data: await blob(await loadFileStream(props.normalizedPath)), + name: options.denormalizedPath, + data: await blob(await loadFileStream(options.normalizedPath)), } const result = await makeCkanApiRequest({ diff --git a/cli/commands/package/copy.ts b/cli/commands/package/copy.ts new file mode 100644 index 00000000..499bc0c2 --- /dev/null +++ b/cli/commands/package/copy.ts @@ -0,0 +1,31 @@ +import { Command } from "@oclif/core" +import { loadPackage, savePackageToFolder } from "dpkit" +import * as options from "../../options/index.ts" +import * as params from "../../params/index.ts" + +export default class CopyPackage extends Command { + static override description = + "Copy a local or remote Data Package to a local folder" + + static override args = { + path: params.requriedDescriptorPath, + } + + static override flags = { + toFolder: options.requiredToFolder, + withRemote: options.withRemote, + } + + public async run() { + const { args, flags } = await this.parse(CopyPackage) + + const dp = await loadPackage(args.path) + + await savePackageToFolder(dp, { + folderPath: flags.toFolder, + withRemote: flags.withRemote, + }) + + this.log(`Package from "${args.path}" copied to "${flags.toFolder}"`) + } +} diff --git a/cli/options/index.ts b/cli/options/index.ts index ea9f8d80..f97054c6 100644 --- a/cli/options/index.ts +++ b/cli/options/index.ts @@ -1,5 +1,6 @@ export * from "./json.ts" export * from "./dialect.ts" +export * from "./package.ts" export * from "./path.ts" export * from "./resource.ts" export * from "./toDialect.ts" diff --git a/cli/options/package.ts b/cli/options/package.ts new file mode 100644 index 00000000..4c2dc69d --- /dev/null +++ b/cli/options/package.ts @@ -0,0 +1,5 @@ +import { Flags } from "@oclif/core" + +export const withRemote = Flags.boolean({ + description: "include remote resources", +}) diff --git a/cli/options/path.ts b/cli/options/path.ts index df4e97b5..d4557547 100644 --- a/cli/options/path.ts +++ b/cli/options/path.ts @@ -3,3 +3,8 @@ import { Flags } from "@oclif/core" export const toPath = Flags.string({ description: "a local output path", }) + +export const requiredToFolder = Flags.string({ + description: "a local output folder", + required: true, +}) diff --git a/folder/package/save.ts b/folder/package/save.ts index 2a81302e..84438028 100644 --- a/folder/package/save.ts +++ b/folder/package/save.ts @@ -28,13 +28,13 @@ export async function savePackageToFolder( await saveResourceFiles(resource, { basepath, withRemote, - saveFile: async props => { + saveFile: async options => { await copyFile({ - sourcePath: props.normalizedPath, - targetPath: props.denormalizedPath, + sourcePath: options.normalizedPath, + targetPath: join(folderPath, options.denormalizedPath), }) - return props.denormalizedPath + return options.denormalizedPath }, }), ) diff --git a/github/package/save.ts b/github/package/save.ts index acaa833d..9a6ad7ff 100644 --- a/github/package/save.ts +++ b/github/package/save.ts @@ -9,7 +9,7 @@ import type { GithubPackage } from "./Package.ts" /** * Save a package to a Github repository - * @param props Object containing the package to save and Github details + * @param options Object containing the package to save and Github details * @returns Object with the repository URL */ export async function savePackageToGithub( @@ -38,23 +38,23 @@ export async function savePackageToGithub( await saveResourceFiles(resource, { basepath, withRemote: false, - saveFile: async props => { - const stream = await loadFileStream(props.normalizedPath) + saveFile: async options => { + const stream = await loadFileStream(options.normalizedPath) const payload = { - path: props.denormalizedPath, + path: options.denormalizedPath, content: Buffer.from(await buffer(stream)).toString("base64"), - message: `Added file "${props.denormalizedPath}"`, + message: `Added file "${options.denormalizedPath}"`, } await makeGithubApiRequest({ - endpoint: `/repos/${githubPackage.owner.login}/${repo}/contents/${props.denormalizedPath}`, + endpoint: `/repos/${githubPackage.owner.login}/${repo}/contents/${options.denormalizedPath}`, method: "PUT", payload, apiKey, }) - return props.denormalizedPath + return options.denormalizedPath }, }), ) diff --git a/zenodo/package/save.ts b/zenodo/package/save.ts index c889f6b0..041c7e1f 100644 --- a/zenodo/package/save.ts +++ b/zenodo/package/save.ts @@ -12,7 +12,7 @@ import { denormalizeZenodoPackage } from "./process/denormalize.ts" /** * Save a package to Zenodo - * @param props Object containing the package to save and Zenodo API details + * @param options Object containing the package to save and Zenodo API details * @returns Object with the deposit URL and DOI */ export async function savePackageToZenodo( @@ -43,10 +43,10 @@ export async function savePackageToZenodo( basepath, withRemote: false, withoutFolders: true, - saveFile: async props => { + saveFile: async options => { const upload = { - name: props.denormalizedPath, - data: await blob(await loadFileStream(props.normalizedPath)), + name: options.denormalizedPath, + data: await blob(await loadFileStream(options.normalizedPath)), } // It seems that record and deposition files have different metadata @@ -59,7 +59,7 @@ export async function savePackageToZenodo( sandbox, }) - return props.denormalizedPath + return options.denormalizedPath }, }), ) diff --git a/zip/package/save.ts b/zip/package/save.ts index 59f6e071..ed6c930f 100644 --- a/zip/package/save.ts +++ b/zip/package/save.ts @@ -30,13 +30,13 @@ export async function savePackageToZip( await saveResourceFiles(resource, { basepath, withRemote, - saveFile: async props => { + saveFile: async options => { zipfile.addReadStream( - await loadFileStream(props.normalizedPath), - props.denormalizedPath, + await loadFileStream(options.normalizedPath), + options.denormalizedPath, ) - return props.denormalizedPath + return options.denormalizedPath }, }), ) From c005b892e344c0ee894f6a71b043ebf3dca46266 Mon Sep 17 00:00:00 2001 From: roll Date: Tue, 12 Aug 2025 16:22:56 +0100 Subject: [PATCH 02/61] Added visidata guide --- docs/content/docs/guides/inline.md | 2 +- docs/content/docs/guides/table.md | 2 +- docs/content/docs/guides/visidata.md | 30 ++++++++++++++++++++++++++++ 3 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 docs/content/docs/guides/visidata.md diff --git a/docs/content/docs/guides/inline.md b/docs/content/docs/guides/inline.md index 0ed632e2..30ffceb5 100644 --- a/docs/content/docs/guides/inline.md +++ b/docs/content/docs/guides/inline.md @@ -1,7 +1,7 @@ --- title: Working with inline data sidebar: - label: Inline data + label: Inline Data order: 5 --- diff --git a/docs/content/docs/guides/table.md b/docs/content/docs/guides/table.md index a996fc20..af9f83e2 100644 --- a/docs/content/docs/guides/table.md +++ b/docs/content/docs/guides/table.md @@ -1,7 +1,7 @@ --- title: Working with tabular data sidebar: - label: Tabular data + label: Tabular Data order: 6 --- diff --git a/docs/content/docs/guides/visidata.md b/docs/content/docs/guides/visidata.md new file mode 100644 index 00000000..0d5a8f92 --- /dev/null +++ b/docs/content/docs/guides/visidata.md @@ -0,0 +1,30 @@ +--- +title: Using dpkit with VisiData +sidebar: + label: VisiData + order: 11 +--- + +[VisiData](https://www.visidata.org/) provide a powerful and flexible environment for data exploration, visualization, and analysis. dpkit can be used as a tool that prepared data for following usage in VisiData. + +## Installation + +- [Install dpkit CLI](../../cli/installation) +- [Install VisiData](https://www.visidata.org/install/) + +## Usage + +For example, we can use dpkit to copy a remote Data Package to a local folder and then load it in VisiData. + +```bash +dpkit copy https://zenodo.org/records/7559361 --toFolder dataset --withRemote +vd dataset/*.csv +``` + +:::tip +Functionality allowing to copy a Data Package to a SQLite database is under development. Once it is ready, it will be possible to prepare data to VisiData in more type-safe manner. +::: + +## References + +- [Visidata](https://www.visidata.org/) From cdcf0827b53c8c6c376b2e3c7a4d93701f77cc69 Mon Sep 17 00:00:00 2001 From: roll Date: Tue, 12 Aug 2025 16:25:08 +0100 Subject: [PATCH 03/61] Renamed package load to show --- cli/commands/package/{load.ts => show.ts} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename cli/commands/package/{load.ts => show.ts} (72%) diff --git a/cli/commands/package/load.ts b/cli/commands/package/show.ts similarity index 72% rename from cli/commands/package/load.ts rename to cli/commands/package/show.ts index ce6c21b0..7c4a05be 100644 --- a/cli/commands/package/load.ts +++ b/cli/commands/package/show.ts @@ -3,8 +3,8 @@ import { loadPackage } from "dpkit" import * as options from "../../options/index.ts" import * as params from "../../params/index.ts" -export default class LoadPackage extends Command { - static override description = "Load a Data Package descriptor" +export default class ShowPackage extends Command { + static override description = "Show a Data Package descriptor" static override args = { path: params.requriedDescriptorPath, @@ -15,7 +15,7 @@ export default class LoadPackage extends Command { } public async run() { - const { args, flags } = await this.parse(LoadPackage) + const { args, flags } = await this.parse(ShowPackage) const dp = await loadPackage(args.path) From 774870e17285b627d855352be16deacffa9f290f Mon Sep 17 00:00:00 2001 From: roll Date: Tue, 12 Aug 2025 16:35:29 +0100 Subject: [PATCH 04/61] Added package archive command --- cli/commands/package/archive.ts | 31 +++++++++++++++++++++++++++++++ cli/options/path.ts | 7 ++++++- 2 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 cli/commands/package/archive.ts diff --git a/cli/commands/package/archive.ts b/cli/commands/package/archive.ts new file mode 100644 index 00000000..99d1109f --- /dev/null +++ b/cli/commands/package/archive.ts @@ -0,0 +1,31 @@ +import { Command } from "@oclif/core" +import { loadPackage, savePackageToZip } from "dpkit" +import * as options from "../../options/index.ts" +import * as params from "../../params/index.ts" + +export default class ArchivePackage extends Command { + static override description = + "Archive a local or remote Data Package to a local zip file" + + static override args = { + path: params.requriedDescriptorPath, + } + + static override flags = { + toArchive: options.requiredToArchive, + withRemote: options.withRemote, + } + + public async run() { + const { args, flags } = await this.parse(ArchivePackage) + + const dp = await loadPackage(args.path) + + await savePackageToZip(dp, { + archivePath: flags.toArchive, + withRemote: flags.withRemote, + }) + + this.log(`Package from "${args.path}" archived to "${flags.toArchive}"`) + } +} diff --git a/cli/options/path.ts b/cli/options/path.ts index d4557547..48f62810 100644 --- a/cli/options/path.ts +++ b/cli/options/path.ts @@ -5,6 +5,11 @@ export const toPath = Flags.string({ }) export const requiredToFolder = Flags.string({ - description: "a local output folder", + description: "a local output folder path", + required: true, +}) + +export const requiredToArchive = Flags.string({ + description: "a local output zip file path", required: true, }) From 10412f518199949a0bbe50196563e7d75a2f4d36 Mon Sep 17 00:00:00 2001 From: roll Date: Wed, 13 Aug 2025 08:37:41 +0100 Subject: [PATCH 05/61] Bootstrapped stricli --- cli/main.ts | 46 ++++++++++++++++++++++++++++++++++++++++++++++ cli/package.json | 1 + pnpm-lock.yaml | 8 ++++++++ 3 files changed, 55 insertions(+) create mode 100644 cli/main.ts diff --git a/cli/main.ts b/cli/main.ts new file mode 100644 index 00000000..a05807c1 --- /dev/null +++ b/cli/main.ts @@ -0,0 +1,46 @@ +import { + type CommandContext, + buildApplication, + buildCommand, + numberParser, +} from "@stricli/core" +import { run } from "@stricli/core" + +interface Flags { + item: string + price: number +} + +export const root = buildCommand({ + func(this: CommandContext, { item, price }: Flags) { + this.process.stdout.write(`${item}s cost $${price.toFixed(2)}`) + }, + parameters: { + flags: { + item: { + kind: "parsed", + parse: String, // Effectively a no-op + brief: "Item to display", + }, + price: { + kind: "parsed", + parse: numberParser, // Like Number() but throws on NaN + brief: "Price of the item", + }, + }, + }, + docs: { + brief: "Example for live playground with parsed flags", + customUsage: [ + "--item apple --price 1", + "--item orange --price 3.5", + "--item grape --price 6.25", + ], + }, +}) + +export const app = buildApplication(root, { + name: "stricli-node-example", +}) + +await run(app, process.argv.slice(2), { process }) diff --git a/cli/package.json b/cli/package.json index 9c4116a9..62e8dd12 100644 --- a/cli/package.json +++ b/cli/package.json @@ -32,6 +32,7 @@ "dependencies": { "@oclif/core": "^4.5.2", "@oclif/plugin-autocomplete": "^3.2.34", + "@stricli/core": "^1.2.0", "dpkit": "workspace:*", "ink": "^6.1.0", "ink-select-input": "^6.2.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8607a30f..b837a21d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -94,6 +94,9 @@ importers: '@oclif/plugin-autocomplete': specifier: ^3.2.34 version: 3.2.34 + '@stricli/core': + specifier: ^1.2.0 + version: 1.2.0 dpkit: specifier: workspace:* version: link:../dpkit @@ -1347,6 +1350,9 @@ packages: resolution: {integrity: sha512-b6heYM9dzZD13t2GOiEQTDE0qX+I1GyOotMwKh9VQqzuNiVdPVT8dM43fe9HNb/3ul+Qwd5oKSEDrDIfhq3bnQ==} engines: {node: '>=8'} + '@stricli/core@1.2.0': + resolution: {integrity: sha512-5b+npntDY0TAB7wAw0daGlh3/R2sf0TDLyrB1By2jCNH+C+lmcSqMtJXOMLVtEGSkIOvqAgIWpLMSs1PXqzt3w==} + '@swc/helpers@0.5.17': resolution: {integrity: sha512-5IKx/Y13RsYd+sauPb2x+U/xZikHjolzfuDgTAl/Tdf3Q8rslRvC19NKDLgAJQ6wsqADk10ntlv08nPFw/gO/A==} @@ -4988,6 +4994,8 @@ snapshots: escape-string-regexp: 1.0.5 lodash.deburr: 4.1.0 + '@stricli/core@1.2.0': {} + '@swc/helpers@0.5.17': dependencies: tslib: 2.8.1 From 9b36bb493445b251ca5b215ef7018438c5c88841 Mon Sep 17 00:00:00 2001 From: roll Date: Wed, 13 Aug 2025 12:05:02 +0100 Subject: [PATCH 06/61] Added new cli stack --- cli/commander.d.ts | 3 ++ cli/main.ts | 66 +++++++++++++---------------------- cli/package.json | 5 ++- cli/scripts/compile.ts | 2 +- pnpm-lock.yaml | 78 +++++++++++++++++++++++++++++++++++++----- 5 files changed, 102 insertions(+), 52 deletions(-) create mode 100644 cli/commander.d.ts diff --git a/cli/commander.d.ts b/cli/commander.d.ts new file mode 100644 index 00000000..abd5017f --- /dev/null +++ b/cli/commander.d.ts @@ -0,0 +1,3 @@ +declare module "commander" { + export * from "@commander-js/extra-typings" +} diff --git a/cli/main.ts b/cli/main.ts index a05807c1..fd127ab5 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -1,46 +1,28 @@ -import { - type CommandContext, - buildApplication, - buildCommand, - numberParser, -} from "@stricli/core" -import { run } from "@stricli/core" +import { styleText } from "node:util" +import { Command, Help, Option, program } from "commander" -interface Flags { - item: string - price: number -} +const testOption = new Option("-t, --test-option ") + +const testCommand = new Command("test").addOption(testOption).action(() => { + const options = testCommand.opts() // smart type + console.log(options) +}) // program type includes chained options and arguments -export const root = buildCommand({ - func(this: CommandContext, { item, price }: Flags) { - this.process.stdout.write(`${item}s cost $${price.toFixed(2)}`) - }, - parameters: { - flags: { - item: { - kind: "parsed", - parse: String, // Effectively a no-op - brief: "Item to display", - }, - price: { - kind: "parsed", - parse: numberParser, // Like Number() but throws on NaN - brief: "Price of the item", - }, - }, - }, - docs: { - brief: "Example for live playground with parsed flags", - customUsage: [ - "--item apple --price 1", - "--item orange --price 3.5", - "--item grape --price 6.25", - ], - }, -}) +const main = new Command("main").addCommand(testCommand) +const sub = new Command("sub").addCommand(testCommand) + +class CustomHelp extends Help { + styleTitle(str: string) { + // Make titles bold and uppercase + return `\x1b[1m${str.toUpperCase()}:\x1b[0m` + } +} -export const app = buildApplication(root, { - name: "stricli-node-example", -}) +const root = program + .addCommand(main) + .addCommand(sub) + .configureHelp({ + styleTitle: str => styleText("bold", str.toUpperCase().slice(0, -1)), + }) -await run(app, process.argv.slice(2), { process }) +root.parse() diff --git a/cli/package.json b/cli/package.json index 62e8dd12..3970c717 100644 --- a/cli/package.json +++ b/cli/package.json @@ -30,9 +30,12 @@ "run": "node ./scripts/run.ts" }, "dependencies": { + "@clack/prompts": "^0.11.0", + "@commander-js/extra-typings": "^14.0.0", + "@inkjs/ui": "^2.0.0", "@oclif/core": "^4.5.2", "@oclif/plugin-autocomplete": "^3.2.34", - "@stricli/core": "^1.2.0", + "commander": "^14.0.0", "dpkit": "workspace:*", "ink": "^6.1.0", "ink-select-input": "^6.2.0", diff --git a/cli/scripts/compile.ts b/cli/scripts/compile.ts index 6b919e34..9ffc9285 100644 --- a/cli/scripts/compile.ts +++ b/cli/scripts/compile.ts @@ -18,4 +18,4 @@ pnpm deploy compile // Compile application -await $({ cwd: "compile" })`deno compile --allow-all entrypoints/run.ts` +await $({ cwd: "compile" })`deno compile --allow-all main.ts` diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b837a21d..43138cbc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -88,15 +88,24 @@ importers: cli: dependencies: + '@clack/prompts': + specifier: ^0.11.0 + version: 0.11.0 + '@commander-js/extra-typings': + specifier: ^14.0.0 + version: 14.0.0(commander@14.0.0) + '@inkjs/ui': + specifier: ^2.0.0 + version: 2.0.0(ink@6.1.0(@types/react@19.1.9)(react@19.1.1)) '@oclif/core': specifier: ^4.5.2 version: 4.5.2 '@oclif/plugin-autocomplete': specifier: ^3.2.34 version: 3.2.34 - '@stricli/core': - specifier: ^1.2.0 - version: 1.2.0 + commander: + specifier: ^14.0.0 + version: 14.0.0 dpkit: specifier: workspace:* version: link:../dpkit @@ -601,6 +610,17 @@ packages: '@changesets/write@0.4.0': resolution: {integrity: sha512-CdTLvIOPiCNuH71pyDu3rA+Q0n65cmAbXnwWH84rKGiFumFzkmHNT8KHTMEchcxN+Kl8I54xGUhJ7l3E7X396Q==} + '@clack/core@0.5.0': + resolution: {integrity: sha512-p3y0FIOwaYRUPRcMO7+dlmLh8PSRcrjuTndsiA0WAFbWES0mLZlrjVoBRZ9DzkPFJZG6KGkJmoEAY0ZcVWTkow==} + + '@clack/prompts@0.11.0': + resolution: {integrity: sha512-pMN5FcrEw9hUkZA4f+zLlzivQSeQf5dRGJjSUbvVYDLvpKCdQx5OaknvKzgbtXOizhP+SJJJjqEbOe55uKKfAw==} + + '@commander-js/extra-typings@14.0.0': + resolution: {integrity: sha512-hIn0ncNaJRLkZrxBIp5AsW/eXEHNKYQBh0aPdoUqNgD+Io3NIykQqpKFyKcuasZhicGaEZJX/JBSIkZ4e5x8Dg==} + peerDependencies: + commander: ~14.0.0 + '@ctrl/tinycolor@4.1.0': resolution: {integrity: sha512-WyOx8cJQ+FQus4Mm4uPIZA64gbk3Wxh0so5Lcii0aJifqwoVOlfFtorjLE0Hen4OYyHZMXDWqMmaQemBhgxFRQ==} engines: {node: '>=14'} @@ -1006,6 +1026,12 @@ packages: cpu: [x64] os: [win32] + '@inkjs/ui@2.0.0': + resolution: {integrity: sha512-5+8fJmwtF9UvikzLfph9sA+LS+l37Ij/szQltkuXLOAXwNkBX9innfzh4pLGXIB59vKEQUtc6D4qGvhD7h3pAg==} + engines: {node: '>=18'} + peerDependencies: + ink: '>=5' + '@isaacs/cliui@8.0.2': resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} engines: {node: '>=12'} @@ -1350,9 +1376,6 @@ packages: resolution: {integrity: sha512-b6heYM9dzZD13t2GOiEQTDE0qX+I1GyOotMwKh9VQqzuNiVdPVT8dM43fe9HNb/3ul+Qwd5oKSEDrDIfhq3bnQ==} engines: {node: '>=8'} - '@stricli/core@1.2.0': - resolution: {integrity: sha512-5b+npntDY0TAB7wAw0daGlh3/R2sf0TDLyrB1By2jCNH+C+lmcSqMtJXOMLVtEGSkIOvqAgIWpLMSs1PXqzt3w==} - '@swc/helpers@0.5.17': resolution: {integrity: sha512-5IKx/Y13RsYd+sauPb2x+U/xZikHjolzfuDgTAl/Tdf3Q8rslRvC19NKDLgAJQ6wsqADk10ntlv08nPFw/gO/A==} @@ -1714,6 +1737,10 @@ packages: resolution: {integrity: sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==} engines: {node: '>=6'} + cli-spinners@3.2.0: + resolution: {integrity: sha512-pXftdQloMZzjCr3pCTIRniDcys6dDzgpgVhAHHk6TKBDbRuP1MkuetTF5KSv4YUutbOPa7+7ZrAJ2kVtbMqyXA==} + engines: {node: '>=18.20'} + cli-truncate@4.0.0: resolution: {integrity: sha512-nPdaFdQ0h/GEigbPClz11D0v/ZJEwxmeVZGeMo3Z5StPtUTkA9o1lD6QwoirYiSDzbcwn2XcjwmCp68W1IS4TA==} engines: {node: '>=18'} @@ -1750,6 +1777,10 @@ packages: comma-separated-tokens@2.0.3: resolution: {integrity: sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==} + commander@14.0.0: + resolution: {integrity: sha512-2uM9rYjPvyq39NwLRqaiLtWHyDC1FvryJDa2ATTVims5YAS4PupsEQsDvP14FqhFr0P49CYDugi59xaxJlTXRA==} + engines: {node: '>=20'} + common-ancestor-path@1.0.1: resolution: {integrity: sha512-L3sHRo1pXXEqX8VU28kfgUY+YGsk09hPqZiZmLacNib6XNTCM8ubYeT7ryXQw8asB1sKgcU5lkB7ONug08aB8w==} @@ -1839,6 +1870,10 @@ packages: resolution: {integrity: sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==} engines: {node: '>=6'} + deepmerge@4.3.1: + resolution: {integrity: sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==} + engines: {node: '>=0.10.0'} + define-data-property@1.1.4: resolution: {integrity: sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==} engines: {node: '>= 0.4'} @@ -4335,6 +4370,21 @@ snapshots: human-id: 4.1.1 prettier: 2.8.8 + '@clack/core@0.5.0': + dependencies: + picocolors: 1.1.1 + sisteransi: 1.0.5 + + '@clack/prompts@0.11.0': + dependencies: + '@clack/core': 0.5.0 + picocolors: 1.1.1 + sisteransi: 1.0.5 + + '@commander-js/extra-typings@14.0.0(commander@14.0.0)': + dependencies: + commander: 14.0.0 + '@ctrl/tinycolor@4.1.0': {} '@emnapi/core@1.4.5': @@ -4620,6 +4670,14 @@ snapshots: '@img/sharp-win32-x64@0.34.2': optional: true + '@inkjs/ui@2.0.0(ink@6.1.0(@types/react@19.1.9)(react@19.1.1))': + dependencies: + chalk: 5.5.0 + cli-spinners: 3.2.0 + deepmerge: 4.3.1 + figures: 6.1.0 + ink: 6.1.0(@types/react@19.1.9)(react@19.1.1) + '@isaacs/cliui@8.0.2': dependencies: string-width: 5.1.2 @@ -4994,8 +5052,6 @@ snapshots: escape-string-regexp: 1.0.5 lodash.deburr: 4.1.0 - '@stricli/core@1.2.0': {} - '@swc/helpers@0.5.17': dependencies: tslib: 2.8.1 @@ -5459,6 +5515,8 @@ snapshots: cli-spinners@2.9.2: {} + cli-spinners@3.2.0: {} + cli-truncate@4.0.0: dependencies: slice-ansi: 5.0.0 @@ -5492,6 +5550,8 @@ snapshots: comma-separated-tokens@2.0.3: {} + commander@14.0.0: {} + common-ancestor-path@1.0.1: {} content-disposition@0.5.4: @@ -5564,6 +5624,8 @@ snapshots: deep-eql@5.0.2: {} + deepmerge@4.3.1: {} + define-data-property@1.1.4: dependencies: es-define-property: 1.0.1 From 1cd3e3c12b988b0dfdbfdff8f3d30549f66b99a7 Mon Sep 17 00:00:00 2001 From: roll Date: Wed, 13 Aug 2025 16:37:35 +0100 Subject: [PATCH 07/61] Migrated to commander --- cli/.oclifrc.js | 6 - cli/commands/dialect/index.ts | 6 + cli/commands/dialect/infer.ts | 31 ++--- cli/commands/package/archive.ts | 36 ++---- cli/commands/package/copy.ts | 36 ++---- cli/commands/package/index.ts | 11 ++ cli/commands/package/show.ts | 31 ++--- cli/commands/table/convert.tsx | 84 ++++++++------ cli/commands/table/describe.tsx | 51 ++++---- cli/commands/table/explore.tsx | 50 ++++---- cli/commands/table/index.ts | 16 +++ cli/commands/table/query.tsx | 47 ++++---- cli/commands/table/script.tsx | 48 ++++---- cli/commands/table/validate.tsx | 67 +++++------ cli/entrypoints/dev.ts | 3 +- cli/entrypoints/run.ts | 3 +- cli/main.ts | 38 +++--- cli/options/dialect.ts | 178 ---------------------------- cli/options/index.ts | 6 - cli/options/json.ts | 6 - cli/options/package.ts | 5 - cli/options/path.ts | 15 --- cli/options/resource.ts | 5 - cli/options/toDialect.ts | 180 ----------------------------- cli/package.json | 4 +- cli/params/dialect.ts | 192 +++++++++++++++++++++++++++++++ cli/params/index.ts | 5 + cli/params/json.ts | 3 + cli/params/package.ts | 6 + cli/params/path.ts | 30 +++-- cli/params/resource.ts | 6 + cli/params/toDialect.ts | 192 +++++++++++++++++++++++++++++++ cli/program.ts | 22 ++++ pnpm-lock.yaml | 198 ++------------------------------ 34 files changed, 743 insertions(+), 874 deletions(-) delete mode 100644 cli/.oclifrc.js create mode 100644 cli/commands/dialect/index.ts create mode 100644 cli/commands/package/index.ts create mode 100644 cli/commands/table/index.ts delete mode 100644 cli/options/dialect.ts delete mode 100644 cli/options/index.ts delete mode 100644 cli/options/json.ts delete mode 100644 cli/options/package.ts delete mode 100644 cli/options/path.ts delete mode 100644 cli/options/resource.ts delete mode 100644 cli/options/toDialect.ts create mode 100644 cli/params/dialect.ts create mode 100644 cli/params/json.ts create mode 100644 cli/params/package.ts create mode 100644 cli/params/resource.ts create mode 100644 cli/params/toDialect.ts create mode 100644 cli/program.ts diff --git a/cli/.oclifrc.js b/cli/.oclifrc.js deleted file mode 100644 index 726cf92b..00000000 --- a/cli/.oclifrc.js +++ /dev/null @@ -1,6 +0,0 @@ -export default { - bin: "dp", - commands: "./commands", - topicSeparator: " ", - plugins: ["@oclif/plugin-autocomplete"], -} diff --git a/cli/commands/dialect/index.ts b/cli/commands/dialect/index.ts new file mode 100644 index 00000000..8e836555 --- /dev/null +++ b/cli/commands/dialect/index.ts @@ -0,0 +1,6 @@ +import { Command } from "commander" +import { inferDialectCommand } from "./infer.ts" + +export const dialectCommand = new Command("dialect") + .description("Table Dialect related commands") + .addCommand(inferDialectCommand) diff --git a/cli/commands/dialect/infer.ts b/cli/commands/dialect/infer.ts index 6c9ca7b8..c812a873 100644 --- a/cli/commands/dialect/infer.ts +++ b/cli/commands/dialect/infer.ts @@ -1,29 +1,18 @@ -import { Command } from "@oclif/core" +import { Command } from "commander" import { inferDialect } from "dpkit" -import * as options from "../../options/index.ts" import * as params from "../../params/index.ts" -export default class InferDialect extends Command { - static override description = "Infer a dialect from a table" +export const inferDialectCommand = new Command("infer") + .description("Infer a dialect from a table") + .addArgument(params.positionalTablePath) + .addOption(params.json) + .action(async (path, options) => { + const dialect = await inferDialect({ path }) - static override args = { - path: params.requriedTablePath, - } - - static override flags = { - json: options.json, - } - - public async run() { - const { args, flags } = await this.parse(InferDialect) - - const dialect = await inferDialect({ path: args.path }) - - if (flags.json) { - this.logJson(dialect) + if (options.json) { + console.log(JSON.stringify(dialect, null, 2)) return } console.log(dialect) - } -} + }) diff --git a/cli/commands/package/archive.ts b/cli/commands/package/archive.ts index 99d1109f..92c5e655 100644 --- a/cli/commands/package/archive.ts +++ b/cli/commands/package/archive.ts @@ -1,31 +1,19 @@ -import { Command } from "@oclif/core" +import { Command } from "commander" import { loadPackage, savePackageToZip } from "dpkit" -import * as options from "../../options/index.ts" import * as params from "../../params/index.ts" -export default class ArchivePackage extends Command { - static override description = - "Archive a local or remote Data Package to a local zip file" - - static override args = { - path: params.requriedDescriptorPath, - } - - static override flags = { - toArchive: options.requiredToArchive, - withRemote: options.withRemote, - } - - public async run() { - const { args, flags } = await this.parse(ArchivePackage) - - const dp = await loadPackage(args.path) +export const archivePackageCommand = new Command("archive") + .description("Archive a local or remote Data Package to a local zip file") + .addArgument(params.positionalDescriptorPath) + .addOption(params.toArchive.makeOptionMandatory()) + .addOption(params.withRemote) + .action(async (path, options) => { + const dp = await loadPackage(path) await savePackageToZip(dp, { - archivePath: flags.toArchive, - withRemote: flags.withRemote, + archivePath: options.toArchive, + withRemote: options.withRemote, }) - this.log(`Package from "${args.path}" archived to "${flags.toArchive}"`) - } -} + console.log(`Package from "${path}" archived to "${options.toArchive}"`) + }) diff --git a/cli/commands/package/copy.ts b/cli/commands/package/copy.ts index 499bc0c2..685ac649 100644 --- a/cli/commands/package/copy.ts +++ b/cli/commands/package/copy.ts @@ -1,31 +1,19 @@ -import { Command } from "@oclif/core" +import { Command } from "commander" import { loadPackage, savePackageToFolder } from "dpkit" -import * as options from "../../options/index.ts" import * as params from "../../params/index.ts" -export default class CopyPackage extends Command { - static override description = - "Copy a local or remote Data Package to a local folder" - - static override args = { - path: params.requriedDescriptorPath, - } - - static override flags = { - toFolder: options.requiredToFolder, - withRemote: options.withRemote, - } - - public async run() { - const { args, flags } = await this.parse(CopyPackage) - - const dp = await loadPackage(args.path) +export const copyPackageCommand = new Command("copy") + .description("Copy a local or remote Data Package to a local folder") + .addArgument(params.positionalDescriptorPath) + .addOption(params.toFolder.makeOptionMandatory()) + .addOption(params.withRemote) + .action(async (path, options) => { + const dp = await loadPackage(path) await savePackageToFolder(dp, { - folderPath: flags.toFolder, - withRemote: flags.withRemote, + folderPath: options.toFolder, + withRemote: options.withRemote, }) - this.log(`Package from "${args.path}" copied to "${flags.toFolder}"`) - } -} + console.log(`Package from "${path}" copied to "${options.toFolder}"`) + }) diff --git a/cli/commands/package/index.ts b/cli/commands/package/index.ts new file mode 100644 index 00000000..45533ad7 --- /dev/null +++ b/cli/commands/package/index.ts @@ -0,0 +1,11 @@ +import { Command } from "commander" +import { archivePackageCommand } from "./archive.ts" +import { copyPackageCommand } from "./copy.ts" +import { showPackageCommand } from "./show.ts" + +export const packageCommand = new Command("package") + .description("Data Package related commands") + .addCommand(showPackageCommand) + .addCommand(archivePackageCommand) + .addCommand(copyPackageCommand) + diff --git a/cli/commands/package/show.ts b/cli/commands/package/show.ts index 7c4a05be..feb0781b 100644 --- a/cli/commands/package/show.ts +++ b/cli/commands/package/show.ts @@ -1,29 +1,18 @@ -import { Command } from "@oclif/core" +import { Command } from "commander" import { loadPackage } from "dpkit" -import * as options from "../../options/index.ts" import * as params from "../../params/index.ts" -export default class ShowPackage extends Command { - static override description = "Show a Data Package descriptor" +export const showPackageCommand = new Command("show") + .description("Show a Data Package descriptor") + .addArgument(params.positionalDescriptorPath) + .addOption(params.json) + .action(async (path, options) => { + const dp = await loadPackage(path) - static override args = { - path: params.requriedDescriptorPath, - } - - static override flags = { - json: options.json, - } - - public async run() { - const { args, flags } = await this.parse(ShowPackage) - - const dp = await loadPackage(args.path) - - if (flags.json) { - this.logJson(dp) + if (options.json) { + console.log(JSON.stringify(dp, null, 2)) return } console.log(dp) - } -} + }) diff --git a/cli/commands/table/convert.tsx b/cli/commands/table/convert.tsx index cfc29fe2..7f3978aa 100644 --- a/cli/commands/table/convert.tsx +++ b/cli/commands/table/convert.tsx @@ -1,46 +1,66 @@ -import { Command } from "@oclif/core" +import { Command } from "commander" import { getTempFilePath, loadFile } from "dpkit" import { readTable, saveTable } from "dpkit" -import * as options from "../../options/index.ts" import * as params from "../../params/index.ts" -export default class ConvertTable extends Command { - static override description = - "Convert a table from a local or remote source path to a target path" +export const convertTableCommand = new Command("convert") + .description( + "Convert a table from a local or remote source path to a target path", + ) + .addArgument(params.positionalTablePath) + .addOption(params.toPath) + .addOption(params.toFormat) + .addOption(params.delimiter) + .addOption(params.header) + .addOption(params.headerRows) + .addOption(params.headerJoin) + .addOption(params.commentRows) + .addOption(params.commentChar) + .addOption(params.quoteChar) + .addOption(params.doubleQuote) + .addOption(params.escapeChar) + .addOption(params.nullSequence) + .addOption(params.skipInitialSpace) + .addOption(params.property) + .addOption(params.itemType) + .addOption(params.itemKeys) + .addOption(params.sheetNumber) + .addOption(params.sheetName) + .addOption(params.table) + .addOption(params.toDelimiter) + .addOption(params.toHeader) + .addOption(params.toHeaderRows) + .addOption(params.toHeaderJoin) + .addOption(params.toCommentRows) + .addOption(params.toCommentChar) + .addOption(params.toQuoteChar) + .addOption(params.toDoubleQuote) + .addOption(params.toEscapeChar) + .addOption(params.toNullSequence) + .addOption(params.toSkipInitialSpace) + .addOption(params.toProperty) + .addOption(params.toItemType) + .addOption(params.toItemKeys) + .addOption(params.toSheetNumber) + .addOption(params.toSheetName) + .addOption(params.toTable) + .action(async (path, options) => { + const dialect = params.createDialectFromOptions(options) + const table = await readTable({ path, dialect }) - static override args = { - path: params.requriedTablePath, - } - - // TODO: support toDialectOptions - static override flags = { - toPath: options.toPath, - toFormat: options.toFormat, - ...options.dialectOptions, - ...options.toDialectOptions, - } - - public async run() { - const { args, flags } = await this.parse(ConvertTable) - - const dialect = options.createDialectFromFlags(flags) - const table = await readTable({ path: args.path, dialect }) - - const toPath = flags.toPath ?? getTempFilePath() - const toDialect = options.createToDialectFromFlags(flags) + const toPath = options.toPath ?? getTempFilePath() + const toDialect = params.createToDialectFromOptions(options) await saveTable(table, { path: toPath, - format: flags.toFormat, + format: options.toFormat, dialect: toDialect, }) - if (!flags.toPath) { - // TODO: stream to stdout + if (!options.toPath) { const buffer = await loadFile(toPath) - this.log(buffer.toString().trim()) + console.log(buffer.toString().trim()) return } - this.log(`Converted table from ${args.path} to ${flags.toPath}`) - } -} + console.log(`Converted table from ${path} to ${options.toPath}`) + }) diff --git a/cli/commands/table/describe.tsx b/cli/commands/table/describe.tsx index 4e7bc690..681bd4b1 100644 --- a/cli/commands/table/describe.tsx +++ b/cli/commands/table/describe.tsx @@ -1,37 +1,42 @@ -import { Command } from "@oclif/core" +import { Command } from "commander" import { readTable } from "dpkit" import { render } from "ink" import React from "react" import { TableGrid } from "../../components/TableGrid.tsx" -import * as options from "../../options/index.ts" import * as params from "../../params/index.ts" -export default class DescribeTable extends Command { - static override description = "Describe a table from a local or remote path" - - static override args = { - path: params.requriedTablePath, - } - - static override flags = { - ...options.dialectOptions, - json: options.json, - } - - public async run() { - const { args, flags } = await this.parse(DescribeTable) - - const dialect = options.createDialectFromFlags(flags) - const table = await readTable({ path: args.path, dialect }) +export const describeTableCommand = new Command("describe") + .description("Describe a table from a local or remote path") + .addArgument(params.positionalTablePath) + .addOption(params.json) + .addOption(params.delimiter) + .addOption(params.header) + .addOption(params.headerRows) + .addOption(params.headerJoin) + .addOption(params.commentRows) + .addOption(params.commentChar) + .addOption(params.quoteChar) + .addOption(params.doubleQuote) + .addOption(params.escapeChar) + .addOption(params.nullSequence) + .addOption(params.skipInitialSpace) + .addOption(params.property) + .addOption(params.itemType) + .addOption(params.itemKeys) + .addOption(params.sheetNumber) + .addOption(params.sheetName) + .addOption(params.table) + .action(async (path, options) => { + const dialect = params.createDialectFromOptions(options) + const table = await readTable({ path, dialect }) const df = await table.collect() const stats = df.describe() - if (flags.json) { - this.logJson(stats) + if (options.json) { + console.log(JSON.stringify(stats, null, 2)) return } render() - } -} + }) diff --git a/cli/commands/table/explore.tsx b/cli/commands/table/explore.tsx index 76bc38ed..b24e2f7e 100644 --- a/cli/commands/table/explore.tsx +++ b/cli/commands/table/explore.tsx @@ -1,35 +1,41 @@ -import { Command } from "@oclif/core" +import { Command } from "commander" import { readTable } from "dpkit" import { render } from "ink" import React from "react" import { TableGrid } from "../../components/TableGrid.tsx" -import * as options from "../../options/index.ts" import * as params from "../../params/index.ts" -export default class ExploreTable extends Command { - static override description = "Explore a table from a local or remote path" +export const exploreTableCommand = new Command("explore") + .description("Explore a table from a local or remote path") + .addArgument(params.positionalTablePath) + .addOption(params.json) + .addOption(params.delimiter) + .addOption(params.header) + .addOption(params.headerRows) + .addOption(params.headerJoin) + .addOption(params.commentRows) + .addOption(params.commentChar) + .addOption(params.quoteChar) + .addOption(params.doubleQuote) + .addOption(params.escapeChar) + .addOption(params.nullSequence) + .addOption(params.skipInitialSpace) + .addOption(params.property) + .addOption(params.itemType) + .addOption(params.itemKeys) + .addOption(params.sheetNumber) + .addOption(params.sheetName) + .addOption(params.table) + .action(async (path, options) => { + const dialect = params.createDialectFromOptions(options) + const table = await readTable({ path, dialect }) - static override args = { - path: params.requriedTablePath, - } - - static override flags = { - ...options.dialectOptions, - } - - public async run() { - const { args, flags } = await this.parse(ExploreTable) - - const dialect = options.createDialectFromFlags(flags) - const table = await readTable({ path: args.path, dialect }) - - if (flags.json) { + if (options.json) { const df = await table.slice(0, 10).collect() const data = df.toRecords() - this.logJson(data) + console.log(JSON.stringify(data, null, 2)) return } render() - } -} + }) diff --git a/cli/commands/table/index.ts b/cli/commands/table/index.ts new file mode 100644 index 00000000..436a40a6 --- /dev/null +++ b/cli/commands/table/index.ts @@ -0,0 +1,16 @@ +import { Command } from "commander" +import { convertTableCommand } from "./convert.tsx" +import { describeTableCommand } from "./describe.tsx" +import { exploreTableCommand } from "./explore.tsx" +import { queryTableCommand } from "./query.tsx" +import { scriptTableCommand } from "./script.tsx" +import { validateTableCommand } from "./validate.tsx" + +export const tableCommand = new Command("table") + .description("Table related commands") + .addCommand(describeTableCommand) + .addCommand(convertTableCommand) + .addCommand(validateTableCommand) + .addCommand(queryTableCommand) + .addCommand(scriptTableCommand) + .addCommand(exploreTableCommand) diff --git a/cli/commands/table/query.tsx b/cli/commands/table/query.tsx index acd278a9..a4696743 100644 --- a/cli/commands/table/query.tsx +++ b/cli/commands/table/query.tsx @@ -1,22 +1,29 @@ -import { Command } from "@oclif/core" -//import { readTable } from "dpkit" -import * as options from "../../options/index.ts" +import { Command } from "commander" import * as params from "../../params/index.ts" -export default class QueryTable extends Command { - static override description = - "Start a querying session for a table from a local or remote path" - - static override args = { - path: params.requriedTablePath, - } - - static override flags = { - ...options.dialectOptions, - } - - public async run() { - //const { args, flags } = await this.parse(QueryTable) - throw new Error("Not implemented") - } -} +export const queryTableCommand = new Command("query") + .description( + "Start a querying session for a table from a local or remote path", + ) + .addArgument(params.positionalTablePath) + .addOption(params.json) + .addOption(params.delimiter) + .addOption(params.header) + .addOption(params.headerRows) + .addOption(params.headerJoin) + .addOption(params.commentRows) + .addOption(params.commentChar) + .addOption(params.quoteChar) + .addOption(params.doubleQuote) + .addOption(params.escapeChar) + .addOption(params.nullSequence) + .addOption(params.skipInitialSpace) + .addOption(params.property) + .addOption(params.itemType) + .addOption(params.itemKeys) + .addOption(params.sheetNumber) + .addOption(params.sheetName) + .addOption(params.table) + .action(async (path, options) => { + throw new Error("Query command not implemented yet") + }) diff --git a/cli/commands/table/script.tsx b/cli/commands/table/script.tsx index 102dd237..36b7e745 100644 --- a/cli/commands/table/script.tsx +++ b/cli/commands/table/script.tsx @@ -1,28 +1,34 @@ import repl from "node:repl" -import { Command } from "@oclif/core" +import { Command } from "commander" import { readTable } from "dpkit" -import * as options from "../../options/index.ts" import * as params from "../../params/index.ts" -export default class ScriptTable extends Command { - static override description = - "Start a scripting session for a table from a local or remote path" - - static override args = { - path: params.requriedTablePath, - } - - static override flags = { - ...options.dialectOptions, - } - - public async run() { - const { args, flags } = await this.parse(ScriptTable) - - const dialect = options.createDialectFromFlags(flags) - const table = await readTable({ path: args.path, dialect }) +export const scriptTableCommand = new Command("script") + .description( + "Start a scripting session for a table from a local or remote path", + ) + .addArgument(params.positionalTablePath) + .addOption(params.delimiter) + .addOption(params.header) + .addOption(params.headerRows) + .addOption(params.headerJoin) + .addOption(params.commentRows) + .addOption(params.commentChar) + .addOption(params.quoteChar) + .addOption(params.doubleQuote) + .addOption(params.escapeChar) + .addOption(params.nullSequence) + .addOption(params.skipInitialSpace) + .addOption(params.property) + .addOption(params.itemType) + .addOption(params.itemKeys) + .addOption(params.sheetNumber) + .addOption(params.sheetName) + .addOption(params.table) + .action(async (path, options) => { + const dialect = params.createDialectFromOptions(options) + const table = await readTable({ path, dialect }) const session = repl.start({ prompt: "dp> " }) session.context.table = table - } -} + }) diff --git a/cli/commands/table/validate.tsx b/cli/commands/table/validate.tsx index d3571d8f..72c83fa9 100644 --- a/cli/commands/table/validate.tsx +++ b/cli/commands/table/validate.tsx @@ -1,46 +1,39 @@ -import { Command } from "@oclif/core" +import { Command } from "commander" import { validateTable } from "dpkit" import { render } from "ink" import React from "react" -import { ErrorGrid } from "../../components/ErrorGrid.tsx" -import * as options from "../../options/index.ts" +import { ErrorGrid } from "../../components/ErrorGrid.jsx" import * as params from "../../params/index.ts" -export default class ExploreTable extends Command { - static override description = "Explore a table from a local or remote path" +export const validateTableCommand = new Command("validate") + .description("Validate a table from a local or remote path") + .addArgument(params.positionalTablePath) + .addOption(params.json) + .addOption(params.delimiter) + .addOption(params.header) + .addOption(params.headerRows) + .addOption(params.headerJoin) + .addOption(params.commentRows) + .addOption(params.commentChar) + .addOption(params.quoteChar) + .addOption(params.doubleQuote) + .addOption(params.escapeChar) + .addOption(params.nullSequence) + .addOption(params.skipInitialSpace) + .addOption(params.property) + .addOption(params.itemType) + .addOption(params.itemKeys) + .addOption(params.sheetNumber) + .addOption(params.sheetName) + .addOption(params.table) + .action(async (path, options) => { + const dialect = params.createDialectFromOptions(options) + const { errors } = await validateTable({ path, dialect }) - static override args = { - path: params.requriedTablePath, - } - - static override flags = { - ...options.dialectOptions, - json: options.json, - } - - public async run() { - const { args, flags } = await this.parse(ExploreTable) - - const dialect = options.createDialectFromFlags(flags) - const { errors } = await validateTable({ path: args.path, dialect }) - - if (flags.json) { - this.logJson(errors) + if (options.json) { + console.log(JSON.stringify(errors, null, 2)) return } - render( - , - ) - } -} + render() + }) diff --git a/cli/entrypoints/dev.ts b/cli/entrypoints/dev.ts index cefc0f18..5a47c26e 100755 --- a/cli/entrypoints/dev.ts +++ b/cli/entrypoints/dev.ts @@ -1,4 +1,3 @@ #!cli/node_modules/.bin/tsx -import { execute } from "@oclif/core" -await execute({ dir: import.meta.url, development: true }) +await import("../main.ts") diff --git a/cli/entrypoints/run.ts b/cli/entrypoints/run.ts index c036c876..9dc8de9e 100755 --- a/cli/entrypoints/run.ts +++ b/cli/entrypoints/run.ts @@ -1,4 +1,3 @@ #!/usr/bin/env node -import { execute } from "@oclif/core" -await execute({ dir: import.meta.url }) +await import("../main.ts") diff --git a/cli/main.ts b/cli/main.ts index fd127ab5..b4d0bf2c 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -1,28 +1,20 @@ import { styleText } from "node:util" -import { Command, Help, Option, program } from "commander" +import { program } from "commander" +import { dialectCommand } from "./commands/dialect/index.ts" +import { packageCommand } from "./commands/package/index.ts" +import { tableCommand } from "./commands/table/index.ts" +import metadata from "./package.json" with { type: "json" } -const testOption = new Option("-t, --test-option ") - -const testCommand = new Command("test").addOption(testOption).action(() => { - const options = testCommand.opts() // smart type - console.log(options) -}) // program type includes chained options and arguments - -const main = new Command("main").addCommand(testCommand) -const sub = new Command("sub").addCommand(testCommand) - -class CustomHelp extends Help { - styleTitle(str: string) { - // Make titles bold and uppercase - return `\x1b[1m${str.toUpperCase()}:\x1b[0m` - } -} - -const root = program - .addCommand(main) - .addCommand(sub) +program + .name("dp") + .description( + "Fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames", + ) + .version(metadata.version) .configureHelp({ styleTitle: str => styleText("bold", str.toUpperCase().slice(0, -1)), }) - -root.parse() + .addCommand(packageCommand) + .addCommand(tableCommand) + .addCommand(dialectCommand) + .parse() diff --git a/cli/options/dialect.ts b/cli/options/dialect.ts deleted file mode 100644 index d5db981e..00000000 --- a/cli/options/dialect.ts +++ /dev/null @@ -1,178 +0,0 @@ -import { Flags } from "@oclif/core" -import type { Dialect } from "dpkit" - -export const header = Flags.boolean({ - description: "whether the file includes a header row with field names", - default: true, -}) - -export const headerRows = Flags.string({ - description: - "comma-separated row numbers (zero-based) that are considered header rows", -}) - -export const headerJoin = Flags.string({ - description: "character used to join multi-line headers", -}) - -export const commentRows = Flags.string({ - description: "comma-separated rows to be excluded from the data (zero-based)", -}) - -export const commentChar = Flags.string({ - description: "character sequence denoting the start of a comment line", -}) - -export const delimiter = Flags.string({ - description: "character used to separate fields in the data", - char: "d", -}) - -export const lineTerminator = Flags.string({ - description: "character sequence used to terminate rows", -}) - -export const quoteChar = Flags.string({ - description: "character used to quote fields", -}) - -export const doubleQuote = Flags.boolean({ - description: - "whether a sequence of two quote characters represents a single quote", -}) - -export const escapeChar = Flags.string({ - description: "character used to escape the delimiter or quote characters", -}) - -export const nullSequence = Flags.string({ - description: - "character sequence representing null or missing values in the data", -}) - -export const skipInitialSpace = Flags.boolean({ - description: - "whether to ignore whitespace immediately following the delimiter", -}) - -export const property = Flags.string({ - description: "for JSON data, the property name containing the data array", -}) - -export const itemType = Flags.string({ - description: "the type of data item in the source", - options: ["array", "object"], -}) - -export const itemKeys = Flags.string({ - description: - "comma-separated object properties to extract as values (for object-based data items)", -}) - -export const sheetNumber = Flags.integer({ - description: "for spreadsheet data, the sheet number to read (zero-based)", -}) - -export const sheetName = Flags.string({ - description: "for spreadsheet data, the sheet name to read", -}) - -export const table = Flags.string({ - description: "for database sources, the table name to read", -}) - -export const dialectOptions = { - delimiter, - header, - headerRows, - headerJoin, - commentRows, - commentChar, - quoteChar, - doubleQuote, - escapeChar, - nullSequence, - skipInitialSpace, - property, - itemType, - itemKeys, - sheetNumber, - table, -} - -// TODO: rebase on types flags -export function createDialectFromFlags(flags: any) { - let dialect: Dialect | undefined - - if (flags.delimiter) { - dialect = { ...dialect, delimiter: flags.delimiter } - } - - if (flags.header === false) { - dialect = { ...dialect, header: flags.header } - } - - if (flags.headerRows) { - dialect = { - ...dialect, - headerRows: flags.headerRows.split(",").map(Number), - } - } - - if (flags.headerJoin) { - dialect = { ...dialect, headerJoin: flags.headerJoin } - } - - if (flags.commentRows) { - dialect = { - ...dialect, - commentRows: flags.commentRows.split(",").map(Number), - } - } - - if (flags.commentChar) { - dialect = { ...dialect, commentChar: flags.commentChar } - } - - if (flags.quoteChar) { - dialect = { ...dialect, quoteChar: flags.quoteChar } - } - - if (flags.doubleQuote) { - dialect = { ...dialect, doubleQuote: flags.doubleQuote } - } - - if (flags.escapeChar) { - dialect = { ...dialect, escapeChar: flags.escapeChar } - } - - if (flags.nullSequence) { - dialect = { ...dialect, nullSequence: flags.nullSequence } - } - - if (flags.skipInitialSpace) { - dialect = { ...dialect, skipInitialSpace: flags.skipInitialSpace } - } - - if (flags.property) { - dialect = { ...dialect, property: flags.property } - } - - if (flags.itemType) { - dialect = { ...dialect, itemType: flags.itemType } - } - - if (flags.itemKeys) { - dialect = { ...dialect, itemKeys: flags.itemKeys.split(",") } - } - - if (flags.sheetNumber) { - dialect = { ...dialect, sheetNumber: flags.sheetNumber } - } - - if (flags.table) { - dialect = { ...dialect, table: flags.table } - } - - return dialect -} diff --git a/cli/options/index.ts b/cli/options/index.ts deleted file mode 100644 index f97054c6..00000000 --- a/cli/options/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -export * from "./json.ts" -export * from "./dialect.ts" -export * from "./package.ts" -export * from "./path.ts" -export * from "./resource.ts" -export * from "./toDialect.ts" diff --git a/cli/options/json.ts b/cli/options/json.ts deleted file mode 100644 index f7d62b3f..00000000 --- a/cli/options/json.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { Flags } from "@oclif/core" - -export const json = Flags.boolean({ - description: "output as JSON", - char: "j", -}) diff --git a/cli/options/package.ts b/cli/options/package.ts deleted file mode 100644 index 4c2dc69d..00000000 --- a/cli/options/package.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { Flags } from "@oclif/core" - -export const withRemote = Flags.boolean({ - description: "include remote resources", -}) diff --git a/cli/options/path.ts b/cli/options/path.ts deleted file mode 100644 index 48f62810..00000000 --- a/cli/options/path.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { Flags } from "@oclif/core" - -export const toPath = Flags.string({ - description: "a local output path", -}) - -export const requiredToFolder = Flags.string({ - description: "a local output folder path", - required: true, -}) - -export const requiredToArchive = Flags.string({ - description: "a local output zip file path", - required: true, -}) diff --git a/cli/options/resource.ts b/cli/options/resource.ts deleted file mode 100644 index f42643f1..00000000 --- a/cli/options/resource.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { Flags } from "@oclif/core" - -export const toFormat = Flags.string({ - description: "output resource format", -}) diff --git a/cli/options/toDialect.ts b/cli/options/toDialect.ts deleted file mode 100644 index bdb2b459..00000000 --- a/cli/options/toDialect.ts +++ /dev/null @@ -1,180 +0,0 @@ -import { Flags } from "@oclif/core" -import type { Dialect } from "dpkit" - -// TODO: merge with dialect.ts - -export const toHeader = Flags.boolean({ - description: "whether the file includes a header row with field names", - default: true, -}) - -export const toHeaderRows = Flags.string({ - description: - "comma-separated row numbers (zero-based) that are considered header rows", -}) - -export const toHeaderJoin = Flags.string({ - description: "character used to join multi-line headers", -}) - -export const toCommentRows = Flags.string({ - description: "comma-separated rows to be excluded from the data (zero-based)", -}) - -export const toCommentChar = Flags.string({ - description: "character sequence denoting the start of a comment line", -}) - -export const toDelimiter = Flags.string({ - description: "character used to separate fields in the data", - char: "d", -}) - -export const toLineTerminator = Flags.string({ - description: "character sequence used to terminate rows", -}) - -export const toQuoteChar = Flags.string({ - description: "character used to quote fields", -}) - -export const toDoubleQuote = Flags.boolean({ - description: - "whether a sequence of two quote characters represents a single quote", -}) - -export const toEscapeChar = Flags.string({ - description: "character used to escape the delimiter or quote characters", -}) - -export const toNullSequence = Flags.string({ - description: - "character sequence representing null or missing values in the data", -}) - -export const toSkipInitialSpace = Flags.boolean({ - description: - "whether to ignore whitespace immediately following the delimiter", -}) - -export const toProperty = Flags.string({ - description: "for JSON data, the property name containing the data array", -}) - -export const toItemType = Flags.string({ - description: "the type of data item in the source", - options: ["array", "object"], -}) - -export const toItemKeys = Flags.string({ - description: - "comma-separated object properties to extract as values (for object-based data items)", -}) - -export const toSheetNumber = Flags.integer({ - description: "for spreadsheet data, the sheet number to read (zero-based)", -}) - -export const toSheetName = Flags.string({ - description: "for spreadsheet data, the sheet name to read", -}) - -export const toTable = Flags.string({ - description: "for database sources, the table name to read", -}) - -export const toDialectOptions = { - toDelimiter, - toHeader, - toHeaderRows, - toHeaderJoin, - toCommentRows, - toCommentChar, - toQuoteChar, - toDoubleQuote, - toEscapeChar, - toNullSequence, - toSkipInitialSpace, - toProperty, - toItemType, - toItemKeys, - toSheetNumber, - toTable, -} - -// TODO: rebase on types flags -export function createToDialectFromFlags(flags: any) { - let dialect: Dialect | undefined - - if (flags.toDelimiter) { - dialect = { ...dialect, delimiter: flags.toDelimiter } - } - - if (flags.toHeader === false) { - dialect = { ...dialect, header: flags.toHeader } - } - - if (flags.toHeaderRows) { - dialect = { - ...dialect, - headerRows: flags.toHeaderRows.split(",").map(Number), - } - } - - if (flags.toHeaderJoin) { - dialect = { ...dialect, headerJoin: flags.toHeaderJoin } - } - - if (flags.toCommentRows) { - dialect = { - ...dialect, - commentRows: flags.toCommentRows.split(",").map(Number), - } - } - - if (flags.toCommentChar) { - dialect = { ...dialect, commentChar: flags.toCommentChar } - } - - if (flags.toQuoteChar) { - dialect = { ...dialect, quoteChar: flags.toQuoteChar } - } - - if (flags.toDoubleQuote) { - dialect = { ...dialect, doubleQuote: flags.toDoubleQuote } - } - - if (flags.toEscapeChar) { - dialect = { ...dialect, escapeChar: flags.toEscapeChar } - } - - if (flags.toNullSequence) { - dialect = { ...dialect, nullSequence: flags.toNullSequence } - } - - if (flags.toSkipInitialSpace) { - dialect = { ...dialect, skipInitialSpace: flags.toSkipInitialSpace } - } - - if (flags.toProperty) { - dialect = { ...dialect, property: flags.toProperty } - } - - if (flags.toItemType) { - dialect = { ...dialect, itemType: flags.toItemType } - } - - if (flags.toItemKeys) { - dialect = { ...dialect, itemKeys: flags.toItemKeys.split(",") } - } - - if (flags.toSheetNumber) { - dialect = { ...dialect, sheetNumber: flags.toSheetNumber } - } - - if (flags.toTable) { - dialect = { ...dialect, table: flags.toTable } - } - - return dialect -} diff --git a/cli/package.json b/cli/package.json index 3970c717..8260adf1 100644 --- a/cli/package.json +++ b/cli/package.json @@ -23,7 +23,7 @@ ], "scripts": { "build": "tsc && pnpm build:copy && pnpm build:mode", - "build:copy": "cp entrypoints/*.cmd build/entrypoints && cp package.json build && cp .oclifrc.js build", + "build:copy": "cp entrypoints/*.cmd build/entrypoints && cp package.json build", "build:mode": "chmod +x ./build/entrypoints/*.js", "compile": "node ./scripts/compile.ts", "dev": "node ./scripts/dev.ts", @@ -33,8 +33,6 @@ "@clack/prompts": "^0.11.0", "@commander-js/extra-typings": "^14.0.0", "@inkjs/ui": "^2.0.0", - "@oclif/core": "^4.5.2", - "@oclif/plugin-autocomplete": "^3.2.34", "commander": "^14.0.0", "dpkit": "workspace:*", "ink": "^6.1.0", diff --git a/cli/params/dialect.ts b/cli/params/dialect.ts new file mode 100644 index 00000000..04d04125 --- /dev/null +++ b/cli/params/dialect.ts @@ -0,0 +1,192 @@ +import { Option } from "commander" +import type { Dialect } from "dpkit" + +export const header = new Option( + "--header", + "whether the file includes a header row with field names", +).default(true) + +export const headerRows = new Option( + "--header-rows ", + "comma-separated row numbers (zero-based) that are considered header rows", +) + +export const headerJoin = new Option( + "--header-join ", + "character used to join multi-line headers", +) + +export const commentRows = new Option( + "--comment-rows ", + "comma-separated rows to be excluded from the data (zero-based)", +) + +export const commentChar = new Option( + "--comment-char ", + "character sequence denoting the start of a comment line", +) + +export const delimiter = new Option( + "-d, --delimiter ", + "character used to separate fields in the data", +) + +export const lineTerminator = new Option( + "--line-terminator ", + "character sequence used to terminate rows", +) + +export const quoteChar = new Option( + "--quote-char ", + "character used to quote fields", +) + +export const doubleQuote = new Option( + "--double-quote", + "whether a sequence of two quote characters represents a single quote", +) + +export const escapeChar = new Option( + "--escape-char ", + "character used to escape the delimiter or quote characters", +) + +export const nullSequence = new Option( + "--null-sequence ", + "character sequence representing null or missing values in the data", +) + +export const skipInitialSpace = new Option( + "--skip-initial-space", + "whether to ignore whitespace immediately following the delimiter", +) + +export const property = new Option( + "--property ", + "for JSON data, the property name containing the data array", +) + +export const itemType = new Option( + "--item-type ", + "the type of data item in the source", +).choices(["array", "object"]) + +export const itemKeys = new Option( + "--item-keys ", + "comma-separated object properties to extract as values (for object-based data items)", +) + +export const sheetNumber = new Option( + "--sheet-number ", + "for spreadsheet data, the sheet number to read (zero-based)", +).argParser(Number.parseInt) + +export const sheetName = new Option( + "--sheet-name ", + "for spreadsheet data, the sheet name to read", +) + +export const table = new Option( + "--table ", + "for database sources, the table name to read", +) + +export const dialectOptions = [ + delimiter, + header, + headerRows, + headerJoin, + commentRows, + commentChar, + quoteChar, + doubleQuote, + escapeChar, + nullSequence, + skipInitialSpace, + property, + itemType, + itemKeys, + sheetNumber, + sheetName, + table, +] + +export function createDialectFromOptions(options: any) { + let dialect: Dialect | undefined + + if (options.delimiter) { + dialect = { ...dialect, delimiter: options.delimiter } + } + + if (options.header === false) { + dialect = { ...dialect, header: options.header } + } + + if (options.headerRows) { + dialect = { + ...dialect, + headerRows: options.headerRows.split(",").map(Number), + } + } + + if (options.headerJoin) { + dialect = { ...dialect, headerJoin: options.headerJoin } + } + + if (options.commentRows) { + dialect = { + ...dialect, + commentRows: options.commentRows.split(",").map(Number), + } + } + + if (options.commentChar) { + dialect = { ...dialect, commentChar: options.commentChar } + } + + if (options.quoteChar) { + dialect = { ...dialect, quoteChar: options.quoteChar } + } + + if (options.doubleQuote) { + dialect = { ...dialect, doubleQuote: options.doubleQuote } + } + + if (options.escapeChar) { + dialect = { ...dialect, escapeChar: options.escapeChar } + } + + if (options.nullSequence) { + dialect = { ...dialect, nullSequence: options.nullSequence } + } + + if (options.skipInitialSpace) { + dialect = { ...dialect, skipInitialSpace: options.skipInitialSpace } + } + + if (options.property) { + dialect = { ...dialect, property: options.property } + } + + if (options.itemType) { + dialect = { ...dialect, itemType: options.itemType } + } + + if (options.itemKeys) { + dialect = { ...dialect, itemKeys: options.itemKeys.split(",") } + } + + if (options.sheetNumber) { + dialect = { ...dialect, sheetNumber: options.sheetNumber } + } + + if (options.sheetName) { + dialect = { ...dialect, sheetName: options.sheetName } + } + + if (options.table) { + dialect = { ...dialect, table: options.table } + } + + return dialect +} diff --git a/cli/params/index.ts b/cli/params/index.ts index 5bec7a0e..f97054c6 100644 --- a/cli/params/index.ts +++ b/cli/params/index.ts @@ -1 +1,6 @@ +export * from "./json.ts" +export * from "./dialect.ts" +export * from "./package.ts" export * from "./path.ts" +export * from "./resource.ts" +export * from "./toDialect.ts" diff --git a/cli/params/json.ts b/cli/params/json.ts new file mode 100644 index 00000000..7d770164 --- /dev/null +++ b/cli/params/json.ts @@ -0,0 +1,3 @@ +import { Option } from "commander" + +export const json = new Option("-j, --json", "output as JSON") diff --git a/cli/params/package.ts b/cli/params/package.ts new file mode 100644 index 00000000..972f89a6 --- /dev/null +++ b/cli/params/package.ts @@ -0,0 +1,6 @@ +import { Option } from "commander" + +export const withRemote = new Option( + "--with-remote", + "include remote resources", +) diff --git a/cli/params/path.ts b/cli/params/path.ts index b94b40b8..9798a8f5 100644 --- a/cli/params/path.ts +++ b/cli/params/path.ts @@ -1,11 +1,23 @@ -import { Args } from "@oclif/core" +import { Argument, Option } from "commander" -export const requriedDescriptorPath = Args.string({ - description: "local or remote path to the descriptor", - required: true, -}) +export const positionalDescriptorPath = new Argument( + "", + "local or remote path to the descriptor", +) -export const requriedTablePath = Args.string({ - description: "local or remote path to the table", - required: true, -}) +export const positionalTablePath = new Argument( + "", + "local or remote path to the table", +) + +export const toPath = new Option("--to-path ", "a local output path") + +export const toFolder = new Option( + "--to-folder ", + "a local output folder path", +) + +export const toArchive = new Option( + "--to-archive ", + "a local output zip file path", +) diff --git a/cli/params/resource.ts b/cli/params/resource.ts new file mode 100644 index 00000000..915f1292 --- /dev/null +++ b/cli/params/resource.ts @@ -0,0 +1,6 @@ +import { Option } from "commander" + +export const toFormat = new Option( + "--to-format ", + "output resource format", +) diff --git a/cli/params/toDialect.ts b/cli/params/toDialect.ts new file mode 100644 index 00000000..394aecc9 --- /dev/null +++ b/cli/params/toDialect.ts @@ -0,0 +1,192 @@ +import { Option } from "commander" +import type { Dialect } from "dpkit" + +export const toHeader = new Option( + "--to-header", + "whether the file includes a header row with field names", +).default(true) + +export const toHeaderRows = new Option( + "--to-header-rows ", + "comma-separated row numbers (zero-based) that are considered header rows", +) + +export const toHeaderJoin = new Option( + "--to-header-join ", + "character used to join multi-line headers", +) + +export const toCommentRows = new Option( + "--to-comment-rows ", + "comma-separated rows to be excluded from the data (zero-based)", +) + +export const toCommentChar = new Option( + "--to-comment-char ", + "character sequence denoting the start of a comment line", +) + +export const toDelimiter = new Option( + "--to-delimiter ", + "character used to separate fields in the data", +) + +export const toLineTerminator = new Option( + "--to-line-terminator ", + "character sequence used to terminate rows", +) + +export const toQuoteChar = new Option( + "--to-quote-char ", + "character used to quote fields", +) + +export const toDoubleQuote = new Option( + "--to-double-quote", + "whether a sequence of two quote characters represents a single quote", +) + +export const toEscapeChar = new Option( + "--to-escape-char ", + "character used to escape the delimiter or quote characters", +) + +export const toNullSequence = new Option( + "--to-null-sequence ", + "character sequence representing null or missing values in the data", +) + +export const toSkipInitialSpace = new Option( + "--to-skip-initial-space", + "whether to ignore whitespace immediately following the delimiter", +) + +export const toProperty = new Option( + "--to-property ", + "for JSON data, the property name containing the data array", +) + +export const toItemType = new Option( + "--to-item-type ", + "the type of data item in the source", +).choices(["array", "object"]) + +export const toItemKeys = new Option( + "--to-item-keys ", + "comma-separated object properties to extract as values (for object-based data items)", +) + +export const toSheetNumber = new Option( + "--to-sheet-number ", + "for spreadsheet data, the sheet number to read (zero-based)", +).argParser(Number.parseInt) + +export const toSheetName = new Option( + "--to-sheet-name ", + "for spreadsheet data, the sheet name to read", +) + +export const toTable = new Option( + "--to-table ", + "for database sources, the table name to read", +) + +export const toDialectOptions = [ + toDelimiter, + toHeader, + toHeaderRows, + toHeaderJoin, + toCommentRows, + toCommentChar, + toQuoteChar, + toDoubleQuote, + toEscapeChar, + toNullSequence, + toSkipInitialSpace, + toProperty, + toItemType, + toItemKeys, + toSheetNumber, + toSheetName, + toTable, +] + +export function createToDialectFromOptions(options: any) { + let dialect: Dialect | undefined + + if (options.toDelimiter) { + dialect = { ...dialect, delimiter: options.toDelimiter } + } + + if (options.toHeader === false) { + dialect = { ...dialect, header: options.toHeader } + } + + if (options.toHeaderRows) { + dialect = { + ...dialect, + headerRows: options.toHeaderRows.split(",").map(Number), + } + } + + if (options.toHeaderJoin) { + dialect = { ...dialect, headerJoin: options.toHeaderJoin } + } + + if (options.toCommentRows) { + dialect = { + ...dialect, + commentRows: options.toCommentRows.split(",").map(Number), + } + } + + if (options.toCommentChar) { + dialect = { ...dialect, commentChar: options.toCommentChar } + } + + if (options.toQuoteChar) { + dialect = { ...dialect, quoteChar: options.toQuoteChar } + } + + if (options.toDoubleQuote) { + dialect = { ...dialect, doubleQuote: options.toDoubleQuote } + } + + if (options.toEscapeChar) { + dialect = { ...dialect, escapeChar: options.toEscapeChar } + } + + if (options.toNullSequence) { + dialect = { ...dialect, nullSequence: options.toNullSequence } + } + + if (options.toSkipInitialSpace) { + dialect = { ...dialect, skipInitialSpace: options.toSkipInitialSpace } + } + + if (options.toProperty) { + dialect = { ...dialect, property: options.toProperty } + } + + if (options.toItemType) { + dialect = { ...dialect, itemType: options.toItemType } + } + + if (options.toItemKeys) { + dialect = { ...dialect, itemKeys: options.toItemKeys.split(",") } + } + + if (options.toSheetNumber) { + dialect = { ...dialect, sheetNumber: options.toSheetNumber } + } + + if (options.toSheetName) { + dialect = { ...dialect, sheetName: options.toSheetName } + } + + if (options.toTable) { + dialect = { ...dialect, table: options.toTable } + } + + return dialect +} diff --git a/cli/program.ts b/cli/program.ts new file mode 100644 index 00000000..8d0d0823 --- /dev/null +++ b/cli/program.ts @@ -0,0 +1,22 @@ +import { styleText } from "node:util" +import { program } from "commander" +import { dialectCommand } from "./commands/dialect/index.js" +import { packageCommand } from "./commands/package/index.js" +import { tableCommand } from "./commands/table/index.js" +import metadata from "./package.json" with { type: "json" } + +const root = program + .name("dp") + .description( + "Fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames", + ) + .version(metadata.version) + .configureHelp({ + styleTitle: str => styleText("bold", str.toUpperCase().slice(0, -1)), + }) + +root.addCommand(packageCommand) +root.addCommand(tableCommand) +root.addCommand(dialectCommand) + +root.parse() diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 43138cbc..164528be 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -97,12 +97,6 @@ importers: '@inkjs/ui': specifier: ^2.0.0 version: 2.0.0(ink@6.1.0(@types/react@19.1.9)(react@19.1.1)) - '@oclif/core': - specifier: ^4.5.2 - version: 4.5.2 - '@oclif/plugin-autocomplete': - specifier: ^3.2.34 - version: 3.2.34 commander: specifier: ^14.0.0 version: 14.0.0 @@ -1164,14 +1158,6 @@ packages: resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==} engines: {node: '>= 8'} - '@oclif/core@4.5.2': - resolution: {integrity: sha512-eQcKyrEcDYeZJKu4vUWiu0ii/1Gfev6GF4FsLSgNez5/+aQyAUCjg3ZWlurf491WiYZTXCWyKAxyPWk8DKv2MA==} - engines: {node: '>=18.0.0'} - - '@oclif/plugin-autocomplete@3.2.34': - resolution: {integrity: sha512-KhbPcNjitAU7jUojMXJ3l7duWVub0L0pEr3r3bLrpJBNuIJhoIJ7p56Ropcb7OMH2xcaz5B8HGq56cTOe1FHEg==} - engines: {node: '>=18.0.0'} - '@oslojs/encoding@1.1.0': resolution: {integrity: sha512-70wQhgYmndg4GCPxPPxPGevRKqTIJ2Nh4OkiMWmDAVYsTQ+Ta7Sq+rPevXyXGdzr30/qZBnyOalCszoMxlyldQ==} @@ -1515,10 +1501,6 @@ packages: resolution: {integrity: sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==} engines: {node: '>=6'} - ansi-escapes@4.3.2: - resolution: {integrity: sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==} - engines: {node: '>=8'} - ansi-escapes@7.0.0: resolution: {integrity: sha512-GdYO7a61mR0fOlAsvC9/rIHf7L96sBc6dEWzeOu+KAea5bZyQRPIpojrVoI4AXGJS/ycu/fBTdLrUkA4ODrvjw==} engines: {node: '>=18'} @@ -1539,10 +1521,6 @@ packages: resolution: {integrity: sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==} engines: {node: '>=12'} - ansis@3.17.0: - resolution: {integrity: sha512-0qWUglt9JEqLFr3w1I1pbrChn1grhaiAR2ocX1PP/flRmxgtwTzPFFFnfIlD6aMOLQZgSuCRlidD70lvx8yhzg==} - engines: {node: '>=14'} - anymatch@3.1.3: resolution: {integrity: sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==} engines: {node: '>= 8'} @@ -1588,9 +1566,6 @@ packages: engines: {node: 18.20.8 || ^20.3.0 || >=22.0.0, npm: '>=9.6.5', pnpm: '>=7.1.0'} hasBin: true - async@3.2.6: - resolution: {integrity: sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==} - auto-bind@5.0.1: resolution: {integrity: sha512-ooviqdwwgfIfNmDwo94wlshcdzfO64XV0Cg6oDsDYBJfITDz1EngD2z7DkbvCWn+XIMsIqW27sEVF6qcpJrRcg==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} @@ -1721,10 +1696,6 @@ packages: resolution: {integrity: sha512-l+2bNRMiQgcfILUi33labAZYIWlH1kWDp+ecNo5iisRKrbm0xcRyCww71/YU0Fkw0mAFpz9bJayXPjey6vkmaQ==} engines: {node: '>=8'} - clean-stack@3.0.1: - resolution: {integrity: sha512-lR9wNiMRcVQjSB3a7xXGLuz4cr4wJuuXlaAEbRutGowQTmlp7R72/DOgN21e8jdwblMWl9UOJMJXarX94pzKdg==} - engines: {node: '>=10'} - cli-boxes@3.0.0: resolution: {integrity: sha512-/lzGpEWL/8PfI0BmBOPRwp0c/wFNX1RdUML3jK/RcSBA9T8mZDdQpqYBKtCFTOfQbwPqWEOpjqW+Fnayc0969g==} engines: {node: '>=10'} @@ -1733,10 +1704,6 @@ packages: resolution: {integrity: sha512-VGtlMu3x/4DOtIUwEkRezxUZ2lBacNJCHash0N0WeZDBS+7Ux1dm3XWAgWYxLJFMMdOeXMHXorshEFhbMSGelg==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - cli-spinners@2.9.2: - resolution: {integrity: sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==} - engines: {node: '>=6'} - cli-spinners@3.2.0: resolution: {integrity: sha512-pXftdQloMZzjCr3pCTIRniDcys6dDzgpgVhAHHk6TKBDbRuP1MkuetTF5KSv4YUutbOPa7+7ZrAJ2kVtbMqyXA==} engines: {node: '>=18.20'} @@ -1950,11 +1917,6 @@ packages: ee-first@1.1.1: resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} - ejs@3.1.10: - resolution: {integrity: sha512-UeJmFfOrAQS8OJWPZ4qtgHyWExa088/MtK5UEyoJGFH67cDEXkZSviOiKRCZ4Xij0zxI3JECgYs3oKx+AizQBA==} - engines: {node: '>=0.10.0'} - hasBin: true - emoji-regex@10.4.0: resolution: {integrity: sha512-EC+0oUMY1Rqm4O6LLrgjtYDvcVYTy7chDnM4Q7030tP4Kwj3u/pR6gP9ygnp2CJMK5Gq+9Q2oqmrFJAz01DXjw==} @@ -2028,10 +1990,6 @@ packages: resolution: {integrity: sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==} engines: {node: '>=8'} - escape-string-regexp@4.0.0: - resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} - engines: {node: '>=10'} - escape-string-regexp@5.0.0: resolution: {integrity: sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==} engines: {node: '>=12'} @@ -2132,9 +2090,6 @@ packages: resolution: {integrity: sha512-d+l3qxjSesT4V7v2fh+QnmFnUWv9lSpjarhShNTgBOfA0ttejbQUAlHLitbjkoRiDulW0OPoQPYIGhIC8ohejg==} engines: {node: '>=18'} - filelist@1.0.4: - resolution: {integrity: sha512-w1cEuf3S+DrLCQL7ET6kz+gmlJdbq9J7yXCSjK/OZCPA+qEN1WyF4ZAf0YYJa4/shHJra2t/d/r8SV4Ji+x+8Q==} - fill-range@7.1.1: resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} engines: {node: '>=8'} @@ -2200,10 +2155,6 @@ packages: resolution: {integrity: sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==} engines: {node: '>= 0.4'} - get-package-type@0.1.0: - resolution: {integrity: sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==} - engines: {node: '>=8.0.0'} - get-proto@1.0.1: resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} engines: {node: '>= 0.4'} @@ -2369,10 +2320,6 @@ packages: import-meta-resolve@4.1.0: resolution: {integrity: sha512-I6fiaX09Xivtk+THaMfAwnA3MVA5Big1WHF1Dfx9hFuvNIWpXnorlkzhcQf6ehrqQiiZECRt1poOAkPmer3ruw==} - indent-string@4.0.0: - resolution: {integrity: sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==} - engines: {node: '>=8'} - indent-string@5.0.0: resolution: {integrity: sha512-m6FAo/spmsW2Ab2fU35JTYwtOKa2yAwXSwgjSv1TJzh4Mh7mC3lzAOVLBprb72XsTrgkEIsl7YrFNAiDiRhIGg==} engines: {node: '>=12'} @@ -2426,11 +2373,6 @@ packages: is-decimal@2.0.1: resolution: {integrity: sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A==} - is-docker@2.2.1: - resolution: {integrity: sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==} - engines: {node: '>=8'} - hasBin: true - is-docker@3.0.0: resolution: {integrity: sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} @@ -2501,10 +2443,6 @@ packages: resolution: {integrity: sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==} engines: {node: '>=0.10.0'} - is-wsl@2.2.0: - resolution: {integrity: sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==} - engines: {node: '>=8'} - is-wsl@3.1.0: resolution: {integrity: sha512-UcVfVfaK4Sc4m7X3dUSoHoozQGBEFeDC+zVo06t98xe8CzHSZZBekNXH+tu0NalHolcJ/QAGqS46Hef7QXBIMw==} engines: {node: '>=16'} @@ -2531,11 +2469,6 @@ packages: jackspeak@3.4.3: resolution: {integrity: sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==} - jake@10.9.4: - resolution: {integrity: sha512-wpHYzhxiVQL+IV05BLE2Xn34zW1S223hvjtqk0+gsPrwd/8JNLXJgZZM/iPFsYc1xyphF+6M6EvdE5E9MBGkDA==} - engines: {node: '>=10'} - hasBin: true - js-tokens@4.0.0: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} @@ -2568,10 +2501,6 @@ packages: resolution: {integrity: sha512-dhG34DXATL5hSxJbIexCft8FChFXtmskoZYnoPWjXQuebWYCNkVeV3KkGegCK9CP1oswI/vQibS2GY7Em/sJJA==} engines: {node: '>= 8'} - lilconfig@3.1.3: - resolution: {integrity: sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==} - engines: {node: '>=14'} - linkify-it@5.0.0: resolution: {integrity: sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ==} @@ -2837,10 +2766,6 @@ packages: resolution: {integrity: sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==} engines: {node: '>=6'} - minimatch@5.1.6: - resolution: {integrity: sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==} - engines: {node: '>=10'} - minimatch@9.0.5: resolution: {integrity: sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==} engines: {node: '>=16 || 14 >=14.17'} @@ -3556,10 +3481,6 @@ packages: resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} engines: {node: '>=8'} - supports-color@8.1.1: - resolution: {integrity: sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==} - engines: {node: '>=10'} - temp-dir@3.0.0: resolution: {integrity: sha512-nHc6S/bwIilKHNRgK/3jlhDoIHcp45YgyiwcAk46Tr0LfEqGBVpmiAyuiuxeVE44m3mXnEeVhaipLOEWmH+Njw==} engines: {node: '>=14.16'} @@ -3658,10 +3579,6 @@ packages: engines: {node: '>=18.0.0'} hasBin: true - type-fest@0.21.3: - resolution: {integrity: sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==} - engines: {node: '>=10'} - type-fest@1.4.0: resolution: {integrity: sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==} engines: {node: '>=10'} @@ -3962,17 +3879,10 @@ packages: engines: {node: '>=8'} hasBin: true - widest-line@3.1.0: - resolution: {integrity: sha512-NsmoXalsWVDMGupxZ5R08ka9flZjjiLvHVAWYOKtiKM8ujtZWr9cRffak+uSE48+Ob8ObalXpwyeUiyDD6QFgg==} - engines: {node: '>=8'} - widest-line@5.0.0: resolution: {integrity: sha512-c9bZp7b5YtRj2wOe6dlj32MK+Bx/M/d+9VB2SHM1OtsUHR0aV0tdP6DWh/iMt0kWi1t5g1Iudu6hQRNd1A4PVA==} engines: {node: '>=18'} - wordwrap@1.0.0: - resolution: {integrity: sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==} - wrap-ansi@7.0.0: resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==} engines: {node: '>=10'} @@ -4159,7 +4069,7 @@ snapshots: '@astrojs/telemetry@3.3.0': dependencies: ci-info: 4.3.0 - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 dlv: 1.1.3 dset: 3.1.4 is-docker: 3.0.0 @@ -4829,36 +4739,6 @@ snapshots: '@nodelib/fs.scandir': 2.1.5 fastq: 1.19.1 - '@oclif/core@4.5.2': - dependencies: - ansi-escapes: 4.3.2 - ansis: 3.17.0 - clean-stack: 3.0.1 - cli-spinners: 2.9.2 - debug: 4.4.1(supports-color@8.1.1) - ejs: 3.1.10 - get-package-type: 0.1.0 - indent-string: 4.0.0 - is-wsl: 2.2.0 - lilconfig: 3.1.3 - minimatch: 9.0.5 - semver: 7.7.2 - string-width: 4.2.3 - supports-color: 8.1.1 - tinyglobby: 0.2.14 - widest-line: 3.1.0 - wordwrap: 1.0.0 - wrap-ansi: 7.0.0 - - '@oclif/plugin-autocomplete@3.2.34': - dependencies: - '@oclif/core': 4.5.2 - ansis: 3.17.0 - debug: 4.4.1(supports-color@8.1.1) - ejs: 3.1.10 - transitivePeerDependencies: - - supports-color - '@oslojs/encoding@1.1.0': {} '@pagefind/darwin-arm64@1.3.0': @@ -5131,7 +5011,7 @@ snapshots: dependencies: '@ampproject/remapping': 2.3.0 '@bcoe/v8-coverage': 1.0.2 - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 istanbul-lib-coverage: 3.2.2 istanbul-lib-report: 3.0.1 istanbul-lib-source-maps: 5.0.6 @@ -5224,10 +5104,6 @@ snapshots: ansi-colors@4.1.3: {} - ansi-escapes@4.3.2: - dependencies: - type-fest: 0.21.3 - ansi-escapes@7.0.0: dependencies: environment: 1.1.0 @@ -5242,8 +5118,6 @@ snapshots: ansi-styles@6.2.1: {} - ansis@3.17.0: {} - anymatch@3.1.3: dependencies: normalize-path: 3.0.0 @@ -5292,7 +5166,7 @@ snapshots: common-ancestor-path: 1.0.1 cookie: 1.0.2 cssesc: 3.0.0 - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 deterministic-object-hash: 2.0.2 devalue: 5.1.1 diff: 5.2.0 @@ -5375,8 +5249,6 @@ snapshots: - uploadthing - yaml - async@3.2.6: {} - auto-bind@5.0.1: {} axobject-query@4.1.0: {} @@ -5503,18 +5375,12 @@ snapshots: ci-info@4.3.0: {} - clean-stack@3.0.1: - dependencies: - escape-string-regexp: 4.0.0 - cli-boxes@3.0.0: {} cli-cursor@4.0.0: dependencies: restore-cursor: 4.0.0 - cli-spinners@2.9.2: {} - cli-spinners@3.2.0: {} cli-truncate@4.0.0: @@ -5612,11 +5478,9 @@ snapshots: dependencies: ms: 2.0.0 - debug@4.4.1(supports-color@8.1.1): + debug@4.4.1: dependencies: ms: 2.1.3 - optionalDependencies: - supports-color: 8.1.1 decode-named-character-reference@1.2.0: dependencies: @@ -5686,10 +5550,6 @@ snapshots: ee-first@1.1.1: {} - ejs@3.1.10: - dependencies: - jake: 10.9.4 - emoji-regex@10.4.0: {} emoji-regex@8.0.0: {} @@ -5772,8 +5632,6 @@ snapshots: escape-string-regexp@2.0.0: {} - escape-string-regexp@4.0.0: {} - escape-string-regexp@5.0.0: {} esprima@4.0.1: {} @@ -5917,10 +5775,6 @@ snapshots: dependencies: is-unicode-supported: 2.1.0 - filelist@1.0.4: - dependencies: - minimatch: 5.1.6 - fill-range@7.1.1: dependencies: to-regex-range: 5.0.1 @@ -6010,8 +5864,6 @@ snapshots: hasown: 2.0.2 math-intrinsics: 1.1.0 - get-package-type@0.1.0: {} - get-proto@1.0.1: dependencies: dunder-proto: 1.0.1 @@ -6292,7 +6144,7 @@ snapshots: http-graceful-shutdown@3.1.14: dependencies: - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 transitivePeerDependencies: - supports-color @@ -6314,8 +6166,6 @@ snapshots: import-meta-resolve@4.1.0: {} - indent-string@4.0.0: {} - indent-string@5.0.0: {} inherits@2.0.4: {} @@ -6379,8 +6229,6 @@ snapshots: is-decimal@2.0.1: {} - is-docker@2.2.1: {} - is-docker@3.0.0: {} is-extglob@2.1.1: {} @@ -6425,10 +6273,6 @@ snapshots: is-windows@1.0.2: {} - is-wsl@2.2.0: - dependencies: - is-docker: 2.2.1 - is-wsl@3.1.0: dependencies: is-inside-container: 1.0.0 @@ -6446,7 +6290,7 @@ snapshots: istanbul-lib-source-maps@5.0.6: dependencies: '@jridgewell/trace-mapping': 0.3.29 - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 istanbul-lib-coverage: 3.2.2 transitivePeerDependencies: - supports-color @@ -6462,12 +6306,6 @@ snapshots: optionalDependencies: '@pkgjs/parseargs': 0.11.0 - jake@10.9.4: - dependencies: - async: 3.2.6 - filelist: 1.0.4 - picocolors: 1.1.1 - js-tokens@4.0.0: {} js-yaml@3.14.1: @@ -6497,8 +6335,6 @@ snapshots: klona@2.0.6: {} - lilconfig@3.1.3: {} - linkify-it@5.0.0: dependencies: uc.micro: 2.1.0 @@ -7006,7 +6842,7 @@ snapshots: micromark@4.0.2: dependencies: '@types/debug': 4.1.12 - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 decode-named-character-reference: 1.2.0 devlop: 1.1.0 micromark-core-commonmark: 2.0.3 @@ -7040,10 +6876,6 @@ snapshots: mimic-fn@2.1.0: {} - minimatch@5.1.6: - dependencies: - brace-expansion: 2.0.2 - minimatch@9.0.5: dependencies: brace-expansion: 2.0.2 @@ -7858,10 +7690,6 @@ snapshots: dependencies: has-flag: 4.0.0 - supports-color@8.1.1: - dependencies: - has-flag: 4.0.0 - temp-dir@3.0.0: {} tempy@3.1.0: @@ -7937,8 +7765,6 @@ snapshots: optionalDependencies: fsevents: 2.3.3 - type-fest@0.21.3: {} - type-fest@1.4.0: {} type-fest@2.19.0: {} @@ -8101,7 +7927,7 @@ snapshots: vite-node@3.1.4(@types/node@24.2.0)(tsx@4.20.3)(yaml@2.8.1): dependencies: cac: 6.7.14 - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 es-module-lexer: 1.7.0 pathe: 2.0.3 vite: 6.3.5(@types/node@24.2.0)(tsx@4.20.3)(yaml@2.8.1) @@ -8147,7 +7973,7 @@ snapshots: '@vitest/spy': 3.1.4 '@vitest/utils': 3.1.4 chai: 5.2.1 - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 expect-type: 1.2.2 magic-string: 0.30.17 pathe: 2.0.3 @@ -8198,16 +8024,10 @@ snapshots: siginfo: 2.0.0 stackback: 0.0.2 - widest-line@3.1.0: - dependencies: - string-width: 4.2.3 - widest-line@5.0.0: dependencies: string-width: 7.2.0 - wordwrap@1.0.0: {} - wrap-ansi@7.0.0: dependencies: ansi-styles: 4.3.0 From 110c20f25501e7e8b9383207d09edc65aaf061f4 Mon Sep 17 00:00:00 2001 From: roll Date: Wed, 13 Aug 2025 17:23:30 +0100 Subject: [PATCH 08/61] Bootstrapped -p/--package option --- cli/commands/table/explore.tsx | 48 +++++++++++++++++++++++++++++++--- cli/helpers/task.ts | 9 +++++++ cli/params/package.ts | 10 +++++++ cli/params/path.ts | 2 +- 4 files changed, 65 insertions(+), 4 deletions(-) create mode 100644 cli/helpers/task.ts diff --git a/cli/commands/table/explore.tsx b/cli/commands/table/explore.tsx index b24e2f7e..8b810adc 100644 --- a/cli/commands/table/explore.tsx +++ b/cli/commands/table/explore.tsx @@ -1,14 +1,19 @@ +import { intro, log, outro, select } from "@clack/prompts" import { Command } from "commander" -import { readTable } from "dpkit" +import { type Resource, loadPackage, readTable } from "dpkit" import { render } from "ink" import React from "react" import { TableGrid } from "../../components/TableGrid.tsx" +import { task } from "../../helpers/task.ts" import * as params from "../../params/index.ts" export const exploreTableCommand = new Command("explore") .description("Explore a table from a local or remote path") .addArgument(params.positionalTablePath) + .addOption(params.fromPackage) .addOption(params.json) + + .optionsGroup("Table Dialect") .addOption(params.delimiter) .addOption(params.header) .addOption(params.headerRows) @@ -26,9 +31,43 @@ export const exploreTableCommand = new Command("explore") .addOption(params.sheetNumber) .addOption(params.sheetName) .addOption(params.table) + .action(async (path, options) => { + intro("Exploring table") + const dialect = params.createDialectFromOptions(options) - const table = await readTable({ path, dialect }) + let resource: Partial | undefined = path + ? { path, dialect } + : undefined + + if (!resource) { + if (!options.package) { + log.error("Please provide a path argument or a package option") + return + } + + const dataPackage = await task( + "Loading package", + loadPackage(options.package), + ) + + const resourceName = await select({ + message: "Select resource", + options: dataPackage.resources.map(resource => ({ + label: resource.name, + value: resource.name, + })), + }) + + resource = dataPackage.resources.find( + resource => resource.name === resourceName, + ) + } + + // @ts-ignore + const table = await task("Loading table", readTable(resource)) + + console.log(table) if (options.json) { const df = await table.slice(0, 10).collect() @@ -37,5 +76,8 @@ export const exploreTableCommand = new Command("explore") return } - render() + const app = render() + await app.waitUntilExit() + + outro("Thanks for using dpkit") }) diff --git a/cli/helpers/task.ts b/cli/helpers/task.ts new file mode 100644 index 00000000..1fc39ebc --- /dev/null +++ b/cli/helpers/task.ts @@ -0,0 +1,9 @@ +import { spinner } from "@clack/prompts" + +export async function task(message: string, promise: Promise) { + const loader = spinner() + loader.start(message) + const result = await promise + loader.stop(message) + return result +} diff --git a/cli/params/package.ts b/cli/params/package.ts index 972f89a6..17d88458 100644 --- a/cli/params/package.ts +++ b/cli/params/package.ts @@ -4,3 +4,13 @@ export const withRemote = new Option( "--with-remote", "include remote resources", ) + +export const fromPackage = new Option( + "-p --package ", + "package to select resource from", +) + +export const fromResource = new Option( + "-r --resource ", + "resource in provided package", +) diff --git a/cli/params/path.ts b/cli/params/path.ts index 9798a8f5..e8e012d7 100644 --- a/cli/params/path.ts +++ b/cli/params/path.ts @@ -6,7 +6,7 @@ export const positionalDescriptorPath = new Argument( ) export const positionalTablePath = new Argument( - "", + "[path]", "local or remote path to the table", ) From 688291b9fddf8298eb660438a1ec8940594edd24 Mon Sep 17 00:00:00 2001 From: roll Date: Wed, 13 Aug 2025 17:38:24 +0100 Subject: [PATCH 09/61] Fixed task --- cli/commands/table/explore.tsx | 4 +--- cli/helpers/task.ts | 3 +++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cli/commands/table/explore.tsx b/cli/commands/table/explore.tsx index 8b810adc..97c6e440 100644 --- a/cli/commands/table/explore.tsx +++ b/cli/commands/table/explore.tsx @@ -67,8 +67,6 @@ export const exploreTableCommand = new Command("explore") // @ts-ignore const table = await task("Loading table", readTable(resource)) - console.log(table) - if (options.json) { const df = await table.slice(0, 10).collect() const data = df.toRecords() @@ -79,5 +77,5 @@ export const exploreTableCommand = new Command("explore") const app = render() await app.waitUntilExit() - outro("Thanks for using dpkit") + outro("Thanks for using dpkit!") }) diff --git a/cli/helpers/task.ts b/cli/helpers/task.ts index 1fc39ebc..a0f1ac35 100644 --- a/cli/helpers/task.ts +++ b/cli/helpers/task.ts @@ -1,3 +1,4 @@ +import { setTimeout } from "node:timers/promises" import { spinner } from "@clack/prompts" export async function task(message: string, promise: Promise) { @@ -5,5 +6,7 @@ export async function task(message: string, promise: Promise) { loader.start(message) const result = await promise loader.stop(message) + // Without this delay, the following ink render will be immidiately terminated + await setTimeout(100) return result } From 55b67cdd5b7a5cd0e2a92ac7868005993bde0d3b Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 14 Aug 2025 10:34:11 +0100 Subject: [PATCH 10/61] Implemented session helpers --- cli/commands/table/explore.tsx | 32 ++++++------------ cli/helpers/session.ts | 61 ++++++++++++++++++++++++++++++++++ cli/helpers/task.ts | 12 ------- cli/package.json | 1 + pnpm-lock.yaml | 3 ++ 5 files changed, 76 insertions(+), 33 deletions(-) create mode 100644 cli/helpers/session.ts delete mode 100644 cli/helpers/task.ts diff --git a/cli/commands/table/explore.tsx b/cli/commands/table/explore.tsx index 97c6e440..6afbfe3e 100644 --- a/cli/commands/table/explore.tsx +++ b/cli/commands/table/explore.tsx @@ -1,17 +1,15 @@ -import { intro, log, outro, select } from "@clack/prompts" import { Command } from "commander" import { type Resource, loadPackage, readTable } from "dpkit" -import { render } from "ink" import React from "react" +import invariant from "tiny-invariant" import { TableGrid } from "../../components/TableGrid.tsx" -import { task } from "../../helpers/task.ts" +import { Session } from "../../helpers/session.ts" import * as params from "../../params/index.ts" export const exploreTableCommand = new Command("explore") .description("Explore a table from a local or remote path") .addArgument(params.positionalTablePath) .addOption(params.fromPackage) - .addOption(params.json) .optionsGroup("Table Dialect") .addOption(params.delimiter) @@ -33,7 +31,8 @@ export const exploreTableCommand = new Command("explore") .addOption(params.table) .action(async (path, options) => { - intro("Exploring table") + const session = Session.create() + session.intro("Exploring table") const dialect = params.createDialectFromOptions(options) let resource: Partial | undefined = path @@ -42,16 +41,15 @@ export const exploreTableCommand = new Command("explore") if (!resource) { if (!options.package) { - log.error("Please provide a path argument or a package option") - return + Session.terminate("Please provide a path argument or a package option") } - const dataPackage = await task( + const dataPackage = await session.task( "Loading package", loadPackage(options.package), ) - const resourceName = await select({ + const resourceName = await session.select({ message: "Select resource", options: dataPackage.resources.map(resource => ({ label: resource.name, @@ -62,20 +60,12 @@ export const exploreTableCommand = new Command("explore") resource = dataPackage.resources.find( resource => resource.name === resourceName, ) - } - - // @ts-ignore - const table = await task("Loading table", readTable(resource)) - if (options.json) { - const df = await table.slice(0, 10).collect() - const data = df.toRecords() - console.log(JSON.stringify(data, null, 2)) - return + invariant(resource, "Resource not found") } - const app = render() - await app.waitUntilExit() + const table = await session.task("Loading table", readTable(resource)) + await session.render() - outro("Thanks for using dpkit!") + session.outro("Thanks for using dpkit!") }) diff --git a/cli/helpers/session.ts b/cli/helpers/session.ts new file mode 100644 index 00000000..3b279156 --- /dev/null +++ b/cli/helpers/session.ts @@ -0,0 +1,61 @@ +import { setImmediate } from "node:timers/promises" +import { spinner } from "@clack/prompts" +import { intro, log, outro, select } from "@clack/prompts" +import type { SelectOptions } from "@clack/prompts" +import { render } from "ink" + +export class Session { + static create(options?: { json?: boolean }) { + return options?.json ? new JsonSession() : new Session() + } + + static terminate(message: string): never { + log.error(message) + process.exit(1) + } + + intro(message: string) { + intro(message) + } + + outro(message: string) { + outro(message) + } + + async select(options: SelectOptions) { + return await select(options) + } + + async task(message: string, promise: Promise) { + const loader = spinner() + + loader.start(message) + const result = await promise + loader.stop(message) + + return result + } + + async render(...args: Parameters) { + // Without waiting for the next tick after clack prompts, + // ink render will be immidiately terminated + await setImmediate() + + const app = render(...args) + await app.waitUntilExit() + } +} + +export class JsonSession extends Session { + intro = () => {} + outro = () => {} + + // @ts-ignore + async select(options: SelectOptions) { + Session.terminate("Interactive mode is not supported with JSON output") + } + + async task(_message: string, promise: Promise) { + return await promise + } +} diff --git a/cli/helpers/task.ts b/cli/helpers/task.ts deleted file mode 100644 index a0f1ac35..00000000 --- a/cli/helpers/task.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { setTimeout } from "node:timers/promises" -import { spinner } from "@clack/prompts" - -export async function task(message: string, promise: Promise) { - const loader = spinner() - loader.start(message) - const result = await promise - loader.stop(message) - // Without this delay, the following ink render will be immidiately terminated - await setTimeout(100) - return result -} diff --git a/cli/package.json b/cli/package.json index 8260adf1..1e00c00a 100644 --- a/cli/package.json +++ b/cli/package.json @@ -39,6 +39,7 @@ "ink-select-input": "^6.2.0", "nodejs-polars": "^0.21.0", "react": "^19.1.1", + "tiny-invariant": "^1.3.3", "ts-extras": "^0.14.0" }, "devDependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 164528be..8d86cb01 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -115,6 +115,9 @@ importers: react: specifier: ^19.1.1 version: 19.1.1 + tiny-invariant: + specifier: ^1.3.3 + version: 1.3.3 ts-extras: specifier: ^0.14.0 version: 0.14.0 From 1887e9c5ba62477da6329ae64c2ade578808a5a2 Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 14 Aug 2025 10:58:15 +0100 Subject: [PATCH 11/61] Implemented selectResource helper --- cli/commands/table/explore.tsx | 36 +++++--------------------------- cli/helpers/resource.ts | 38 ++++++++++++++++++++++++++++++++++ cli/helpers/session.ts | 12 +++++++---- 3 files changed, 51 insertions(+), 35 deletions(-) create mode 100644 cli/helpers/resource.ts diff --git a/cli/commands/table/explore.tsx b/cli/commands/table/explore.tsx index 6afbfe3e..9d3077ee 100644 --- a/cli/commands/table/explore.tsx +++ b/cli/commands/table/explore.tsx @@ -1,8 +1,8 @@ import { Command } from "commander" -import { type Resource, loadPackage, readTable } from "dpkit" +import { readTable } from "dpkit" import React from "react" -import invariant from "tiny-invariant" import { TableGrid } from "../../components/TableGrid.tsx" +import { selectResource } from "../../helpers/resource.ts" import { Session } from "../../helpers/session.ts" import * as params from "../../params/index.ts" @@ -34,35 +34,9 @@ export const exploreTableCommand = new Command("explore") const session = Session.create() session.intro("Exploring table") - const dialect = params.createDialectFromOptions(options) - let resource: Partial | undefined = path - ? { path, dialect } - : undefined - - if (!resource) { - if (!options.package) { - Session.terminate("Please provide a path argument or a package option") - } - - const dataPackage = await session.task( - "Loading package", - loadPackage(options.package), - ) - - const resourceName = await session.select({ - message: "Select resource", - options: dataPackage.resources.map(resource => ({ - label: resource.name, - value: resource.name, - })), - }) - - resource = dataPackage.resources.find( - resource => resource.name === resourceName, - ) - - invariant(resource, "Resource not found") - } + const resource = path + ? { path, dialect: params.createDialectFromOptions(options) } + : await selectResource(session, options) const table = await session.task("Loading table", readTable(resource)) await session.render() diff --git a/cli/helpers/resource.ts b/cli/helpers/resource.ts new file mode 100644 index 00000000..92fa037d --- /dev/null +++ b/cli/helpers/resource.ts @@ -0,0 +1,38 @@ +import { loadPackage } from "dpkit" +import { Session } from "./session.ts" + +export async function selectResource( + session: Session, + options: { package?: string; resource?: string }, +) { + if (!options.package) { + Session.terminate("Please provide a path argument or a package option") + } + + const dataPackage = await session.task( + "Loading package", + loadPackage(options.package), + ) + + const resourceName = + options.resource ?? + (await session.select({ + message: "Select resource", + options: dataPackage.resources.map(resource => ({ + label: resource.name, + value: resource.name, + })), + })) + + const resource = dataPackage.resources.find( + resource => resource.name === resourceName, + ) + + if (!resource) { + Session.terminate( + `Resource "${resourceName}" is not found in the provided data package`, + ) + } + + return resource +} diff --git a/cli/helpers/session.ts b/cli/helpers/session.ts index 3b279156..5e3310d7 100644 --- a/cli/helpers/session.ts +++ b/cli/helpers/session.ts @@ -3,6 +3,7 @@ import { spinner } from "@clack/prompts" import { intro, log, outro, select } from "@clack/prompts" import type { SelectOptions } from "@clack/prompts" import { render } from "ink" +import invariant from "tiny-invariant" export class Session { static create(options?: { json?: boolean }) { @@ -23,7 +24,7 @@ export class Session { } async select(options: SelectOptions) { - return await select(options) + return String(await select(options)) } async task(message: string, promise: Promise) { @@ -50,9 +51,12 @@ export class JsonSession extends Session { intro = () => {} outro = () => {} - // @ts-ignore - async select(options: SelectOptions) { - Session.terminate("Interactive mode is not supported with JSON output") + async select(_options: SelectOptions): Promise { + Session.terminate("Selection is not supported in JSON mode") + } + + async render(..._args: Parameters) { + invariant(false, "Render must not be used in JSON mode") } async task(_message: string, promise: Promise) { From 2f8e6f021407018aec0cbf8be251b00318d210a0 Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 14 Aug 2025 11:05:01 +0100 Subject: [PATCH 12/61] Added dialect helpers --- cli/commands/table/convert.tsx | 12 +- cli/commands/table/describe.tsx | 3 +- cli/commands/table/explore.tsx | 5 +- cli/commands/table/query.tsx | 2 +- cli/commands/table/script.tsx | 3 +- cli/commands/table/validate.tsx | 3 +- cli/helpers/dialect.ts | 163 +++++++++++++++++++++++++++ cli/main.ts | 2 +- cli/params/dialect.ts | 191 +++++++++++++++---------------- cli/params/index.ts | 1 - cli/params/toDialect.ts | 192 -------------------------------- 11 files changed, 275 insertions(+), 302 deletions(-) create mode 100644 cli/helpers/dialect.ts delete mode 100644 cli/params/toDialect.ts diff --git a/cli/commands/table/convert.tsx b/cli/commands/table/convert.tsx index 7f3978aa..234442a5 100644 --- a/cli/commands/table/convert.tsx +++ b/cli/commands/table/convert.tsx @@ -1,15 +1,20 @@ import { Command } from "commander" import { getTempFilePath, loadFile } from "dpkit" import { readTable, saveTable } from "dpkit" +import { createDialectFromOptions } from "../../helpers/dialect.ts" +import { createToDialectFromOptions } from "../../helpers/dialect.ts" import * as params from "../../params/index.ts" export const convertTableCommand = new Command("convert") .description( "Convert a table from a local or remote source path to a target path", ) + .addArgument(params.positionalTablePath) .addOption(params.toPath) .addOption(params.toFormat) + + .optionsGroup("Table Dialect") .addOption(params.delimiter) .addOption(params.header) .addOption(params.headerRows) @@ -27,6 +32,8 @@ export const convertTableCommand = new Command("convert") .addOption(params.sheetNumber) .addOption(params.sheetName) .addOption(params.table) + + .optionsGroup("Table Dialect (output)") .addOption(params.toDelimiter) .addOption(params.toHeader) .addOption(params.toHeaderRows) @@ -44,12 +51,13 @@ export const convertTableCommand = new Command("convert") .addOption(params.toSheetNumber) .addOption(params.toSheetName) .addOption(params.toTable) + .action(async (path, options) => { - const dialect = params.createDialectFromOptions(options) + const dialect = createDialectFromOptions(options) const table = await readTable({ path, dialect }) const toPath = options.toPath ?? getTempFilePath() - const toDialect = params.createToDialectFromOptions(options) + const toDialect = createToDialectFromOptions(options) await saveTable(table, { path: toPath, format: options.toFormat, diff --git a/cli/commands/table/describe.tsx b/cli/commands/table/describe.tsx index 681bd4b1..de8108ea 100644 --- a/cli/commands/table/describe.tsx +++ b/cli/commands/table/describe.tsx @@ -3,6 +3,7 @@ import { readTable } from "dpkit" import { render } from "ink" import React from "react" import { TableGrid } from "../../components/TableGrid.tsx" +import { createDialectFromOptions } from "../../helpers/dialect.ts" import * as params from "../../params/index.ts" export const describeTableCommand = new Command("describe") @@ -27,7 +28,7 @@ export const describeTableCommand = new Command("describe") .addOption(params.sheetName) .addOption(params.table) .action(async (path, options) => { - const dialect = params.createDialectFromOptions(options) + const dialect = createDialectFromOptions(options) const table = await readTable({ path, dialect }) const df = await table.collect() diff --git a/cli/commands/table/explore.tsx b/cli/commands/table/explore.tsx index 9d3077ee..6ce1b888 100644 --- a/cli/commands/table/explore.tsx +++ b/cli/commands/table/explore.tsx @@ -2,14 +2,17 @@ import { Command } from "commander" import { readTable } from "dpkit" import React from "react" import { TableGrid } from "../../components/TableGrid.tsx" +import { createDialectFromOptions } from "../../helpers/dialect.ts" import { selectResource } from "../../helpers/resource.ts" import { Session } from "../../helpers/session.ts" import * as params from "../../params/index.ts" export const exploreTableCommand = new Command("explore") .description("Explore a table from a local or remote path") + .addArgument(params.positionalTablePath) .addOption(params.fromPackage) + .addOption(params.fromResource) .optionsGroup("Table Dialect") .addOption(params.delimiter) @@ -35,7 +38,7 @@ export const exploreTableCommand = new Command("explore") session.intro("Exploring table") const resource = path - ? { path, dialect: params.createDialectFromOptions(options) } + ? { path, dialect: createDialectFromOptions(options) } : await selectResource(session, options) const table = await session.task("Loading table", readTable(resource)) diff --git a/cli/commands/table/query.tsx b/cli/commands/table/query.tsx index a4696743..c4145990 100644 --- a/cli/commands/table/query.tsx +++ b/cli/commands/table/query.tsx @@ -24,6 +24,6 @@ export const queryTableCommand = new Command("query") .addOption(params.sheetNumber) .addOption(params.sheetName) .addOption(params.table) - .action(async (path, options) => { + .action(async (_path, _options) => { throw new Error("Query command not implemented yet") }) diff --git a/cli/commands/table/script.tsx b/cli/commands/table/script.tsx index 36b7e745..ca0c0e38 100644 --- a/cli/commands/table/script.tsx +++ b/cli/commands/table/script.tsx @@ -1,6 +1,7 @@ import repl from "node:repl" import { Command } from "commander" import { readTable } from "dpkit" +import { createDialectFromOptions } from "../../helpers/dialect.ts" import * as params from "../../params/index.ts" export const scriptTableCommand = new Command("script") @@ -26,7 +27,7 @@ export const scriptTableCommand = new Command("script") .addOption(params.sheetName) .addOption(params.table) .action(async (path, options) => { - const dialect = params.createDialectFromOptions(options) + const dialect = createDialectFromOptions(options) const table = await readTable({ path, dialect }) const session = repl.start({ prompt: "dp> " }) diff --git a/cli/commands/table/validate.tsx b/cli/commands/table/validate.tsx index 72c83fa9..e3069934 100644 --- a/cli/commands/table/validate.tsx +++ b/cli/commands/table/validate.tsx @@ -3,6 +3,7 @@ import { validateTable } from "dpkit" import { render } from "ink" import React from "react" import { ErrorGrid } from "../../components/ErrorGrid.jsx" +import { createDialectFromOptions } from "../../helpers/dialect.ts" import * as params from "../../params/index.ts" export const validateTableCommand = new Command("validate") @@ -27,7 +28,7 @@ export const validateTableCommand = new Command("validate") .addOption(params.sheetName) .addOption(params.table) .action(async (path, options) => { - const dialect = params.createDialectFromOptions(options) + const dialect = createDialectFromOptions(options) const { errors } = await validateTable({ path, dialect }) if (options.json) { diff --git a/cli/helpers/dialect.ts b/cli/helpers/dialect.ts new file mode 100644 index 00000000..47660411 --- /dev/null +++ b/cli/helpers/dialect.ts @@ -0,0 +1,163 @@ +import type { Dialect } from "dpkit" + +// TODO: Find a better way to construct dialects + +export function createDialectFromOptions(options: any) { + let dialect: Dialect | undefined + + if (options.delimiter) { + dialect = { ...dialect, delimiter: options.delimiter } + } + + if (options.header === false) { + dialect = { ...dialect, header: options.header } + } + + if (options.headerRows) { + dialect = { + ...dialect, + headerRows: options.headerRows.split(",").map(Number), + } + } + + if (options.headerJoin) { + dialect = { ...dialect, headerJoin: options.headerJoin } + } + + if (options.commentRows) { + dialect = { + ...dialect, + commentRows: options.commentRows.split(",").map(Number), + } + } + + if (options.commentChar) { + dialect = { ...dialect, commentChar: options.commentChar } + } + + if (options.quoteChar) { + dialect = { ...dialect, quoteChar: options.quoteChar } + } + + if (options.doubleQuote) { + dialect = { ...dialect, doubleQuote: options.doubleQuote } + } + + if (options.escapeChar) { + dialect = { ...dialect, escapeChar: options.escapeChar } + } + + if (options.nullSequence) { + dialect = { ...dialect, nullSequence: options.nullSequence } + } + + if (options.skipInitialSpace) { + dialect = { ...dialect, skipInitialSpace: options.skipInitialSpace } + } + + if (options.property) { + dialect = { ...dialect, property: options.property } + } + + if (options.itemType) { + dialect = { ...dialect, itemType: options.itemType } + } + + if (options.itemKeys) { + dialect = { ...dialect, itemKeys: options.itemKeys.split(",") } + } + + if (options.sheetNumber) { + dialect = { ...dialect, sheetNumber: options.sheetNumber } + } + + if (options.sheetName) { + dialect = { ...dialect, sheetName: options.sheetName } + } + + if (options.table) { + dialect = { ...dialect, table: options.table } + } + + return dialect +} + +export function createToDialectFromOptions(options: any) { + let dialect: Dialect | undefined + + if (options.toDelimiter) { + dialect = { ...dialect, delimiter: options.toDelimiter } + } + + if (options.toHeader === false) { + dialect = { ...dialect, header: options.toHeader } + } + + if (options.toHeaderRows) { + dialect = { + ...dialect, + headerRows: options.toHeaderRows.split(",").map(Number), + } + } + + if (options.toHeaderJoin) { + dialect = { ...dialect, headerJoin: options.toHeaderJoin } + } + + if (options.toCommentRows) { + dialect = { + ...dialect, + commentRows: options.toCommentRows.split(",").map(Number), + } + } + + if (options.toCommentChar) { + dialect = { ...dialect, commentChar: options.toCommentChar } + } + + if (options.toQuoteChar) { + dialect = { ...dialect, quoteChar: options.toQuoteChar } + } + + if (options.toDoubleQuote) { + dialect = { ...dialect, doubleQuote: options.toDoubleQuote } + } + + if (options.toEscapeChar) { + dialect = { ...dialect, escapeChar: options.toEscapeChar } + } + + if (options.toNullSequence) { + dialect = { ...dialect, nullSequence: options.toNullSequence } + } + + if (options.toSkipInitialSpace) { + dialect = { ...dialect, skipInitialSpace: options.toSkipInitialSpace } + } + + if (options.toProperty) { + dialect = { ...dialect, property: options.toProperty } + } + + if (options.toItemType) { + dialect = { ...dialect, itemType: options.toItemType } + } + + if (options.toItemKeys) { + dialect = { ...dialect, itemKeys: options.toItemKeys.split(",") } + } + + if (options.toSheetNumber) { + dialect = { ...dialect, sheetNumber: options.toSheetNumber } + } + + if (options.toSheetName) { + dialect = { ...dialect, sheetName: options.toSheetName } + } + + if (options.toTable) { + dialect = { ...dialect, table: options.toTable } + } + + return dialect +} diff --git a/cli/main.ts b/cli/main.ts index b4d0bf2c..b7a21ca0 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -8,7 +8,7 @@ import metadata from "./package.json" with { type: "json" } program .name("dp") .description( - "Fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames", + "Fast data management CLI built on top of the Data Package standard and Polars DataFrames", ) .version(metadata.version) .configureHelp({ diff --git a/cli/params/dialect.ts b/cli/params/dialect.ts index 04d04125..919195ae 100644 --- a/cli/params/dialect.ts +++ b/cli/params/dialect.ts @@ -1,5 +1,4 @@ import { Option } from "commander" -import type { Dialect } from "dpkit" export const header = new Option( "--header", @@ -27,7 +26,7 @@ export const commentChar = new Option( ) export const delimiter = new Option( - "-d, --delimiter ", + "--delimiter ", "character used to separate fields in the data", ) @@ -91,102 +90,92 @@ export const table = new Option( "for database sources, the table name to read", ) -export const dialectOptions = [ - delimiter, - header, - headerRows, - headerJoin, - commentRows, - commentChar, - quoteChar, - doubleQuote, - escapeChar, - nullSequence, - skipInitialSpace, - property, - itemType, - itemKeys, - sheetNumber, - sheetName, - table, -] - -export function createDialectFromOptions(options: any) { - let dialect: Dialect | undefined - - if (options.delimiter) { - dialect = { ...dialect, delimiter: options.delimiter } - } - - if (options.header === false) { - dialect = { ...dialect, header: options.header } - } - - if (options.headerRows) { - dialect = { - ...dialect, - headerRows: options.headerRows.split(",").map(Number), - } - } - - if (options.headerJoin) { - dialect = { ...dialect, headerJoin: options.headerJoin } - } - - if (options.commentRows) { - dialect = { - ...dialect, - commentRows: options.commentRows.split(",").map(Number), - } - } - - if (options.commentChar) { - dialect = { ...dialect, commentChar: options.commentChar } - } - - if (options.quoteChar) { - dialect = { ...dialect, quoteChar: options.quoteChar } - } - - if (options.doubleQuote) { - dialect = { ...dialect, doubleQuote: options.doubleQuote } - } - - if (options.escapeChar) { - dialect = { ...dialect, escapeChar: options.escapeChar } - } - - if (options.nullSequence) { - dialect = { ...dialect, nullSequence: options.nullSequence } - } - - if (options.skipInitialSpace) { - dialect = { ...dialect, skipInitialSpace: options.skipInitialSpace } - } - - if (options.property) { - dialect = { ...dialect, property: options.property } - } - - if (options.itemType) { - dialect = { ...dialect, itemType: options.itemType } - } - - if (options.itemKeys) { - dialect = { ...dialect, itemKeys: options.itemKeys.split(",") } - } - - if (options.sheetNumber) { - dialect = { ...dialect, sheetNumber: options.sheetNumber } - } - - if (options.sheetName) { - dialect = { ...dialect, sheetName: options.sheetName } - } - - if (options.table) { - dialect = { ...dialect, table: options.table } - } - - return dialect -} +export const toHeader = new Option( + "--to-header", + "whether the file includes a header row with field names", +).default(true) + +export const toHeaderRows = new Option( + "--to-header-rows ", + "comma-separated row numbers (zero-based) that are considered header rows", +) + +export const toHeaderJoin = new Option( + "--to-header-join ", + "character used to join multi-line headers", +) + +export const toCommentRows = new Option( + "--to-comment-rows ", + "comma-separated rows to be excluded from the data (zero-based)", +) + +export const toCommentChar = new Option( + "--to-comment-char ", + "character sequence denoting the start of a comment line", +) + +export const toDelimiter = new Option( + "--to-delimiter ", + "character used to separate fields in the data", +) + +export const toLineTerminator = new Option( + "--to-line-terminator ", + "character sequence used to terminate rows", +) + +export const toQuoteChar = new Option( + "--to-quote-char ", + "character used to quote fields", +) + +export const toDoubleQuote = new Option( + "--to-double-quote", + "whether a sequence of two quote characters represents a single quote", +) + +export const toEscapeChar = new Option( + "--to-escape-char ", + "character used to escape the delimiter or quote characters", +) + +export const toNullSequence = new Option( + "--to-null-sequence ", + "character sequence representing null or missing values in the data", +) + +export const toSkipInitialSpace = new Option( + "--to-skip-initial-space", + "whether to ignore whitespace immediately following the delimiter", +) + +export const toProperty = new Option( + "--to-property ", + "for JSON data, the property name containing the data array", +) + +export const toItemType = new Option( + "--to-item-type ", + "the type of data item in the source", +).choices(["array", "object"]) + +export const toItemKeys = new Option( + "--to-item-keys ", + "comma-separated object properties to extract as values (for object-based data items)", +) + +export const toSheetNumber = new Option( + "--to-sheet-number ", + "for spreadsheet data, the sheet number to read (zero-based)", +).argParser(Number.parseInt) + +export const toSheetName = new Option( + "--to-sheet-name ", + "for spreadsheet data, the sheet name to read", +) + +export const toTable = new Option( + "--to-table ", + "for database sources, the table name to read", +) diff --git a/cli/params/index.ts b/cli/params/index.ts index f97054c6..28f8eced 100644 --- a/cli/params/index.ts +++ b/cli/params/index.ts @@ -3,4 +3,3 @@ export * from "./dialect.ts" export * from "./package.ts" export * from "./path.ts" export * from "./resource.ts" -export * from "./toDialect.ts" diff --git a/cli/params/toDialect.ts b/cli/params/toDialect.ts deleted file mode 100644 index 394aecc9..00000000 --- a/cli/params/toDialect.ts +++ /dev/null @@ -1,192 +0,0 @@ -import { Option } from "commander" -import type { Dialect } from "dpkit" - -export const toHeader = new Option( - "--to-header", - "whether the file includes a header row with field names", -).default(true) - -export const toHeaderRows = new Option( - "--to-header-rows ", - "comma-separated row numbers (zero-based) that are considered header rows", -) - -export const toHeaderJoin = new Option( - "--to-header-join ", - "character used to join multi-line headers", -) - -export const toCommentRows = new Option( - "--to-comment-rows ", - "comma-separated rows to be excluded from the data (zero-based)", -) - -export const toCommentChar = new Option( - "--to-comment-char ", - "character sequence denoting the start of a comment line", -) - -export const toDelimiter = new Option( - "--to-delimiter ", - "character used to separate fields in the data", -) - -export const toLineTerminator = new Option( - "--to-line-terminator ", - "character sequence used to terminate rows", -) - -export const toQuoteChar = new Option( - "--to-quote-char ", - "character used to quote fields", -) - -export const toDoubleQuote = new Option( - "--to-double-quote", - "whether a sequence of two quote characters represents a single quote", -) - -export const toEscapeChar = new Option( - "--to-escape-char ", - "character used to escape the delimiter or quote characters", -) - -export const toNullSequence = new Option( - "--to-null-sequence ", - "character sequence representing null or missing values in the data", -) - -export const toSkipInitialSpace = new Option( - "--to-skip-initial-space", - "whether to ignore whitespace immediately following the delimiter", -) - -export const toProperty = new Option( - "--to-property ", - "for JSON data, the property name containing the data array", -) - -export const toItemType = new Option( - "--to-item-type ", - "the type of data item in the source", -).choices(["array", "object"]) - -export const toItemKeys = new Option( - "--to-item-keys ", - "comma-separated object properties to extract as values (for object-based data items)", -) - -export const toSheetNumber = new Option( - "--to-sheet-number ", - "for spreadsheet data, the sheet number to read (zero-based)", -).argParser(Number.parseInt) - -export const toSheetName = new Option( - "--to-sheet-name ", - "for spreadsheet data, the sheet name to read", -) - -export const toTable = new Option( - "--to-table ", - "for database sources, the table name to read", -) - -export const toDialectOptions = [ - toDelimiter, - toHeader, - toHeaderRows, - toHeaderJoin, - toCommentRows, - toCommentChar, - toQuoteChar, - toDoubleQuote, - toEscapeChar, - toNullSequence, - toSkipInitialSpace, - toProperty, - toItemType, - toItemKeys, - toSheetNumber, - toSheetName, - toTable, -] - -export function createToDialectFromOptions(options: any) { - let dialect: Dialect | undefined - - if (options.toDelimiter) { - dialect = { ...dialect, delimiter: options.toDelimiter } - } - - if (options.toHeader === false) { - dialect = { ...dialect, header: options.toHeader } - } - - if (options.toHeaderRows) { - dialect = { - ...dialect, - headerRows: options.toHeaderRows.split(",").map(Number), - } - } - - if (options.toHeaderJoin) { - dialect = { ...dialect, headerJoin: options.toHeaderJoin } - } - - if (options.toCommentRows) { - dialect = { - ...dialect, - commentRows: options.toCommentRows.split(",").map(Number), - } - } - - if (options.toCommentChar) { - dialect = { ...dialect, commentChar: options.toCommentChar } - } - - if (options.toQuoteChar) { - dialect = { ...dialect, quoteChar: options.toQuoteChar } - } - - if (options.toDoubleQuote) { - dialect = { ...dialect, doubleQuote: options.toDoubleQuote } - } - - if (options.toEscapeChar) { - dialect = { ...dialect, escapeChar: options.toEscapeChar } - } - - if (options.toNullSequence) { - dialect = { ...dialect, nullSequence: options.toNullSequence } - } - - if (options.toSkipInitialSpace) { - dialect = { ...dialect, skipInitialSpace: options.toSkipInitialSpace } - } - - if (options.toProperty) { - dialect = { ...dialect, property: options.toProperty } - } - - if (options.toItemType) { - dialect = { ...dialect, itemType: options.toItemType } - } - - if (options.toItemKeys) { - dialect = { ...dialect, itemKeys: options.toItemKeys.split(",") } - } - - if (options.toSheetNumber) { - dialect = { ...dialect, sheetNumber: options.toSheetNumber } - } - - if (options.toSheetName) { - dialect = { ...dialect, sheetName: options.toSheetName } - } - - if (options.toTable) { - dialect = { ...dialect, table: options.toTable } - } - - return dialect -} From 803b5f1212b28d5b9d24eb339dd7b90fe90fef6e Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 14 Aug 2025 11:45:20 +0100 Subject: [PATCH 13/61] Fixed help messages --- cli/commands/dialect/infer.ts | 16 ++++++++++------ cli/commands/package/archive.ts | 4 ++++ cli/commands/package/copy.ts | 4 ++++ cli/commands/package/show.ts | 4 ++++ cli/commands/resource/.keep | 0 cli/commands/resource/infer.ts | 1 + cli/commands/schema/.keep | 0 cli/commands/schema/infer.ts | 1 + cli/commands/table/convert.tsx | 2 ++ cli/commands/table/describe.tsx | 5 +++++ cli/commands/table/explore.tsx | 2 ++ cli/commands/table/index.ts | 3 +++ cli/commands/table/query.tsx | 5 +++++ cli/commands/table/script.tsx | 6 ++++++ cli/commands/table/validate.tsx | 6 ++++++ cli/helpers/help.ts | 6 ++++++ cli/helpers/session.ts | 8 ++++++++ cli/main.ts | 11 ++++++----- 18 files changed, 73 insertions(+), 11 deletions(-) delete mode 100644 cli/commands/resource/.keep create mode 100644 cli/commands/resource/infer.ts delete mode 100644 cli/commands/schema/.keep create mode 100644 cli/commands/schema/infer.ts create mode 100644 cli/helpers/help.ts diff --git a/cli/commands/dialect/infer.ts b/cli/commands/dialect/infer.ts index c812a873..30f8d8de 100644 --- a/cli/commands/dialect/infer.ts +++ b/cli/commands/dialect/infer.ts @@ -1,18 +1,22 @@ import { Command } from "commander" import { inferDialect } from "dpkit" +import { helpConfiguration } from "../../helpers/help.ts" +import { Session } from "../../helpers/session.ts" import * as params from "../../params/index.ts" export const inferDialectCommand = new Command("infer") + .configureHelp(helpConfiguration) .description("Infer a dialect from a table") + .addArgument(params.positionalTablePath) .addOption(params.json) + .action(async (path, options) => { - const dialect = await inferDialect({ path }) + const session = Session.create(options) + session.intro("Inferring dialect") - if (options.json) { - console.log(JSON.stringify(dialect, null, 2)) - return - } + const dialect = await session.task("Loading sample", inferDialect({ path })) - console.log(dialect) + session.object(dialect) + session.outro("Thanks for using dpkit!") }) diff --git a/cli/commands/package/archive.ts b/cli/commands/package/archive.ts index 92c5e655..c3f7db57 100644 --- a/cli/commands/package/archive.ts +++ b/cli/commands/package/archive.ts @@ -1,12 +1,16 @@ import { Command } from "commander" import { loadPackage, savePackageToZip } from "dpkit" +import { helpConfiguration } from "../../helpers/help.ts" import * as params from "../../params/index.ts" export const archivePackageCommand = new Command("archive") + .configureHelp(helpConfiguration) .description("Archive a local or remote Data Package to a local zip file") + .addArgument(params.positionalDescriptorPath) .addOption(params.toArchive.makeOptionMandatory()) .addOption(params.withRemote) + .action(async (path, options) => { const dp = await loadPackage(path) diff --git a/cli/commands/package/copy.ts b/cli/commands/package/copy.ts index 685ac649..fc8e3399 100644 --- a/cli/commands/package/copy.ts +++ b/cli/commands/package/copy.ts @@ -1,12 +1,16 @@ import { Command } from "commander" import { loadPackage, savePackageToFolder } from "dpkit" +import { helpConfiguration } from "../../helpers/help.ts" import * as params from "../../params/index.ts" export const copyPackageCommand = new Command("copy") + .configureHelp(helpConfiguration) .description("Copy a local or remote Data Package to a local folder") + .addArgument(params.positionalDescriptorPath) .addOption(params.toFolder.makeOptionMandatory()) .addOption(params.withRemote) + .action(async (path, options) => { const dp = await loadPackage(path) diff --git a/cli/commands/package/show.ts b/cli/commands/package/show.ts index feb0781b..0606ef8e 100644 --- a/cli/commands/package/show.ts +++ b/cli/commands/package/show.ts @@ -1,11 +1,15 @@ import { Command } from "commander" import { loadPackage } from "dpkit" +import { helpConfiguration } from "../../helpers/help.ts" import * as params from "../../params/index.ts" export const showPackageCommand = new Command("show") + .configureHelp(helpConfiguration) .description("Show a Data Package descriptor") + .addArgument(params.positionalDescriptorPath) .addOption(params.json) + .action(async (path, options) => { const dp = await loadPackage(path) diff --git a/cli/commands/resource/.keep b/cli/commands/resource/.keep deleted file mode 100644 index e69de29b..00000000 diff --git a/cli/commands/resource/infer.ts b/cli/commands/resource/infer.ts new file mode 100644 index 00000000..1039d4c5 --- /dev/null +++ b/cli/commands/resource/infer.ts @@ -0,0 +1 @@ +// TODO: implement diff --git a/cli/commands/schema/.keep b/cli/commands/schema/.keep deleted file mode 100644 index e69de29b..00000000 diff --git a/cli/commands/schema/infer.ts b/cli/commands/schema/infer.ts new file mode 100644 index 00000000..1039d4c5 --- /dev/null +++ b/cli/commands/schema/infer.ts @@ -0,0 +1 @@ +// TODO: implement diff --git a/cli/commands/table/convert.tsx b/cli/commands/table/convert.tsx index 234442a5..9f066580 100644 --- a/cli/commands/table/convert.tsx +++ b/cli/commands/table/convert.tsx @@ -3,9 +3,11 @@ import { getTempFilePath, loadFile } from "dpkit" import { readTable, saveTable } from "dpkit" import { createDialectFromOptions } from "../../helpers/dialect.ts" import { createToDialectFromOptions } from "../../helpers/dialect.ts" +import { helpConfiguration } from "../../helpers/help.ts" import * as params from "../../params/index.ts" export const convertTableCommand = new Command("convert") + .configureHelp(helpConfiguration) .description( "Convert a table from a local or remote source path to a target path", ) diff --git a/cli/commands/table/describe.tsx b/cli/commands/table/describe.tsx index de8108ea..e6ab9d03 100644 --- a/cli/commands/table/describe.tsx +++ b/cli/commands/table/describe.tsx @@ -4,12 +4,17 @@ import { render } from "ink" import React from "react" import { TableGrid } from "../../components/TableGrid.tsx" import { createDialectFromOptions } from "../../helpers/dialect.ts" +import { helpConfiguration } from "../../helpers/help.ts" import * as params from "../../params/index.ts" export const describeTableCommand = new Command("describe") + .configureHelp(helpConfiguration) .description("Describe a table from a local or remote path") + .addArgument(params.positionalTablePath) .addOption(params.json) + + .optionsGroup("Table Dialect") .addOption(params.delimiter) .addOption(params.header) .addOption(params.headerRows) diff --git a/cli/commands/table/explore.tsx b/cli/commands/table/explore.tsx index 6ce1b888..588ecd96 100644 --- a/cli/commands/table/explore.tsx +++ b/cli/commands/table/explore.tsx @@ -3,11 +3,13 @@ import { readTable } from "dpkit" import React from "react" import { TableGrid } from "../../components/TableGrid.tsx" import { createDialectFromOptions } from "../../helpers/dialect.ts" +import { helpConfiguration } from "../../helpers/help.ts" import { selectResource } from "../../helpers/resource.ts" import { Session } from "../../helpers/session.ts" import * as params from "../../params/index.ts" export const exploreTableCommand = new Command("explore") + .configureHelp(helpConfiguration) .description("Explore a table from a local or remote path") .addArgument(params.positionalTablePath) diff --git a/cli/commands/table/index.ts b/cli/commands/table/index.ts index 436a40a6..ee7b1d18 100644 --- a/cli/commands/table/index.ts +++ b/cli/commands/table/index.ts @@ -1,4 +1,5 @@ import { Command } from "commander" +import { helpConfiguration } from "../../helpers/help.ts" import { convertTableCommand } from "./convert.tsx" import { describeTableCommand } from "./describe.tsx" import { exploreTableCommand } from "./explore.tsx" @@ -8,6 +9,8 @@ import { validateTableCommand } from "./validate.tsx" export const tableCommand = new Command("table") .description("Table related commands") + .configureHelp(helpConfiguration) + .addCommand(describeTableCommand) .addCommand(convertTableCommand) .addCommand(validateTableCommand) diff --git a/cli/commands/table/query.tsx b/cli/commands/table/query.tsx index c4145990..66eabf79 100644 --- a/cli/commands/table/query.tsx +++ b/cli/commands/table/query.tsx @@ -1,12 +1,16 @@ import { Command } from "commander" +import { helpConfiguration } from "../../helpers/help.ts" import * as params from "../../params/index.ts" export const queryTableCommand = new Command("query") + .configureHelp(helpConfiguration) .description( "Start a querying session for a table from a local or remote path", ) .addArgument(params.positionalTablePath) .addOption(params.json) + + .optionsGroup("Table Dialect") .addOption(params.delimiter) .addOption(params.header) .addOption(params.headerRows) @@ -24,6 +28,7 @@ export const queryTableCommand = new Command("query") .addOption(params.sheetNumber) .addOption(params.sheetName) .addOption(params.table) + .action(async (_path, _options) => { throw new Error("Query command not implemented yet") }) diff --git a/cli/commands/table/script.tsx b/cli/commands/table/script.tsx index ca0c0e38..70b7fb2c 100644 --- a/cli/commands/table/script.tsx +++ b/cli/commands/table/script.tsx @@ -2,13 +2,18 @@ import repl from "node:repl" import { Command } from "commander" import { readTable } from "dpkit" import { createDialectFromOptions } from "../../helpers/dialect.ts" +import { helpConfiguration } from "../../helpers/help.ts" import * as params from "../../params/index.ts" export const scriptTableCommand = new Command("script") + .configureHelp(helpConfiguration) .description( "Start a scripting session for a table from a local or remote path", ) + .addArgument(params.positionalTablePath) + + .optionsGroup("Table Dialect") .addOption(params.delimiter) .addOption(params.header) .addOption(params.headerRows) @@ -26,6 +31,7 @@ export const scriptTableCommand = new Command("script") .addOption(params.sheetNumber) .addOption(params.sheetName) .addOption(params.table) + .action(async (path, options) => { const dialect = createDialectFromOptions(options) const table = await readTable({ path, dialect }) diff --git a/cli/commands/table/validate.tsx b/cli/commands/table/validate.tsx index e3069934..ef0b3b6a 100644 --- a/cli/commands/table/validate.tsx +++ b/cli/commands/table/validate.tsx @@ -4,12 +4,17 @@ import { render } from "ink" import React from "react" import { ErrorGrid } from "../../components/ErrorGrid.jsx" import { createDialectFromOptions } from "../../helpers/dialect.ts" +import { helpConfiguration } from "../../helpers/help.ts" import * as params from "../../params/index.ts" export const validateTableCommand = new Command("validate") + .configureHelp(helpConfiguration) .description("Validate a table from a local or remote path") + .addArgument(params.positionalTablePath) .addOption(params.json) + + .optionsGroup("Table Dialect") .addOption(params.delimiter) .addOption(params.header) .addOption(params.headerRows) @@ -27,6 +32,7 @@ export const validateTableCommand = new Command("validate") .addOption(params.sheetNumber) .addOption(params.sheetName) .addOption(params.table) + .action(async (path, options) => { const dialect = createDialectFromOptions(options) const { errors } = await validateTable({ path, dialect }) diff --git a/cli/helpers/help.ts b/cli/helpers/help.ts new file mode 100644 index 00000000..32916488 --- /dev/null +++ b/cli/helpers/help.ts @@ -0,0 +1,6 @@ +import { styleText } from "node:util" +import type { HelpConfiguration } from "commander" + +export const helpConfiguration: HelpConfiguration = { + styleTitle: str => styleText("bold", str.toUpperCase().slice(0, -1)), +} diff --git a/cli/helpers/session.ts b/cli/helpers/session.ts index 5e3310d7..b80ff666 100644 --- a/cli/helpers/session.ts +++ b/cli/helpers/session.ts @@ -23,6 +23,10 @@ export class Session { outro(message) } + object(object: Record) { + console.log(object) + } + async select(options: SelectOptions) { return String(await select(options)) } @@ -51,6 +55,10 @@ export class JsonSession extends Session { intro = () => {} outro = () => {} + object(object: Record) { + console.log(JSON.stringify(object, null, 2)) + } + async select(_options: SelectOptions): Promise { Session.terminate("Selection is not supported in JSON mode") } diff --git a/cli/main.ts b/cli/main.ts index b7a21ca0..0c8146a1 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -1,8 +1,8 @@ -import { styleText } from "node:util" import { program } from "commander" import { dialectCommand } from "./commands/dialect/index.ts" import { packageCommand } from "./commands/package/index.ts" import { tableCommand } from "./commands/table/index.ts" +import { helpConfiguration } from "./helpers/help.ts" import metadata from "./package.json" with { type: "json" } program @@ -10,11 +10,12 @@ program .description( "Fast data management CLI built on top of the Data Package standard and Polars DataFrames", ) - .version(metadata.version) - .configureHelp({ - styleTitle: str => styleText("bold", str.toUpperCase().slice(0, -1)), - }) + + .version(metadata.version, "-v, --version") + .configureHelp(helpConfiguration) + .addCommand(packageCommand) .addCommand(tableCommand) .addCommand(dialectCommand) + .parse() From ecfd332b82dd76785b21aa678a3ff1d550eaa7a4 Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 14 Aug 2025 11:56:33 +0100 Subject: [PATCH 14/61] Updated dialect infer --- cli/commands/dialect/index.ts | 2 +- cli/commands/dialect/{infer.ts => infer.tsx} | 4 +++- cli/commands/dialect/show.ts | 1 + cli/commands/package/infer.ts | 1 + cli/commands/resource/show.ts | 1 + cli/commands/schema/show.ts | 1 + cli/components/DataGrid.tsx | 5 +++-- cli/components/DialectGrid.tsx | 7 +++++++ cli/components/TableGrid.tsx | 2 +- 9 files changed, 19 insertions(+), 5 deletions(-) rename cli/commands/dialect/{infer.ts => infer.tsx} (82%) create mode 100644 cli/commands/dialect/show.ts create mode 100644 cli/commands/package/infer.ts create mode 100644 cli/commands/resource/show.ts create mode 100644 cli/commands/schema/show.ts create mode 100644 cli/components/DialectGrid.tsx diff --git a/cli/commands/dialect/index.ts b/cli/commands/dialect/index.ts index 8e836555..e00a91af 100644 --- a/cli/commands/dialect/index.ts +++ b/cli/commands/dialect/index.ts @@ -1,5 +1,5 @@ import { Command } from "commander" -import { inferDialectCommand } from "./infer.ts" +import { inferDialectCommand } from "./infer.tsx" export const dialectCommand = new Command("dialect") .description("Table Dialect related commands") diff --git a/cli/commands/dialect/infer.ts b/cli/commands/dialect/infer.tsx similarity index 82% rename from cli/commands/dialect/infer.ts rename to cli/commands/dialect/infer.tsx index 30f8d8de..7e7e770d 100644 --- a/cli/commands/dialect/infer.ts +++ b/cli/commands/dialect/infer.tsx @@ -1,5 +1,7 @@ import { Command } from "commander" import { inferDialect } from "dpkit" +import React from "react" +import { DialectGrid } from "../../components/DialectGrid.tsx" import { helpConfiguration } from "../../helpers/help.ts" import { Session } from "../../helpers/session.ts" import * as params from "../../params/index.ts" @@ -16,7 +18,7 @@ export const inferDialectCommand = new Command("infer") session.intro("Inferring dialect") const dialect = await session.task("Loading sample", inferDialect({ path })) + await session.render() - session.object(dialect) session.outro("Thanks for using dpkit!") }) diff --git a/cli/commands/dialect/show.ts b/cli/commands/dialect/show.ts new file mode 100644 index 00000000..1039d4c5 --- /dev/null +++ b/cli/commands/dialect/show.ts @@ -0,0 +1 @@ +// TODO: implement diff --git a/cli/commands/package/infer.ts b/cli/commands/package/infer.ts new file mode 100644 index 00000000..1039d4c5 --- /dev/null +++ b/cli/commands/package/infer.ts @@ -0,0 +1 @@ +// TODO: implement diff --git a/cli/commands/resource/show.ts b/cli/commands/resource/show.ts new file mode 100644 index 00000000..1039d4c5 --- /dev/null +++ b/cli/commands/resource/show.ts @@ -0,0 +1 @@ +// TODO: implement diff --git a/cli/commands/schema/show.ts b/cli/commands/schema/show.ts new file mode 100644 index 00000000..1039d4c5 --- /dev/null +++ b/cli/commands/schema/show.ts @@ -0,0 +1 @@ +// TODO: implement diff --git a/cli/components/DataGrid.tsx b/cli/components/DataGrid.tsx index 398da95a..0658bd76 100644 --- a/cli/components/DataGrid.tsx +++ b/cli/components/DataGrid.tsx @@ -8,8 +8,9 @@ export function DataGrid(props: { data: Record[] col?: number order?: Order + rowHeight?: number }) { - const { data, col, order } = props + const { data, col, order, rowHeight } = props const colNames = Object.keys(data[0] ?? {}) const colWidth = Math.min( @@ -64,7 +65,7 @@ export function DataGrid(props: { ? "#444" : undefined } - height={2} + height={rowHeight} overflow="hidden" > {(row[name] ?? "").toString()} diff --git a/cli/components/DialectGrid.tsx b/cli/components/DialectGrid.tsx new file mode 100644 index 00000000..f3b72dc3 --- /dev/null +++ b/cli/components/DialectGrid.tsx @@ -0,0 +1,7 @@ +import type { Dialect } from "dpkit" +import React from "react" +import { DataGrid } from "./DataGrid.tsx" + +export function DialectGrid(props: { dialect: Dialect }) { + return +} diff --git a/cli/components/TableGrid.tsx b/cli/components/TableGrid.tsx index a261af21..53ebeddc 100644 --- a/cli/components/TableGrid.tsx +++ b/cli/components/TableGrid.tsx @@ -92,7 +92,7 @@ export function TableGrid(props: { table: Table }) { return ( - + ) From 5c99fdb6530e6ccae4c3e6837c4d3cbb415fd7a1 Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 14 Aug 2025 12:06:58 +0100 Subject: [PATCH 15/61] Rebased on joint render --- cli/commands/dialect/infer.tsx | 6 +++++- cli/helpers/object.ts | 3 +++ cli/helpers/session.ts | 17 +++++------------ 3 files changed, 13 insertions(+), 13 deletions(-) create mode 100644 cli/helpers/object.ts diff --git a/cli/commands/dialect/infer.tsx b/cli/commands/dialect/infer.tsx index 7e7e770d..2bf1c478 100644 --- a/cli/commands/dialect/infer.tsx +++ b/cli/commands/dialect/infer.tsx @@ -3,6 +3,7 @@ import { inferDialect } from "dpkit" import React from "react" import { DialectGrid } from "../../components/DialectGrid.tsx" import { helpConfiguration } from "../../helpers/help.ts" +import { isEmptyObject } from "../../helpers/object.ts" import { Session } from "../../helpers/session.ts" import * as params from "../../params/index.ts" @@ -18,7 +19,10 @@ export const inferDialectCommand = new Command("infer") session.intro("Inferring dialect") const dialect = await session.task("Loading sample", inferDialect({ path })) - await session.render() + if (isEmptyObject(dialect)) { + Session.terminate("Could not infer dialect") + } + await session.render(dialect, ) session.outro("Thanks for using dpkit!") }) diff --git a/cli/helpers/object.ts b/cli/helpers/object.ts new file mode 100644 index 00000000..13a712c5 --- /dev/null +++ b/cli/helpers/object.ts @@ -0,0 +1,3 @@ +export function isEmptyObject(obj: Record): boolean { + return Object.keys(obj).length === 0 +} \ No newline at end of file diff --git a/cli/helpers/session.ts b/cli/helpers/session.ts index b80ff666..a689b452 100644 --- a/cli/helpers/session.ts +++ b/cli/helpers/session.ts @@ -3,6 +3,7 @@ import { spinner } from "@clack/prompts" import { intro, log, outro, select } from "@clack/prompts" import type { SelectOptions } from "@clack/prompts" import { render } from "ink" +import type React from "react" import invariant from "tiny-invariant" export class Session { @@ -23,10 +24,6 @@ export class Session { outro(message) } - object(object: Record) { - console.log(object) - } - async select(options: SelectOptions) { return String(await select(options)) } @@ -41,12 +38,12 @@ export class Session { return result } - async render(...args: Parameters) { + async render(_object: Record, node: React.ReactNode) { // Without waiting for the next tick after clack prompts, // ink render will be immidiately terminated await setImmediate() - const app = render(...args) + const app = render(node) await app.waitUntilExit() } } @@ -55,16 +52,12 @@ export class JsonSession extends Session { intro = () => {} outro = () => {} - object(object: Record) { - console.log(JSON.stringify(object, null, 2)) - } - async select(_options: SelectOptions): Promise { Session.terminate("Selection is not supported in JSON mode") } - async render(..._args: Parameters) { - invariant(false, "Render must not be used in JSON mode") + async render(object: Record, _node: React.ReactNode) { + console.log(JSON.stringify(object, null, 2)) } async task(_message: string, promise: Promise) { From 5ff65863e1d9579ea20275fdb90e0e1afd23926c Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 14 Aug 2025 12:38:25 +0100 Subject: [PATCH 16/61] Improved session into/outro --- cli/commands/dialect/infer.tsx | 14 +++++++++---- cli/commands/table/explore.tsx | 9 ++++----- cli/components/DialectGrid.tsx | 2 ++ cli/helpers/help.ts | 4 ++-- cli/helpers/resource.ts | 4 ++++ cli/helpers/session.ts | 36 ++++++++++++++++++++++------------ cli/package.json | 2 ++ pnpm-lock.yaml | 6 ++++++ 8 files changed, 54 insertions(+), 23 deletions(-) diff --git a/cli/commands/dialect/infer.tsx b/cli/commands/dialect/infer.tsx index 2bf1c478..66e26364 100644 --- a/cli/commands/dialect/infer.tsx +++ b/cli/commands/dialect/infer.tsx @@ -4,6 +4,7 @@ import React from "react" import { DialectGrid } from "../../components/DialectGrid.tsx" import { helpConfiguration } from "../../helpers/help.ts" import { isEmptyObject } from "../../helpers/object.ts" +import { selectResource } from "../../helpers/resource.ts" import { Session } from "../../helpers/session.ts" import * as params from "../../params/index.ts" @@ -12,17 +13,22 @@ export const inferDialectCommand = new Command("infer") .description("Infer a dialect from a table") .addArgument(params.positionalTablePath) + .addOption(params.fromPackage) + .addOption(params.fromResource) .addOption(params.json) .action(async (path, options) => { - const session = Session.create(options) - session.intro("Inferring dialect") + const session = Session.create({ + title: "Inferring dialect", + json: options.json, + }) - const dialect = await session.task("Loading sample", inferDialect({ path })) + const resource = path ? { path } : await selectResource(session, options) + + const dialect = await session.task("Loading sample", inferDialect(resource)) if (isEmptyObject(dialect)) { Session.terminate("Could not infer dialect") } await session.render(dialect, ) - session.outro("Thanks for using dpkit!") }) diff --git a/cli/commands/table/explore.tsx b/cli/commands/table/explore.tsx index 588ecd96..9014579d 100644 --- a/cli/commands/table/explore.tsx +++ b/cli/commands/table/explore.tsx @@ -36,15 +36,14 @@ export const exploreTableCommand = new Command("explore") .addOption(params.table) .action(async (path, options) => { - const session = Session.create() - session.intro("Exploring table") + const session = Session.create({ + title: "Exploring table", + }) const resource = path ? { path, dialect: createDialectFromOptions(options) } : await selectResource(session, options) const table = await session.task("Loading table", readTable(resource)) - await session.render() - - session.outro("Thanks for using dpkit!") + await session.render(table, ) }) diff --git a/cli/components/DialectGrid.tsx b/cli/components/DialectGrid.tsx index f3b72dc3..e687b9a4 100644 --- a/cli/components/DialectGrid.tsx +++ b/cli/components/DialectGrid.tsx @@ -2,6 +2,8 @@ import type { Dialect } from "dpkit" import React from "react" import { DataGrid } from "./DataGrid.tsx" +// TODO: Support non-visible chars like TAB and CR + export function DialectGrid(props: { dialect: Dialect }) { return } diff --git a/cli/helpers/help.ts b/cli/helpers/help.ts index 32916488..68a955cf 100644 --- a/cli/helpers/help.ts +++ b/cli/helpers/help.ts @@ -1,6 +1,6 @@ -import { styleText } from "node:util" import type { HelpConfiguration } from "commander" +import pc from "picocolors" export const helpConfiguration: HelpConfiguration = { - styleTitle: str => styleText("bold", str.toUpperCase().slice(0, -1)), + styleTitle: str => pc.bold(str.toUpperCase().replace(":", "")), } diff --git a/cli/helpers/resource.ts b/cli/helpers/resource.ts index 92fa037d..3fd0714c 100644 --- a/cli/helpers/resource.ts +++ b/cli/helpers/resource.ts @@ -29,6 +29,10 @@ export async function selectResource( ) if (!resource) { + if (typeof resourceName !== "string") { + Session.terminate("Resource selection cancelled") + } + Session.terminate( `Resource "${resourceName}" is not found in the provided data package`, ) diff --git a/cli/helpers/session.ts b/cli/helpers/session.ts index a689b452..b573a176 100644 --- a/cli/helpers/session.ts +++ b/cli/helpers/session.ts @@ -2,13 +2,21 @@ import { setImmediate } from "node:timers/promises" import { spinner } from "@clack/prompts" import { intro, log, outro, select } from "@clack/prompts" import type { SelectOptions } from "@clack/prompts" +import exitHook from "exit-hook" import { render } from "ink" +import pc from "picocolors" import type React from "react" -import invariant from "tiny-invariant" export class Session { - static create(options?: { json?: boolean }) { - return options?.json ? new JsonSession() : new Session() + title: string + + static create(options: { title: string; json?: boolean }) { + const session = options.json + ? new JsonSession(options) + : new Session(options) + + session.start() + return session } static terminate(message: string): never { @@ -16,16 +24,21 @@ export class Session { process.exit(1) } - intro(message: string) { - intro(message) + constructor(options: { title: string }) { + this.title = options.title } - outro(message: string) { - outro(message) + start() { + intro(pc.bold(this.title)) + exitHook(() => { + outro( + `Problems? ${pc.underline(pc.cyan("https://github.com/datisthq/dpkit/issues"))}`, + ) + }) } async select(options: SelectOptions) { - return String(await select(options)) + return await select(options) } async task(message: string, promise: Promise) { @@ -38,7 +51,7 @@ export class Session { return result } - async render(_object: Record, node: React.ReactNode) { + async render(_object: any, node: React.ReactNode) { // Without waiting for the next tick after clack prompts, // ink render will be immidiately terminated await setImmediate() @@ -49,14 +62,13 @@ export class Session { } export class JsonSession extends Session { - intro = () => {} - outro = () => {} + start = () => {} async select(_options: SelectOptions): Promise { Session.terminate("Selection is not supported in JSON mode") } - async render(object: Record, _node: React.ReactNode) { + async render(object: any, _node: React.ReactNode) { console.log(JSON.stringify(object, null, 2)) } diff --git a/cli/package.json b/cli/package.json index 1e00c00a..e7afb8e1 100644 --- a/cli/package.json +++ b/cli/package.json @@ -35,9 +35,11 @@ "@inkjs/ui": "^2.0.0", "commander": "^14.0.0", "dpkit": "workspace:*", + "exit-hook": "^4.0.0", "ink": "^6.1.0", "ink-select-input": "^6.2.0", "nodejs-polars": "^0.21.0", + "picocolors": "^1.1.1", "react": "^19.1.1", "tiny-invariant": "^1.3.3", "ts-extras": "^0.14.0" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8d86cb01..1483fa5a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -103,6 +103,9 @@ importers: dpkit: specifier: workspace:* version: link:../dpkit + exit-hook: + specifier: ^4.0.0 + version: 4.0.0 ink: specifier: ^6.1.0 version: 6.1.0(@types/react@19.1.9)(react@19.1.1) @@ -112,6 +115,9 @@ importers: nodejs-polars: specifier: ^0.21.0 version: 0.21.0 + picocolors: + specifier: ^1.1.1 + version: 1.1.1 react: specifier: ^19.1.1 version: 19.1.1 From 856a85f1a25f34aca1018bd17c1ca6e2236813b7 Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 14 Aug 2025 13:04:35 +0100 Subject: [PATCH 17/61] Bootstrapped validate commands --- cli/commands/dialect/validate.ts | 1 + cli/commands/package/validate.ts | 1 + cli/commands/resource/validate.ts | 1 + cli/commands/schema/validate.ts | 1 + 4 files changed, 4 insertions(+) create mode 100644 cli/commands/dialect/validate.ts create mode 100644 cli/commands/package/validate.ts create mode 100644 cli/commands/resource/validate.ts create mode 100644 cli/commands/schema/validate.ts diff --git a/cli/commands/dialect/validate.ts b/cli/commands/dialect/validate.ts new file mode 100644 index 00000000..1039d4c5 --- /dev/null +++ b/cli/commands/dialect/validate.ts @@ -0,0 +1 @@ +// TODO: implement diff --git a/cli/commands/package/validate.ts b/cli/commands/package/validate.ts new file mode 100644 index 00000000..1039d4c5 --- /dev/null +++ b/cli/commands/package/validate.ts @@ -0,0 +1 @@ +// TODO: implement diff --git a/cli/commands/resource/validate.ts b/cli/commands/resource/validate.ts new file mode 100644 index 00000000..1039d4c5 --- /dev/null +++ b/cli/commands/resource/validate.ts @@ -0,0 +1 @@ +// TODO: implement diff --git a/cli/commands/schema/validate.ts b/cli/commands/schema/validate.ts new file mode 100644 index 00000000..1039d4c5 --- /dev/null +++ b/cli/commands/schema/validate.ts @@ -0,0 +1 @@ +// TODO: implement From f19463b0c452aebc74411e3deb72ba9c83f57530 Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 14 Aug 2025 13:06:58 +0100 Subject: [PATCH 18/61] Exlude compile from tsconfig --- cli/commands/package/index.ts | 1 - cli/helpers/object.ts | 2 +- cli/helpers/session.ts | 2 +- tsconfig.json | 7 ++++++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/cli/commands/package/index.ts b/cli/commands/package/index.ts index 45533ad7..06ef1b53 100644 --- a/cli/commands/package/index.ts +++ b/cli/commands/package/index.ts @@ -8,4 +8,3 @@ export const packageCommand = new Command("package") .addCommand(showPackageCommand) .addCommand(archivePackageCommand) .addCommand(copyPackageCommand) - diff --git a/cli/helpers/object.ts b/cli/helpers/object.ts index 13a712c5..ea92287e 100644 --- a/cli/helpers/object.ts +++ b/cli/helpers/object.ts @@ -1,3 +1,3 @@ export function isEmptyObject(obj: Record): boolean { return Object.keys(obj).length === 0 -} \ No newline at end of file +} diff --git a/cli/helpers/session.ts b/cli/helpers/session.ts index b573a176..6175a71e 100644 --- a/cli/helpers/session.ts +++ b/cli/helpers/session.ts @@ -64,7 +64,7 @@ export class Session { export class JsonSession extends Session { start = () => {} - async select(_options: SelectOptions): Promise { + async select(_options: SelectOptions): Promise { Session.terminate("Selection is not supported in JSON mode") } diff --git a/tsconfig.json b/tsconfig.json index 8b1e1a39..b076abc1 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,7 +1,12 @@ { "$schema": "https://json.schemastore.org/tsconfig", + "include": ["${configDir}/**/*.ts", "${configDir}/**/*.tsx"], - "exclude": ["${configDir}/**/build/", "${configDir}/**/docs/"], + "exclude": [ + "${configDir}/**/docs/", + "${configDir}/**/build/", + "${configDir}/**/compile/" + ], "compilerOptions": { "target": "ES2022", From 0672d19034214b7d43b6fe68f8ad991f78c26729 Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 14 Aug 2025 16:12:50 +0100 Subject: [PATCH 19/61] Added dialect show command --- cli/commands/dialect/index.ts | 2 ++ cli/commands/dialect/infer.tsx | 4 +-- cli/commands/dialect/show.ts | 1 - cli/commands/dialect/show.tsx | 48 ++++++++++++++++++++++++++++++++++ cli/commands/table/explore.tsx | 2 +- cli/components/DataGrid.tsx | 6 ++++- 6 files changed, 58 insertions(+), 5 deletions(-) delete mode 100644 cli/commands/dialect/show.ts create mode 100644 cli/commands/dialect/show.tsx diff --git a/cli/commands/dialect/index.ts b/cli/commands/dialect/index.ts index e00a91af..1665d46b 100644 --- a/cli/commands/dialect/index.ts +++ b/cli/commands/dialect/index.ts @@ -1,6 +1,8 @@ import { Command } from "commander" import { inferDialectCommand } from "./infer.tsx" +import { showDialectCommand } from "./show.tsx" export const dialectCommand = new Command("dialect") .description("Table Dialect related commands") .addCommand(inferDialectCommand) + .addCommand(showDialectCommand) diff --git a/cli/commands/dialect/infer.tsx b/cli/commands/dialect/infer.tsx index 66e26364..f263b443 100644 --- a/cli/commands/dialect/infer.tsx +++ b/cli/commands/dialect/infer.tsx @@ -10,7 +10,7 @@ import * as params from "../../params/index.ts" export const inferDialectCommand = new Command("infer") .configureHelp(helpConfiguration) - .description("Infer a dialect from a table") + .description("Infer a table dialect from a table") .addArgument(params.positionalTablePath) .addOption(params.fromPackage) @@ -19,7 +19,7 @@ export const inferDialectCommand = new Command("infer") .action(async (path, options) => { const session = Session.create({ - title: "Inferring dialect", + title: "Infer dialect", json: options.json, }) diff --git a/cli/commands/dialect/show.ts b/cli/commands/dialect/show.ts deleted file mode 100644 index 1039d4c5..00000000 --- a/cli/commands/dialect/show.ts +++ /dev/null @@ -1 +0,0 @@ -// TODO: implement diff --git a/cli/commands/dialect/show.tsx b/cli/commands/dialect/show.tsx new file mode 100644 index 00000000..efb69800 --- /dev/null +++ b/cli/commands/dialect/show.tsx @@ -0,0 +1,48 @@ +import { Command } from "commander" +import { loadDialect } from "dpkit" +import type { Dialect } from "dpkit" +import React from "react" +import { DialectGrid } from "../../components/DialectGrid.tsx" +import { helpConfiguration } from "../../helpers/help.ts" +import { isEmptyObject } from "../../helpers/object.ts" +import { selectResource } from "../../helpers/resource.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const showDialectCommand = new Command("show") + .configureHelp(helpConfiguration) + .description("Show a table dialect from a local or remote path") + + .addArgument(params.positionalTablePath) + .addOption(params.fromPackage) + .addOption(params.fromResource) + .addOption(params.json) + + .action(async (path, options) => { + let dialect: Dialect | undefined + + const session = Session.create({ + title: "Show dialect", + json: options.json, + }) + + if (!path) { + const resource = await selectResource(session, options) + if (typeof resource.dialect !== "string") { + dialect = resource.dialect + } else { + path = resource.dialect + } + } + + if (!dialect) { + // @ts-ignore + dialect = await session.task("Loading dialect", loadDialect(path)) + } + + if (isEmptyObject(dialect)) { + Session.terminate("Dialect is not available") + } + + await session.render(dialect, ) + }) diff --git a/cli/commands/table/explore.tsx b/cli/commands/table/explore.tsx index 9014579d..32dc70d8 100644 --- a/cli/commands/table/explore.tsx +++ b/cli/commands/table/explore.tsx @@ -37,7 +37,7 @@ export const exploreTableCommand = new Command("explore") .action(async (path, options) => { const session = Session.create({ - title: "Exploring table", + title: "Explore table", }) const resource = path diff --git a/cli/components/DataGrid.tsx b/cli/components/DataGrid.tsx index 0658bd76..eac5d266 100644 --- a/cli/components/DataGrid.tsx +++ b/cli/components/DataGrid.tsx @@ -1,6 +1,9 @@ import { Box, Text } from "ink" import React from "react" +// TODO: Accept column names as an option +// TODO: Autocalculate geometry (e.g. row height etc) + const MIN_COLUMN_WIDTH = 15 export type Order = { col: number; dir: "asc" | "desc" } @@ -12,7 +15,8 @@ export function DataGrid(props: { }) { const { data, col, order, rowHeight } = props - const colNames = Object.keys(data[0] ?? {}) + // TODO: fix $schema related cludge + const colNames = Object.keys(data[0] ?? {}).filter(name => name !== "$schema") const colWidth = Math.min( process.stdout.columns / colNames.length, MIN_COLUMN_WIDTH, From f5cd17933db4f0f99c3b3db86024c696126b2572 Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 14 Aug 2025 16:28:56 +0100 Subject: [PATCH 20/61] Implemented validate dialect --- cli/commands/dialect/index.ts | 2 ++ cli/commands/dialect/show.tsx | 4 +++ cli/commands/dialect/validate.ts | 1 - cli/commands/dialect/validate.tsx | 57 +++++++++++++++++++++++++++++++ cli/helpers/session.ts | 11 +++++- 5 files changed, 73 insertions(+), 2 deletions(-) delete mode 100644 cli/commands/dialect/validate.ts create mode 100644 cli/commands/dialect/validate.tsx diff --git a/cli/commands/dialect/index.ts b/cli/commands/dialect/index.ts index 1665d46b..268515a0 100644 --- a/cli/commands/dialect/index.ts +++ b/cli/commands/dialect/index.ts @@ -1,8 +1,10 @@ import { Command } from "commander" import { inferDialectCommand } from "./infer.tsx" import { showDialectCommand } from "./show.tsx" +import { validateDialectCommand } from "./validate.tsx" export const dialectCommand = new Command("dialect") .description("Table Dialect related commands") .addCommand(inferDialectCommand) .addCommand(showDialectCommand) + .addCommand(validateDialectCommand) diff --git a/cli/commands/dialect/show.tsx b/cli/commands/dialect/show.tsx index efb69800..43ac4056 100644 --- a/cli/commands/dialect/show.tsx +++ b/cli/commands/dialect/show.tsx @@ -28,6 +28,10 @@ export const showDialectCommand = new Command("show") if (!path) { const resource = await selectResource(session, options) + if (!resource.dialect) { + Session.terminate("Dialect is not available") + } + if (typeof resource.dialect !== "string") { dialect = resource.dialect } else { diff --git a/cli/commands/dialect/validate.ts b/cli/commands/dialect/validate.ts deleted file mode 100644 index 1039d4c5..00000000 --- a/cli/commands/dialect/validate.ts +++ /dev/null @@ -1 +0,0 @@ -// TODO: implement diff --git a/cli/commands/dialect/validate.tsx b/cli/commands/dialect/validate.tsx new file mode 100644 index 00000000..724e7589 --- /dev/null +++ b/cli/commands/dialect/validate.tsx @@ -0,0 +1,57 @@ +import { Command } from "commander" +import { loadDescriptor, validateDialect } from "dpkit" +import type { Descriptor } from "dpkit" +import { helpConfiguration } from "../../helpers/help.ts" +import { selectResource } from "../../helpers/resource.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const validateDialectCommand = new Command("validate") + .configureHelp(helpConfiguration) + .description("Validate a table dialect from a local or remote path") + + .addArgument(params.positionalTablePath) + .addOption(params.fromPackage) + .addOption(params.fromResource) + .addOption(params.json) + + .action(async (path, options) => { + let descriptor: Descriptor | undefined + + const session = Session.create({ + title: "Validate dialect", + json: options.json, + }) + + if (!path) { + const resource = await selectResource(session, options) + + if (!resource.dialect) { + Session.terminate("Dialect is not available") + } + + if (typeof resource.dialect !== "string") { + descriptor = resource.dialect as Descriptor + } else { + path = resource.dialect + } + } + + if (!descriptor) { + descriptor = await session.task( + "Loading descriptor", + // @ts-ignore + loadDescriptor(path), + ) + } + + const { valid } = await session.task( + "Validating descriptor", + // @ts-ignore + validateDialect(descriptor), + ) + + valid + ? session.success("Dialect is valid") + : session.error("Dialect is not valid") + }) diff --git a/cli/helpers/session.ts b/cli/helpers/session.ts index 6175a71e..05450ce8 100644 --- a/cli/helpers/session.ts +++ b/cli/helpers/session.ts @@ -27,7 +27,6 @@ export class Session { constructor(options: { title: string }) { this.title = options.title } - start() { intro(pc.bold(this.title)) exitHook(() => { @@ -37,6 +36,14 @@ export class Session { }) } + success(message: string) { + log.success(message) + } + + error(message: string) { + log.error(message) + } + async select(options: SelectOptions) { return await select(options) } @@ -63,6 +70,8 @@ export class Session { export class JsonSession extends Session { start = () => {} + success = () => {} + error = () => {} async select(_options: SelectOptions): Promise { Session.terminate("Selection is not supported in JSON mode") From 1b6a5cdc2f2d53f15fe0e5469eeb44f9c8c4c3c2 Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 14 Aug 2025 16:34:17 +0100 Subject: [PATCH 21/61] Added more grids --- cli/components/DialectGrid.tsx | 4 +++- cli/components/PackageGrid.tsx | 15 +++++++++++++++ cli/components/ResourceGrid.tsx | 11 +++++++++++ cli/components/SchemaGrid.tsx | 15 +++++++++++++++ 4 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 cli/components/PackageGrid.tsx create mode 100644 cli/components/ResourceGrid.tsx create mode 100644 cli/components/SchemaGrid.tsx diff --git a/cli/components/DialectGrid.tsx b/cli/components/DialectGrid.tsx index e687b9a4..8995c4b7 100644 --- a/cli/components/DialectGrid.tsx +++ b/cli/components/DialectGrid.tsx @@ -5,5 +5,7 @@ import { DataGrid } from "./DataGrid.tsx" // TODO: Support non-visible chars like TAB and CR export function DialectGrid(props: { dialect: Dialect }) { - return + const data = [props.dialect] + + return } diff --git a/cli/components/PackageGrid.tsx b/cli/components/PackageGrid.tsx new file mode 100644 index 00000000..c2ec5a75 --- /dev/null +++ b/cli/components/PackageGrid.tsx @@ -0,0 +1,15 @@ +import type { Package } from "dpkit" +import React from "react" +import { DataGrid } from "./DataGrid.tsx" + +// TODO: Support showing other package/resource properties + +export function PackageGrid(props: { schema: Package }) { + const data = [ + Object.fromEntries( + props.schema.resources.map(resource => [resource.name, resource.path]), + ), + ] + + return +} diff --git a/cli/components/ResourceGrid.tsx b/cli/components/ResourceGrid.tsx new file mode 100644 index 00000000..cb218225 --- /dev/null +++ b/cli/components/ResourceGrid.tsx @@ -0,0 +1,11 @@ +import type { Resource } from "dpkit" +import React from "react" +import { DataGrid } from "./DataGrid.tsx" + +// TODO: Support better display of resource properties + +export function ResourceGrid(props: { resource: Resource }) { + const data = [props.resource] + + return +} diff --git a/cli/components/SchemaGrid.tsx b/cli/components/SchemaGrid.tsx new file mode 100644 index 00000000..ec5d302e --- /dev/null +++ b/cli/components/SchemaGrid.tsx @@ -0,0 +1,15 @@ +import type { Schema } from "dpkit" +import React from "react" +import { DataGrid } from "./DataGrid.tsx" + +// TODO: Support showing other schema/field properties + +export function SchemaGrid(props: { schema: Schema }) { + const data = [ + Object.fromEntries( + props.schema.fields.map(field => [field.name, field.type]), + ), + ] + + return +} From 3a6884e3a3040fe8b03c85c9de3e5895e7d37882 Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 14 Aug 2025 16:47:12 +0100 Subject: [PATCH 22/61] Implemented resource commands --- cli/commands/dialect/validate.tsx | 4 ++- cli/commands/resource/index.ts | 10 +++++++ cli/commands/resource/infer.ts | 1 - cli/commands/resource/infer.tsx | 41 +++++++++++++++++++++++++ cli/commands/resource/show.ts | 1 - cli/commands/resource/show.tsx | 44 +++++++++++++++++++++++++++ cli/commands/resource/validate.ts | 1 - cli/commands/resource/validate.tsx | 48 ++++++++++++++++++++++++++++++ cli/main.ts | 2 ++ dpkit/index.ts | 1 + dpkit/resource/index.ts | 1 + dpkit/resource/infer.ts | 10 +++++++ dpkit/table/index.ts | 1 + 13 files changed, 161 insertions(+), 4 deletions(-) create mode 100644 cli/commands/resource/index.ts delete mode 100644 cli/commands/resource/infer.ts create mode 100644 cli/commands/resource/infer.tsx delete mode 100644 cli/commands/resource/show.ts create mode 100644 cli/commands/resource/show.tsx delete mode 100644 cli/commands/resource/validate.ts create mode 100644 cli/commands/resource/validate.tsx create mode 100644 dpkit/resource/index.ts create mode 100644 dpkit/resource/infer.ts diff --git a/cli/commands/dialect/validate.tsx b/cli/commands/dialect/validate.tsx index 724e7589..900505de 100644 --- a/cli/commands/dialect/validate.tsx +++ b/cli/commands/dialect/validate.tsx @@ -38,11 +38,13 @@ export const validateDialectCommand = new Command("validate") } if (!descriptor) { - descriptor = await session.task( + const result = await session.task( "Loading descriptor", // @ts-ignore loadDescriptor(path), ) + + descriptor = result.descriptor } const { valid } = await session.task( diff --git a/cli/commands/resource/index.ts b/cli/commands/resource/index.ts new file mode 100644 index 00000000..f165747c --- /dev/null +++ b/cli/commands/resource/index.ts @@ -0,0 +1,10 @@ +import { Command } from "commander" +import { inferResourceCommand } from "./infer.tsx" +import { showResourceCommand } from "./show.tsx" +import { validateResourceCommand } from "./validate.tsx" + +export const resourceCommand = new Command("resource") + .description("Data Resource related commands") + .addCommand(inferResourceCommand) + .addCommand(showResourceCommand) + .addCommand(validateResourceCommand) diff --git a/cli/commands/resource/infer.ts b/cli/commands/resource/infer.ts deleted file mode 100644 index 1039d4c5..00000000 --- a/cli/commands/resource/infer.ts +++ /dev/null @@ -1 +0,0 @@ -// TODO: implement diff --git a/cli/commands/resource/infer.tsx b/cli/commands/resource/infer.tsx new file mode 100644 index 00000000..39f6d477 --- /dev/null +++ b/cli/commands/resource/infer.tsx @@ -0,0 +1,41 @@ +import { Command } from "commander" +import { inferResource } from "dpkit" +import React from "react" +import { ResourceGrid } from "../../components/ResourceGrid.tsx" +import { helpConfiguration } from "../../helpers/help.ts" +import { isEmptyObject } from "../../helpers/object.ts" +import { selectResource } from "../../helpers/resource.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const inferResourceCommand = new Command("infer") + .configureHelp(helpConfiguration) + .description("Infer a data resource from a table") + + .addArgument(params.positionalTablePath) + .addOption(params.fromPackage) + .addOption(params.fromResource) + .addOption(params.json) + + .action(async (path, options) => { + const session = Session.create({ + title: "Infer resource", + json: options.json, + }) + + const resource = path ? { path } : await selectResource(session, options) + + const inferredResource = await session.task( + "Loading sample", + inferResource(resource), + ) + if (isEmptyObject(inferredResource)) { + Session.terminate("Could not infer resource") + } + + await session.render( + inferredResource, + // @ts-ignore + , + ) + }) diff --git a/cli/commands/resource/show.ts b/cli/commands/resource/show.ts deleted file mode 100644 index 1039d4c5..00000000 --- a/cli/commands/resource/show.ts +++ /dev/null @@ -1 +0,0 @@ -// TODO: implement diff --git a/cli/commands/resource/show.tsx b/cli/commands/resource/show.tsx new file mode 100644 index 00000000..e7ff2177 --- /dev/null +++ b/cli/commands/resource/show.tsx @@ -0,0 +1,44 @@ +import { Command } from "commander" +import { loadResourceDescriptor } from "dpkit" +import type { Resource } from "dpkit" +import React from "react" +import { ResourceGrid } from "../../components/ResourceGrid.tsx" +import { helpConfiguration } from "../../helpers/help.ts" +import { isEmptyObject } from "../../helpers/object.ts" +import { selectResource } from "../../helpers/resource.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const showResourceCommand = new Command("show") + .configureHelp(helpConfiguration) + .description("Show a data resource from a local or remote path") + + .addArgument(params.positionalTablePath) + .addOption(params.fromPackage) + .addOption(params.fromResource) + .addOption(params.json) + + .action(async (path, options) => { + let resource: Resource | undefined + + const session = Session.create({ + title: "Show resource", + json: options.json, + }) + + if (!path) { + resource = await selectResource(session, options) + } else { + // @ts-ignore + resource = await session.task( + "Loading resource", + loadResourceDescriptor(path), + ) + } + + if (isEmptyObject(resource)) { + Session.terminate("Resource is not available") + } + + await session.render(resource, ) + }) diff --git a/cli/commands/resource/validate.ts b/cli/commands/resource/validate.ts deleted file mode 100644 index 1039d4c5..00000000 --- a/cli/commands/resource/validate.ts +++ /dev/null @@ -1 +0,0 @@ -// TODO: implement diff --git a/cli/commands/resource/validate.tsx b/cli/commands/resource/validate.tsx new file mode 100644 index 00000000..4b41af56 --- /dev/null +++ b/cli/commands/resource/validate.tsx @@ -0,0 +1,48 @@ +import { Command } from "commander" +import { loadDescriptor, validateResourceDescriptor } from "dpkit" +import type { Descriptor } from "dpkit" +import { helpConfiguration } from "../../helpers/help.ts" +import { selectResource } from "../../helpers/resource.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const validateResourceCommand = new Command("validate") + .configureHelp(helpConfiguration) + .description("Validate a data resource from a local or remote path") + + .addArgument(params.positionalTablePath) + .addOption(params.fromPackage) + .addOption(params.fromResource) + .addOption(params.json) + + .action(async (path, options) => { + let descriptor: Descriptor | undefined + + const session = Session.create({ + title: "Validate resource", + json: options.json, + }) + + if (!path) { + const resource = await selectResource(session, options) + descriptor = resource as unknown as Descriptor + } else { + const result = await session.task( + "Loading descriptor", + // @ts-ignore + loadDescriptor(path), + ) + + descriptor = result.descriptor + } + + const { valid } = await session.task( + "Validating descriptor", + // @ts-ignore + validateResourceDescriptor(descriptor), + ) + + valid + ? session.success("Resource is valid") + : session.error("Resource is not valid") + }) diff --git a/cli/main.ts b/cli/main.ts index 0c8146a1..655677be 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -1,6 +1,7 @@ import { program } from "commander" import { dialectCommand } from "./commands/dialect/index.ts" import { packageCommand } from "./commands/package/index.ts" +import { resourceCommand } from "./commands/resource/index.ts" import { tableCommand } from "./commands/table/index.ts" import { helpConfiguration } from "./helpers/help.ts" import metadata from "./package.json" with { type: "json" } @@ -15,6 +16,7 @@ program .configureHelp(helpConfiguration) .addCommand(packageCommand) + .addCommand(resourceCommand) .addCommand(tableCommand) .addCommand(dialectCommand) diff --git a/dpkit/index.ts b/dpkit/index.ts index a28b8c61..d1e59557 100644 --- a/dpkit/index.ts +++ b/dpkit/index.ts @@ -13,5 +13,6 @@ export * from "@dpkit/zip" export * from "./dialect/index.ts" export * from "./package/index.ts" +export * from "./resource/index.ts" export * from "./table/index.ts" export * from "./plugin.ts" diff --git a/dpkit/resource/index.ts b/dpkit/resource/index.ts new file mode 100644 index 00000000..329b8170 --- /dev/null +++ b/dpkit/resource/index.ts @@ -0,0 +1 @@ +export { inferResource } from "./infer.ts" diff --git a/dpkit/resource/infer.ts b/dpkit/resource/infer.ts new file mode 100644 index 00000000..43e95bae --- /dev/null +++ b/dpkit/resource/infer.ts @@ -0,0 +1,10 @@ +import type { Resource } from "@dpkit/core" +import { inferTable } from "../table/index.ts" + +// TODO: Implement properly + +export async function inferResource(resource: Partial) { + const { dialect, schema } = await inferTable(resource) + + return { ...resource, dialect, schema } +} diff --git a/dpkit/table/index.ts b/dpkit/table/index.ts index 09df76ca..5fb5a1f7 100644 --- a/dpkit/table/index.ts +++ b/dpkit/table/index.ts @@ -1,4 +1,5 @@ export { loadTable } from "./load.ts" export { readTable } from "./read.ts" export { saveTable } from "./save.ts" +export { inferTable } from "./infer.ts" export { validateTable } from "./validate.ts" From 723f41a1683bdb0a97554bd97529d1b32b031b48 Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 14 Aug 2025 16:53:25 +0100 Subject: [PATCH 23/61] Implemented schema commands --- cli/commands/schema/index.ts | 10 ++++++ cli/commands/schema/infer.ts | 1 - cli/commands/schema/infer.tsx | 44 ++++++++++++++++++++++++ cli/commands/schema/show.ts | 1 - cli/commands/schema/show.tsx | 52 ++++++++++++++++++++++++++++ cli/commands/schema/validate.ts | 1 - cli/commands/schema/validate.tsx | 59 ++++++++++++++++++++++++++++++++ cli/main.ts | 4 ++- 8 files changed, 168 insertions(+), 4 deletions(-) create mode 100644 cli/commands/schema/index.ts delete mode 100644 cli/commands/schema/infer.ts create mode 100644 cli/commands/schema/infer.tsx delete mode 100644 cli/commands/schema/show.ts create mode 100644 cli/commands/schema/show.tsx delete mode 100644 cli/commands/schema/validate.ts create mode 100644 cli/commands/schema/validate.tsx diff --git a/cli/commands/schema/index.ts b/cli/commands/schema/index.ts new file mode 100644 index 00000000..0d6ec5b5 --- /dev/null +++ b/cli/commands/schema/index.ts @@ -0,0 +1,10 @@ +import { Command } from "commander" +import { inferSchemaCommand } from "./infer.tsx" +import { showSchemaCommand } from "./show.tsx" +import { validateSchemaCommand } from "./validate.tsx" + +export const schemaCommand = new Command("schema") + .description("Table Schema related commands") + .addCommand(inferSchemaCommand) + .addCommand(showSchemaCommand) + .addCommand(validateSchemaCommand) diff --git a/cli/commands/schema/infer.ts b/cli/commands/schema/infer.ts deleted file mode 100644 index 1039d4c5..00000000 --- a/cli/commands/schema/infer.ts +++ /dev/null @@ -1 +0,0 @@ -// TODO: implement diff --git a/cli/commands/schema/infer.tsx b/cli/commands/schema/infer.tsx new file mode 100644 index 00000000..a6680aaf --- /dev/null +++ b/cli/commands/schema/infer.tsx @@ -0,0 +1,44 @@ +import { Command } from "commander" +import { inferSchema, loadTable } from "dpkit" +import React from "react" +import { SchemaGrid } from "../../components/SchemaGrid.tsx" +import { helpConfiguration } from "../../helpers/help.ts" +import { isEmptyObject } from "../../helpers/object.ts" +import { selectResource } from "../../helpers/resource.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const inferSchemaCommand = new Command("infer") + .configureHelp(helpConfiguration) + .description("Infer a table schema from a table") + + .addArgument(params.positionalTablePath) + .addOption(params.fromPackage) + .addOption(params.fromResource) + .addOption(params.json) + + .action(async (path, options) => { + const session = Session.create({ + title: "Infer schema", + json: options.json, + }) + + const resource = path ? { path } : await selectResource(session, options) + + const table = await session.task("Loading table", loadTable(resource)) + + const inferredSchema = await session.task( + "Inferring schema", + inferSchema(table), + ) + + if (isEmptyObject(inferredSchema)) { + Session.terminate("Could not infer schema") + } + + await session.render( + inferredSchema, + // @ts-ignore + , + ) + }) diff --git a/cli/commands/schema/show.ts b/cli/commands/schema/show.ts deleted file mode 100644 index 1039d4c5..00000000 --- a/cli/commands/schema/show.ts +++ /dev/null @@ -1 +0,0 @@ -// TODO: implement diff --git a/cli/commands/schema/show.tsx b/cli/commands/schema/show.tsx new file mode 100644 index 00000000..fb7c6c75 --- /dev/null +++ b/cli/commands/schema/show.tsx @@ -0,0 +1,52 @@ +import { Command } from "commander" +import { loadSchema } from "dpkit" +import type { Schema } from "dpkit" +import React from "react" +import { SchemaGrid } from "../../components/SchemaGrid.tsx" +import { helpConfiguration } from "../../helpers/help.ts" +import { isEmptyObject } from "../../helpers/object.ts" +import { selectResource } from "../../helpers/resource.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const showSchemaCommand = new Command("show") + .configureHelp(helpConfiguration) + .description("Show a table schema from a local or remote path") + + .addArgument(params.positionalTablePath) + .addOption(params.fromPackage) + .addOption(params.fromResource) + .addOption(params.json) + + .action(async (path, options) => { + let schema: Schema | undefined + + const session = Session.create({ + title: "Show schema", + json: options.json, + }) + + if (!path) { + const resource = await selectResource(session, options) + if (!resource.schema) { + Session.terminate("Schema is not available") + } + + if (typeof resource.schema !== "string") { + schema = resource.schema + } else { + path = resource.schema + } + } + + if (!schema) { + // @ts-ignore + schema = await session.task("Loading schema", loadSchema(path)) + } + + if (isEmptyObject(schema)) { + Session.terminate("Schema is not available") + } + + await session.render(schema, ) + }) diff --git a/cli/commands/schema/validate.ts b/cli/commands/schema/validate.ts deleted file mode 100644 index 1039d4c5..00000000 --- a/cli/commands/schema/validate.ts +++ /dev/null @@ -1 +0,0 @@ -// TODO: implement diff --git a/cli/commands/schema/validate.tsx b/cli/commands/schema/validate.tsx new file mode 100644 index 00000000..ca5d74dd --- /dev/null +++ b/cli/commands/schema/validate.tsx @@ -0,0 +1,59 @@ +import { Command } from "commander" +import { loadDescriptor, validateSchema } from "dpkit" +import type { Descriptor } from "dpkit" +import { helpConfiguration } from "../../helpers/help.ts" +import { selectResource } from "../../helpers/resource.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const validateSchemaCommand = new Command("validate") + .configureHelp(helpConfiguration) + .description("Validate a table schema from a local or remote path") + + .addArgument(params.positionalTablePath) + .addOption(params.fromPackage) + .addOption(params.fromResource) + .addOption(params.json) + + .action(async (path, options) => { + let descriptor: Descriptor | undefined + + const session = Session.create({ + title: "Validate schema", + json: options.json, + }) + + if (!path) { + const resource = await selectResource(session, options) + + if (!resource.schema) { + Session.terminate("Schema is not available") + } + + if (typeof resource.schema !== "string") { + descriptor = resource.schema as unknown as Descriptor + } else { + path = resource.schema + } + } + + if (!descriptor) { + const result = await session.task( + "Loading descriptor", + // @ts-ignore + loadDescriptor(path), + ) + + descriptor = result.descriptor + } + + const { valid } = await session.task( + "Validating descriptor", + // @ts-ignore + validateSchema(descriptor), + ) + + valid + ? session.success("Schema is valid") + : session.error("Schema is not valid") + }) diff --git a/cli/main.ts b/cli/main.ts index 655677be..ef04c354 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -2,6 +2,7 @@ import { program } from "commander" import { dialectCommand } from "./commands/dialect/index.ts" import { packageCommand } from "./commands/package/index.ts" import { resourceCommand } from "./commands/resource/index.ts" +import { schemaCommand } from "./commands/schema/index.ts" import { tableCommand } from "./commands/table/index.ts" import { helpConfiguration } from "./helpers/help.ts" import metadata from "./package.json" with { type: "json" } @@ -17,7 +18,8 @@ program .addCommand(packageCommand) .addCommand(resourceCommand) - .addCommand(tableCommand) .addCommand(dialectCommand) + .addCommand(schemaCommand) + .addCommand(tableCommand) .parse() From 65229ebf24e9f2bf7f74e0bd83d77e052f9473d6 Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 14 Aug 2025 17:02:08 +0100 Subject: [PATCH 24/61] Renamed describe to stats --- cli/commands/table/index.ts | 4 +-- .../table/{describe.tsx => stats.tsx} | 31 +++++++++++-------- 2 files changed, 20 insertions(+), 15 deletions(-) rename cli/commands/table/{describe.tsx => stats.tsx} (55%) diff --git a/cli/commands/table/index.ts b/cli/commands/table/index.ts index ee7b1d18..9648ff80 100644 --- a/cli/commands/table/index.ts +++ b/cli/commands/table/index.ts @@ -1,17 +1,17 @@ import { Command } from "commander" import { helpConfiguration } from "../../helpers/help.ts" import { convertTableCommand } from "./convert.tsx" -import { describeTableCommand } from "./describe.tsx" import { exploreTableCommand } from "./explore.tsx" import { queryTableCommand } from "./query.tsx" import { scriptTableCommand } from "./script.tsx" +import { statsTableCommand } from "./stats.tsx" import { validateTableCommand } from "./validate.tsx" export const tableCommand = new Command("table") .description("Table related commands") .configureHelp(helpConfiguration) - .addCommand(describeTableCommand) + .addCommand(statsTableCommand) .addCommand(convertTableCommand) .addCommand(validateTableCommand) .addCommand(queryTableCommand) diff --git a/cli/commands/table/describe.tsx b/cli/commands/table/stats.tsx similarity index 55% rename from cli/commands/table/describe.tsx rename to cli/commands/table/stats.tsx index e6ab9d03..1a37d649 100644 --- a/cli/commands/table/describe.tsx +++ b/cli/commands/table/stats.tsx @@ -1,17 +1,20 @@ import { Command } from "commander" import { readTable } from "dpkit" -import { render } from "ink" import React from "react" -import { TableGrid } from "../../components/TableGrid.tsx" +import { DataGrid } from "../../components/DataGrid.tsx" import { createDialectFromOptions } from "../../helpers/dialect.ts" import { helpConfiguration } from "../../helpers/help.ts" +import { selectResource } from "../../helpers/resource.ts" +import { Session } from "../../helpers/session.ts" import * as params from "../../params/index.ts" -export const describeTableCommand = new Command("describe") +export const statsTableCommand = new Command("stats") .configureHelp(helpConfiguration) - .description("Describe a table from a local or remote path") + .description("Show stats for a table from a local or remote path") .addArgument(params.positionalTablePath) + .addOption(params.fromPackage) + .addOption(params.fromResource) .addOption(params.json) .optionsGroup("Table Dialect") @@ -33,16 +36,18 @@ export const describeTableCommand = new Command("describe") .addOption(params.sheetName) .addOption(params.table) .action(async (path, options) => { - const dialect = createDialectFromOptions(options) - const table = await readTable({ path, dialect }) + const session = Session.create({ + title: "Table Stats", + }) - const df = await table.collect() - const stats = df.describe() + const resource = path + ? { path, dialect: createDialectFromOptions(options) } + : await selectResource(session, options) - if (options.json) { - console.log(JSON.stringify(stats, null, 2)) - return - } + const table = await session.task("Loading table", readTable(resource)) + const df = await session.task("Calculating stats", table.collect()) - render() + const stats = df.describe().toRecords() + + session.render(stats, ) }) From 64448f036a598b5c42ed3505741e8f2022cd670b Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 14 Aug 2025 17:12:35 +0100 Subject: [PATCH 25/61] Split table errors/validate --- cli/commands/table/errors.tsx | 54 +++++++++++++++++++++++++++++++++ cli/commands/table/index.ts | 8 +++-- cli/commands/table/stats.tsx | 2 +- cli/commands/table/validate.tsx | 49 ++++++++++++------------------ cli/components/ErrorGrid.tsx | 32 ++----------------- 5 files changed, 82 insertions(+), 63 deletions(-) create mode 100644 cli/commands/table/errors.tsx diff --git a/cli/commands/table/errors.tsx b/cli/commands/table/errors.tsx new file mode 100644 index 00000000..f06929b3 --- /dev/null +++ b/cli/commands/table/errors.tsx @@ -0,0 +1,54 @@ +import { Command } from "commander" +import { validateTable } from "dpkit" +import React from "react" +import { ErrorGrid } from "../../components/ErrorGrid.jsx" +import { createDialectFromOptions } from "../../helpers/dialect.ts" +import { helpConfiguration } from "../../helpers/help.ts" +import { selectResource } from "../../helpers/resource.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const errorsTableCommand = new Command("errors") + .configureHelp(helpConfiguration) + .description("Validate a table from a local or remote path") + + .addArgument(params.positionalTablePath) + .addOption(params.fromPackage) + .addOption(params.fromResource) + .addOption(params.json) + + .optionsGroup("Table Dialect") + .addOption(params.delimiter) + .addOption(params.header) + .addOption(params.headerRows) + .addOption(params.headerJoin) + .addOption(params.commentRows) + .addOption(params.commentChar) + .addOption(params.quoteChar) + .addOption(params.doubleQuote) + .addOption(params.escapeChar) + .addOption(params.nullSequence) + .addOption(params.skipInitialSpace) + .addOption(params.property) + .addOption(params.itemType) + .addOption(params.itemKeys) + .addOption(params.sheetNumber) + .addOption(params.sheetName) + .addOption(params.table) + + .action(async (path, options) => { + const session = Session.create({ + title: "Table errors", + }) + + const resource = path + ? { path, dialect: createDialectFromOptions(options) } + : await selectResource(session, options) + + const { errors } = await session.task( + "Validating table", + validateTable(resource), + ) + + session.render(errors, ) + }) diff --git a/cli/commands/table/index.ts b/cli/commands/table/index.ts index 9648ff80..78ca2b7b 100644 --- a/cli/commands/table/index.ts +++ b/cli/commands/table/index.ts @@ -1,6 +1,7 @@ import { Command } from "commander" import { helpConfiguration } from "../../helpers/help.ts" import { convertTableCommand } from "./convert.tsx" +import { errorsTableCommand } from "./errors.tsx" import { exploreTableCommand } from "./explore.tsx" import { queryTableCommand } from "./query.tsx" import { scriptTableCommand } from "./script.tsx" @@ -11,9 +12,10 @@ export const tableCommand = new Command("table") .description("Table related commands") .configureHelp(helpConfiguration) - .addCommand(statsTableCommand) .addCommand(convertTableCommand) - .addCommand(validateTableCommand) + .addCommand(errorsTableCommand) + .addCommand(exploreTableCommand) .addCommand(queryTableCommand) .addCommand(scriptTableCommand) - .addCommand(exploreTableCommand) + .addCommand(statsTableCommand) + .addCommand(validateTableCommand) diff --git a/cli/commands/table/stats.tsx b/cli/commands/table/stats.tsx index 1a37d649..927d27e4 100644 --- a/cli/commands/table/stats.tsx +++ b/cli/commands/table/stats.tsx @@ -37,7 +37,7 @@ export const statsTableCommand = new Command("stats") .addOption(params.table) .action(async (path, options) => { const session = Session.create({ - title: "Table Stats", + title: "Table stats", }) const resource = path diff --git a/cli/commands/table/validate.tsx b/cli/commands/table/validate.tsx index ef0b3b6a..35072ea5 100644 --- a/cli/commands/table/validate.tsx +++ b/cli/commands/table/validate.tsx @@ -1,10 +1,9 @@ import { Command } from "commander" import { validateTable } from "dpkit" -import { render } from "ink" -import React from "react" -import { ErrorGrid } from "../../components/ErrorGrid.jsx" import { createDialectFromOptions } from "../../helpers/dialect.ts" import { helpConfiguration } from "../../helpers/help.ts" +import { selectResource } from "../../helpers/resource.ts" +import { Session } from "../../helpers/session.ts" import * as params from "../../params/index.ts" export const validateTableCommand = new Command("validate") @@ -12,35 +11,27 @@ export const validateTableCommand = new Command("validate") .description("Validate a table from a local or remote path") .addArgument(params.positionalTablePath) + .addOption(params.fromPackage) + .addOption(params.fromResource) .addOption(params.json) - .optionsGroup("Table Dialect") - .addOption(params.delimiter) - .addOption(params.header) - .addOption(params.headerRows) - .addOption(params.headerJoin) - .addOption(params.commentRows) - .addOption(params.commentChar) - .addOption(params.quoteChar) - .addOption(params.doubleQuote) - .addOption(params.escapeChar) - .addOption(params.nullSequence) - .addOption(params.skipInitialSpace) - .addOption(params.property) - .addOption(params.itemType) - .addOption(params.itemKeys) - .addOption(params.sheetNumber) - .addOption(params.sheetName) - .addOption(params.table) - .action(async (path, options) => { - const dialect = createDialectFromOptions(options) - const { errors } = await validateTable({ path, dialect }) + const session = Session.create({ + title: "Validate Table", + }) + + const resource = path + ? { path, dialect: createDialectFromOptions(options) } + : await selectResource(session, options) + + const { valid } = await session.task( + "Validating table", + validateTable(resource), + ) - if (options.json) { - console.log(JSON.stringify(errors, null, 2)) - return - } + // TODO: Show errors/count by type grid if not valid - render() + valid + ? session.success("Table is valid") + : session.error("Table is not valid") }) diff --git a/cli/components/ErrorGrid.tsx b/cli/components/ErrorGrid.tsx index 71818bfc..1c1b603b 100644 --- a/cli/components/ErrorGrid.tsx +++ b/cli/components/ErrorGrid.tsx @@ -1,38 +1,10 @@ import type { TableError } from "dpkit" -import { Box } from "ink" -import SelectInput from "ink-select-input" import { DataFrame } from "nodejs-polars" -import { useState } from "react" import React from "react" -import { objectKeys } from "ts-extras" import { TableGrid } from "./TableGrid.tsx" export function ErrorGrid(props: { errors: TableError[] }) { - const { errors } = props - const [errorType, setErrorType] = useState(errors[0]?.type ?? "") + const table = DataFrame(props.errors).lazy() - const errorsByType = Object.groupBy(errors, error => error.type) - // @ts-ignore - const selectErrors = errorsByType[errorType] - - const table = DataFrame(selectErrors).lazy() - - const handleSelect = async (item: any) => { - setErrorType(item.value) - } - - return ( - - - ({ - label: type, - value: type, - }))} - /> - - - - ) + return } From aa8811b993d74e88f35d83edfccbc24feb3c3013 Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 14 Aug 2025 17:17:47 +0100 Subject: [PATCH 26/61] Added dialect errors command --- cli/commands/dialect/errors.tsx | 59 +++++++++++++++++++++++++++++++++ cli/commands/dialect/index.ts | 5 +++ cli/commands/package/index.ts | 3 ++ cli/commands/resource/index.ts | 3 ++ cli/commands/schema/index.ts | 3 ++ cli/commands/table/index.ts | 2 +- cli/components/ErrorGrid.tsx | 6 ++-- 7 files changed, 78 insertions(+), 3 deletions(-) create mode 100644 cli/commands/dialect/errors.tsx diff --git a/cli/commands/dialect/errors.tsx b/cli/commands/dialect/errors.tsx new file mode 100644 index 00000000..71010cdb --- /dev/null +++ b/cli/commands/dialect/errors.tsx @@ -0,0 +1,59 @@ +import { Command } from "commander" +import { loadDescriptor, validateDialect } from "dpkit" +import type { Descriptor } from "dpkit" +import React from "react" +import { ErrorGrid } from "../../components/ErrorGrid.jsx" +import { helpConfiguration } from "../../helpers/help.ts" +import { selectResource } from "../../helpers/resource.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const errorsDialectCommand = new Command("errors") + .configureHelp(helpConfiguration) + .description("Show errors for a table dialect from a local or remote path") + + .addArgument(params.positionalTablePath) + .addOption(params.fromPackage) + .addOption(params.fromResource) + .addOption(params.json) + + .action(async (path, options) => { + let descriptor: Descriptor | undefined + + const session = Session.create({ + title: "Validate dialect", + json: options.json, + }) + + if (!path) { + const resource = await selectResource(session, options) + + if (!resource.dialect) { + Session.terminate("Dialect is not available") + } + + if (typeof resource.dialect !== "string") { + descriptor = resource.dialect as Descriptor + } else { + path = resource.dialect + } + } + + if (!descriptor) { + const result = await session.task( + "Loading descriptor", + // @ts-ignore + loadDescriptor(path), + ) + + descriptor = result.descriptor + } + + const { errors } = await session.task( + "Validating descriptor", + // @ts-ignore + validateDialect(descriptor), + ) + + session.render(errors, ) + }) diff --git a/cli/commands/dialect/index.ts b/cli/commands/dialect/index.ts index 268515a0..43e27fcf 100644 --- a/cli/commands/dialect/index.ts +++ b/cli/commands/dialect/index.ts @@ -1,10 +1,15 @@ import { Command } from "commander" +import { helpConfiguration } from "../../helpers/help.ts" +import { errorsDialectCommand } from "./errors.tsx" import { inferDialectCommand } from "./infer.tsx" import { showDialectCommand } from "./show.tsx" import { validateDialectCommand } from "./validate.tsx" export const dialectCommand = new Command("dialect") + .configureHelp(helpConfiguration) .description("Table Dialect related commands") + + .addCommand(errorsDialectCommand) .addCommand(inferDialectCommand) .addCommand(showDialectCommand) .addCommand(validateDialectCommand) diff --git a/cli/commands/package/index.ts b/cli/commands/package/index.ts index 06ef1b53..6779e468 100644 --- a/cli/commands/package/index.ts +++ b/cli/commands/package/index.ts @@ -1,10 +1,13 @@ import { Command } from "commander" +import { helpConfiguration } from "../../helpers/help.ts" import { archivePackageCommand } from "./archive.ts" import { copyPackageCommand } from "./copy.ts" import { showPackageCommand } from "./show.ts" export const packageCommand = new Command("package") + .configureHelp(helpConfiguration) .description("Data Package related commands") + .addCommand(showPackageCommand) .addCommand(archivePackageCommand) .addCommand(copyPackageCommand) diff --git a/cli/commands/resource/index.ts b/cli/commands/resource/index.ts index f165747c..d647a2b0 100644 --- a/cli/commands/resource/index.ts +++ b/cli/commands/resource/index.ts @@ -1,10 +1,13 @@ import { Command } from "commander" +import { helpConfiguration } from "../../helpers/help.ts" import { inferResourceCommand } from "./infer.tsx" import { showResourceCommand } from "./show.tsx" import { validateResourceCommand } from "./validate.tsx" export const resourceCommand = new Command("resource") + .configureHelp(helpConfiguration) .description("Data Resource related commands") + .addCommand(inferResourceCommand) .addCommand(showResourceCommand) .addCommand(validateResourceCommand) diff --git a/cli/commands/schema/index.ts b/cli/commands/schema/index.ts index 0d6ec5b5..8484f9b8 100644 --- a/cli/commands/schema/index.ts +++ b/cli/commands/schema/index.ts @@ -1,10 +1,13 @@ import { Command } from "commander" +import { helpConfiguration } from "../../helpers/help.ts" import { inferSchemaCommand } from "./infer.tsx" import { showSchemaCommand } from "./show.tsx" import { validateSchemaCommand } from "./validate.tsx" export const schemaCommand = new Command("schema") + .configureHelp(helpConfiguration) .description("Table Schema related commands") + .addCommand(inferSchemaCommand) .addCommand(showSchemaCommand) .addCommand(validateSchemaCommand) diff --git a/cli/commands/table/index.ts b/cli/commands/table/index.ts index 78ca2b7b..a56294e3 100644 --- a/cli/commands/table/index.ts +++ b/cli/commands/table/index.ts @@ -9,8 +9,8 @@ import { statsTableCommand } from "./stats.tsx" import { validateTableCommand } from "./validate.tsx" export const tableCommand = new Command("table") - .description("Table related commands") .configureHelp(helpConfiguration) + .description("Table related commands") .addCommand(convertTableCommand) .addCommand(errorsTableCommand) diff --git a/cli/components/ErrorGrid.tsx b/cli/components/ErrorGrid.tsx index 1c1b603b..e17f49a1 100644 --- a/cli/components/ErrorGrid.tsx +++ b/cli/components/ErrorGrid.tsx @@ -1,9 +1,11 @@ -import type { TableError } from "dpkit" +//import type { TableError } from "dpkit" import { DataFrame } from "nodejs-polars" import React from "react" import { TableGrid } from "./TableGrid.tsx" -export function ErrorGrid(props: { errors: TableError[] }) { +// TODO: Improve implementation (esp. typing) + +export function ErrorGrid(props: { errors: Record[] }) { const table = DataFrame(props.errors).lazy() return From c8d394458612173cfb097d2e5ef19eca478c6f2a Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 14 Aug 2025 17:21:54 +0100 Subject: [PATCH 27/61] Added other error commands --- cli/commands/dialect/errors.tsx | 4 +- cli/commands/dialect/show.tsx | 4 +- cli/commands/dialect/validate.tsx | 4 +- cli/commands/package/errors.tsx | 47 +++++++++++++++++++++++ cli/commands/package/index.ts | 2 + cli/commands/resource/errors.tsx | 47 +++++++++++++++++++++++ cli/commands/resource/index.ts | 2 + cli/commands/resource/show.tsx | 4 +- cli/commands/resource/validate.tsx | 4 +- cli/commands/schema/errors.tsx | 60 ++++++++++++++++++++++++++++++ cli/commands/schema/index.ts | 2 + cli/commands/schema/show.tsx | 4 +- cli/commands/schema/validate.tsx | 4 +- 13 files changed, 174 insertions(+), 14 deletions(-) create mode 100644 cli/commands/package/errors.tsx create mode 100644 cli/commands/resource/errors.tsx create mode 100644 cli/commands/schema/errors.tsx diff --git a/cli/commands/dialect/errors.tsx b/cli/commands/dialect/errors.tsx index 71010cdb..762dc6a9 100644 --- a/cli/commands/dialect/errors.tsx +++ b/cli/commands/dialect/errors.tsx @@ -18,13 +18,13 @@ export const errorsDialectCommand = new Command("errors") .addOption(params.json) .action(async (path, options) => { - let descriptor: Descriptor | undefined - const session = Session.create({ title: "Validate dialect", json: options.json, }) + let descriptor: Descriptor | undefined + if (!path) { const resource = await selectResource(session, options) diff --git a/cli/commands/dialect/show.tsx b/cli/commands/dialect/show.tsx index 43ac4056..eb802049 100644 --- a/cli/commands/dialect/show.tsx +++ b/cli/commands/dialect/show.tsx @@ -19,13 +19,13 @@ export const showDialectCommand = new Command("show") .addOption(params.json) .action(async (path, options) => { - let dialect: Dialect | undefined - const session = Session.create({ title: "Show dialect", json: options.json, }) + let dialect: Dialect | undefined + if (!path) { const resource = await selectResource(session, options) if (!resource.dialect) { diff --git a/cli/commands/dialect/validate.tsx b/cli/commands/dialect/validate.tsx index 900505de..d3e570c9 100644 --- a/cli/commands/dialect/validate.tsx +++ b/cli/commands/dialect/validate.tsx @@ -16,13 +16,13 @@ export const validateDialectCommand = new Command("validate") .addOption(params.json) .action(async (path, options) => { - let descriptor: Descriptor | undefined - const session = Session.create({ title: "Validate dialect", json: options.json, }) + let descriptor: Descriptor | undefined + if (!path) { const resource = await selectResource(session, options) diff --git a/cli/commands/package/errors.tsx b/cli/commands/package/errors.tsx new file mode 100644 index 00000000..0ec558be --- /dev/null +++ b/cli/commands/package/errors.tsx @@ -0,0 +1,47 @@ +import { Command } from "commander" +import { loadDescriptor, validatePackageDescriptor } from "dpkit" +import type { Descriptor } from "dpkit" +import React from "react" +import { ErrorGrid } from "../../components/ErrorGrid.jsx" +import { helpConfiguration } from "../../helpers/help.ts" +import { selectResource } from "../../helpers/resource.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const errorsPackageCommand = new Command("errors") + .configureHelp(helpConfiguration) + .description("Show errors for a data package from a local or remote path") + + .addArgument(params.positionalTablePath) + .addOption(params.fromPackage) + .addOption(params.fromResource) + .addOption(params.json) + + .action(async (path, options) => { + let descriptor: Descriptor | undefined + + const session = Session.create({ + title: "Validate package", + json: options.json, + }) + + if (!path) { + const resource = await selectResource(session, options) + descriptor = resource as unknown as Descriptor + } else { + const result = await session.task( + "Loading descriptor", + // @ts-ignore + loadDescriptor(path), + ) + + descriptor = result.descriptor + } + + const { errors } = await session.task( + "Validating descriptor", + validatePackageDescriptor(descriptor), + ) + + session.render(errors, ) + }) diff --git a/cli/commands/package/index.ts b/cli/commands/package/index.ts index 6779e468..dd9413ae 100644 --- a/cli/commands/package/index.ts +++ b/cli/commands/package/index.ts @@ -2,12 +2,14 @@ import { Command } from "commander" import { helpConfiguration } from "../../helpers/help.ts" import { archivePackageCommand } from "./archive.ts" import { copyPackageCommand } from "./copy.ts" +import { errorsPackageCommand } from "./errors.tsx" import { showPackageCommand } from "./show.ts" export const packageCommand = new Command("package") .configureHelp(helpConfiguration) .description("Data Package related commands") + .addCommand(errorsPackageCommand) .addCommand(showPackageCommand) .addCommand(archivePackageCommand) .addCommand(copyPackageCommand) diff --git a/cli/commands/resource/errors.tsx b/cli/commands/resource/errors.tsx new file mode 100644 index 00000000..48e70e64 --- /dev/null +++ b/cli/commands/resource/errors.tsx @@ -0,0 +1,47 @@ +import { Command } from "commander" +import { loadDescriptor, validateResourceDescriptor } from "dpkit" +import type { Descriptor } from "dpkit" +import React from "react" +import { ErrorGrid } from "../../components/ErrorGrid.jsx" +import { helpConfiguration } from "../../helpers/help.ts" +import { selectResource } from "../../helpers/resource.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const errorsResourceCommand = new Command("errors") + .configureHelp(helpConfiguration) + .description("Show errors for a data resource from a local or remote path") + + .addArgument(params.positionalTablePath) + .addOption(params.fromPackage) + .addOption(params.fromResource) + .addOption(params.json) + + .action(async (path, options) => { + const session = Session.create({ + title: "Validate resource", + json: options.json, + }) + + let descriptor: Descriptor | undefined + + if (!path) { + const resource = await selectResource(session, options) + descriptor = resource as unknown as Descriptor + } else { + const result = await session.task( + "Loading descriptor", + // @ts-ignore + loadDescriptor(path), + ) + + descriptor = result.descriptor + } + + const { errors } = await session.task( + "Validating descriptor", + validateResourceDescriptor(descriptor), + ) + + session.render(errors, ) + }) diff --git a/cli/commands/resource/index.ts b/cli/commands/resource/index.ts index d647a2b0..c2e66d69 100644 --- a/cli/commands/resource/index.ts +++ b/cli/commands/resource/index.ts @@ -1,5 +1,6 @@ import { Command } from "commander" import { helpConfiguration } from "../../helpers/help.ts" +import { errorsResourceCommand } from "./errors.tsx" import { inferResourceCommand } from "./infer.tsx" import { showResourceCommand } from "./show.tsx" import { validateResourceCommand } from "./validate.tsx" @@ -8,6 +9,7 @@ export const resourceCommand = new Command("resource") .configureHelp(helpConfiguration) .description("Data Resource related commands") + .addCommand(errorsResourceCommand) .addCommand(inferResourceCommand) .addCommand(showResourceCommand) .addCommand(validateResourceCommand) diff --git a/cli/commands/resource/show.tsx b/cli/commands/resource/show.tsx index e7ff2177..43b336c4 100644 --- a/cli/commands/resource/show.tsx +++ b/cli/commands/resource/show.tsx @@ -19,13 +19,13 @@ export const showResourceCommand = new Command("show") .addOption(params.json) .action(async (path, options) => { - let resource: Resource | undefined - const session = Session.create({ title: "Show resource", json: options.json, }) + let resource: Resource | undefined + if (!path) { resource = await selectResource(session, options) } else { diff --git a/cli/commands/resource/validate.tsx b/cli/commands/resource/validate.tsx index 4b41af56..43d28631 100644 --- a/cli/commands/resource/validate.tsx +++ b/cli/commands/resource/validate.tsx @@ -16,13 +16,13 @@ export const validateResourceCommand = new Command("validate") .addOption(params.json) .action(async (path, options) => { - let descriptor: Descriptor | undefined - const session = Session.create({ title: "Validate resource", json: options.json, }) + let descriptor: Descriptor | undefined + if (!path) { const resource = await selectResource(session, options) descriptor = resource as unknown as Descriptor diff --git a/cli/commands/schema/errors.tsx b/cli/commands/schema/errors.tsx new file mode 100644 index 00000000..ca9cb9aa --- /dev/null +++ b/cli/commands/schema/errors.tsx @@ -0,0 +1,60 @@ +import { Command } from "commander" +import { loadDescriptor, validateSchema } from "dpkit" +import type { Descriptor } from "dpkit" +import React from "react" +import { ErrorGrid } from "../../components/ErrorGrid.jsx" +import { helpConfiguration } from "../../helpers/help.ts" +import { selectResource } from "../../helpers/resource.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const errorsSchemaCommand = new Command("errors") + .configureHelp(helpConfiguration) + .description("Show errors for a table schema from a local or remote path") + + .addArgument(params.positionalTablePath) + .addOption(params.fromPackage) + .addOption(params.fromResource) + .addOption(params.json) + + .action(async (path, options) => { + const session = Session.create({ + title: "Validate schema", + json: options.json, + }) + + let descriptor: Descriptor | undefined + + if (!path) { + const resource = await selectResource(session, options) + + if (!resource.schema) { + Session.terminate("Schema is not available") + } + + if (typeof resource.schema !== "string") { + descriptor = resource.schema as Descriptor + } else { + path = resource.schema + } + } + + if (!descriptor) { + const result = await session.task( + "Loading descriptor", + // @ts-ignore + loadDescriptor(path), + ) + + descriptor = result.descriptor + } + + const { errors } = await session.task( + "Validating descriptor", + // @ts-ignore + validateSchema(descriptor), + ) + + session.render(errors, ) + }) + diff --git a/cli/commands/schema/index.ts b/cli/commands/schema/index.ts index 8484f9b8..200f757c 100644 --- a/cli/commands/schema/index.ts +++ b/cli/commands/schema/index.ts @@ -1,5 +1,6 @@ import { Command } from "commander" import { helpConfiguration } from "../../helpers/help.ts" +import { errorsSchemaCommand } from "./errors.tsx" import { inferSchemaCommand } from "./infer.tsx" import { showSchemaCommand } from "./show.tsx" import { validateSchemaCommand } from "./validate.tsx" @@ -8,6 +9,7 @@ export const schemaCommand = new Command("schema") .configureHelp(helpConfiguration) .description("Table Schema related commands") + .addCommand(errorsSchemaCommand) .addCommand(inferSchemaCommand) .addCommand(showSchemaCommand) .addCommand(validateSchemaCommand) diff --git a/cli/commands/schema/show.tsx b/cli/commands/schema/show.tsx index fb7c6c75..50963339 100644 --- a/cli/commands/schema/show.tsx +++ b/cli/commands/schema/show.tsx @@ -19,13 +19,13 @@ export const showSchemaCommand = new Command("show") .addOption(params.json) .action(async (path, options) => { - let schema: Schema | undefined - const session = Session.create({ title: "Show schema", json: options.json, }) + let schema: Schema | undefined + if (!path) { const resource = await selectResource(session, options) if (!resource.schema) { diff --git a/cli/commands/schema/validate.tsx b/cli/commands/schema/validate.tsx index ca5d74dd..cb17090b 100644 --- a/cli/commands/schema/validate.tsx +++ b/cli/commands/schema/validate.tsx @@ -16,13 +16,13 @@ export const validateSchemaCommand = new Command("validate") .addOption(params.json) .action(async (path, options) => { - let descriptor: Descriptor | undefined - const session = Session.create({ title: "Validate schema", json: options.json, }) + let descriptor: Descriptor | undefined + if (!path) { const resource = await selectResource(session, options) From 82c6a894562fe9dac58acbad63c19acb53cafb7b Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 14 Aug 2025 17:28:16 +0100 Subject: [PATCH 28/61] Rebased other table commands on session --- cli/commands/table/convert.tsx | 29 ++++++++++++++++++++++------- cli/commands/table/query.tsx | 7 ++++++- cli/commands/table/script.tsx | 19 +++++++++++++++---- 3 files changed, 43 insertions(+), 12 deletions(-) diff --git a/cli/commands/table/convert.tsx b/cli/commands/table/convert.tsx index 9f066580..6a6a4e7d 100644 --- a/cli/commands/table/convert.tsx +++ b/cli/commands/table/convert.tsx @@ -4,6 +4,8 @@ import { readTable, saveTable } from "dpkit" import { createDialectFromOptions } from "../../helpers/dialect.ts" import { createToDialectFromOptions } from "../../helpers/dialect.ts" import { helpConfiguration } from "../../helpers/help.ts" +import { selectResource } from "../../helpers/resource.ts" +import { Session } from "../../helpers/session.ts" import * as params from "../../params/index.ts" export const convertTableCommand = new Command("convert") @@ -13,6 +15,8 @@ export const convertTableCommand = new Command("convert") ) .addArgument(params.positionalTablePath) + .addOption(params.fromPackage) + .addOption(params.fromResource) .addOption(params.toPath) .addOption(params.toFormat) @@ -55,16 +59,27 @@ export const convertTableCommand = new Command("convert") .addOption(params.toTable) .action(async (path, options) => { - const dialect = createDialectFromOptions(options) - const table = await readTable({ path, dialect }) + const session = Session.create({ + title: "Table errors", + }) + + const resource = path + ? { path, dialect: createDialectFromOptions(options) } + : await selectResource(session, options) + + const table = await session.task("Loading table", readTable(resource)) const toPath = options.toPath ?? getTempFilePath() const toDialect = createToDialectFromOptions(options) - await saveTable(table, { - path: toPath, - format: options.toFormat, - dialect: toDialect, - }) + + await session.task( + "Saving table", + saveTable(table, { + path: toPath, + format: options.toFormat, + dialect: toDialect, + }), + ) if (!options.toPath) { const buffer = await loadFile(toPath) diff --git a/cli/commands/table/query.tsx b/cli/commands/table/query.tsx index 66eabf79..3642d1a8 100644 --- a/cli/commands/table/query.tsx +++ b/cli/commands/table/query.tsx @@ -1,5 +1,6 @@ import { Command } from "commander" import { helpConfiguration } from "../../helpers/help.ts" +import { Session } from "../../helpers/session.ts" import * as params from "../../params/index.ts" export const queryTableCommand = new Command("query") @@ -30,5 +31,9 @@ export const queryTableCommand = new Command("query") .addOption(params.table) .action(async (_path, _options) => { - throw new Error("Query command not implemented yet") + const session = Session.create({ + title: "Query table", + }) + + Session.terminate("Query command not implemented yet") }) diff --git a/cli/commands/table/script.tsx b/cli/commands/table/script.tsx index 70b7fb2c..d6df4b0f 100644 --- a/cli/commands/table/script.tsx +++ b/cli/commands/table/script.tsx @@ -3,6 +3,8 @@ import { Command } from "commander" import { readTable } from "dpkit" import { createDialectFromOptions } from "../../helpers/dialect.ts" import { helpConfiguration } from "../../helpers/help.ts" +import { selectResource } from "../../helpers/resource.ts" +import { Session } from "../../helpers/session.ts" import * as params from "../../params/index.ts" export const scriptTableCommand = new Command("script") @@ -12,6 +14,8 @@ export const scriptTableCommand = new Command("script") ) .addArgument(params.positionalTablePath) + .addOption(params.fromPackage) + .addOption(params.fromResource) .optionsGroup("Table Dialect") .addOption(params.delimiter) @@ -33,9 +37,16 @@ export const scriptTableCommand = new Command("script") .addOption(params.table) .action(async (path, options) => { - const dialect = createDialectFromOptions(options) - const table = await readTable({ path, dialect }) + const session = Session.create({ + title: "Explore table", + }) - const session = repl.start({ prompt: "dp> " }) - session.context.table = table + const resource = path + ? { path, dialect: createDialectFromOptions(options) } + : await selectResource(session, options) + + const table = await session.task("Loading table", readTable(resource)) + + const replSession = repl.start({ prompt: "dp> " }) + replSession.context.table = table }) From 6a59135241ba14c7b945b5d944da870fb65f92f5 Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 14 Aug 2025 17:35:55 +0100 Subject: [PATCH 29/61] Update package commands --- cli/commands/package/archive.ts | 20 ++++++++++++++------ cli/commands/package/copy.ts | 20 ++++++++++++++------ cli/commands/package/errors.tsx | 4 ++-- cli/commands/package/{show.ts => show.tsx} | 15 +++++++++------ cli/commands/resource/copy.ts | 1 + cli/commands/schema/errors.tsx | 3 +-- cli/commands/table/errors.tsx | 1 + cli/commands/table/query.tsx | 2 +- cli/commands/table/stats.tsx | 1 + cli/commands/table/validate.tsx | 1 + cli/components/PackageGrid.tsx | 7 +++++-- 11 files changed, 50 insertions(+), 25 deletions(-) rename cli/commands/package/{show.ts => show.tsx} (53%) create mode 100644 cli/commands/resource/copy.ts diff --git a/cli/commands/package/archive.ts b/cli/commands/package/archive.ts index c3f7db57..439ca926 100644 --- a/cli/commands/package/archive.ts +++ b/cli/commands/package/archive.ts @@ -1,6 +1,7 @@ import { Command } from "commander" import { loadPackage, savePackageToZip } from "dpkit" import { helpConfiguration } from "../../helpers/help.ts" +import { Session } from "../../helpers/session.ts" import * as params from "../../params/index.ts" export const archivePackageCommand = new Command("archive") @@ -12,12 +13,19 @@ export const archivePackageCommand = new Command("archive") .addOption(params.withRemote) .action(async (path, options) => { - const dp = await loadPackage(path) - - await savePackageToZip(dp, { - archivePath: options.toArchive, - withRemote: options.withRemote, + const session = Session.create({ + title: "Validate package", }) - console.log(`Package from "${path}" archived to "${options.toArchive}"`) + const dp = await session.task("Loading package", loadPackage(path)) + + session.task( + "Archiving package", + savePackageToZip(dp, { + archivePath: options.toArchive, + withRemote: options.withRemote, + }), + ) + + session.success(`Package from "${path}" archived to "${options.toArchive}"`) }) diff --git a/cli/commands/package/copy.ts b/cli/commands/package/copy.ts index fc8e3399..9d2d6d2c 100644 --- a/cli/commands/package/copy.ts +++ b/cli/commands/package/copy.ts @@ -1,6 +1,7 @@ import { Command } from "commander" import { loadPackage, savePackageToFolder } from "dpkit" import { helpConfiguration } from "../../helpers/help.ts" +import { Session } from "../../helpers/session.ts" import * as params from "../../params/index.ts" export const copyPackageCommand = new Command("copy") @@ -12,12 +13,19 @@ export const copyPackageCommand = new Command("copy") .addOption(params.withRemote) .action(async (path, options) => { - const dp = await loadPackage(path) - - await savePackageToFolder(dp, { - folderPath: options.toFolder, - withRemote: options.withRemote, + const session = Session.create({ + title: "Validate package", }) - console.log(`Package from "${path}" copied to "${options.toFolder}"`) + const dp = await session.task("Loading package", loadPackage(path)) + + await session.task( + "Copying package", + savePackageToFolder(dp, { + folderPath: options.toFolder, + withRemote: options.withRemote, + }), + ) + + session.success(`Package from "${path}" copied to "${options.toFolder}"`) }) diff --git a/cli/commands/package/errors.tsx b/cli/commands/package/errors.tsx index 0ec558be..e1afb12a 100644 --- a/cli/commands/package/errors.tsx +++ b/cli/commands/package/errors.tsx @@ -18,13 +18,13 @@ export const errorsPackageCommand = new Command("errors") .addOption(params.json) .action(async (path, options) => { - let descriptor: Descriptor | undefined - const session = Session.create({ title: "Validate package", json: options.json, }) + let descriptor: Descriptor | undefined + if (!path) { const resource = await selectResource(session, options) descriptor = resource as unknown as Descriptor diff --git a/cli/commands/package/show.ts b/cli/commands/package/show.tsx similarity index 53% rename from cli/commands/package/show.ts rename to cli/commands/package/show.tsx index 0606ef8e..8bedc204 100644 --- a/cli/commands/package/show.ts +++ b/cli/commands/package/show.tsx @@ -1,6 +1,9 @@ import { Command } from "commander" import { loadPackage } from "dpkit" +import React from "react" +import { PackageGrid } from "../../components/PackageGrid.tsx" import { helpConfiguration } from "../../helpers/help.ts" +import { Session } from "../../helpers/session.ts" import * as params from "../../params/index.ts" export const showPackageCommand = new Command("show") @@ -11,12 +14,12 @@ export const showPackageCommand = new Command("show") .addOption(params.json) .action(async (path, options) => { - const dp = await loadPackage(path) + const session = Session.create({ + title: "Show package", + json: options.json, + }) - if (options.json) { - console.log(JSON.stringify(dp, null, 2)) - return - } + const dp = await session.task("Loading package", loadPackage(path)) - console.log(dp) + await session.render(dp, ) }) diff --git a/cli/commands/resource/copy.ts b/cli/commands/resource/copy.ts new file mode 100644 index 00000000..1039d4c5 --- /dev/null +++ b/cli/commands/resource/copy.ts @@ -0,0 +1 @@ +// TODO: implement diff --git a/cli/commands/schema/errors.tsx b/cli/commands/schema/errors.tsx index ca9cb9aa..e66cc1f9 100644 --- a/cli/commands/schema/errors.tsx +++ b/cli/commands/schema/errors.tsx @@ -33,7 +33,7 @@ export const errorsSchemaCommand = new Command("errors") } if (typeof resource.schema !== "string") { - descriptor = resource.schema as Descriptor + descriptor = resource.schema as unknown as Descriptor } else { path = resource.schema } @@ -57,4 +57,3 @@ export const errorsSchemaCommand = new Command("errors") session.render(errors, ) }) - diff --git a/cli/commands/table/errors.tsx b/cli/commands/table/errors.tsx index f06929b3..94f68a3b 100644 --- a/cli/commands/table/errors.tsx +++ b/cli/commands/table/errors.tsx @@ -39,6 +39,7 @@ export const errorsTableCommand = new Command("errors") .action(async (path, options) => { const session = Session.create({ title: "Table errors", + json: options.json, }) const resource = path diff --git a/cli/commands/table/query.tsx b/cli/commands/table/query.tsx index 3642d1a8..6886c660 100644 --- a/cli/commands/table/query.tsx +++ b/cli/commands/table/query.tsx @@ -9,7 +9,6 @@ export const queryTableCommand = new Command("query") "Start a querying session for a table from a local or remote path", ) .addArgument(params.positionalTablePath) - .addOption(params.json) .optionsGroup("Table Dialect") .addOption(params.delimiter) @@ -31,6 +30,7 @@ export const queryTableCommand = new Command("query") .addOption(params.table) .action(async (_path, _options) => { + // @ts-ignore const session = Session.create({ title: "Query table", }) diff --git a/cli/commands/table/stats.tsx b/cli/commands/table/stats.tsx index 927d27e4..e85a5827 100644 --- a/cli/commands/table/stats.tsx +++ b/cli/commands/table/stats.tsx @@ -38,6 +38,7 @@ export const statsTableCommand = new Command("stats") .action(async (path, options) => { const session = Session.create({ title: "Table stats", + json: options.json, }) const resource = path diff --git a/cli/commands/table/validate.tsx b/cli/commands/table/validate.tsx index 35072ea5..150d9b92 100644 --- a/cli/commands/table/validate.tsx +++ b/cli/commands/table/validate.tsx @@ -18,6 +18,7 @@ export const validateTableCommand = new Command("validate") .action(async (path, options) => { const session = Session.create({ title: "Validate Table", + json: options.json, }) const resource = path diff --git a/cli/components/PackageGrid.tsx b/cli/components/PackageGrid.tsx index c2ec5a75..7c52f552 100644 --- a/cli/components/PackageGrid.tsx +++ b/cli/components/PackageGrid.tsx @@ -4,10 +4,13 @@ import { DataGrid } from "./DataGrid.tsx" // TODO: Support showing other package/resource properties -export function PackageGrid(props: { schema: Package }) { +export function PackageGrid(props: { dataPackage: Package }) { const data = [ Object.fromEntries( - props.schema.resources.map(resource => [resource.name, resource.path]), + props.dataPackage.resources.map(resource => [ + resource.name, + resource.path, + ]), ), ] From 8a8e4abdafcd290265d02854fb36a2d836e320ee Mon Sep 17 00:00:00 2001 From: roll Date: Fri, 15 Aug 2025 09:09:13 +0100 Subject: [PATCH 30/61] Implemented file copy command --- cli/commands/file/copy.ts | 41 +++++++++++++++++++++++++++++++++++ cli/commands/file/index.ts | 9 ++++++++ cli/commands/file/stats.ts | 0 cli/commands/package/index.ts | 6 ++--- cli/commands/resource/copy.ts | 1 - cli/main.ts | 2 ++ cli/params/path.ts | 5 +++++ 7 files changed, 60 insertions(+), 4 deletions(-) create mode 100644 cli/commands/file/copy.ts create mode 100644 cli/commands/file/index.ts create mode 100644 cli/commands/file/stats.ts delete mode 100644 cli/commands/resource/copy.ts diff --git a/cli/commands/file/copy.ts b/cli/commands/file/copy.ts new file mode 100644 index 00000000..98fcc798 --- /dev/null +++ b/cli/commands/file/copy.ts @@ -0,0 +1,41 @@ +import { Command } from "commander" +import { copyFile } from "dpkit" +import { helpConfiguration } from "../../helpers/help.ts" +import { selectResource } from "../../helpers/resource.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const copyFileCommand = new Command("copy") + .configureHelp(helpConfiguration) + .description("Copy a local or remote file to a local path") + + .addArgument(params.positionalFilePath) + .addOption(params.toPath.makeOptionMandatory()) + .addOption(params.fromPackage) + .addOption(params.fromResource) + + .action(async (path, options) => { + const session = Session.create({ + title: "Copy file", + }) + + if (!path) { + const resource = await selectResource(session, options) + + if (typeof resource.path !== "string") { + Session.terminate("Only single file resources are supported") + } + + path = resource.path + } + + await session.task( + "Copying file", + copyFile({ + sourcePath: path, + targetPath: options.toPath, + }), + ) + + session.success(`File from "${path}" copied to "${options.toPath}"`) + }) diff --git a/cli/commands/file/index.ts b/cli/commands/file/index.ts new file mode 100644 index 00000000..d925064e --- /dev/null +++ b/cli/commands/file/index.ts @@ -0,0 +1,9 @@ +import { Command } from "commander" +import { helpConfiguration } from "../../helpers/help.ts" +import { copyFileCommand } from "./copy.ts" + +export const fileCommand = new Command("file") + .configureHelp(helpConfiguration) + .description("File related commands") + + .addCommand(copyFileCommand) diff --git a/cli/commands/file/stats.ts b/cli/commands/file/stats.ts new file mode 100644 index 00000000..e69de29b diff --git a/cli/commands/package/index.ts b/cli/commands/package/index.ts index dd9413ae..220196f7 100644 --- a/cli/commands/package/index.ts +++ b/cli/commands/package/index.ts @@ -3,13 +3,13 @@ import { helpConfiguration } from "../../helpers/help.ts" import { archivePackageCommand } from "./archive.ts" import { copyPackageCommand } from "./copy.ts" import { errorsPackageCommand } from "./errors.tsx" -import { showPackageCommand } from "./show.ts" +import { showPackageCommand } from "./show.tsx" export const packageCommand = new Command("package") .configureHelp(helpConfiguration) .description("Data Package related commands") - .addCommand(errorsPackageCommand) - .addCommand(showPackageCommand) .addCommand(archivePackageCommand) .addCommand(copyPackageCommand) + .addCommand(errorsPackageCommand) + .addCommand(showPackageCommand) diff --git a/cli/commands/resource/copy.ts b/cli/commands/resource/copy.ts deleted file mode 100644 index 1039d4c5..00000000 --- a/cli/commands/resource/copy.ts +++ /dev/null @@ -1 +0,0 @@ -// TODO: implement diff --git a/cli/main.ts b/cli/main.ts index ef04c354..0462aa75 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -1,5 +1,6 @@ import { program } from "commander" import { dialectCommand } from "./commands/dialect/index.ts" +import { fileCommand } from "./commands/file/index.ts" import { packageCommand } from "./commands/package/index.ts" import { resourceCommand } from "./commands/resource/index.ts" import { schemaCommand } from "./commands/schema/index.ts" @@ -21,5 +22,6 @@ program .addCommand(dialectCommand) .addCommand(schemaCommand) .addCommand(tableCommand) + .addCommand(fileCommand) .parse() diff --git a/cli/params/path.ts b/cli/params/path.ts index e8e012d7..e851b95f 100644 --- a/cli/params/path.ts +++ b/cli/params/path.ts @@ -10,6 +10,11 @@ export const positionalTablePath = new Argument( "local or remote path to the table", ) +export const positionalFilePath = new Argument( + "[path]", + "local or remote path to the file", +) + export const toPath = new Option("--to-path ", "a local output path") export const toFolder = new Option( From 8625015b149b8b4013392dc5d195d5cf69f96ca1 Mon Sep 17 00:00:00 2001 From: roll Date: Fri, 15 Aug 2025 09:49:16 +0100 Subject: [PATCH 31/61] Implemened describeFile --- file/file/describe.ts | 16 ++++++++++++++++ file/file/index.ts | 1 + file/package.json | 1 + pnpm-lock.yaml | 12 ++++++++++++ 4 files changed, 30 insertions(+) create mode 100644 file/file/describe.ts diff --git a/file/file/describe.ts b/file/file/describe.ts new file mode 100644 index 00000000..e0be1004 --- /dev/null +++ b/file/file/describe.ts @@ -0,0 +1,16 @@ +import { stat } from "node:fs/promises" +import { hashFile } from "hasha" +import { prefetchFile } from "./fetch.ts" + +export async function describeFile( + path: string, + options?: { hashType?: "sha256" }, +) { + const hashType = options?.hashType ?? "sha256" + const localPath = await prefetchFile(path) + + const bytes = (await stat(localPath)).size + const hash = await hashFile(localPath, { algorithm: hashType }) + + return { bytes, hash } +} diff --git a/file/file/index.ts b/file/file/index.ts index da2aea7a..6228afd1 100644 --- a/file/file/index.ts +++ b/file/file/index.ts @@ -4,3 +4,4 @@ export { saveFile } from "./save.ts" export { getTempFilePath, writeTempFile } from "./temp.ts" export { assertLocalPathVacant, isLocalPathExist } from "./path.ts" export { prefetchFile, prefetchFiles } from "./fetch.ts" +export { describeFile } from "./describe.ts" diff --git a/file/package.json b/file/package.json index 62014daa..6dd5a32d 100644 --- a/file/package.json +++ b/file/package.json @@ -25,6 +25,7 @@ "dependencies": { "@dpkit/core": "workspace:*", "exit-hook": "^4.0.0", + "hasha": "^6.0.0", "tempy": "3.1.0", "tiny-invariant": "^1.3.3" } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1483fa5a..d63a12ea 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -280,6 +280,9 @@ importers: exit-hook: specifier: ^4.0.0 version: 4.0.0 + hasha: + specifier: ^6.0.0 + version: 6.0.0 tempy: specifier: 3.1.0 version: 3.1.0 @@ -2215,6 +2218,10 @@ packages: resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==} engines: {node: '>= 0.4'} + hasha@6.0.0: + resolution: {integrity: sha512-MLydoyGp9QJcjlhE5lsLHXYpWayjjWqkavzju2ZWD2tYa1CgmML1K1gWAu22BLFa2eZ0OfvJ/DlfoVjaD54U2Q==} + engines: {node: '>=18'} + hasown@2.0.2: resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} engines: {node: '>= 0.4'} @@ -5940,6 +5947,11 @@ snapshots: has-symbols@1.1.0: {} + hasha@6.0.0: + dependencies: + is-stream: 3.0.0 + type-fest: 4.41.0 + hasown@2.0.2: dependencies: function-bind: 1.1.2 From bf45ce55daeef611e5066d2fa1773a4800af6830 Mon Sep 17 00:00:00 2001 From: roll Date: Fri, 15 Aug 2025 10:09:07 +0100 Subject: [PATCH 32/61] Implemented file stats command --- cli/commands/file/index.ts | 2 ++ cli/commands/file/stats.ts | 0 cli/commands/file/stats.tsx | 42 +++++++++++++++++++++++++++++++++++++ cli/params/file.ts | 5 +++++ cli/params/index.ts | 3 ++- file/file/describe.ts | 8 ++++--- 6 files changed, 56 insertions(+), 4 deletions(-) delete mode 100644 cli/commands/file/stats.ts create mode 100644 cli/commands/file/stats.tsx create mode 100644 cli/params/file.ts diff --git a/cli/commands/file/index.ts b/cli/commands/file/index.ts index d925064e..12df8b04 100644 --- a/cli/commands/file/index.ts +++ b/cli/commands/file/index.ts @@ -1,9 +1,11 @@ import { Command } from "commander" import { helpConfiguration } from "../../helpers/help.ts" import { copyFileCommand } from "./copy.ts" +import { statsFileCommand } from "./stats.tsx" export const fileCommand = new Command("file") .configureHelp(helpConfiguration) .description("File related commands") .addCommand(copyFileCommand) + .addCommand(statsFileCommand) diff --git a/cli/commands/file/stats.ts b/cli/commands/file/stats.ts deleted file mode 100644 index e69de29b..00000000 diff --git a/cli/commands/file/stats.tsx b/cli/commands/file/stats.tsx new file mode 100644 index 00000000..d7402339 --- /dev/null +++ b/cli/commands/file/stats.tsx @@ -0,0 +1,42 @@ +import { Command } from "commander" +import { describeFile } from "dpkit" +import React from "react" +import { DataGrid } from "../../components/DataGrid.tsx" +import { helpConfiguration } from "../../helpers/help.ts" +import { selectResource } from "../../helpers/resource.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const statsFileCommand = new Command("stats") + .configureHelp(helpConfiguration) + .description("Show stats for a local or remote file") + + .addArgument(params.positionalFilePath) + .addOption(params.fromPackage) + .addOption(params.fromResource) + .addOption(params.hashType) + .addOption(params.json) + + .action(async (path, options) => { + const session = Session.create({ + title: "File stats", + json: options.json, + }) + + if (!path) { + const resource = await selectResource(session, options) + + if (typeof resource.path !== "string") { + Session.terminate("Only single file resources are supported") + } + + path = resource.path + } + + const stats = await session.task( + "Calculating stats", + describeFile(path, { hashType: options.hashType }), + ) + + session.render(stats, ) + }) diff --git a/cli/params/file.ts b/cli/params/file.ts new file mode 100644 index 00000000..78599138 --- /dev/null +++ b/cli/params/file.ts @@ -0,0 +1,5 @@ +import { Option } from "commander" + +export const hashType = new Option("--hash-type ", "hash type") + .choices(["md5", "sha1", "sha256", "sha512"]) + .default("sha256") diff --git a/cli/params/index.ts b/cli/params/index.ts index 28f8eced..a013ffb7 100644 --- a/cli/params/index.ts +++ b/cli/params/index.ts @@ -1,5 +1,6 @@ -export * from "./json.ts" export * from "./dialect.ts" +export * from "./file.ts" +export * from "./json.ts" export * from "./package.ts" export * from "./path.ts" export * from "./resource.ts" diff --git a/file/file/describe.ts b/file/file/describe.ts index e0be1004..bdd89020 100644 --- a/file/file/describe.ts +++ b/file/file/describe.ts @@ -2,15 +2,17 @@ import { stat } from "node:fs/promises" import { hashFile } from "hasha" import { prefetchFile } from "./fetch.ts" +type HashType = "md5" | "sha1" | "sha256" | "sha512" + export async function describeFile( path: string, - options?: { hashType?: "sha256" }, + options: { hashType: HashType }, ) { - const hashType = options?.hashType ?? "sha256" + const algorithm = options.hashType const localPath = await prefetchFile(path) const bytes = (await stat(localPath)).size - const hash = await hashFile(localPath, { algorithm: hashType }) + const hash = await hashFile(localPath, { algorithm }) return { bytes, hash } } From e4c93938c53eb44b5d503fc7ddbb5cf8a95cdd56 Mon Sep 17 00:00:00 2001 From: roll Date: Fri, 15 Aug 2025 11:04:39 +0100 Subject: [PATCH 33/61] Removed extend/ui packages --- extend/CHANGELOG.md | 13 ------------- extend/README.md | 3 --- extend/index.ts | 0 extend/package.json | 25 ------------------------- extend/tsconfig.json | 3 --- pnpm-lock.yaml | 4 ---- ui/CHANGELOG.md | 13 ------------- ui/README.md | 3 --- ui/index.ts | 0 ui/package.json | 25 ------------------------- ui/tsconfig.json | 3 --- 11 files changed, 92 deletions(-) delete mode 100644 extend/CHANGELOG.md delete mode 100644 extend/README.md delete mode 100644 extend/index.ts delete mode 100644 extend/package.json delete mode 100644 extend/tsconfig.json delete mode 100644 ui/CHANGELOG.md delete mode 100644 ui/README.md delete mode 100644 ui/index.ts delete mode 100644 ui/package.json delete mode 100644 ui/tsconfig.json diff --git a/extend/CHANGELOG.md b/extend/CHANGELOG.md deleted file mode 100644 index e20bdb21..00000000 --- a/extend/CHANGELOG.md +++ /dev/null @@ -1,13 +0,0 @@ -# @dpkit/extend - -## 0.3.0 - -### Minor Changes - -- 313a275: Added Json/Jsonl/Arrow/Parquet support; Bootstrapped CLI - -## 0.2.0 - -### Minor Changes - -- edfde49: General framework improvements diff --git a/extend/README.md b/extend/README.md deleted file mode 100644 index 757f15fc..00000000 --- a/extend/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# @dpkit/extend - -dpkit is a fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/extend/index.ts b/extend/index.ts deleted file mode 100644 index e69de29b..00000000 diff --git a/extend/package.json b/extend/package.json deleted file mode 100644 index 48787cd5..00000000 --- a/extend/package.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "name": "@dpkit/extend", - "type": "module", - "version": "0.3.0", - "exports": "./build/index.js", - "license": "MIT", - "author": "Evgeny Karev", - "repository": "https://github.com/datisthq/dpkit", - "description": "Fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames", - "keywords": [ - "data", - "polars", - "dataframe", - "datapackage", - "tableschema", - "typescript", - "validation", - "quality", - "fair", - "extend" - ], - "scripts": { - "build": "tsc" - } -} diff --git a/extend/tsconfig.json b/extend/tsconfig.json deleted file mode 100644 index 3c43903c..00000000 --- a/extend/tsconfig.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "extends": "../tsconfig.json" -} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d63a12ea..c17c55de 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -270,8 +270,6 @@ importers: excel: {} - extend: {} - file: dependencies: '@dpkit/core': @@ -394,8 +392,6 @@ importers: specifier: ^6.0.6 version: 6.0.6 - ui: {} - zenodo: dependencies: '@dpkit/core': diff --git a/ui/CHANGELOG.md b/ui/CHANGELOG.md deleted file mode 100644 index 67b728ad..00000000 --- a/ui/CHANGELOG.md +++ /dev/null @@ -1,13 +0,0 @@ -# @dpkit/ui - -## 0.3.0 - -### Minor Changes - -- 313a275: Added Json/Jsonl/Arrow/Parquet support; Bootstrapped CLI - -## 0.2.0 - -### Minor Changes - -- edfde49: General framework improvements diff --git a/ui/README.md b/ui/README.md deleted file mode 100644 index 5b3240e3..00000000 --- a/ui/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# @dpkit/ui - -dpkit is a fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/ui/index.ts b/ui/index.ts deleted file mode 100644 index e69de29b..00000000 diff --git a/ui/package.json b/ui/package.json deleted file mode 100644 index 9c0764bc..00000000 --- a/ui/package.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "name": "@dpkit/ui", - "type": "module", - "version": "0.3.0", - "exports": "./build/index.js", - "license": "MIT", - "author": "Evgeny Karev", - "repository": "https://github.com/datisthq/dpkit", - "description": "Fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames", - "keywords": [ - "data", - "polars", - "dataframe", - "datapackage", - "tableschema", - "typescript", - "validation", - "quality", - "fair", - "ui" - ], - "scripts": { - "build": "tsc" - } -} diff --git a/ui/tsconfig.json b/ui/tsconfig.json deleted file mode 100644 index 3c43903c..00000000 --- a/ui/tsconfig.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "extends": "../tsconfig.json" -} From 5ce4e6d989e5bec8fe208da3a19416df492b2b6e Mon Sep 17 00:00:00 2001 From: roll Date: Fri, 15 Aug 2025 11:10:06 +0100 Subject: [PATCH 34/61] Renamed to inferResourceFormat --- arrow/plugin.ts | 4 ++-- core/resource/index.ts | 2 +- core/resource/infer.ts | 2 +- csv/plugin.ts | 4 ++-- json/plugin.ts | 4 ++-- parquet/plugin.ts | 4 ++-- pnpm-workspace.yaml | 2 -- 7 files changed, 10 insertions(+), 12 deletions(-) diff --git a/arrow/plugin.ts b/arrow/plugin.ts index 133ee938..c678b6d5 100644 --- a/arrow/plugin.ts +++ b/arrow/plugin.ts @@ -1,5 +1,5 @@ import type { Resource } from "@dpkit/core" -import { inferFormat } from "@dpkit/core" +import { inferResourceFormat } from "@dpkit/core" import type { TablePlugin } from "@dpkit/table" import type { SaveTableOptions, Table } from "@dpkit/table" import { loadArrowTable, saveArrowTable } from "./table/index.ts" @@ -21,6 +21,6 @@ export class ArrowPlugin implements TablePlugin { } function getIsArrow(resource: Partial) { - const format = inferFormat(resource) + const format = inferResourceFormat(resource) return format === "arrow" || format === "feather" } diff --git a/core/resource/index.ts b/core/resource/index.ts index c17827f1..8d3ac4b2 100644 --- a/core/resource/index.ts +++ b/core/resource/index.ts @@ -1,5 +1,5 @@ export type { Resource } from "./Resource.ts" -export { inferFormat } from "./infer.ts" +export { inferResourceFormat } from "./infer.ts" export { assertResource } from "./assert.ts" export { loadResourceDescriptor } from "./load.ts" export { saveResourceDescriptor } from "./save.ts" diff --git a/core/resource/infer.ts b/core/resource/infer.ts index 64416d1f..961319b7 100644 --- a/core/resource/infer.ts +++ b/core/resource/infer.ts @@ -1,7 +1,7 @@ import { getFilename, getFormat } from "../general/index.ts" import type { Resource } from "./Resource.ts" -export function inferFormat(resource: Partial) { +export function inferResourceFormat(resource: Partial) { let format = resource.format if (!format) { diff --git a/csv/plugin.ts b/csv/plugin.ts index d34bc16f..a8be7407 100644 --- a/csv/plugin.ts +++ b/csv/plugin.ts @@ -1,5 +1,5 @@ import type { Resource } from "@dpkit/core" -import { inferFormat } from "@dpkit/core" +import { inferResourceFormat } from "@dpkit/core" import type { InferDialectOptions, TablePlugin } from "@dpkit/table" import type { SaveTableOptions, Table } from "@dpkit/table" import { inferCsvDialect } from "./dialect/index.ts" @@ -32,6 +32,6 @@ export class CsvPlugin implements TablePlugin { } function getIsCsv(resource: Partial) { - const format = inferFormat(resource) + const format = inferResourceFormat(resource) return ["csv", "tsv"].includes(format ?? "") } diff --git a/json/plugin.ts b/json/plugin.ts index eab7b74e..d57a9ead 100644 --- a/json/plugin.ts +++ b/json/plugin.ts @@ -1,5 +1,5 @@ import type { Resource } from "@dpkit/core" -import { inferFormat } from "@dpkit/core" +import { inferResourceFormat } from "@dpkit/core" import type { TablePlugin } from "@dpkit/table" import type { SaveTableOptions, Table } from "@dpkit/table" import { loadJsonTable, loadJsonlTable } from "./table/index.ts" @@ -36,7 +36,7 @@ export class JsonPlugin implements TablePlugin { } function getFormatInfo(resource: Partial) { - const format = inferFormat(resource) + const format = inferResourceFormat(resource) const isJson = format === "json" const isJsonl = format === "jsonl" || format === "ndjson" return { isJson, isJsonl } diff --git a/parquet/plugin.ts b/parquet/plugin.ts index 4aacf435..d978d341 100644 --- a/parquet/plugin.ts +++ b/parquet/plugin.ts @@ -1,5 +1,5 @@ import type { Resource } from "@dpkit/core" -import { inferFormat } from "@dpkit/core" +import { inferResourceFormat } from "@dpkit/core" import type { TablePlugin } from "@dpkit/table" import type { SaveTableOptions, Table } from "@dpkit/table" import { loadParquetTable, saveParquetTable } from "./table/index.ts" @@ -21,6 +21,6 @@ export class ParquetPlugin implements TablePlugin { } function getIsParquet(resource: Partial) { - const format = inferFormat(resource) + const format = inferResourceFormat(resource) return format === "parquet" } diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 717b560a..bfa5aa97 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -11,7 +11,6 @@ packages: - docs - dpkit - excel - - extend - file - folder - github @@ -21,6 +20,5 @@ packages: - parquet - table - test - - ui - zenodo - zip From 383d15536b2b5c72328819c45d3937f49d9cda64 Mon Sep 17 00:00:00 2001 From: roll Date: Fri, 15 Aug 2025 11:18:33 +0100 Subject: [PATCH 35/61] Implemented inferResourceName --- core/resource/index.ts | 2 +- core/resource/infer.ts | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/core/resource/index.ts b/core/resource/index.ts index 8d3ac4b2..531c1774 100644 --- a/core/resource/index.ts +++ b/core/resource/index.ts @@ -1,5 +1,5 @@ export type { Resource } from "./Resource.ts" -export { inferResourceFormat } from "./infer.ts" +export { inferResourceName, inferResourceFormat } from "./infer.ts" export { assertResource } from "./assert.ts" export { loadResourceDescriptor } from "./load.ts" export { saveResourceDescriptor } from "./save.ts" diff --git a/core/resource/infer.ts b/core/resource/infer.ts index 961319b7..9da3c280 100644 --- a/core/resource/infer.ts +++ b/core/resource/infer.ts @@ -1,6 +1,22 @@ -import { getFilename, getFormat } from "../general/index.ts" +import { getFilename, getFormat, getName } from "../general/index.ts" import type { Resource } from "./Resource.ts" +export function inferResourceName(resource: Partial) { + let name = resource.name + + if (!name) { + const path = Array.isArray(resource.path) ? resource.path[0] : resource.path + if (path) { + const filename = getFilename(path) + name = getName(filename) + } else { + name = "resource" + } + } + + return name +} + export function inferResourceFormat(resource: Partial) { let format = resource.format From da1fd7df618e284eb95de33579ebb7ebec5f2276 Mon Sep 17 00:00:00 2001 From: roll Date: Fri, 15 Aug 2025 12:14:38 +0100 Subject: [PATCH 36/61] Implemented proper infer resource --- cli/commands/resource/infer.tsx | 11 +++---- dpkit/dialect/infer.ts | 2 +- dpkit/resource/infer.ts | 52 +++++++++++++++++++++++++++++++-- dpkit/table/infer.ts | 2 ++ file/file/describe.ts | 13 ++++----- file/file/index.ts | 1 + file/file/infer.ts | 49 +++++++++++++++++++++++++++++++ file/package.json | 2 ++ pnpm-lock.yaml | 17 +++++++++++ table/schema/infer.ts | 2 ++ 10 files changed, 134 insertions(+), 17 deletions(-) create mode 100644 file/file/infer.ts diff --git a/cli/commands/resource/infer.tsx b/cli/commands/resource/infer.tsx index 39f6d477..e2f35cd9 100644 --- a/cli/commands/resource/infer.tsx +++ b/cli/commands/resource/infer.tsx @@ -25,17 +25,18 @@ export const inferResourceCommand = new Command("infer") const resource = path ? { path } : await selectResource(session, options) - const inferredResource = await session.task( - "Loading sample", + const result = await session.task( + "Inferring resource", inferResource(resource), ) - if (isEmptyObject(inferredResource)) { + + if (isEmptyObject(result)) { Session.terminate("Could not infer resource") } await session.render( - inferredResource, + result, // @ts-ignore - , + , ) }) diff --git a/dpkit/dialect/infer.ts b/dpkit/dialect/infer.ts index c37b8087..fdf55d27 100644 --- a/dpkit/dialect/infer.ts +++ b/dpkit/dialect/infer.ts @@ -2,7 +2,7 @@ import type { Dialect, Resource } from "@dpkit/core" import type { InferDialectOptions } from "@dpkit/table" import { dpkit } from "../plugin.ts" -// TODO: implement inferDialect +// TODO: review default values being {} vs undefined export async function inferDialect( resource: Partial, diff --git a/dpkit/resource/infer.ts b/dpkit/resource/infer.ts index 43e95bae..4bffe612 100644 --- a/dpkit/resource/infer.ts +++ b/dpkit/resource/infer.ts @@ -1,10 +1,56 @@ import type { Resource } from "@dpkit/core" +import { inferResourceFormat, inferResourceName } from "@dpkit/core" +import { prefetchFile } from "@dpkit/file" +import { inferFileBytes, inferFileEncoding, inferFileHash } from "@dpkit/file" import { inferTable } from "../table/index.ts" -// TODO: Implement properly +// TODO: Support multipart resources? (clarify on the specs level) export async function inferResource(resource: Partial) { - const { dialect, schema } = await inferTable(resource) + const result = { ...resource } - return { ...resource, dialect, schema } + if (!result.name) { + result.name = inferResourceName(resource) + } + + if (!result.format) { + result.format = inferResourceFormat(resource) + } + + if (typeof resource.path === "string") { + const localPath = await prefetchFile(resource.path) + + if (!result.encoding) { + const encoding = await inferFileEncoding(localPath) + if (encoding) { + result.encoding = encoding + } + } + + if (!result.bytes) { + result.bytes = await inferFileBytes(localPath) + } + + if (!result.hash) { + result.hash = await inferFileHash(localPath) + } + + if (!result.schema) { + try { + const localResource = { ...resource, path: localPath } + const { dialect, schema } = await inferTable(localResource) + + result.dialect = dialect + result.schema = schema + } catch {} + } + } + + if (!result.type) { + if (result.schema) { + result.type = "table" + } + } + + return result } diff --git a/dpkit/table/infer.ts b/dpkit/table/infer.ts index 49213a03..e9ed01af 100644 --- a/dpkit/table/infer.ts +++ b/dpkit/table/infer.ts @@ -5,6 +5,8 @@ import { inferSchema } from "@dpkit/table" import { inferDialect } from "../dialect/index.ts" import { loadTable } from "./load.ts" +// TODO: Allow non-tabular resources returning undefineds? + export async function inferTable( resource: Partial, ): Promise<{ dialect: Dialect; schema: Schema; table: Table }> { diff --git a/file/file/describe.ts b/file/file/describe.ts index bdd89020..4188a2b6 100644 --- a/file/file/describe.ts +++ b/file/file/describe.ts @@ -1,18 +1,15 @@ -import { stat } from "node:fs/promises" -import { hashFile } from "hasha" import { prefetchFile } from "./fetch.ts" - -type HashType = "md5" | "sha1" | "sha256" | "sha512" +import { inferFileBytes, inferFileHash } from "./infer.ts" +import type { HashType } from "./infer.ts" export async function describeFile( path: string, - options: { hashType: HashType }, + options?: { hashType?: HashType }, ) { - const algorithm = options.hashType const localPath = await prefetchFile(path) - const bytes = (await stat(localPath)).size - const hash = await hashFile(localPath, { algorithm }) + const bytes = await inferFileBytes(localPath) + const hash = await inferFileHash(localPath, { hashType: options?.hashType }) return { bytes, hash } } diff --git a/file/file/index.ts b/file/file/index.ts index 6228afd1..8721b19d 100644 --- a/file/file/index.ts +++ b/file/file/index.ts @@ -4,4 +4,5 @@ export { saveFile } from "./save.ts" export { getTempFilePath, writeTempFile } from "./temp.ts" export { assertLocalPathVacant, isLocalPathExist } from "./path.ts" export { prefetchFile, prefetchFiles } from "./fetch.ts" +export { inferFileEncoding, inferFileBytes, inferFileHash } from "./infer.ts" export { describeFile } from "./describe.ts" diff --git a/file/file/infer.ts b/file/file/infer.ts new file mode 100644 index 00000000..a69d08f9 --- /dev/null +++ b/file/file/infer.ts @@ -0,0 +1,49 @@ +import { stat } from "node:fs/promises" +import chardet from "chardet" +import { hashFile } from "hasha" +import { isBinaryFile } from "isbinaryfile" +import { prefetchFile } from "./fetch.ts" +import { loadFile } from "./load.ts" + +export type HashType = "md5" | "sha1" | "sha256" | "sha512" + +export async function inferFileHash( + path: string, + options?: { hashType?: HashType }, +) { + const localPath = await prefetchFile(path) + const algorithm = options?.hashType ?? "sha256" + + const result = await hashFile(localPath, { algorithm }) + return `${algorithm}:${result}` +} + +export async function inferFileBytes(path: string) { + const localPath = await prefetchFile(path) + + const result = await stat(localPath) + return result.size +} + +export async function inferFileEncoding( + path: string, + options?: { sampleBytes?: number; confidencePercent?: number }, +) { + const maxBytes = options?.sampleBytes ?? 10_000 + const confidencePercent = options?.confidencePercent ?? 75 + + const buffer = await loadFile(path, { maxBytes }) + const isBinary = await isBinaryFile(buffer) + + if (!isBinary) { + const matches = chardet.analyse(buffer) + console.log(matches) + for (const match of matches) { + if (match.confidence >= confidencePercent) { + return match.name.toLowerCase() + } + } + } + + return undefined +} diff --git a/file/package.json b/file/package.json index 6dd5a32d..30e93ceb 100644 --- a/file/package.json +++ b/file/package.json @@ -24,8 +24,10 @@ }, "dependencies": { "@dpkit/core": "workspace:*", + "chardet": "^2.1.0", "exit-hook": "^4.0.0", "hasha": "^6.0.0", + "isbinaryfile": "^5.0.4", "tempy": "3.1.0", "tiny-invariant": "^1.3.3" } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c17c55de..a556b5b4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -275,12 +275,18 @@ importers: '@dpkit/core': specifier: workspace:* version: link:../core + chardet: + specifier: ^2.1.0 + version: 2.1.0 exit-hook: specifier: ^4.0.0 version: 4.0.0 hasha: specifier: ^6.0.0 version: 6.0.0 + isbinaryfile: + specifier: ^5.0.4 + version: 5.0.4 tempy: specifier: 3.1.0 version: 3.1.0 @@ -1688,6 +1694,9 @@ packages: chardet@0.7.0: resolution: {integrity: sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==} + chardet@2.1.0: + resolution: {integrity: sha512-bNFETTG/pM5ryzQ9Ad0lJOTa6HWD/YsScAR3EnCPZRPlQh77JocYktSHOUHelyhm8IARL+o4c4F1bP5KVOjiRA==} + check-error@2.1.1: resolution: {integrity: sha512-OAlb+T7V4Op9OwdkjmguYRqncdlx5JiofwOAUkmTF+jNdHwzTaTs4sRAGpzLF3oOz5xAyDGrPgeIDFQmDOTiJw==} engines: {node: '>= 16'} @@ -2459,6 +2468,10 @@ packages: resolution: {integrity: sha512-UcVfVfaK4Sc4m7X3dUSoHoozQGBEFeDC+zVo06t98xe8CzHSZZBekNXH+tu0NalHolcJ/QAGqS46Hef7QXBIMw==} engines: {node: '>=16'} + isbinaryfile@5.0.4: + resolution: {integrity: sha512-YKBKVkKhty7s8rxddb40oOkuP0NbaeXrQvLin6QMHL7Ypiy2RW9LwOVrVgZRyOrhQlayMd9t+D8yDy8MKFTSDQ==} + engines: {node: '>= 18.0.0'} + isexe@2.0.0: resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} @@ -5377,6 +5390,8 @@ snapshots: chardet@0.7.0: {} + chardet@2.1.0: {} + check-error@2.1.1: {} chokidar@4.0.3: @@ -6294,6 +6309,8 @@ snapshots: dependencies: is-inside-container: 1.0.0 + isbinaryfile@5.0.4: {} + isexe@2.0.0: {} istanbul-lib-coverage@3.2.2: {} diff --git a/table/schema/infer.ts b/table/schema/infer.ts index 7354fcce..e7611819 100644 --- a/table/schema/infer.ts +++ b/table/schema/infer.ts @@ -3,6 +3,8 @@ import { col } from "nodejs-polars" import { getPolarsSchema } from "../schema/index.ts" import type { Table } from "../table/index.ts" +// TODO: Review default values being {fields: []} vs undefined + export type InferSchemaOptions = { sampleRows?: number confidence?: number From 5c469f84e6b510fa2c1eb23ae5b57ff2feb8d79a Mon Sep 17 00:00:00 2001 From: roll Date: Fri, 15 Aug 2025 15:42:41 +0100 Subject: [PATCH 37/61] Implemented inferPackage --- core/resource/infer.ts | 4 +--- dpkit/package/index.ts | 1 + dpkit/package/infer.ts | 14 ++++++++++++++ dpkit/resource/infer.ts | 7 +++---- 4 files changed, 19 insertions(+), 7 deletions(-) create mode 100644 dpkit/package/infer.ts diff --git a/core/resource/infer.ts b/core/resource/infer.ts index 9da3c280..ceecbf8a 100644 --- a/core/resource/infer.ts +++ b/core/resource/infer.ts @@ -9,12 +9,10 @@ export function inferResourceName(resource: Partial) { if (path) { const filename = getFilename(path) name = getName(filename) - } else { - name = "resource" } } - return name + return name ?? "resource" } export function inferResourceFormat(resource: Partial) { diff --git a/dpkit/package/index.ts b/dpkit/package/index.ts index 3a5cbd54..c8b5c385 100644 --- a/dpkit/package/index.ts +++ b/dpkit/package/index.ts @@ -1,2 +1,3 @@ export { loadPackage } from "./load.ts" export { savePackage } from "./save.ts" +export { inferPackage } from "./infer.ts" diff --git a/dpkit/package/infer.ts b/dpkit/package/infer.ts new file mode 100644 index 00000000..57784660 --- /dev/null +++ b/dpkit/package/infer.ts @@ -0,0 +1,14 @@ +import type { Package } from "@dpkit/core" +import { inferResource } from "../resource/index.ts" + +export async function inferPackage(dataPackage: Package) { + const result = { ...dataPackage } + + result.resources = await Promise.all( + dataPackage.resources.map(async resource => { + return await inferResource(resource) + }), + ) + + return result +} diff --git a/dpkit/resource/infer.ts b/dpkit/resource/infer.ts index 4bffe612..6eccd130 100644 --- a/dpkit/resource/infer.ts +++ b/dpkit/resource/infer.ts @@ -7,10 +7,9 @@ import { inferTable } from "../table/index.ts" // TODO: Support multipart resources? (clarify on the specs level) export async function inferResource(resource: Partial) { - const result = { ...resource } - - if (!result.name) { - result.name = inferResourceName(resource) + const result = { + ...resource, + name: resource.name ?? inferResourceName(resource), } if (!result.format) { From a456b844f279c6797c95e0b62f67c59e7a79f866 Mon Sep 17 00:00:00 2001 From: roll Date: Fri, 15 Aug 2025 15:54:55 +0100 Subject: [PATCH 38/61] Implemented infer package --- cli/commands/package/index.ts | 2 ++ cli/commands/package/infer.ts | 1 - cli/commands/package/infer.tsx | 35 ++++++++++++++++++++++++++++++++++ cli/params/path.ts | 5 +++++ dpkit/package/infer.ts | 23 ++++++++++++++-------- file/file/infer.ts | 1 - 6 files changed, 57 insertions(+), 10 deletions(-) delete mode 100644 cli/commands/package/infer.ts create mode 100644 cli/commands/package/infer.tsx diff --git a/cli/commands/package/index.ts b/cli/commands/package/index.ts index 220196f7..a97c7e6c 100644 --- a/cli/commands/package/index.ts +++ b/cli/commands/package/index.ts @@ -3,6 +3,7 @@ import { helpConfiguration } from "../../helpers/help.ts" import { archivePackageCommand } from "./archive.ts" import { copyPackageCommand } from "./copy.ts" import { errorsPackageCommand } from "./errors.tsx" +import { inferPackageCommand } from "./infer.tsx" import { showPackageCommand } from "./show.tsx" export const packageCommand = new Command("package") @@ -12,4 +13,5 @@ export const packageCommand = new Command("package") .addCommand(archivePackageCommand) .addCommand(copyPackageCommand) .addCommand(errorsPackageCommand) + .addCommand(inferPackageCommand) .addCommand(showPackageCommand) diff --git a/cli/commands/package/infer.ts b/cli/commands/package/infer.ts deleted file mode 100644 index 1039d4c5..00000000 --- a/cli/commands/package/infer.ts +++ /dev/null @@ -1 +0,0 @@ -// TODO: implement diff --git a/cli/commands/package/infer.tsx b/cli/commands/package/infer.tsx new file mode 100644 index 00000000..9a1d21a0 --- /dev/null +++ b/cli/commands/package/infer.tsx @@ -0,0 +1,35 @@ +import { Command } from "commander" +import { inferPackage } from "dpkit" +import React from "react" +import { PackageGrid } from "../../components/PackageGrid.tsx" +import { helpConfiguration } from "../../helpers/help.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const inferPackageCommand = new Command("infer") + .configureHelp(helpConfiguration) + .description("Infer a data package from local or remote file paths") + + .addArgument(params.positionalFilePaths) + .addOption(params.json) + + .action(async (paths, options) => { + const session = Session.create({ + title: "Infer resource", + json: options.json, + }) + + const sourcePackage = { + resources: paths.map(path => ({ path })), + } + + const targetPackage = await session.task( + "Inferring package", + inferPackage(sourcePackage), + ) + + await session.render( + targetPackage, + , + ) + }) diff --git a/cli/params/path.ts b/cli/params/path.ts index e851b95f..cf591669 100644 --- a/cli/params/path.ts +++ b/cli/params/path.ts @@ -15,6 +15,11 @@ export const positionalFilePath = new Argument( "local or remote path to the file", ) +export const positionalFilePaths = new Argument( + "", + "local paths to files", +) + export const toPath = new Option("--to-path ", "a local output path") export const toFolder = new Option( diff --git a/dpkit/package/infer.ts b/dpkit/package/infer.ts index 57784660..e942c879 100644 --- a/dpkit/package/infer.ts +++ b/dpkit/package/infer.ts @@ -1,14 +1,21 @@ -import type { Package } from "@dpkit/core" +import type { Package, Resource } from "@dpkit/core" import { inferResource } from "../resource/index.ts" -export async function inferPackage(dataPackage: Package) { - const result = { ...dataPackage } +// TODO: Move PartialPackage/Resource to @dpkit/core? - result.resources = await Promise.all( - dataPackage.resources.map(async resource => { - return await inferResource(resource) - }), - ) +interface PartialPackage extends Omit { + resources: Partial[] +} + +export async function inferPackage(dataPackage: PartialPackage) { + const result = { + ...dataPackage, + resources: await Promise.all( + dataPackage.resources.map(async resource => { + return await inferResource(resource) + }), + ), + } return result } diff --git a/file/file/infer.ts b/file/file/infer.ts index a69d08f9..6aaaf087 100644 --- a/file/file/infer.ts +++ b/file/file/infer.ts @@ -37,7 +37,6 @@ export async function inferFileEncoding( if (!isBinary) { const matches = chardet.analyse(buffer) - console.log(matches) for (const match of matches) { if (match.confidence >= confidencePercent) { return match.name.toLowerCase() From 22ec12428bf2067ef0c3618b51d951e4dad9407d Mon Sep 17 00:00:00 2001 From: roll Date: Fri, 15 Aug 2025 16:05:52 +0100 Subject: [PATCH 39/61] Boostrapped validateResource --- dpkit/resource/validate.ts | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 dpkit/resource/validate.ts diff --git a/dpkit/resource/validate.ts b/dpkit/resource/validate.ts new file mode 100644 index 00000000..23accb94 --- /dev/null +++ b/dpkit/resource/validate.ts @@ -0,0 +1,5 @@ +import type { Resource } from "@dpkit/core" + +export async function validateResource(resource: Partial) { + return resource +} From 6a5a9da4e03743e619f98ab8c18a61e98e5f12ad Mon Sep 17 00:00:00 2001 From: roll Date: Fri, 15 Aug 2025 16:31:46 +0100 Subject: [PATCH 40/61] Moved metadata erorrs to its own folder --- core/dialect/assert.spec.ts | 2 +- core/dialect/assert.ts | 3 +- core/{general/Error.ts => error/Assertion.ts} | 9 +----- core/error/Metadata.ts | 8 +++++ core/error/index.ts | 2 ++ core/general/descriptor/validate.ts | 2 +- core/general/index.ts | 1 - core/index.ts | 1 + core/package/assert.spec.ts | 2 +- core/package/assert.ts | 3 +- core/resource/assert.spec.ts | 2 +- core/resource/assert.ts | 3 +- core/resource/validate.ts | 2 +- core/schema/assert.spec.ts | 2 +- core/schema/assert.ts | 3 +- dpkit/resource/validate.ts | 29 +++++++++++++++++-- 16 files changed, 52 insertions(+), 22 deletions(-) rename core/{general/Error.ts => error/Assertion.ts} (62%) create mode 100644 core/error/Metadata.ts create mode 100644 core/error/index.ts diff --git a/core/dialect/assert.spec.ts b/core/dialect/assert.spec.ts index f0fc9eb5..86b46829 100644 --- a/core/dialect/assert.spec.ts +++ b/core/dialect/assert.spec.ts @@ -1,5 +1,5 @@ import { describe, expect, expectTypeOf, it } from "vitest" -import { AssertionError } from "../general/index.ts" +import { AssertionError } from "../error/index.ts" import type { Dialect } from "./Dialect.ts" import { assertDialect } from "./assert.ts" diff --git a/core/dialect/assert.ts b/core/dialect/assert.ts index b87140eb..b7446758 100644 --- a/core/dialect/assert.ts +++ b/core/dialect/assert.ts @@ -1,4 +1,5 @@ -import { AssertionError, type Descriptor } from "../general/index.ts" +import { AssertionError } from "../error/index.ts" +import type { Descriptor } from "../general/index.ts" import type { Dialect } from "./Dialect.ts" import { validateDialect } from "./validate.ts" diff --git a/core/general/Error.ts b/core/error/Assertion.ts similarity index 62% rename from core/general/Error.ts rename to core/error/Assertion.ts index 542186bc..4287e235 100644 --- a/core/general/Error.ts +++ b/core/error/Assertion.ts @@ -1,11 +1,4 @@ -import type { ErrorObject } from "ajv" - -/** - * A descriptor error - */ -export interface MetadataError extends ErrorObject { - type: "metadata" -} +import type { MetadataError } from "./Metadata.ts" /** * Thrown when a descriptor assertion fails diff --git a/core/error/Metadata.ts b/core/error/Metadata.ts new file mode 100644 index 00000000..2be7983f --- /dev/null +++ b/core/error/Metadata.ts @@ -0,0 +1,8 @@ +import type { ErrorObject } from "ajv" + +/** + * A descriptor error + */ +export interface MetadataError extends ErrorObject { + type: "metadata" +} diff --git a/core/error/index.ts b/core/error/index.ts new file mode 100644 index 00000000..790584e4 --- /dev/null +++ b/core/error/index.ts @@ -0,0 +1,2 @@ +export { AssertionError } from "./Assertion.ts" +export type { MetadataError } from "./Metadata.ts" diff --git a/core/general/descriptor/validate.ts b/core/general/descriptor/validate.ts index b046c164..92c575c4 100644 --- a/core/general/descriptor/validate.ts +++ b/core/general/descriptor/validate.ts @@ -1,4 +1,4 @@ -import type { MetadataError } from "../Error.ts" +import type { MetadataError } from "../../error/index.ts" import { ajv } from "../profile/ajv.ts" import type { Descriptor } from "./Descriptor.ts" diff --git a/core/general/index.ts b/core/general/index.ts index 57c1b3e9..20eee670 100644 --- a/core/general/index.ts +++ b/core/general/index.ts @@ -8,7 +8,6 @@ export type { Descriptor } from "./descriptor/Descriptor.ts" export type { Metadata } from "./metadata/Metadata.ts" export { isDescriptor } from "./descriptor/Descriptor.ts" export { loadProfile } from "./profile/load.ts" -export { AssertionError, type MetadataError } from "./Error.ts" export { getName, getFormat, diff --git a/core/index.ts b/core/index.ts index 735a4049..807f6b4d 100644 --- a/core/index.ts +++ b/core/index.ts @@ -1,4 +1,5 @@ export * from "./general/index.ts" +export * from "./error/index.ts" export * from "./dialect/index.ts" export * from "./field/index.ts" export * from "./package/index.ts" diff --git a/core/package/assert.spec.ts b/core/package/assert.spec.ts index c167e8c6..e5253b77 100644 --- a/core/package/assert.spec.ts +++ b/core/package/assert.spec.ts @@ -1,5 +1,5 @@ import { describe, expect, expectTypeOf, it } from "vitest" -import { AssertionError } from "../general/index.ts" +import { AssertionError } from "../error/index.ts" import type { Package } from "./Package.ts" import { assertPackage } from "./assert.ts" diff --git a/core/package/assert.ts b/core/package/assert.ts index 92570015..1392feee 100644 --- a/core/package/assert.ts +++ b/core/package/assert.ts @@ -1,4 +1,5 @@ -import { AssertionError, type Descriptor } from "../general/index.ts" +import { AssertionError } from "../error/index.ts" +import type { Descriptor } from "../general/index.ts" import type { Package } from "./Package.ts" import { validatePackageDescriptor } from "./validate.ts" diff --git a/core/resource/assert.spec.ts b/core/resource/assert.spec.ts index 0299c8ad..d3a58358 100644 --- a/core/resource/assert.spec.ts +++ b/core/resource/assert.spec.ts @@ -1,5 +1,5 @@ import { describe, expect, expectTypeOf, it } from "vitest" -import { AssertionError } from "../general/index.ts" +import { AssertionError } from "../error/index.ts" import type { Resource } from "./Resource.ts" import { assertResource } from "./assert.ts" diff --git a/core/resource/assert.ts b/core/resource/assert.ts index 68375be2..dddd2954 100644 --- a/core/resource/assert.ts +++ b/core/resource/assert.ts @@ -1,4 +1,5 @@ -import { AssertionError, type Descriptor } from "../general/index.ts" +import { AssertionError } from "../error/index.ts" +import type { Descriptor } from "../general/index.ts" import type { Resource } from "./Resource.ts" import { validateResourceDescriptor } from "./validate.ts" diff --git a/core/resource/validate.ts b/core/resource/validate.ts index f5a80da5..06abe1db 100644 --- a/core/resource/validate.ts +++ b/core/resource/validate.ts @@ -1,6 +1,6 @@ import { loadDialect } from "../dialect/index.ts" +import { AssertionError } from "../error/index.ts" import { type Descriptor, validateDescriptor } from "../general/index.ts" -import { AssertionError } from "../general/index.ts" import { loadProfile } from "../general/index.ts" import { loadSchema } from "../schema/index.ts" import type { Resource } from "./Resource.ts" diff --git a/core/schema/assert.spec.ts b/core/schema/assert.spec.ts index db0a8cb0..3ba795f2 100644 --- a/core/schema/assert.spec.ts +++ b/core/schema/assert.spec.ts @@ -1,5 +1,5 @@ import { describe, expect, expectTypeOf, it } from "vitest" -import { AssertionError } from "../general/index.ts" +import { AssertionError } from "../error/index.ts" import type { Schema } from "./Schema.ts" import { assertSchema } from "./assert.ts" diff --git a/core/schema/assert.ts b/core/schema/assert.ts index 0a078986..230ac22b 100644 --- a/core/schema/assert.ts +++ b/core/schema/assert.ts @@ -1,4 +1,5 @@ -import { AssertionError, type Descriptor } from "../general/index.ts" +import { AssertionError } from "../error/index.ts" +import type { Descriptor } from "../general/index.ts" import type { Schema } from "./Schema.ts" import { validateSchema } from "./validate.ts" diff --git a/dpkit/resource/validate.ts b/dpkit/resource/validate.ts index 23accb94..e26d830e 100644 --- a/dpkit/resource/validate.ts +++ b/dpkit/resource/validate.ts @@ -1,5 +1,28 @@ -import type { Resource } from "@dpkit/core" +import type { Descriptor, Resource } from "@dpkit/core" +import { validateResourceDescriptor } from "@dpkit/core" +import { validateTable } from "../table/index.ts" -export async function validateResource(resource: Partial) { - return resource +export async function validateResource( + descriptorOrResource: Descriptor | Partial, + options?: { + basepath?: string + }, +) { + const { valid, errors, resource } = await validateResourceDescriptor( + descriptorOrResource, + { basepath: options?.basepath }, + ) + + if (!resource) { + return { valid, errors } + } + + if (resource.bytes && resource.hash) { + } + + try { + return await validateTable(resource) + } catch { + return { valid: true, errors: [] } + } } From 2b477932121000cdfb7b8558c315517b66f2309b Mon Sep 17 00:00:00 2001 From: roll Date: Fri, 15 Aug 2025 16:35:20 +0100 Subject: [PATCH 41/61] Added file errors --- file/error/Base.ts | 3 +++ file/error/Bytes.ts | 7 +++++++ file/error/File.ts | 4 ++++ file/error/Hash.ts | 7 +++++++ file/error/index.ts | 1 + file/index.ts | 1 + 6 files changed, 23 insertions(+) create mode 100644 file/error/Base.ts create mode 100644 file/error/Bytes.ts create mode 100644 file/error/File.ts create mode 100644 file/error/Hash.ts create mode 100644 file/error/index.ts diff --git a/file/error/Base.ts b/file/error/Base.ts new file mode 100644 index 00000000..89943662 --- /dev/null +++ b/file/error/Base.ts @@ -0,0 +1,3 @@ +export interface BaseFileError { + type: string +} diff --git a/file/error/Bytes.ts b/file/error/Bytes.ts new file mode 100644 index 00000000..fbf5cce0 --- /dev/null +++ b/file/error/Bytes.ts @@ -0,0 +1,7 @@ +import type { BaseFileError } from "./Base.ts" + +export interface BytesError extends BaseFileError { + type: "file/bytes" + bytes: number + actualBytes: number +} diff --git a/file/error/File.ts b/file/error/File.ts new file mode 100644 index 00000000..997380d7 --- /dev/null +++ b/file/error/File.ts @@ -0,0 +1,4 @@ +import type { BytesError } from "./Bytes.ts" +import type { HashError } from "./Hash.ts" + +export type FileError = BytesError | HashError diff --git a/file/error/Hash.ts b/file/error/Hash.ts new file mode 100644 index 00000000..44a1ab04 --- /dev/null +++ b/file/error/Hash.ts @@ -0,0 +1,7 @@ +import type { BaseFileError } from "./Base.ts" + +export interface HashError extends BaseFileError { + type: "file/hash" + hash: string + actualHahs: string +} diff --git a/file/error/index.ts b/file/error/index.ts new file mode 100644 index 00000000..822f4c00 --- /dev/null +++ b/file/error/index.ts @@ -0,0 +1 @@ +export type { FileError } from "./File.ts" diff --git a/file/index.ts b/file/index.ts index d7427de1..00ec5705 100644 --- a/file/index.ts +++ b/file/index.ts @@ -1,3 +1,4 @@ +export * from "./error/index.ts" export * from "./file/index.ts" export * from "./package/index.ts" export * from "./resource/index.ts" From 2b197fb6748a66c2f062391add391a78dc202b9e Mon Sep 17 00:00:00 2001 From: roll Date: Fri, 15 Aug 2025 16:43:48 +0100 Subject: [PATCH 42/61] Implemente validateFile --- file/file/index.ts | 1 + file/file/validate.ts | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 file/file/validate.ts diff --git a/file/file/index.ts b/file/file/index.ts index 8721b19d..ed15776b 100644 --- a/file/file/index.ts +++ b/file/file/index.ts @@ -6,3 +6,4 @@ export { assertLocalPathVacant, isLocalPathExist } from "./path.ts" export { prefetchFile, prefetchFiles } from "./fetch.ts" export { inferFileEncoding, inferFileBytes, inferFileHash } from "./infer.ts" export { describeFile } from "./describe.ts" +export { validateFile } from "./validate.ts" diff --git a/file/file/validate.ts b/file/file/validate.ts new file mode 100644 index 00000000..6f770b40 --- /dev/null +++ b/file/file/validate.ts @@ -0,0 +1,39 @@ +import type { FileError } from "../error/index.ts" +import { prefetchFile } from "./fetch.ts" +import { inferFileBytes, inferFileHash } from "./infer.ts" + +export async function validateFile( + path: string, + options?: { bytes?: number; hash?: string }, +) { + const errors: FileError[] = [] + const localPath = await prefetchFile(path) + + if (options?.bytes) { + const bytes = await inferFileBytes(localPath) + if (bytes !== options.bytes) { + errors.push({ + type: "file/bytes", + bytes: options.bytes, + actualBytes: bytes, + }) + } + } + + if (options?.hash) { + const [hashValue, hashType = "md5"] = options.hash.split(":").toReversed() + // TODO: figure out how we should handle this + // @ts-ignore + const hash = await inferFileHash(localPath, { hashType }) + if (hash !== hashValue) { + errors.push({ + type: "file/hash", + hash: options.hash, + actualHahs: hash, + }) + } + } + + const valid = errors.length === 0 + return { valid, errors } +} From 4f5cda27abf7cb0d7dc55089d90a034d9c2f46f5 Mon Sep 17 00:00:00 2001 From: roll Date: Fri, 15 Aug 2025 16:46:47 +0100 Subject: [PATCH 43/61] Implemented validateResource --- dpkit/resource/index.ts | 1 + dpkit/resource/validate.ts | 19 +++++++++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/dpkit/resource/index.ts b/dpkit/resource/index.ts index 329b8170..ea494c50 100644 --- a/dpkit/resource/index.ts +++ b/dpkit/resource/index.ts @@ -1 +1,2 @@ export { inferResource } from "./infer.ts" +export { validateResource } from "./validate.ts" diff --git a/dpkit/resource/validate.ts b/dpkit/resource/validate.ts index e26d830e..33d781bd 100644 --- a/dpkit/resource/validate.ts +++ b/dpkit/resource/validate.ts @@ -1,7 +1,10 @@ import type { Descriptor, Resource } from "@dpkit/core" import { validateResourceDescriptor } from "@dpkit/core" +import { validateFile } from "@dpkit/file" import { validateTable } from "../table/index.ts" +// TODO: Support multipart resources? (clarify on the specs level) + export async function validateResource( descriptorOrResource: Descriptor | Partial, options?: { @@ -17,12 +20,20 @@ export async function validateResource( return { valid, errors } } - if (resource.bytes && resource.hash) { + if (resource.bytes || resource.hash) { + if (typeof resource.path === "string") { + return await validateFile(resource.path, { + bytes: resource.bytes, + hash: resource.hash, + }) + } } try { + // TODO: rebase on not-rasing? + // It will raise if the resource is not a table return await validateTable(resource) - } catch { - return { valid: true, errors: [] } - } + } catch {} + + return { valid: true, errors: [] } } From 777016c6f214f61c89683c08360cd8b116b9507e Mon Sep 17 00:00:00 2001 From: roll Date: Fri, 15 Aug 2025 17:13:20 +0100 Subject: [PATCH 44/61] Implemented resource validate command --- cli/commands/resource/errors.tsx | 22 ++++------------------ cli/commands/resource/validate.tsx | 25 +++++-------------------- dpkit/resource/validate.ts | 21 ++++++++++++++------- 3 files changed, 23 insertions(+), 45 deletions(-) diff --git a/cli/commands/resource/errors.tsx b/cli/commands/resource/errors.tsx index 48e70e64..23d1750a 100644 --- a/cli/commands/resource/errors.tsx +++ b/cli/commands/resource/errors.tsx @@ -1,6 +1,5 @@ import { Command } from "commander" -import { loadDescriptor, validateResourceDescriptor } from "dpkit" -import type { Descriptor } from "dpkit" +import { validateResource } from "dpkit" import React from "react" import { ErrorGrid } from "../../components/ErrorGrid.jsx" import { helpConfiguration } from "../../helpers/help.ts" @@ -23,24 +22,11 @@ export const errorsResourceCommand = new Command("errors") json: options.json, }) - let descriptor: Descriptor | undefined - - if (!path) { - const resource = await selectResource(session, options) - descriptor = resource as unknown as Descriptor - } else { - const result = await session.task( - "Loading descriptor", - // @ts-ignore - loadDescriptor(path), - ) - - descriptor = result.descriptor - } + const descriptor = path ? path : await selectResource(session, options) const { errors } = await session.task( - "Validating descriptor", - validateResourceDescriptor(descriptor), + "Validating resource", + validateResource(descriptor), ) session.render(errors, ) diff --git a/cli/commands/resource/validate.tsx b/cli/commands/resource/validate.tsx index 43d28631..413efc04 100644 --- a/cli/commands/resource/validate.tsx +++ b/cli/commands/resource/validate.tsx @@ -1,6 +1,5 @@ import { Command } from "commander" -import { loadDescriptor, validateResourceDescriptor } from "dpkit" -import type { Descriptor } from "dpkit" +import { validateResource } from "dpkit" import { helpConfiguration } from "../../helpers/help.ts" import { selectResource } from "../../helpers/resource.ts" import { Session } from "../../helpers/session.ts" @@ -10,7 +9,7 @@ export const validateResourceCommand = new Command("validate") .configureHelp(helpConfiguration) .description("Validate a data resource from a local or remote path") - .addArgument(params.positionalTablePath) + .addArgument(params.positionalDescriptorPath) .addOption(params.fromPackage) .addOption(params.fromResource) .addOption(params.json) @@ -21,25 +20,11 @@ export const validateResourceCommand = new Command("validate") json: options.json, }) - let descriptor: Descriptor | undefined - - if (!path) { - const resource = await selectResource(session, options) - descriptor = resource as unknown as Descriptor - } else { - const result = await session.task( - "Loading descriptor", - // @ts-ignore - loadDescriptor(path), - ) - - descriptor = result.descriptor - } + const descriptor = path ? path : await selectResource(session, options) const { valid } = await session.task( - "Validating descriptor", - // @ts-ignore - validateResourceDescriptor(descriptor), + "Validating resource", + validateResource(descriptor), ) valid diff --git a/dpkit/resource/validate.ts b/dpkit/resource/validate.ts index 33d781bd..cffe661e 100644 --- a/dpkit/resource/validate.ts +++ b/dpkit/resource/validate.ts @@ -1,19 +1,26 @@ import type { Descriptor, Resource } from "@dpkit/core" -import { validateResourceDescriptor } from "@dpkit/core" +import { loadDescriptor, validateResourceDescriptor } from "@dpkit/core" import { validateFile } from "@dpkit/file" import { validateTable } from "../table/index.ts" // TODO: Support multipart resources? (clarify on the specs level) export async function validateResource( - descriptorOrResource: Descriptor | Partial, - options?: { - basepath?: string - }, + pathOrDescriptorOrResource: string | Descriptor | Partial, + options?: { basepath?: string }, ) { + let descriptor = pathOrDescriptorOrResource + let basepath = options?.basepath + + if (typeof descriptor === "string") { + const result = await loadDescriptor(descriptor) + descriptor = result.descriptor + basepath = result.basepath + } + const { valid, errors, resource } = await validateResourceDescriptor( - descriptorOrResource, - { basepath: options?.basepath }, + descriptor, + { basepath }, ) if (!resource) { From e98160360072e95a2d68e7c655c833adb07c0d68 Mon Sep 17 00:00:00 2001 From: roll Date: Fri, 15 Aug 2025 17:23:28 +0100 Subject: [PATCH 45/61] Fixed validateFile --- cli/commands/resource/validate.tsx | 2 +- cli/params/path.ts | 5 +++++ file/error/Hash.ts | 2 +- file/file/validate.ts | 8 ++++---- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/cli/commands/resource/validate.tsx b/cli/commands/resource/validate.tsx index 413efc04..303c0ce1 100644 --- a/cli/commands/resource/validate.tsx +++ b/cli/commands/resource/validate.tsx @@ -9,7 +9,7 @@ export const validateResourceCommand = new Command("validate") .configureHelp(helpConfiguration) .description("Validate a data resource from a local or remote path") - .addArgument(params.positionalDescriptorPath) + .addArgument(params.optionalPositionalDescriptorPath) .addOption(params.fromPackage) .addOption(params.fromResource) .addOption(params.json) diff --git a/cli/params/path.ts b/cli/params/path.ts index cf591669..f9f7b9f8 100644 --- a/cli/params/path.ts +++ b/cli/params/path.ts @@ -1,5 +1,10 @@ import { Argument, Option } from "commander" +export const optionalPositionalDescriptorPath = new Argument( + "[path]", + "local or remote path to the descriptor", +) + export const positionalDescriptorPath = new Argument( "", "local or remote path to the descriptor", diff --git a/file/error/Hash.ts b/file/error/Hash.ts index 44a1ab04..ed8bc4ab 100644 --- a/file/error/Hash.ts +++ b/file/error/Hash.ts @@ -3,5 +3,5 @@ import type { BaseFileError } from "./Base.ts" export interface HashError extends BaseFileError { type: "file/hash" hash: string - actualHahs: string + actualHash: string } diff --git a/file/file/validate.ts b/file/file/validate.ts index 6f770b40..fc892dc6 100644 --- a/file/file/validate.ts +++ b/file/file/validate.ts @@ -21,15 +21,15 @@ export async function validateFile( } if (options?.hash) { - const [hashValue, hashType = "md5"] = options.hash.split(":").toReversed() - // TODO: figure out how we should handle this + const [_hashValue, hashType = "md5"] = options.hash.split(":").toReversed() + // TODO: figure out how we should handle other hash types // @ts-ignore const hash = await inferFileHash(localPath, { hashType }) - if (hash !== hashValue) { + if (hash !== options.hash) { errors.push({ type: "file/hash", hash: options.hash, - actualHahs: hash, + actualHash: hash, }) } } From 6617b25d0cb02fe88d4d3e5e82fb15a2a814d8ab Mon Sep 17 00:00:00 2001 From: roll Date: Fri, 15 Aug 2025 17:29:23 +0100 Subject: [PATCH 46/61] Implemented validatePackage --- dpkit/package/index.ts | 1 + dpkit/package/validate.ts | 41 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 dpkit/package/validate.ts diff --git a/dpkit/package/index.ts b/dpkit/package/index.ts index c8b5c385..39f8f5db 100644 --- a/dpkit/package/index.ts +++ b/dpkit/package/index.ts @@ -1,3 +1,4 @@ export { loadPackage } from "./load.ts" export { savePackage } from "./save.ts" export { inferPackage } from "./infer.ts" +export { validatePackage } from "./validate.ts" diff --git a/dpkit/package/validate.ts b/dpkit/package/validate.ts new file mode 100644 index 00000000..6cd0a178 --- /dev/null +++ b/dpkit/package/validate.ts @@ -0,0 +1,41 @@ +import type { Descriptor, Package } from "@dpkit/core" +import { loadDescriptor, validatePackageDescriptor } from "@dpkit/core" +import { validateResource } from "../resource/index.ts" + +// TODO: Handle error addressing (to which resource an error belongs) +// TODO: Support multipart resources? (clarify on the specs level) + +export async function validatePackage( + pathOrDescriptorOrPackage: string | Descriptor | Partial, + options?: { basepath?: string }, +) { + let descriptor = pathOrDescriptorOrPackage + let basepath = options?.basepath + + if (typeof descriptor === "string") { + const result = await loadDescriptor(descriptor) + descriptor = result.descriptor + basepath = result.basepath + } + + const { valid, errors, dataPackage } = await validatePackageDescriptor( + descriptor, + { basepath }, + ) + + if (!dataPackage) { + return { valid, errors } + } + + const results = await Promise.all( + dataPackage.resources.map(async resource => { + return await validateResource(resource) + }), + ) + + // @ts-ignore + const allErrors = results.flatMap(result => result.errors) + const allValid = allErrors.length === 0 + + return { valid: allValid, errors: allErrors } +} From d555fe9857867cdd71612f5360458685750fa829 Mon Sep 17 00:00:00 2001 From: roll Date: Fri, 15 Aug 2025 17:33:59 +0100 Subject: [PATCH 47/61] Implemented package validate command --- cli/commands/package/errors.tsx | 28 +++++----------------------- cli/commands/package/index.ts | 2 ++ cli/commands/package/validate.ts | 29 ++++++++++++++++++++++++++++- cli/commands/resource/errors.tsx | 2 +- 4 files changed, 36 insertions(+), 25 deletions(-) diff --git a/cli/commands/package/errors.tsx b/cli/commands/package/errors.tsx index e1afb12a..dfcc4ee3 100644 --- a/cli/commands/package/errors.tsx +++ b/cli/commands/package/errors.tsx @@ -1,10 +1,8 @@ import { Command } from "commander" -import { loadDescriptor, validatePackageDescriptor } from "dpkit" -import type { Descriptor } from "dpkit" +import { validatePackage } from "dpkit" import React from "react" import { ErrorGrid } from "../../components/ErrorGrid.jsx" import { helpConfiguration } from "../../helpers/help.ts" -import { selectResource } from "../../helpers/resource.ts" import { Session } from "../../helpers/session.ts" import * as params from "../../params/index.ts" @@ -12,9 +10,7 @@ export const errorsPackageCommand = new Command("errors") .configureHelp(helpConfiguration) .description("Show errors for a data package from a local or remote path") - .addArgument(params.positionalTablePath) - .addOption(params.fromPackage) - .addOption(params.fromResource) + .addArgument(params.positionalDescriptorPath) .addOption(params.json) .action(async (path, options) => { @@ -23,25 +19,11 @@ export const errorsPackageCommand = new Command("errors") json: options.json, }) - let descriptor: Descriptor | undefined - - if (!path) { - const resource = await selectResource(session, options) - descriptor = resource as unknown as Descriptor - } else { - const result = await session.task( - "Loading descriptor", - // @ts-ignore - loadDescriptor(path), - ) - - descriptor = result.descriptor - } - const { errors } = await session.task( - "Validating descriptor", - validatePackageDescriptor(descriptor), + "Validating package", + validatePackage(path), ) + // @ts-ignore session.render(errors, ) }) diff --git a/cli/commands/package/index.ts b/cli/commands/package/index.ts index a97c7e6c..d740deab 100644 --- a/cli/commands/package/index.ts +++ b/cli/commands/package/index.ts @@ -5,6 +5,7 @@ import { copyPackageCommand } from "./copy.ts" import { errorsPackageCommand } from "./errors.tsx" import { inferPackageCommand } from "./infer.tsx" import { showPackageCommand } from "./show.tsx" +import { validatePackageCommand } from "./validate.ts" export const packageCommand = new Command("package") .configureHelp(helpConfiguration) @@ -15,3 +16,4 @@ export const packageCommand = new Command("package") .addCommand(errorsPackageCommand) .addCommand(inferPackageCommand) .addCommand(showPackageCommand) + .addCommand(validatePackageCommand) diff --git a/cli/commands/package/validate.ts b/cli/commands/package/validate.ts index 1039d4c5..61a59240 100644 --- a/cli/commands/package/validate.ts +++ b/cli/commands/package/validate.ts @@ -1 +1,28 @@ -// TODO: implement +import { Command } from "commander" +import { validatePackage } from "dpkit" +import { helpConfiguration } from "../../helpers/help.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const validatePackageCommand = new Command("validate") + .configureHelp(helpConfiguration) + .description("Validate a data package from a local or remote path") + + .addArgument(params.positionalDescriptorPath) + .addOption(params.json) + + .action(async (path, options) => { + const session = Session.create({ + title: "Validate package", + json: options.json, + }) + + const { valid } = await session.task( + "Validating package", + validatePackage(path), + ) + + valid + ? session.success("Package is valid") + : session.error("Package is not valid") + }) diff --git a/cli/commands/resource/errors.tsx b/cli/commands/resource/errors.tsx index 23d1750a..260646d7 100644 --- a/cli/commands/resource/errors.tsx +++ b/cli/commands/resource/errors.tsx @@ -11,7 +11,7 @@ export const errorsResourceCommand = new Command("errors") .configureHelp(helpConfiguration) .description("Show errors for a data resource from a local or remote path") - .addArgument(params.positionalTablePath) + .addArgument(params.optionalPositionalDescriptorPath) .addOption(params.fromPackage) .addOption(params.fromResource) .addOption(params.json) From ebcc97053e234aa9a0ca23bb977789b3948b9b16 Mon Sep 17 00:00:00 2001 From: roll Date: Mon, 18 Aug 2025 15:36:01 +0100 Subject: [PATCH 48/61] Fixed spinner cancel behavior --- cli/helpers/session.ts | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/cli/helpers/session.ts b/cli/helpers/session.ts index 05450ce8..b8e6496f 100644 --- a/cli/helpers/session.ts +++ b/cli/helpers/session.ts @@ -29,11 +29,7 @@ export class Session { } start() { intro(pc.bold(this.title)) - exitHook(() => { - outro( - `Problems? ${pc.underline(pc.cyan("https://github.com/datisthq/dpkit/issues"))}`, - ) - }) + this.#enableExitHook() } success(message: string) { @@ -49,11 +45,17 @@ export class Session { } async task(message: string, promise: Promise) { + // TODO: Consider spinner's onCancel or other solution when @clack/prompts@1.0 is released + // We disable/enable the exit hook to friend it with spinner's "Cancel" event const loader = spinner() + this.#disableExitHook?.() loader.start(message) + const result = await promise + loader.stop(message) + this.#enableExitHook() return result } @@ -66,6 +68,17 @@ export class Session { const app = render(node) await app.waitUntilExit() } + + #disableExitHook?: ReturnType + #enableExitHook() { + this.#disableExitHook = exitHook(() => this.#handleExit()) + } + + #handleExit() { + outro( + `Problems? ${pc.underline(pc.cyan("https://github.com/datisthq/dpkit/issues"))}`, + ) + } } export class JsonSession extends Session { From 192fe8848bc31e654c916b7a9829c3aaef6e4151 Mon Sep 17 00:00:00 2001 From: roll Date: Mon, 18 Aug 2025 15:50:18 +0100 Subject: [PATCH 49/61] Added completion todo --- cli/main.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cli/main.ts b/cli/main.ts index 0462aa75..2cfaf5eb 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -1,3 +1,5 @@ +// TODO: Support tab completion when @bombsh/tab is released +//import tab from "@bombsh/tab/commander" import { program } from "commander" import { dialectCommand } from "./commands/dialect/index.ts" import { fileCommand } from "./commands/file/index.ts" @@ -8,7 +10,7 @@ import { tableCommand } from "./commands/table/index.ts" import { helpConfiguration } from "./helpers/help.ts" import metadata from "./package.json" with { type: "json" } -program +const main = program .name("dp") .description( "Fast data management CLI built on top of the Data Package standard and Polars DataFrames", @@ -24,4 +26,5 @@ program .addCommand(tableCommand) .addCommand(fileCommand) - .parse() +//tab(main) +main.parse() From af6b78afe6af7d10603c89598a28cfe01b5dff2f Mon Sep 17 00:00:00 2001 From: roll Date: Mon, 18 Aug 2025 16:42:49 +0100 Subject: [PATCH 50/61] Implemented publish command --- cli/commands/package/index.ts | 2 + cli/commands/package/publish/ckan.ts | 59 ++++++++++++++++++++++++++ cli/commands/package/publish/github.ts | 52 +++++++++++++++++++++++ cli/commands/package/publish/index.ts | 13 ++++++ cli/commands/package/publish/zenodo.ts | 45 ++++++++++++++++++++ cli/params/ckan.ts | 18 ++++++++ cli/params/github.ts | 16 +++++++ cli/params/index.ts | 1 + cli/params/zenodo.ts | 11 +++++ 9 files changed, 217 insertions(+) create mode 100644 cli/commands/package/publish/ckan.ts create mode 100644 cli/commands/package/publish/github.ts create mode 100644 cli/commands/package/publish/index.ts create mode 100644 cli/commands/package/publish/zenodo.ts create mode 100644 cli/params/ckan.ts create mode 100644 cli/params/github.ts create mode 100644 cli/params/zenodo.ts diff --git a/cli/commands/package/index.ts b/cli/commands/package/index.ts index d740deab..821dae40 100644 --- a/cli/commands/package/index.ts +++ b/cli/commands/package/index.ts @@ -4,6 +4,7 @@ import { archivePackageCommand } from "./archive.ts" import { copyPackageCommand } from "./copy.ts" import { errorsPackageCommand } from "./errors.tsx" import { inferPackageCommand } from "./infer.tsx" +import { publishPackageCommand } from "./publish/index.ts" import { showPackageCommand } from "./show.tsx" import { validatePackageCommand } from "./validate.ts" @@ -16,4 +17,5 @@ export const packageCommand = new Command("package") .addCommand(errorsPackageCommand) .addCommand(inferPackageCommand) .addCommand(showPackageCommand) + .addCommand(publishPackageCommand) .addCommand(validatePackageCommand) diff --git a/cli/commands/package/publish/ckan.ts b/cli/commands/package/publish/ckan.ts new file mode 100644 index 00000000..19c14ddc --- /dev/null +++ b/cli/commands/package/publish/ckan.ts @@ -0,0 +1,59 @@ +import { Option } from "commander" +import { Command } from "commander" +import { loadPackage, savePackageToCkan } from "dpkit" +import { helpConfiguration } from "../../../helpers/help.ts" +import { Session } from "../../../helpers/session.ts" +import * as params from "../../../params/index.ts" + +export const toCkanApiKey = new Option( + "--to-api-key ", + "API key for CKAN API", +) + +export const toCkanUrl = new Option( + "--to-ckan-url ", + "Base CKAN url to publish to", +) + +export const toCkanOwnerOrg = new Option( + "--to-owner-org ", + "Owner organization for the CKAN dataset", +) + +export const toCkanDatasetName = new Option( + "--to-dataset-name ", + "Name for the CKAN dataset", +) + +export const ckanPublishPackageCommand = new Command("ckan") + .configureHelp(helpConfiguration) + .description("Publish a data package from a local or remote path to CKAN") + + .addArgument(params.positionalDescriptorPath) + .addOption(params.withRemote) + + .optionsGroup("CKAN") + .addOption(toCkanApiKey.makeOptionMandatory()) + .addOption(toCkanUrl.makeOptionMandatory()) + .addOption(toCkanOwnerOrg.makeOptionMandatory()) + .addOption(toCkanDatasetName.makeOptionMandatory()) + + .action(async (path, options) => { + const session = Session.create({ + title: "Publish package", + }) + + const dp = await session.task("Loading package", loadPackage(path)) + + const result = await session.task( + "Publishing package", + savePackageToCkan(dp, { + ckanUrl: options.toCkanUrl, + apiKey: options.toApiKey, + ownerOrg: options.toOwnerOrg, + datasetName: options.toDatasetName, + }), + ) + + session.success(`Package from "${path}" published to "${result.path}"`) + }) diff --git a/cli/commands/package/publish/github.ts b/cli/commands/package/publish/github.ts new file mode 100644 index 00000000..3117109a --- /dev/null +++ b/cli/commands/package/publish/github.ts @@ -0,0 +1,52 @@ +import { Option } from "commander" +import { Command } from "commander" +import { loadPackage, savePackageToGithub } from "dpkit" +import { helpConfiguration } from "../../../helpers/help.ts" +import { Session } from "../../../helpers/session.ts" +import * as params from "../../../params/index.ts" + +export const toGithubApiKey = new Option( + "--to-api-key ", + "API key for GitHub API", +) + +export const toGithubRepo = new Option( + "--to-repo ", + "GitHub repository name", +) + +export const toGithubOrg = new Option( + "--to-org ", + "GitHub organization (optional, defaults to user repositories)", +) + +export const githubPublishPackageCommand = new Command("github") + .configureHelp(helpConfiguration) + .description("Publish a data package from a local or remote path to GitHub") + + .addArgument(params.positionalDescriptorPath) + .addOption(params.withRemote) + + .optionsGroup("GitHub") + .addOption(toGithubApiKey.makeOptionMandatory()) + .addOption(toGithubRepo.makeOptionMandatory()) + .addOption(toGithubOrg) + + .action(async (path, options) => { + const session = Session.create({ + title: "Publish package", + }) + + const dp = await session.task("Loading package", loadPackage(path)) + + const result = await session.task( + "Publishing package", + savePackageToGithub(dp, { + apiKey: options.toApiKey, + repo: options.toRepo, + org: options.toOrg, + }), + ) + + session.success(`Package from "${path}" published to "${result.path}"`) + }) diff --git a/cli/commands/package/publish/index.ts b/cli/commands/package/publish/index.ts new file mode 100644 index 00000000..4a3311e7 --- /dev/null +++ b/cli/commands/package/publish/index.ts @@ -0,0 +1,13 @@ +import { Command } from "commander" +import { helpConfiguration } from "../../../helpers/help.ts" +import { ckanPublishPackageCommand } from "./ckan.ts" +import { githubPublishPackageCommand } from "./github.ts" +import { zenodoPublishPackageCommand } from "./zenodo.ts" + +export const publishPackageCommand = new Command("publish") + .configureHelp(helpConfiguration) + .description("Publish data packages to various platforms") + + .addCommand(ckanPublishPackageCommand) + .addCommand(githubPublishPackageCommand) + .addCommand(zenodoPublishPackageCommand) diff --git a/cli/commands/package/publish/zenodo.ts b/cli/commands/package/publish/zenodo.ts new file mode 100644 index 00000000..674f1a3c --- /dev/null +++ b/cli/commands/package/publish/zenodo.ts @@ -0,0 +1,45 @@ +import { Option } from "commander" +import { Command } from "commander" +import { loadPackage, savePackageToZenodo } from "dpkit" +import { helpConfiguration } from "../../../helpers/help.ts" +import { Session } from "../../../helpers/session.ts" +import * as params from "../../../params/index.ts" + +export const toZenodoApiKey = new Option( + "--to-api-key ", + "API key for Zenodo API", +) + +export const toZenodoSandbox = new Option( + "--to-sandbox", + "Use Zenodo sandbox environment", +).default(false) + +export const zenodoPublishPackageCommand = new Command("zenodo") + .configureHelp(helpConfiguration) + .description("Publish a data package from a local or remote path to Zenodo") + + .addArgument(params.positionalDescriptorPath) + .addOption(params.withRemote) + + .optionsGroup("Zenodo") + .addOption(toZenodoApiKey.makeOptionMandatory()) + .addOption(toZenodoSandbox) + + .action(async (path, options) => { + const session = Session.create({ + title: "Publish package", + }) + + const dp = await session.task("Loading package", loadPackage(path)) + + const result = await session.task( + "Publishing package", + savePackageToZenodo(dp, { + apiKey: options.toApiKey, + sandbox: options.toSandbox, + }), + ) + + session.success(`Package from "${path}" published to "${result.path}"`) + }) diff --git a/cli/params/ckan.ts b/cli/params/ckan.ts new file mode 100644 index 00000000..cb8ff119 --- /dev/null +++ b/cli/params/ckan.ts @@ -0,0 +1,18 @@ +import { Option } from "commander" + +export const toCkanApiKey = new Option( + "--to-ckan-api-key ", + "API key for CKAN API", +) + +export const toCkanUrl = new Option("--to-ckan-url ", "CKAN URL") + +export const toCkanOwnerOrg = new Option( + "--to-ckan-owner-org ", + "Owner organization for the CKAN dataset", +) + +export const toCkanDatasetName = new Option( + "--to-ckan-dataset-name ", + "Name for the CKAN dataset", +) diff --git a/cli/params/github.ts b/cli/params/github.ts new file mode 100644 index 00000000..63a81e3f --- /dev/null +++ b/cli/params/github.ts @@ -0,0 +1,16 @@ +import { Option } from "commander" + +export const toGithubApiKey = new Option( + "--to-github-api-key ", + "API key for GitHub API", +) + +export const toGithubRepo = new Option( + "--to-github-repo ", + "GitHub repository name", +) + +export const toGithubOrg = new Option( + "--to-github-org ", + "GitHub organization (optional, defaults to user repositories)", +) diff --git a/cli/params/index.ts b/cli/params/index.ts index a013ffb7..010f53cb 100644 --- a/cli/params/index.ts +++ b/cli/params/index.ts @@ -1,3 +1,4 @@ +export * from "./ckan.ts" export * from "./dialect.ts" export * from "./file.ts" export * from "./json.ts" diff --git a/cli/params/zenodo.ts b/cli/params/zenodo.ts new file mode 100644 index 00000000..fb2f6a49 --- /dev/null +++ b/cli/params/zenodo.ts @@ -0,0 +1,11 @@ +import { Option } from "commander" + +export const toZenodoApiKey = new Option( + "--to-zenodo-api-key ", + "API key for Zenodo API", +) + +export const toZenodoSandbox = new Option( + "--to-zenodo-sandbox", + "Use Zenodo sandbox environment", +).default(false) From 4112d1937a0aeaf7e3ffeb5647f73a569d281d4c Mon Sep 17 00:00:00 2001 From: roll Date: Tue, 19 Aug 2025 10:36:54 +0100 Subject: [PATCH 51/61] Show errors by type in validate --- cli/commands/dialect/errors.tsx | 6 +++--- cli/commands/dialect/validate.tsx | 17 ++++++++++------ cli/commands/file/errors.tsx | 0 cli/commands/file/validate.tsx | 0 cli/commands/package/errors.tsx | 4 ++-- cli/commands/package/index.ts | 2 +- .../package/{validate.ts => validate.tsx} | 14 +++++++++---- cli/commands/resource/errors.tsx | 4 ++-- cli/commands/resource/validate.tsx | 13 ++++++++---- cli/commands/schema/errors.tsx | 4 ++-- cli/commands/schema/validate.tsx | 13 ++++++++---- cli/commands/table/errors.tsx | 2 +- cli/commands/table/validate.tsx | 13 +++++++----- cli/components/DataGrid.tsx | 5 +++-- cli/components/ErrorGrid.tsx | 20 ++++++++++++++++--- cli/components/TableGrid.tsx | 16 ++++++++++++--- cli/package.json | 1 + pnpm-lock.yaml | 11 ++++++---- 18 files changed, 99 insertions(+), 46 deletions(-) create mode 100644 cli/commands/file/errors.tsx create mode 100644 cli/commands/file/validate.tsx rename cli/commands/package/{validate.ts => validate.tsx} (69%) diff --git a/cli/commands/dialect/errors.tsx b/cli/commands/dialect/errors.tsx index 762dc6a9..9d415a2a 100644 --- a/cli/commands/dialect/errors.tsx +++ b/cli/commands/dialect/errors.tsx @@ -19,7 +19,7 @@ export const errorsDialectCommand = new Command("errors") .action(async (path, options) => { const session = Session.create({ - title: "Validate dialect", + title: "Dialect errors", json: options.json, }) @@ -41,7 +41,7 @@ export const errorsDialectCommand = new Command("errors") if (!descriptor) { const result = await session.task( - "Loading descriptor", + "Loading dialect", // @ts-ignore loadDescriptor(path), ) @@ -50,7 +50,7 @@ export const errorsDialectCommand = new Command("errors") } const { errors } = await session.task( - "Validating descriptor", + "Finding errors", // @ts-ignore validateDialect(descriptor), ) diff --git a/cli/commands/dialect/validate.tsx b/cli/commands/dialect/validate.tsx index d3e570c9..670875a4 100644 --- a/cli/commands/dialect/validate.tsx +++ b/cli/commands/dialect/validate.tsx @@ -1,6 +1,8 @@ import { Command } from "commander" import { loadDescriptor, validateDialect } from "dpkit" import type { Descriptor } from "dpkit" +import React from "react" +import { ErrorGrid } from "../../components/ErrorGrid.tsx" import { helpConfiguration } from "../../helpers/help.ts" import { selectResource } from "../../helpers/resource.ts" import { Session } from "../../helpers/session.ts" @@ -39,7 +41,7 @@ export const validateDialectCommand = new Command("validate") if (!descriptor) { const result = await session.task( - "Loading descriptor", + "Loading dialect", // @ts-ignore loadDescriptor(path), ) @@ -47,13 +49,16 @@ export const validateDialectCommand = new Command("validate") descriptor = result.descriptor } - const { valid } = await session.task( - "Validating descriptor", + const { valid, errors } = await session.task( + "Validating dialect", // @ts-ignore validateDialect(descriptor), ) - valid - ? session.success("Dialect is valid") - : session.error("Dialect is not valid") + if (valid) { + session.success("Dialect is valid") + return + } + + session.render(errors, ) }) diff --git a/cli/commands/file/errors.tsx b/cli/commands/file/errors.tsx new file mode 100644 index 00000000..e69de29b diff --git a/cli/commands/file/validate.tsx b/cli/commands/file/validate.tsx new file mode 100644 index 00000000..e69de29b diff --git a/cli/commands/package/errors.tsx b/cli/commands/package/errors.tsx index dfcc4ee3..198bcb03 100644 --- a/cli/commands/package/errors.tsx +++ b/cli/commands/package/errors.tsx @@ -15,12 +15,12 @@ export const errorsPackageCommand = new Command("errors") .action(async (path, options) => { const session = Session.create({ - title: "Validate package", + title: "Package errors", json: options.json, }) const { errors } = await session.task( - "Validating package", + "Finding errors", validatePackage(path), ) diff --git a/cli/commands/package/index.ts b/cli/commands/package/index.ts index 821dae40..ec040217 100644 --- a/cli/commands/package/index.ts +++ b/cli/commands/package/index.ts @@ -6,7 +6,7 @@ import { errorsPackageCommand } from "./errors.tsx" import { inferPackageCommand } from "./infer.tsx" import { publishPackageCommand } from "./publish/index.ts" import { showPackageCommand } from "./show.tsx" -import { validatePackageCommand } from "./validate.ts" +import { validatePackageCommand } from "./validate.tsx" export const packageCommand = new Command("package") .configureHelp(helpConfiguration) diff --git a/cli/commands/package/validate.ts b/cli/commands/package/validate.tsx similarity index 69% rename from cli/commands/package/validate.ts rename to cli/commands/package/validate.tsx index 61a59240..c5ff0d73 100644 --- a/cli/commands/package/validate.ts +++ b/cli/commands/package/validate.tsx @@ -1,5 +1,7 @@ import { Command } from "commander" import { validatePackage } from "dpkit" +import React from "react" +import { ErrorGrid } from "../../components/ErrorGrid.tsx" import { helpConfiguration } from "../../helpers/help.ts" import { Session } from "../../helpers/session.ts" import * as params from "../../params/index.ts" @@ -17,12 +19,16 @@ export const validatePackageCommand = new Command("validate") json: options.json, }) - const { valid } = await session.task( + const { valid, errors } = await session.task( "Validating package", validatePackage(path), ) - valid - ? session.success("Package is valid") - : session.error("Package is not valid") + if (valid) { + session.success("Package is valid") + return + } + + // @ts-ignore + session.render(errors, ) }) diff --git a/cli/commands/resource/errors.tsx b/cli/commands/resource/errors.tsx index 260646d7..28ef73eb 100644 --- a/cli/commands/resource/errors.tsx +++ b/cli/commands/resource/errors.tsx @@ -18,14 +18,14 @@ export const errorsResourceCommand = new Command("errors") .action(async (path, options) => { const session = Session.create({ - title: "Validate resource", + title: "Resource errors", json: options.json, }) const descriptor = path ? path : await selectResource(session, options) const { errors } = await session.task( - "Validating resource", + "Finding errors", validateResource(descriptor), ) diff --git a/cli/commands/resource/validate.tsx b/cli/commands/resource/validate.tsx index 303c0ce1..9cd40695 100644 --- a/cli/commands/resource/validate.tsx +++ b/cli/commands/resource/validate.tsx @@ -1,5 +1,7 @@ import { Command } from "commander" import { validateResource } from "dpkit" +import React from "react" +import { ErrorGrid } from "../../components/ErrorGrid.tsx" import { helpConfiguration } from "../../helpers/help.ts" import { selectResource } from "../../helpers/resource.ts" import { Session } from "../../helpers/session.ts" @@ -22,12 +24,15 @@ export const validateResourceCommand = new Command("validate") const descriptor = path ? path : await selectResource(session, options) - const { valid } = await session.task( + const { valid, errors } = await session.task( "Validating resource", validateResource(descriptor), ) - valid - ? session.success("Resource is valid") - : session.error("Resource is not valid") + if (valid) { + session.success("Resource is valid") + return + } + + session.render(errors, ) }) diff --git a/cli/commands/schema/errors.tsx b/cli/commands/schema/errors.tsx index e66cc1f9..d51230f8 100644 --- a/cli/commands/schema/errors.tsx +++ b/cli/commands/schema/errors.tsx @@ -19,7 +19,7 @@ export const errorsSchemaCommand = new Command("errors") .action(async (path, options) => { const session = Session.create({ - title: "Validate schema", + title: "Schema errors", json: options.json, }) @@ -41,7 +41,7 @@ export const errorsSchemaCommand = new Command("errors") if (!descriptor) { const result = await session.task( - "Loading descriptor", + "Loading schema", // @ts-ignore loadDescriptor(path), ) diff --git a/cli/commands/schema/validate.tsx b/cli/commands/schema/validate.tsx index cb17090b..2af2b245 100644 --- a/cli/commands/schema/validate.tsx +++ b/cli/commands/schema/validate.tsx @@ -1,6 +1,8 @@ import { Command } from "commander" import { loadDescriptor, validateSchema } from "dpkit" import type { Descriptor } from "dpkit" +import React from "react" +import { ErrorGrid } from "../../components/ErrorGrid.tsx" import { helpConfiguration } from "../../helpers/help.ts" import { selectResource } from "../../helpers/resource.ts" import { Session } from "../../helpers/session.ts" @@ -47,13 +49,16 @@ export const validateSchemaCommand = new Command("validate") descriptor = result.descriptor } - const { valid } = await session.task( + const { valid, errors } = await session.task( "Validating descriptor", // @ts-ignore validateSchema(descriptor), ) - valid - ? session.success("Schema is valid") - : session.error("Schema is not valid") + if (valid) { + session.success("Schema is valid") + return + } + + session.render(errors, ) }) diff --git a/cli/commands/table/errors.tsx b/cli/commands/table/errors.tsx index 94f68a3b..e09e395a 100644 --- a/cli/commands/table/errors.tsx +++ b/cli/commands/table/errors.tsx @@ -47,7 +47,7 @@ export const errorsTableCommand = new Command("errors") : await selectResource(session, options) const { errors } = await session.task( - "Validating table", + "Finding errors", validateTable(resource), ) diff --git a/cli/commands/table/validate.tsx b/cli/commands/table/validate.tsx index 150d9b92..93298210 100644 --- a/cli/commands/table/validate.tsx +++ b/cli/commands/table/validate.tsx @@ -1,5 +1,7 @@ import { Command } from "commander" import { validateTable } from "dpkit" +import React from "react" +import { ErrorGrid } from "../../components/ErrorGrid.tsx" import { createDialectFromOptions } from "../../helpers/dialect.ts" import { helpConfiguration } from "../../helpers/help.ts" import { selectResource } from "../../helpers/resource.ts" @@ -25,14 +27,15 @@ export const validateTableCommand = new Command("validate") ? { path, dialect: createDialectFromOptions(options) } : await selectResource(session, options) - const { valid } = await session.task( + const { valid, errors } = await session.task( "Validating table", validateTable(resource), ) - // TODO: Show errors/count by type grid if not valid + if (valid) { + session.success("Table is valid") + return + } - valid - ? session.success("Table is valid") - : session.error("Table is not valid") + session.render(errors, ) }) diff --git a/cli/components/DataGrid.tsx b/cli/components/DataGrid.tsx index eac5d266..ec67e814 100644 --- a/cli/components/DataGrid.tsx +++ b/cli/components/DataGrid.tsx @@ -12,8 +12,9 @@ export function DataGrid(props: { col?: number order?: Order rowHeight?: number + borderColor?: "green" | "red" }) { - const { data, col, order, rowHeight } = props + const { data, col, order, rowHeight, borderColor = "green" } = props // TODO: fix $schema related cludge const colNames = Object.keys(data[0] ?? {}).filter(name => name !== "$schema") @@ -32,7 +33,7 @@ export function DataGrid(props: { diff --git a/cli/components/ErrorGrid.tsx b/cli/components/ErrorGrid.tsx index e17f49a1..8cb99a4b 100644 --- a/cli/components/ErrorGrid.tsx +++ b/cli/components/ErrorGrid.tsx @@ -1,12 +1,26 @@ //import type { TableError } from "dpkit" +import { countBy } from "es-toolkit" import { DataFrame } from "nodejs-polars" import React from "react" +import { DataGrid } from "./DataGrid.tsx" import { TableGrid } from "./TableGrid.tsx" // TODO: Improve implementation (esp. typing) -export function ErrorGrid(props: { errors: Record[] }) { - const table = DataFrame(props.errors).lazy() +export function ErrorGrid(props: { + errors: Record[] + byType?: boolean +}) { + if (props.byType) { + const errorsByType = countBy(props.errors, error => error.type) + const data = Object.entries(errorsByType).map(([error, count]) => ({ + error, + count, + })) + + return + } - return + const table = DataFrame(props.errors).lazy() + return } diff --git a/cli/components/TableGrid.tsx b/cli/components/TableGrid.tsx index 53ebeddc..954a6640 100644 --- a/cli/components/TableGrid.tsx +++ b/cli/components/TableGrid.tsx @@ -9,8 +9,12 @@ import { DataGrid } from "./DataGrid.tsx" const PAGE_SIZE = 10 type Data = Record[] -export function TableGrid(props: { table: Table }) { - const { table } = props +export function TableGrid(props: { + table: Table + borderColor?: "green" | "red" +}) { + const { table, borderColor } = props + const { exit } = useApp() const [col, setCol] = useState(0) const [page, setPage] = useState(1) @@ -92,7 +96,13 @@ export function TableGrid(props: { table: Table }) { return ( - + ) diff --git a/cli/package.json b/cli/package.json index e7afb8e1..08819489 100644 --- a/cli/package.json +++ b/cli/package.json @@ -35,6 +35,7 @@ "@inkjs/ui": "^2.0.0", "commander": "^14.0.0", "dpkit": "workspace:*", + "es-toolkit": "^1.39.10", "exit-hook": "^4.0.0", "ink": "^6.1.0", "ink-select-input": "^6.2.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a556b5b4..46474524 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -103,6 +103,9 @@ importers: dpkit: specifier: workspace:* version: link:../dpkit + es-toolkit: + specifier: ^1.39.10 + version: 1.39.10 exit-hook: specifier: ^4.0.0 version: 4.0.0 @@ -1982,8 +1985,8 @@ packages: resolution: {integrity: sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==} engines: {node: '>= 0.4'} - es-toolkit@1.39.8: - resolution: {integrity: sha512-A8QO9TfF+rltS8BXpdu8OS+rpGgEdnRhqIVxO/ZmNvnXBYgOdSsxukT55ELyP94gZIntWJ+Li9QRrT2u1Kitpg==} + es-toolkit@1.39.10: + resolution: {integrity: sha512-E0iGnTtbDhkeczB0T+mxmoVlT4YNweEKBLq7oaU4p11mecdsZpNWOglI4895Vh4usbQ+LsJiuLuI2L0Vdmfm2w==} esast-util-from-estree@2.0.0: resolution: {integrity: sha512-4CyanoAudUSBAn5K13H4JhsMH6L9ZP7XbLVe/dKybkxMO7eDyLsT8UHl9TRNrU2Gr9nz+FovfSIjuXWJ81uVwQ==} @@ -5608,7 +5611,7 @@ snapshots: dependencies: es-errors: 1.3.0 - es-toolkit@1.39.8: {} + es-toolkit@1.39.10: {} esast-util-from-estree@2.0.0: dependencies: @@ -6220,7 +6223,7 @@ snapshots: cli-cursor: 4.0.0 cli-truncate: 4.0.0 code-excerpt: 4.0.0 - es-toolkit: 1.39.8 + es-toolkit: 1.39.10 indent-string: 5.0.0 is-in-ci: 1.0.0 patch-console: 2.0.0 From a85f6c9806bd01be58658f2c1dd56198a405babc Mon Sep 17 00:00:00 2001 From: roll Date: Tue, 19 Aug 2025 10:47:07 +0100 Subject: [PATCH 52/61] Implemented file validate command --- cli/commands/file/index.ts | 2 ++ cli/commands/file/validate.tsx | 42 ++++++++++++++++++++++++++++++++++ cli/params/file.ts | 4 ++++ cli/params/path.ts | 5 ++++ 4 files changed, 53 insertions(+) diff --git a/cli/commands/file/index.ts b/cli/commands/file/index.ts index 12df8b04..99d7c7f1 100644 --- a/cli/commands/file/index.ts +++ b/cli/commands/file/index.ts @@ -2,6 +2,7 @@ import { Command } from "commander" import { helpConfiguration } from "../../helpers/help.ts" import { copyFileCommand } from "./copy.ts" import { statsFileCommand } from "./stats.tsx" +import { validateFileCommand } from "./validate.tsx" export const fileCommand = new Command("file") .configureHelp(helpConfiguration) @@ -9,3 +10,4 @@ export const fileCommand = new Command("file") .addCommand(copyFileCommand) .addCommand(statsFileCommand) + .addCommand(validateFileCommand) diff --git a/cli/commands/file/validate.tsx b/cli/commands/file/validate.tsx index e69de29b..5d279c77 100644 --- a/cli/commands/file/validate.tsx +++ b/cli/commands/file/validate.tsx @@ -0,0 +1,42 @@ +import { Command } from "commander" +import { validateFile } from "dpkit" +import React from "react" +import { ErrorGrid } from "../../components/ErrorGrid.tsx" +import { helpConfiguration } from "../../helpers/help.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const validateFileCommand = new Command("validate") + .configureHelp(helpConfiguration) + .description("Validate a file from a local or remote path") + + .addArgument(params.requiredPositionalFilePath) + .addOption(params.bytes) + .addOption(params.hash) + .addOption(params.json) + + .action(async (path, options) => { + const session = Session.create({ + title: "Validate file", + json: options.json, + }) + + if (!options.bytes && !options.hash) { + Session.terminate("You must specify either --bytes or --hash") + } + + const { valid, errors } = await session.task( + "Validating file", + validateFile(path, { + bytes: options.bytes ? Number.parseInt(options.bytes) : undefined, + hash: options.hash, + }), + ) + + if (valid) { + session.success("File is valid") + return + } + + session.render(errors, ) + }) diff --git a/cli/params/file.ts b/cli/params/file.ts index 78599138..4a1349c5 100644 --- a/cli/params/file.ts +++ b/cli/params/file.ts @@ -3,3 +3,7 @@ import { Option } from "commander" export const hashType = new Option("--hash-type ", "hash type") .choices(["md5", "sha1", "sha256", "sha512"]) .default("sha256") + +export const bytes = new Option("--bytes ", "expected file size in bytes") + +export const hash = new Option("--hash ", "expected file hash") diff --git a/cli/params/path.ts b/cli/params/path.ts index f9f7b9f8..132334b9 100644 --- a/cli/params/path.ts +++ b/cli/params/path.ts @@ -20,6 +20,11 @@ export const positionalFilePath = new Argument( "local or remote path to the file", ) +export const requiredPositionalFilePath = new Argument( + "", + "local or remote path to the file", +) + export const positionalFilePaths = new Argument( "", "local paths to files", From 73faa70b5e8306c4f6d9a09937d764d4e875086a Mon Sep 17 00:00:00 2001 From: roll Date: Tue, 19 Aug 2025 10:51:41 +0100 Subject: [PATCH 53/61] Implemented file errors command --- cli/commands/file/errors.tsx | 38 ++++++++++++++++++++++++++++++++++++ cli/commands/file/index.ts | 2 ++ cli/params/file.ts | 5 ++++- 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/cli/commands/file/errors.tsx b/cli/commands/file/errors.tsx index e69de29b..de159611 100644 --- a/cli/commands/file/errors.tsx +++ b/cli/commands/file/errors.tsx @@ -0,0 +1,38 @@ +import { Command } from "commander" +import { validateFile } from "dpkit" +import React from "react" +import { ErrorGrid } from "../../components/ErrorGrid.jsx" +import { helpConfiguration } from "../../helpers/help.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const errorsFileCommand = new Command("errors") + .configureHelp(helpConfiguration) + .description("Show errors for a file from a local or remote path") + + .addArgument(params.requiredPositionalFilePath) + .addOption(params.bytes) + .addOption(params.hash) + .addOption(params.json) + + .action(async (path, options) => { + const session = Session.create({ + title: "File errors", + json: options.json, + }) + + if (!options.bytes && !options.hash) { + Session.terminate("You must specify either --bytes or --hash") + } + + const { errors } = await session.task( + "Finding errors", + validateFile(path, { + bytes: options.bytes ? Number.parseInt(options.bytes) : undefined, + hash: options.hash, + }), + ) + + // @ts-ignore + session.render(errors, ) + }) diff --git a/cli/commands/file/index.ts b/cli/commands/file/index.ts index 99d7c7f1..d41f3c94 100644 --- a/cli/commands/file/index.ts +++ b/cli/commands/file/index.ts @@ -1,6 +1,7 @@ import { Command } from "commander" import { helpConfiguration } from "../../helpers/help.ts" import { copyFileCommand } from "./copy.ts" +import { errorsFileCommand } from "./errors.tsx" import { statsFileCommand } from "./stats.tsx" import { validateFileCommand } from "./validate.tsx" @@ -9,5 +10,6 @@ export const fileCommand = new Command("file") .description("File related commands") .addCommand(copyFileCommand) + .addCommand(errorsFileCommand) .addCommand(statsFileCommand) .addCommand(validateFileCommand) diff --git a/cli/params/file.ts b/cli/params/file.ts index 4a1349c5..4db3d8a6 100644 --- a/cli/params/file.ts +++ b/cli/params/file.ts @@ -4,6 +4,9 @@ export const hashType = new Option("--hash-type ", "hash type") .choices(["md5", "sha1", "sha256", "sha512"]) .default("sha256") -export const bytes = new Option("--bytes ", "expected file size in bytes") +export const bytes = new Option( + "--bytes ", + "expected file size in bytes", +) export const hash = new Option("--hash ", "expected file hash") From 03262c14a42e058514d2f01be381e8c99ff5f2bb Mon Sep 17 00:00:00 2001 From: roll Date: Tue, 19 Aug 2025 11:28:10 +0100 Subject: [PATCH 54/61] Updated validatePackage --- dpkit/package/validate.ts | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/dpkit/package/validate.ts b/dpkit/package/validate.ts index 6cd0a178..cca3ac23 100644 --- a/dpkit/package/validate.ts +++ b/dpkit/package/validate.ts @@ -27,15 +27,19 @@ export async function validatePackage( return { valid, errors } } - const results = await Promise.all( - dataPackage.resources.map(async resource => { - return await validateResource(resource) - }), + const errorsByResource = Object.fromEntries( + await Promise.all( + dataPackage.resources.map(async resource => { + const errors = await validateResource(resource) + return [resource.name, errors] + }), + ), ) - // @ts-ignore - const allErrors = results.flatMap(result => result.errors) - const allValid = allErrors.length === 0 + const allValid = !Object.values(errorsByResource).find( + // @ts-ignore + errors => !!errors.length, + ) - return { valid: allValid, errors: allErrors } + return { valid: allValid, errorsByResource } } From d3aa87e265e126c31cacf531f1a5f981f4b3bedc Mon Sep 17 00:00:00 2001 From: roll Date: Tue, 19 Aug 2025 11:49:03 +0100 Subject: [PATCH 55/61] Bootstrapped validatePackage update --- cli/commands/package/validate.tsx | 18 +++++++++++++++--- cli/components/ErrorGrid.tsx | 18 ++++++++++++++---- dpkit/package.json | 5 +++-- dpkit/package/validate.ts | 9 +++++---- pnpm-lock.yaml | 3 +++ 5 files changed, 40 insertions(+), 13 deletions(-) diff --git a/cli/commands/package/validate.tsx b/cli/commands/package/validate.tsx index c5ff0d73..415d9957 100644 --- a/cli/commands/package/validate.tsx +++ b/cli/commands/package/validate.tsx @@ -19,7 +19,7 @@ export const validatePackageCommand = new Command("validate") json: options.json, }) - const { valid, errors } = await session.task( + const { valid, errors, resourceErrors } = await session.task( "Validating package", validatePackage(path), ) @@ -29,6 +29,18 @@ export const validatePackageCommand = new Command("validate") return } - // @ts-ignore - session.render(errors, ) + if (errors) { + session.render(errors, ) + } else if (resourceErrors) { + const allErrors = Object.entries(resourceErrors).map( + ([resource, errors]) => { + return errors.map(error => ({ ...error, resource })) + }, + ) + + session.render( + allErrors, + , + ) + } }) diff --git a/cli/components/ErrorGrid.tsx b/cli/components/ErrorGrid.tsx index 8cb99a4b..2351364a 100644 --- a/cli/components/ErrorGrid.tsx +++ b/cli/components/ErrorGrid.tsx @@ -9,11 +9,21 @@ import { TableGrid } from "./TableGrid.tsx" export function ErrorGrid(props: { errors: Record[] - byType?: boolean + groupBy?: "type" | "resource" }) { - if (props.byType) { - const errorsByType = countBy(props.errors, error => error.type) - const data = Object.entries(errorsByType).map(([error, count]) => ({ + if (props.groupBy === "resource") { + const groups = countBy(props.errors, error => error.resource) + const data = Object.entries(groups).map(([resource, count]) => ({ + resource, + count, + })) + + return + } + + if (props.groupBy === "type") { + const groups = countBy(props.errors, error => error.type) + const data = Object.entries(groups).map(([error, count]) => ({ error, count, })) diff --git a/dpkit/package.json b/dpkit/package.json index 362e8b87..dd615f96 100644 --- a/dpkit/package.json +++ b/dpkit/package.json @@ -23,8 +23,8 @@ "@dpkit/arrow": "workspace:*", "@dpkit/camtrap": "workspace:*", "@dpkit/ckan": "workspace:*", - "@dpkit/csv": "workspace:*", "@dpkit/core": "workspace:*", + "@dpkit/csv": "workspace:*", "@dpkit/datahub": "workspace:*", "@dpkit/file": "workspace:*", "@dpkit/folder": "workspace:*", @@ -34,7 +34,8 @@ "@dpkit/parquet": "workspace:*", "@dpkit/table": "workspace:*", "@dpkit/zenodo": "workspace:*", - "@dpkit/zip": "workspace:*" + "@dpkit/zip": "workspace:*", + "ts-extras": "^0.14.0" }, "scripts": { "build": "tsc" diff --git a/dpkit/package/validate.ts b/dpkit/package/validate.ts index cca3ac23..d4bf8c72 100644 --- a/dpkit/package/validate.ts +++ b/dpkit/package/validate.ts @@ -1,5 +1,6 @@ import type { Descriptor, Package } from "@dpkit/core" import { loadDescriptor, validatePackageDescriptor } from "@dpkit/core" +import { objectFromEntries } from "ts-extras" import { validateResource } from "../resource/index.ts" // TODO: Handle error addressing (to which resource an error belongs) @@ -27,19 +28,19 @@ export async function validatePackage( return { valid, errors } } - const errorsByResource = Object.fromEntries( + const resourceErrors = objectFromEntries( await Promise.all( dataPackage.resources.map(async resource => { - const errors = await validateResource(resource) + const { errors } = await validateResource(resource) return [resource.name, errors] }), ), ) - const allValid = !Object.values(errorsByResource).find( + const allValid = !Object.values(resourceErrors).find( // @ts-ignore errors => !!errors.length, ) - return { valid: allValid, errorsByResource } + return { valid: allValid, resourceErrors } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 46474524..56e22d63 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -270,6 +270,9 @@ importers: '@dpkit/zip': specifier: workspace:* version: link:../zip + ts-extras: + specifier: ^0.14.0 + version: 0.14.0 excel: {} From 49f3a25c10201d2a47f41624173cb93ae94dd406 Mon Sep 17 00:00:00 2001 From: roll Date: Tue, 19 Aug 2025 12:18:44 +0100 Subject: [PATCH 56/61] Added resource to package validation --- cli/commands/dialect/validate.tsx | 2 +- cli/commands/file/validate.tsx | 2 +- cli/commands/package/validate.tsx | 17 ++--------------- cli/commands/resource/validate.tsx | 2 +- cli/commands/schema/validate.tsx | 2 +- cli/commands/table/validate.tsx | 2 +- core/error/Base.ts | 3 +++ core/error/Metadata.ts | 13 +++++++++++-- core/error/index.ts | 1 + core/general/descriptor/validate.spec.ts | 1 + dpkit/package.json | 3 +-- dpkit/package/validate.ts | 22 ++++++++++------------ file/error/Base.ts | 6 +++--- pnpm-lock.yaml | 3 --- table/error/Base.ts | 6 +++--- 15 files changed, 40 insertions(+), 45 deletions(-) create mode 100644 core/error/Base.ts diff --git a/cli/commands/dialect/validate.tsx b/cli/commands/dialect/validate.tsx index 670875a4..e7af52e0 100644 --- a/cli/commands/dialect/validate.tsx +++ b/cli/commands/dialect/validate.tsx @@ -60,5 +60,5 @@ export const validateDialectCommand = new Command("validate") return } - session.render(errors, ) + session.render(errors, ) }) diff --git a/cli/commands/file/validate.tsx b/cli/commands/file/validate.tsx index 5d279c77..5652e198 100644 --- a/cli/commands/file/validate.tsx +++ b/cli/commands/file/validate.tsx @@ -38,5 +38,5 @@ export const validateFileCommand = new Command("validate") return } - session.render(errors, ) + session.render(errors, ) }) diff --git a/cli/commands/package/validate.tsx b/cli/commands/package/validate.tsx index 415d9957..133f895f 100644 --- a/cli/commands/package/validate.tsx +++ b/cli/commands/package/validate.tsx @@ -19,7 +19,7 @@ export const validatePackageCommand = new Command("validate") json: options.json, }) - const { valid, errors, resourceErrors } = await session.task( + const { valid, errors } = await session.task( "Validating package", validatePackage(path), ) @@ -29,18 +29,5 @@ export const validatePackageCommand = new Command("validate") return } - if (errors) { - session.render(errors, ) - } else if (resourceErrors) { - const allErrors = Object.entries(resourceErrors).map( - ([resource, errors]) => { - return errors.map(error => ({ ...error, resource })) - }, - ) - - session.render( - allErrors, - , - ) - } + session.render(errors, ) }) diff --git a/cli/commands/resource/validate.tsx b/cli/commands/resource/validate.tsx index 9cd40695..2eb4daf2 100644 --- a/cli/commands/resource/validate.tsx +++ b/cli/commands/resource/validate.tsx @@ -34,5 +34,5 @@ export const validateResourceCommand = new Command("validate") return } - session.render(errors, ) + session.render(errors, ) }) diff --git a/cli/commands/schema/validate.tsx b/cli/commands/schema/validate.tsx index 2af2b245..ca38947c 100644 --- a/cli/commands/schema/validate.tsx +++ b/cli/commands/schema/validate.tsx @@ -60,5 +60,5 @@ export const validateSchemaCommand = new Command("validate") return } - session.render(errors, ) + session.render(errors, ) }) diff --git a/cli/commands/table/validate.tsx b/cli/commands/table/validate.tsx index 93298210..9c00bb34 100644 --- a/cli/commands/table/validate.tsx +++ b/cli/commands/table/validate.tsx @@ -37,5 +37,5 @@ export const validateTableCommand = new Command("validate") return } - session.render(errors, ) + session.render(errors, ) }) diff --git a/core/error/Base.ts b/core/error/Base.ts new file mode 100644 index 00000000..cde68ce5 --- /dev/null +++ b/core/error/Base.ts @@ -0,0 +1,3 @@ +export interface BaseError { + type: string +} diff --git a/core/error/Metadata.ts b/core/error/Metadata.ts index 2be7983f..91512062 100644 --- a/core/error/Metadata.ts +++ b/core/error/Metadata.ts @@ -1,8 +1,17 @@ -import type { ErrorObject } from "ajv" +import type { BaseError } from "./Base.ts" /** * A descriptor error */ -export interface MetadataError extends ErrorObject { +export interface MetadataError extends BaseError { type: "metadata" + keyword: string + instancePath: string + schemaPath: string + params: object + propertyName?: string + message?: string + schema?: any + parentSchema?: object + data?: any } diff --git a/core/error/index.ts b/core/error/index.ts index 790584e4..414f1fee 100644 --- a/core/error/index.ts +++ b/core/error/index.ts @@ -1,2 +1,3 @@ export { AssertionError } from "./Assertion.ts" +export type { BaseError } from "./Base.ts" export type { MetadataError } from "./Metadata.ts" diff --git a/core/general/descriptor/validate.spec.ts b/core/general/descriptor/validate.spec.ts index accf7f2c..bc03fa9d 100644 --- a/core/general/descriptor/validate.spec.ts +++ b/core/general/descriptor/validate.spec.ts @@ -82,6 +82,7 @@ describe("validateDescriptor", () => { expect(error.keyword).toBe("required") expect(error.params).toBeDefined() if (error.params) { + // @ts-ignore expect(error.params.missingProperty).toBe("required_field") } } diff --git a/dpkit/package.json b/dpkit/package.json index dd615f96..b2af2b4b 100644 --- a/dpkit/package.json +++ b/dpkit/package.json @@ -34,8 +34,7 @@ "@dpkit/parquet": "workspace:*", "@dpkit/table": "workspace:*", "@dpkit/zenodo": "workspace:*", - "@dpkit/zip": "workspace:*", - "ts-extras": "^0.14.0" + "@dpkit/zip": "workspace:*" }, "scripts": { "build": "tsc" diff --git a/dpkit/package/validate.ts b/dpkit/package/validate.ts index d4bf8c72..8464d059 100644 --- a/dpkit/package/validate.ts +++ b/dpkit/package/validate.ts @@ -1,6 +1,5 @@ import type { Descriptor, Package } from "@dpkit/core" import { loadDescriptor, validatePackageDescriptor } from "@dpkit/core" -import { objectFromEntries } from "ts-extras" import { validateResource } from "../resource/index.ts" // TODO: Handle error addressing (to which resource an error belongs) @@ -25,22 +24,21 @@ export async function validatePackage( ) if (!dataPackage) { - return { valid, errors } + return { + valid, + errors: errors.map(error => ({ ...error, resource: undefined })), + } } - const resourceErrors = objectFromEntries( + const resourceErrors = ( await Promise.all( dataPackage.resources.map(async resource => { const { errors } = await validateResource(resource) - return [resource.name, errors] + return errors.map(error => ({ ...error, resource: resource.name })) }), - ), - ) - - const allValid = !Object.values(resourceErrors).find( - // @ts-ignore - errors => !!errors.length, - ) + ) + ).flat() - return { valid: allValid, resourceErrors } + const resourceValid = !resourceErrors.length + return { valid: resourceValid, errors: resourceErrors } } diff --git a/file/error/Base.ts b/file/error/Base.ts index 89943662..a517c9cf 100644 --- a/file/error/Base.ts +++ b/file/error/Base.ts @@ -1,3 +1,3 @@ -export interface BaseFileError { - type: string -} +import type { BaseError } from "@dpkit/core" + +export interface BaseFileError extends BaseError {} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 56e22d63..46474524 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -270,9 +270,6 @@ importers: '@dpkit/zip': specifier: workspace:* version: link:../zip - ts-extras: - specifier: ^0.14.0 - version: 0.14.0 excel: {} diff --git a/table/error/Base.ts b/table/error/Base.ts index 51539707..e0d2fcb6 100644 --- a/table/error/Base.ts +++ b/table/error/Base.ts @@ -1,3 +1,3 @@ -export interface BaseTableError { - type: string -} +import type { BaseError } from "@dpkit/core" + +export interface BaseTableError extends BaseError {} From ab32599c88e1d4adb98f3fd52cb47e3339d21c2a Mon Sep 17 00:00:00 2001 From: roll Date: Tue, 19 Aug 2025 12:51:52 +0100 Subject: [PATCH 57/61] Normalized validation json output --- cli/commands/dialect/errors.tsx | 11 ++++++---- cli/commands/dialect/validate.tsx | 9 ++++----- cli/commands/file/errors.tsx | 11 ++++++---- cli/commands/file/validate.tsx | 9 ++++----- cli/commands/package/errors.tsx | 14 ++++++------- cli/commands/package/validate.tsx | 9 ++++----- cli/commands/resource/errors.tsx | 10 +++++++--- cli/commands/resource/validate.tsx | 9 ++++----- cli/commands/schema/errors.tsx | 13 +++++++----- cli/commands/schema/validate.tsx | 9 ++++----- cli/commands/table/errors.tsx | 13 ++++++------ cli/commands/table/validate.tsx | 9 ++++----- .../{ErrorGrid.tsx => ReportGrid.tsx} | 20 ++++++++++++------- 13 files changed, 80 insertions(+), 66 deletions(-) rename cli/components/{ErrorGrid.tsx => ReportGrid.tsx} (62%) diff --git a/cli/commands/dialect/errors.tsx b/cli/commands/dialect/errors.tsx index 9d415a2a..5a5195c3 100644 --- a/cli/commands/dialect/errors.tsx +++ b/cli/commands/dialect/errors.tsx @@ -2,7 +2,7 @@ import { Command } from "commander" import { loadDescriptor, validateDialect } from "dpkit" import type { Descriptor } from "dpkit" import React from "react" -import { ErrorGrid } from "../../components/ErrorGrid.jsx" +import { ReportGrid } from "../../components/ReportGrid.tsx" import { helpConfiguration } from "../../helpers/help.ts" import { selectResource } from "../../helpers/resource.ts" import { Session } from "../../helpers/session.ts" @@ -49,11 +49,14 @@ export const errorsDialectCommand = new Command("errors") descriptor = result.descriptor } - const { errors } = await session.task( + const report = await session.task( "Finding errors", - // @ts-ignore validateDialect(descriptor), ) - session.render(errors, ) + if (report.valid) { + session.success("Dialect is valid") + } + + session.render(report, ) }) diff --git a/cli/commands/dialect/validate.tsx b/cli/commands/dialect/validate.tsx index e7af52e0..43e23466 100644 --- a/cli/commands/dialect/validate.tsx +++ b/cli/commands/dialect/validate.tsx @@ -2,7 +2,7 @@ import { Command } from "commander" import { loadDescriptor, validateDialect } from "dpkit" import type { Descriptor } from "dpkit" import React from "react" -import { ErrorGrid } from "../../components/ErrorGrid.tsx" +import { ReportGrid } from "../../components/ReportGrid.tsx" import { helpConfiguration } from "../../helpers/help.ts" import { selectResource } from "../../helpers/resource.ts" import { Session } from "../../helpers/session.ts" @@ -49,16 +49,15 @@ export const validateDialectCommand = new Command("validate") descriptor = result.descriptor } - const { valid, errors } = await session.task( + const report = await session.task( "Validating dialect", // @ts-ignore validateDialect(descriptor), ) - if (valid) { + if (report.valid) { session.success("Dialect is valid") - return } - session.render(errors, ) + session.render(report, ) }) diff --git a/cli/commands/file/errors.tsx b/cli/commands/file/errors.tsx index de159611..dd47a6b7 100644 --- a/cli/commands/file/errors.tsx +++ b/cli/commands/file/errors.tsx @@ -1,7 +1,7 @@ import { Command } from "commander" import { validateFile } from "dpkit" import React from "react" -import { ErrorGrid } from "../../components/ErrorGrid.jsx" +import { ReportGrid } from "../../components/ReportGrid.tsx" import { helpConfiguration } from "../../helpers/help.ts" import { Session } from "../../helpers/session.ts" import * as params from "../../params/index.ts" @@ -25,7 +25,7 @@ export const errorsFileCommand = new Command("errors") Session.terminate("You must specify either --bytes or --hash") } - const { errors } = await session.task( + const report = await session.task( "Finding errors", validateFile(path, { bytes: options.bytes ? Number.parseInt(options.bytes) : undefined, @@ -33,6 +33,9 @@ export const errorsFileCommand = new Command("errors") }), ) - // @ts-ignore - session.render(errors, ) + if (report.valid) { + session.success("File is valid") + } + + session.render(report, ) }) diff --git a/cli/commands/file/validate.tsx b/cli/commands/file/validate.tsx index 5652e198..72cb756d 100644 --- a/cli/commands/file/validate.tsx +++ b/cli/commands/file/validate.tsx @@ -1,7 +1,7 @@ import { Command } from "commander" import { validateFile } from "dpkit" import React from "react" -import { ErrorGrid } from "../../components/ErrorGrid.tsx" +import { ReportGrid } from "../../components/ReportGrid.tsx" import { helpConfiguration } from "../../helpers/help.ts" import { Session } from "../../helpers/session.ts" import * as params from "../../params/index.ts" @@ -25,7 +25,7 @@ export const validateFileCommand = new Command("validate") Session.terminate("You must specify either --bytes or --hash") } - const { valid, errors } = await session.task( + const report = await session.task( "Validating file", validateFile(path, { bytes: options.bytes ? Number.parseInt(options.bytes) : undefined, @@ -33,10 +33,9 @@ export const validateFileCommand = new Command("validate") }), ) - if (valid) { + if (report.valid) { session.success("File is valid") - return } - session.render(errors, ) + session.render(report, ) }) diff --git a/cli/commands/package/errors.tsx b/cli/commands/package/errors.tsx index 198bcb03..9eedb797 100644 --- a/cli/commands/package/errors.tsx +++ b/cli/commands/package/errors.tsx @@ -1,7 +1,7 @@ import { Command } from "commander" import { validatePackage } from "dpkit" import React from "react" -import { ErrorGrid } from "../../components/ErrorGrid.jsx" +import { ReportGrid } from "../../components/ReportGrid.tsx" import { helpConfiguration } from "../../helpers/help.ts" import { Session } from "../../helpers/session.ts" import * as params from "../../params/index.ts" @@ -19,11 +19,11 @@ export const errorsPackageCommand = new Command("errors") json: options.json, }) - const { errors } = await session.task( - "Finding errors", - validatePackage(path), - ) + const report = await session.task("Finding errors", validatePackage(path)) - // @ts-ignore - session.render(errors, ) + if (report.valid) { + session.success("Package is valid") + } + + session.render(report, ) }) diff --git a/cli/commands/package/validate.tsx b/cli/commands/package/validate.tsx index 133f895f..5d88ccb0 100644 --- a/cli/commands/package/validate.tsx +++ b/cli/commands/package/validate.tsx @@ -1,7 +1,7 @@ import { Command } from "commander" import { validatePackage } from "dpkit" import React from "react" -import { ErrorGrid } from "../../components/ErrorGrid.tsx" +import { ReportGrid } from "../../components/ReportGrid.tsx" import { helpConfiguration } from "../../helpers/help.ts" import { Session } from "../../helpers/session.ts" import * as params from "../../params/index.ts" @@ -19,15 +19,14 @@ export const validatePackageCommand = new Command("validate") json: options.json, }) - const { valid, errors } = await session.task( + const report = await session.task( "Validating package", validatePackage(path), ) - if (valid) { + if (report.valid) { session.success("Package is valid") - return } - session.render(errors, ) + session.render(report, ) }) diff --git a/cli/commands/resource/errors.tsx b/cli/commands/resource/errors.tsx index 28ef73eb..5ad7da42 100644 --- a/cli/commands/resource/errors.tsx +++ b/cli/commands/resource/errors.tsx @@ -1,7 +1,7 @@ import { Command } from "commander" import { validateResource } from "dpkit" import React from "react" -import { ErrorGrid } from "../../components/ErrorGrid.jsx" +import { ReportGrid } from "../../components/ReportGrid.tsx" import { helpConfiguration } from "../../helpers/help.ts" import { selectResource } from "../../helpers/resource.ts" import { Session } from "../../helpers/session.ts" @@ -24,10 +24,14 @@ export const errorsResourceCommand = new Command("errors") const descriptor = path ? path : await selectResource(session, options) - const { errors } = await session.task( + const report = await session.task( "Finding errors", validateResource(descriptor), ) - session.render(errors, ) + if (report.valid) { + session.success("Resource is valid") + } + + session.render(report, ) }) diff --git a/cli/commands/resource/validate.tsx b/cli/commands/resource/validate.tsx index 2eb4daf2..b3ae7795 100644 --- a/cli/commands/resource/validate.tsx +++ b/cli/commands/resource/validate.tsx @@ -1,7 +1,7 @@ import { Command } from "commander" import { validateResource } from "dpkit" import React from "react" -import { ErrorGrid } from "../../components/ErrorGrid.tsx" +import { ReportGrid } from "../../components/ReportGrid.tsx" import { helpConfiguration } from "../../helpers/help.ts" import { selectResource } from "../../helpers/resource.ts" import { Session } from "../../helpers/session.ts" @@ -24,15 +24,14 @@ export const validateResourceCommand = new Command("validate") const descriptor = path ? path : await selectResource(session, options) - const { valid, errors } = await session.task( + const report = await session.task( "Validating resource", validateResource(descriptor), ) - if (valid) { + if (report.valid) { session.success("Resource is valid") - return } - session.render(errors, ) + session.render(report, ) }) diff --git a/cli/commands/schema/errors.tsx b/cli/commands/schema/errors.tsx index d51230f8..5955e83c 100644 --- a/cli/commands/schema/errors.tsx +++ b/cli/commands/schema/errors.tsx @@ -2,7 +2,7 @@ import { Command } from "commander" import { loadDescriptor, validateSchema } from "dpkit" import type { Descriptor } from "dpkit" import React from "react" -import { ErrorGrid } from "../../components/ErrorGrid.jsx" +import { ReportGrid } from "../../components/ReportGrid.tsx" import { helpConfiguration } from "../../helpers/help.ts" import { selectResource } from "../../helpers/resource.ts" import { Session } from "../../helpers/session.ts" @@ -49,11 +49,14 @@ export const errorsSchemaCommand = new Command("errors") descriptor = result.descriptor } - const { errors } = await session.task( - "Validating descriptor", - // @ts-ignore + const report = await session.task( + "Validating schema", validateSchema(descriptor), ) - session.render(errors, ) + if (report.valid) { + session.success("Schema is valid") + } + + session.render(report, ) }) diff --git a/cli/commands/schema/validate.tsx b/cli/commands/schema/validate.tsx index ca38947c..e8d773c2 100644 --- a/cli/commands/schema/validate.tsx +++ b/cli/commands/schema/validate.tsx @@ -2,7 +2,7 @@ import { Command } from "commander" import { loadDescriptor, validateSchema } from "dpkit" import type { Descriptor } from "dpkit" import React from "react" -import { ErrorGrid } from "../../components/ErrorGrid.tsx" +import { ReportGrid } from "../../components/ReportGrid.tsx" import { helpConfiguration } from "../../helpers/help.ts" import { selectResource } from "../../helpers/resource.ts" import { Session } from "../../helpers/session.ts" @@ -49,16 +49,15 @@ export const validateSchemaCommand = new Command("validate") descriptor = result.descriptor } - const { valid, errors } = await session.task( + const report = await session.task( "Validating descriptor", // @ts-ignore validateSchema(descriptor), ) - if (valid) { + if (report.valid) { session.success("Schema is valid") - return } - session.render(errors, ) + session.render(report, ) }) diff --git a/cli/commands/table/errors.tsx b/cli/commands/table/errors.tsx index e09e395a..abdefeed 100644 --- a/cli/commands/table/errors.tsx +++ b/cli/commands/table/errors.tsx @@ -1,7 +1,7 @@ import { Command } from "commander" import { validateTable } from "dpkit" import React from "react" -import { ErrorGrid } from "../../components/ErrorGrid.jsx" +import { ReportGrid } from "../../components/ReportGrid.tsx" import { createDialectFromOptions } from "../../helpers/dialect.ts" import { helpConfiguration } from "../../helpers/help.ts" import { selectResource } from "../../helpers/resource.ts" @@ -46,10 +46,11 @@ export const errorsTableCommand = new Command("errors") ? { path, dialect: createDialectFromOptions(options) } : await selectResource(session, options) - const { errors } = await session.task( - "Finding errors", - validateTable(resource), - ) + const report = await session.task("Finding errors", validateTable(resource)) - session.render(errors, ) + if (report.valid) { + session.success("Table is valid") + } + + session.render(report, ) }) diff --git a/cli/commands/table/validate.tsx b/cli/commands/table/validate.tsx index 9c00bb34..c2323cad 100644 --- a/cli/commands/table/validate.tsx +++ b/cli/commands/table/validate.tsx @@ -1,7 +1,7 @@ import { Command } from "commander" import { validateTable } from "dpkit" import React from "react" -import { ErrorGrid } from "../../components/ErrorGrid.tsx" +import { ReportGrid } from "../../components/ReportGrid.tsx" import { createDialectFromOptions } from "../../helpers/dialect.ts" import { helpConfiguration } from "../../helpers/help.ts" import { selectResource } from "../../helpers/resource.ts" @@ -27,15 +27,14 @@ export const validateTableCommand = new Command("validate") ? { path, dialect: createDialectFromOptions(options) } : await selectResource(session, options) - const { valid, errors } = await session.task( + const report = await session.task( "Validating table", validateTable(resource), ) - if (valid) { + if (report.valid) { session.success("Table is valid") - return } - session.render(errors, ) + session.render(report, ) }) diff --git a/cli/components/ErrorGrid.tsx b/cli/components/ReportGrid.tsx similarity index 62% rename from cli/components/ErrorGrid.tsx rename to cli/components/ReportGrid.tsx index 2351364a..982c0627 100644 --- a/cli/components/ErrorGrid.tsx +++ b/cli/components/ReportGrid.tsx @@ -7,12 +7,18 @@ import { TableGrid } from "./TableGrid.tsx" // TODO: Improve implementation (esp. typing) -export function ErrorGrid(props: { - errors: Record[] +export function ReportGrid(props: { + report: { valid: boolean; errors: Record[] } groupBy?: "type" | "resource" }) { - if (props.groupBy === "resource") { - const groups = countBy(props.errors, error => error.resource) + const { report, groupBy } = props + + if (report.valid) { + return null + } + + if (groupBy === "resource") { + const groups = countBy(report.errors, error => error.resource) const data = Object.entries(groups).map(([resource, count]) => ({ resource, count, @@ -21,8 +27,8 @@ export function ErrorGrid(props: { return } - if (props.groupBy === "type") { - const groups = countBy(props.errors, error => error.type) + if (groupBy === "type") { + const groups = countBy(report.errors, error => error.type) const data = Object.entries(groups).map(([error, count]) => ({ error, count, @@ -31,6 +37,6 @@ export function ErrorGrid(props: { return } - const table = DataFrame(props.errors).lazy() + const table = DataFrame(report.errors).lazy() return } From 599205fbb18ca84a87654ed0b3453dee19ee6659 Mon Sep 17 00:00:00 2001 From: roll Date: Tue, 19 Aug 2025 15:12:24 +0100 Subject: [PATCH 58/61] Added plugins support for validatePackage --- cli/components/ReportGrid.tsx | 3 ++- dpkit/package/validate.ts | 27 ++++++++++++++++++++------- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/cli/components/ReportGrid.tsx b/cli/components/ReportGrid.tsx index 982c0627..10c371e8 100644 --- a/cli/components/ReportGrid.tsx +++ b/cli/components/ReportGrid.tsx @@ -6,6 +6,7 @@ import { DataGrid } from "./DataGrid.tsx" import { TableGrid } from "./TableGrid.tsx" // TODO: Improve implementation (esp. typing) +// TODO: Rebase on resource/type grouping? export function ReportGrid(props: { report: { valid: boolean; errors: Record[] } @@ -18,7 +19,7 @@ export function ReportGrid(props: { } if (groupBy === "resource") { - const groups = countBy(report.errors, error => error.resource) + const groups = countBy(report.errors, error => error.resource ?? "") const data = Object.entries(groups).map(([resource, count]) => ({ resource, count, diff --git a/dpkit/package/validate.ts b/dpkit/package/validate.ts index 8464d059..56941400 100644 --- a/dpkit/package/validate.ts +++ b/dpkit/package/validate.ts @@ -1,21 +1,34 @@ import type { Descriptor, Package } from "@dpkit/core" import { loadDescriptor, validatePackageDescriptor } from "@dpkit/core" +import { dpkit } from "../plugin.ts" import { validateResource } from "../resource/index.ts" -// TODO: Handle error addressing (to which resource an error belongs) +// TODO: Improve implementation // TODO: Support multipart resources? (clarify on the specs level) export async function validatePackage( - pathOrDescriptorOrPackage: string | Descriptor | Partial, + source: string | Descriptor | Partial, options?: { basepath?: string }, ) { - let descriptor = pathOrDescriptorOrPackage + let descriptor: Descriptor | undefined let basepath = options?.basepath - if (typeof descriptor === "string") { - const result = await loadDescriptor(descriptor) - descriptor = result.descriptor - basepath = result.basepath + if (typeof source !== "string") { + descriptor = source + } else { + for (const plugin of dpkit.plugins) { + const result = await plugin.loadPackage?.(source) + if (result) { + descriptor = result as unknown as Descriptor + break + } + } + + if (!descriptor) { + const result = await loadDescriptor(source) + descriptor = result.descriptor + basepath = result.basepath + } } const { valid, errors, dataPackage } = await validatePackageDescriptor( From 902bf7b9c673223f72bb7ec4093e82114bfe76b7 Mon Sep 17 00:00:00 2001 From: roll Date: Tue, 19 Aug 2025 15:29:36 +0100 Subject: [PATCH 59/61] Rebased on resource/type grouping --- cli/commands/package/validate.tsx | 5 ++++- cli/components/DataGrid.tsx | 13 +++++++++++++ cli/components/ReportGrid.tsx | 15 +++++++++------ 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/cli/commands/package/validate.tsx b/cli/commands/package/validate.tsx index 5d88ccb0..9f0bf413 100644 --- a/cli/commands/package/validate.tsx +++ b/cli/commands/package/validate.tsx @@ -28,5 +28,8 @@ export const validatePackageCommand = new Command("validate") session.success("Package is valid") } - session.render(report, ) + session.render( + report, + , + ) }) diff --git a/cli/components/DataGrid.tsx b/cli/components/DataGrid.tsx index ec67e814..c504248f 100644 --- a/cli/components/DataGrid.tsx +++ b/cli/components/DataGrid.tsx @@ -18,6 +18,19 @@ export function DataGrid(props: { // TODO: fix $schema related cludge const colNames = Object.keys(data[0] ?? {}).filter(name => name !== "$schema") + + // TODO: fix type related cludge + if (colNames.includes("type")) { + colNames.splice(colNames.indexOf("type"), 1) + colNames.unshift("type") + } + + // TODO: fix resource related cludge + if (colNames.includes("resource")) { + colNames.splice(colNames.indexOf("resource"), 1) + colNames.unshift("resource") + } + const colWidth = Math.min( process.stdout.columns / colNames.length, MIN_COLUMN_WIDTH, diff --git a/cli/components/ReportGrid.tsx b/cli/components/ReportGrid.tsx index 10c371e8..b897a668 100644 --- a/cli/components/ReportGrid.tsx +++ b/cli/components/ReportGrid.tsx @@ -6,11 +6,10 @@ import { DataGrid } from "./DataGrid.tsx" import { TableGrid } from "./TableGrid.tsx" // TODO: Improve implementation (esp. typing) -// TODO: Rebase on resource/type grouping? export function ReportGrid(props: { report: { valid: boolean; errors: Record[] } - groupBy?: "type" | "resource" + groupBy?: "type" | "resource/type" }) { const { report, groupBy } = props @@ -18,10 +17,14 @@ export function ReportGrid(props: { return null } - if (groupBy === "resource") { - const groups = countBy(report.errors, error => error.resource ?? "") - const data = Object.entries(groups).map(([resource, count]) => ({ - resource, + if (groupBy === "resource/type") { + const groups = countBy(report.errors, error => + [error.resource ?? "", error.type].join("/"), + ) + + const data = Object.entries(groups).map(([resourceType, count]) => ({ + resource: resourceType.split("/")[0], + type: resourceType.split("/")[1], count, })) From 195b2bab7542ac51650c4703c661f51802384419 Mon Sep 17 00:00:00 2001 From: roll Date: Tue, 19 Aug 2025 16:21:01 +0100 Subject: [PATCH 60/61] Fixed cli compilation --- cli/package.json | 1 + cli/scripts/compile.ts | 5 ++++- pnpm-lock.yaml | 3 +++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/cli/package.json b/cli/package.json index 08819489..0cd687b9 100644 --- a/cli/package.json +++ b/cli/package.json @@ -46,6 +46,7 @@ "ts-extras": "^0.14.0" }, "devDependencies": { + "@types/node": "24.2.0", "@types/react": "19.1.9", "tsx": "4.20.3" } diff --git a/cli/scripts/compile.ts b/cli/scripts/compile.ts index 9ffc9285..b7db5709 100644 --- a/cli/scripts/compile.ts +++ b/cli/scripts/compile.ts @@ -1,4 +1,5 @@ import { execa } from "execa" +import metadata from "../package.json" with { type: "json" } const $ = execa({ preferLocal: true, stdout: ["inherit"] }) // Cleanup @@ -18,4 +19,6 @@ pnpm deploy compile // Compile application -await $({ cwd: "compile" })`deno compile --allow-all main.ts` +await $({ + cwd: "compile", +})`deno compile --allow-all --no-check --output build/dp-${metadata.version}-x86_64-unknown-linux-gnu main.ts` diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 46474524..49aa62f1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -131,6 +131,9 @@ importers: specifier: ^0.14.0 version: 0.14.0 devDependencies: + '@types/node': + specifier: 24.2.0 + version: 24.2.0 '@types/react': specifier: 19.1.9 version: 19.1.9 From 33c279a05f745c09e0f6d3c37594af0dcd3fb582 Mon Sep 17 00:00:00 2001 From: roll Date: Tue, 19 Aug 2025 16:27:27 +0100 Subject: [PATCH 61/61] Added changeset --- .changeset/eleven-pillows-eat.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .changeset/eleven-pillows-eat.md diff --git a/.changeset/eleven-pillows-eat.md b/.changeset/eleven-pillows-eat.md new file mode 100644 index 00000000..8d6776ea --- /dev/null +++ b/.changeset/eleven-pillows-eat.md @@ -0,0 +1,18 @@ +--- +"@dpkit/parquet": minor +"@dpkit/folder": minor +"@dpkit/github": minor +"@dpkit/zenodo": minor +"@dpkit/arrow": minor +"dpkit": minor +"@dpkit/table": minor +"@dpkit/ckan": minor +"@dpkit/core": minor +"@dpkit/file": minor +"@dpkit/json": minor +"@dpkit/cli": minor +"@dpkit/csv": minor +"@dpkit/zip": minor +--- + +Improved CLI implementation