diff --git a/.changeset/eleven-pillows-eat.md b/.changeset/eleven-pillows-eat.md new file mode 100644 index 00000000..8d6776ea --- /dev/null +++ b/.changeset/eleven-pillows-eat.md @@ -0,0 +1,18 @@ +--- +"@dpkit/parquet": minor +"@dpkit/folder": minor +"@dpkit/github": minor +"@dpkit/zenodo": minor +"@dpkit/arrow": minor +"dpkit": minor +"@dpkit/table": minor +"@dpkit/ckan": minor +"@dpkit/core": minor +"@dpkit/file": minor +"@dpkit/json": minor +"@dpkit/cli": minor +"@dpkit/csv": minor +"@dpkit/zip": minor +--- + +Improved CLI implementation diff --git a/arrow/plugin.ts b/arrow/plugin.ts index 133ee938..c678b6d5 100644 --- a/arrow/plugin.ts +++ b/arrow/plugin.ts @@ -1,5 +1,5 @@ import type { Resource } from "@dpkit/core" -import { inferFormat } from "@dpkit/core" +import { inferResourceFormat } from "@dpkit/core" import type { TablePlugin } from "@dpkit/table" import type { SaveTableOptions, Table } from "@dpkit/table" import { loadArrowTable, saveArrowTable } from "./table/index.ts" @@ -21,6 +21,6 @@ export class ArrowPlugin implements TablePlugin { } function getIsArrow(resource: Partial) { - const format = inferFormat(resource) + const format = inferResourceFormat(resource) return format === "arrow" || format === "feather" } diff --git a/ckan/package/save.ts b/ckan/package/save.ts index 72fd4b81..34bc3cbe 100644 --- a/ckan/package/save.ts +++ b/ckan/package/save.ts @@ -54,20 +54,20 @@ export async function savePackageToCkan( basepath, withRemote: true, withoutFolders: true, - saveFile: async props => { - const filename = getFilename(props.normalizedPath) + saveFile: async options => { + const filename = getFilename(options.normalizedPath) const ckanResource = denormalizeCkanResource(resource) const payload = { ...ckanResource, package_id: datasetName, - name: props.denormalizedPath, + name: options.denormalizedPath, format: getFormat(filename)?.toUpperCase(), } const upload = { - name: props.denormalizedPath, - data: await blob(await loadFileStream(props.normalizedPath)), + name: options.denormalizedPath, + data: await blob(await loadFileStream(options.normalizedPath)), } const result = await makeCkanApiRequest({ diff --git a/cli/.oclifrc.js b/cli/.oclifrc.js deleted file mode 100644 index 726cf92b..00000000 --- a/cli/.oclifrc.js +++ /dev/null @@ -1,6 +0,0 @@ -export default { - bin: "dp", - commands: "./commands", - topicSeparator: " ", - plugins: ["@oclif/plugin-autocomplete"], -} diff --git a/cli/commander.d.ts b/cli/commander.d.ts new file mode 100644 index 00000000..abd5017f --- /dev/null +++ b/cli/commander.d.ts @@ -0,0 +1,3 @@ +declare module "commander" { + export * from "@commander-js/extra-typings" +} diff --git a/cli/commands/dialect/errors.tsx b/cli/commands/dialect/errors.tsx new file mode 100644 index 00000000..5a5195c3 --- /dev/null +++ b/cli/commands/dialect/errors.tsx @@ -0,0 +1,62 @@ +import { Command } from "commander" +import { loadDescriptor, validateDialect } from "dpkit" +import type { Descriptor } from "dpkit" +import React from "react" +import { ReportGrid } from "../../components/ReportGrid.tsx" +import { helpConfiguration } from "../../helpers/help.ts" +import { selectResource } from "../../helpers/resource.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const errorsDialectCommand = new Command("errors") + .configureHelp(helpConfiguration) + .description("Show errors for a table dialect from a local or remote path") + + .addArgument(params.positionalTablePath) + .addOption(params.fromPackage) + .addOption(params.fromResource) + .addOption(params.json) + + .action(async (path, options) => { + const session = Session.create({ + title: "Dialect errors", + json: options.json, + }) + + let descriptor: Descriptor | undefined + + if (!path) { + const resource = await selectResource(session, options) + + if (!resource.dialect) { + Session.terminate("Dialect is not available") + } + + if (typeof resource.dialect !== "string") { + descriptor = resource.dialect as Descriptor + } else { + path = resource.dialect + } + } + + if (!descriptor) { + const result = await session.task( + "Loading dialect", + // @ts-ignore + loadDescriptor(path), + ) + + descriptor = result.descriptor + } + + const report = await session.task( + "Finding errors", + validateDialect(descriptor), + ) + + if (report.valid) { + session.success("Dialect is valid") + } + + session.render(report, ) + }) diff --git a/cli/commands/dialect/index.ts b/cli/commands/dialect/index.ts new file mode 100644 index 00000000..43e27fcf --- /dev/null +++ b/cli/commands/dialect/index.ts @@ -0,0 +1,15 @@ +import { Command } from "commander" +import { helpConfiguration } from "../../helpers/help.ts" +import { errorsDialectCommand } from "./errors.tsx" +import { inferDialectCommand } from "./infer.tsx" +import { showDialectCommand } from "./show.tsx" +import { validateDialectCommand } from "./validate.tsx" + +export const dialectCommand = new Command("dialect") + .configureHelp(helpConfiguration) + .description("Table Dialect related commands") + + .addCommand(errorsDialectCommand) + .addCommand(inferDialectCommand) + .addCommand(showDialectCommand) + .addCommand(validateDialectCommand) diff --git a/cli/commands/dialect/infer.ts b/cli/commands/dialect/infer.ts deleted file mode 100644 index 6c9ca7b8..00000000 --- a/cli/commands/dialect/infer.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { Command } from "@oclif/core" -import { inferDialect } from "dpkit" -import * as options from "../../options/index.ts" -import * as params from "../../params/index.ts" - -export default class InferDialect extends Command { - static override description = "Infer a dialect from a table" - - static override args = { - path: params.requriedTablePath, - } - - static override flags = { - json: options.json, - } - - public async run() { - const { args, flags } = await this.parse(InferDialect) - - const dialect = await inferDialect({ path: args.path }) - - if (flags.json) { - this.logJson(dialect) - return - } - - console.log(dialect) - } -} diff --git a/cli/commands/dialect/infer.tsx b/cli/commands/dialect/infer.tsx new file mode 100644 index 00000000..f263b443 --- /dev/null +++ b/cli/commands/dialect/infer.tsx @@ -0,0 +1,34 @@ +import { Command } from "commander" +import { inferDialect } from "dpkit" +import React from "react" +import { DialectGrid } from "../../components/DialectGrid.tsx" +import { helpConfiguration } from "../../helpers/help.ts" +import { isEmptyObject } from "../../helpers/object.ts" +import { selectResource } from "../../helpers/resource.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const inferDialectCommand = new Command("infer") + .configureHelp(helpConfiguration) + .description("Infer a table dialect from a table") + + .addArgument(params.positionalTablePath) + .addOption(params.fromPackage) + .addOption(params.fromResource) + .addOption(params.json) + + .action(async (path, options) => { + const session = Session.create({ + title: "Infer dialect", + json: options.json, + }) + + const resource = path ? { path } : await selectResource(session, options) + + const dialect = await session.task("Loading sample", inferDialect(resource)) + if (isEmptyObject(dialect)) { + Session.terminate("Could not infer dialect") + } + + await session.render(dialect, ) + }) diff --git a/cli/commands/dialect/show.tsx b/cli/commands/dialect/show.tsx new file mode 100644 index 00000000..eb802049 --- /dev/null +++ b/cli/commands/dialect/show.tsx @@ -0,0 +1,52 @@ +import { Command } from "commander" +import { loadDialect } from "dpkit" +import type { Dialect } from "dpkit" +import React from "react" +import { DialectGrid } from "../../components/DialectGrid.tsx" +import { helpConfiguration } from "../../helpers/help.ts" +import { isEmptyObject } from "../../helpers/object.ts" +import { selectResource } from "../../helpers/resource.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const showDialectCommand = new Command("show") + .configureHelp(helpConfiguration) + .description("Show a table dialect from a local or remote path") + + .addArgument(params.positionalTablePath) + .addOption(params.fromPackage) + .addOption(params.fromResource) + .addOption(params.json) + + .action(async (path, options) => { + const session = Session.create({ + title: "Show dialect", + json: options.json, + }) + + let dialect: Dialect | undefined + + if (!path) { + const resource = await selectResource(session, options) + if (!resource.dialect) { + Session.terminate("Dialect is not available") + } + + if (typeof resource.dialect !== "string") { + dialect = resource.dialect + } else { + path = resource.dialect + } + } + + if (!dialect) { + // @ts-ignore + dialect = await session.task("Loading dialect", loadDialect(path)) + } + + if (isEmptyObject(dialect)) { + Session.terminate("Dialect is not available") + } + + await session.render(dialect, ) + }) diff --git a/cli/commands/dialect/validate.tsx b/cli/commands/dialect/validate.tsx new file mode 100644 index 00000000..43e23466 --- /dev/null +++ b/cli/commands/dialect/validate.tsx @@ -0,0 +1,63 @@ +import { Command } from "commander" +import { loadDescriptor, validateDialect } from "dpkit" +import type { Descriptor } from "dpkit" +import React from "react" +import { ReportGrid } from "../../components/ReportGrid.tsx" +import { helpConfiguration } from "../../helpers/help.ts" +import { selectResource } from "../../helpers/resource.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const validateDialectCommand = new Command("validate") + .configureHelp(helpConfiguration) + .description("Validate a table dialect from a local or remote path") + + .addArgument(params.positionalTablePath) + .addOption(params.fromPackage) + .addOption(params.fromResource) + .addOption(params.json) + + .action(async (path, options) => { + const session = Session.create({ + title: "Validate dialect", + json: options.json, + }) + + let descriptor: Descriptor | undefined + + if (!path) { + const resource = await selectResource(session, options) + + if (!resource.dialect) { + Session.terminate("Dialect is not available") + } + + if (typeof resource.dialect !== "string") { + descriptor = resource.dialect as Descriptor + } else { + path = resource.dialect + } + } + + if (!descriptor) { + const result = await session.task( + "Loading dialect", + // @ts-ignore + loadDescriptor(path), + ) + + descriptor = result.descriptor + } + + const report = await session.task( + "Validating dialect", + // @ts-ignore + validateDialect(descriptor), + ) + + if (report.valid) { + session.success("Dialect is valid") + } + + session.render(report, ) + }) diff --git a/cli/commands/file/copy.ts b/cli/commands/file/copy.ts new file mode 100644 index 00000000..98fcc798 --- /dev/null +++ b/cli/commands/file/copy.ts @@ -0,0 +1,41 @@ +import { Command } from "commander" +import { copyFile } from "dpkit" +import { helpConfiguration } from "../../helpers/help.ts" +import { selectResource } from "../../helpers/resource.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const copyFileCommand = new Command("copy") + .configureHelp(helpConfiguration) + .description("Copy a local or remote file to a local path") + + .addArgument(params.positionalFilePath) + .addOption(params.toPath.makeOptionMandatory()) + .addOption(params.fromPackage) + .addOption(params.fromResource) + + .action(async (path, options) => { + const session = Session.create({ + title: "Copy file", + }) + + if (!path) { + const resource = await selectResource(session, options) + + if (typeof resource.path !== "string") { + Session.terminate("Only single file resources are supported") + } + + path = resource.path + } + + await session.task( + "Copying file", + copyFile({ + sourcePath: path, + targetPath: options.toPath, + }), + ) + + session.success(`File from "${path}" copied to "${options.toPath}"`) + }) diff --git a/cli/commands/file/errors.tsx b/cli/commands/file/errors.tsx new file mode 100644 index 00000000..dd47a6b7 --- /dev/null +++ b/cli/commands/file/errors.tsx @@ -0,0 +1,41 @@ +import { Command } from "commander" +import { validateFile } from "dpkit" +import React from "react" +import { ReportGrid } from "../../components/ReportGrid.tsx" +import { helpConfiguration } from "../../helpers/help.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const errorsFileCommand = new Command("errors") + .configureHelp(helpConfiguration) + .description("Show errors for a file from a local or remote path") + + .addArgument(params.requiredPositionalFilePath) + .addOption(params.bytes) + .addOption(params.hash) + .addOption(params.json) + + .action(async (path, options) => { + const session = Session.create({ + title: "File errors", + json: options.json, + }) + + if (!options.bytes && !options.hash) { + Session.terminate("You must specify either --bytes or --hash") + } + + const report = await session.task( + "Finding errors", + validateFile(path, { + bytes: options.bytes ? Number.parseInt(options.bytes) : undefined, + hash: options.hash, + }), + ) + + if (report.valid) { + session.success("File is valid") + } + + session.render(report, ) + }) diff --git a/cli/commands/file/index.ts b/cli/commands/file/index.ts new file mode 100644 index 00000000..d41f3c94 --- /dev/null +++ b/cli/commands/file/index.ts @@ -0,0 +1,15 @@ +import { Command } from "commander" +import { helpConfiguration } from "../../helpers/help.ts" +import { copyFileCommand } from "./copy.ts" +import { errorsFileCommand } from "./errors.tsx" +import { statsFileCommand } from "./stats.tsx" +import { validateFileCommand } from "./validate.tsx" + +export const fileCommand = new Command("file") + .configureHelp(helpConfiguration) + .description("File related commands") + + .addCommand(copyFileCommand) + .addCommand(errorsFileCommand) + .addCommand(statsFileCommand) + .addCommand(validateFileCommand) diff --git a/cli/commands/file/stats.tsx b/cli/commands/file/stats.tsx new file mode 100644 index 00000000..d7402339 --- /dev/null +++ b/cli/commands/file/stats.tsx @@ -0,0 +1,42 @@ +import { Command } from "commander" +import { describeFile } from "dpkit" +import React from "react" +import { DataGrid } from "../../components/DataGrid.tsx" +import { helpConfiguration } from "../../helpers/help.ts" +import { selectResource } from "../../helpers/resource.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const statsFileCommand = new Command("stats") + .configureHelp(helpConfiguration) + .description("Show stats for a local or remote file") + + .addArgument(params.positionalFilePath) + .addOption(params.fromPackage) + .addOption(params.fromResource) + .addOption(params.hashType) + .addOption(params.json) + + .action(async (path, options) => { + const session = Session.create({ + title: "File stats", + json: options.json, + }) + + if (!path) { + const resource = await selectResource(session, options) + + if (typeof resource.path !== "string") { + Session.terminate("Only single file resources are supported") + } + + path = resource.path + } + + const stats = await session.task( + "Calculating stats", + describeFile(path, { hashType: options.hashType }), + ) + + session.render(stats, ) + }) diff --git a/cli/commands/file/validate.tsx b/cli/commands/file/validate.tsx new file mode 100644 index 00000000..72cb756d --- /dev/null +++ b/cli/commands/file/validate.tsx @@ -0,0 +1,41 @@ +import { Command } from "commander" +import { validateFile } from "dpkit" +import React from "react" +import { ReportGrid } from "../../components/ReportGrid.tsx" +import { helpConfiguration } from "../../helpers/help.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const validateFileCommand = new Command("validate") + .configureHelp(helpConfiguration) + .description("Validate a file from a local or remote path") + + .addArgument(params.requiredPositionalFilePath) + .addOption(params.bytes) + .addOption(params.hash) + .addOption(params.json) + + .action(async (path, options) => { + const session = Session.create({ + title: "Validate file", + json: options.json, + }) + + if (!options.bytes && !options.hash) { + Session.terminate("You must specify either --bytes or --hash") + } + + const report = await session.task( + "Validating file", + validateFile(path, { + bytes: options.bytes ? Number.parseInt(options.bytes) : undefined, + hash: options.hash, + }), + ) + + if (report.valid) { + session.success("File is valid") + } + + session.render(report, ) + }) diff --git a/cli/commands/package/archive.ts b/cli/commands/package/archive.ts new file mode 100644 index 00000000..439ca926 --- /dev/null +++ b/cli/commands/package/archive.ts @@ -0,0 +1,31 @@ +import { Command } from "commander" +import { loadPackage, savePackageToZip } from "dpkit" +import { helpConfiguration } from "../../helpers/help.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const archivePackageCommand = new Command("archive") + .configureHelp(helpConfiguration) + .description("Archive a local or remote Data Package to a local zip file") + + .addArgument(params.positionalDescriptorPath) + .addOption(params.toArchive.makeOptionMandatory()) + .addOption(params.withRemote) + + .action(async (path, options) => { + const session = Session.create({ + title: "Validate package", + }) + + const dp = await session.task("Loading package", loadPackage(path)) + + session.task( + "Archiving package", + savePackageToZip(dp, { + archivePath: options.toArchive, + withRemote: options.withRemote, + }), + ) + + session.success(`Package from "${path}" archived to "${options.toArchive}"`) + }) diff --git a/cli/commands/package/copy.ts b/cli/commands/package/copy.ts new file mode 100644 index 00000000..9d2d6d2c --- /dev/null +++ b/cli/commands/package/copy.ts @@ -0,0 +1,31 @@ +import { Command } from "commander" +import { loadPackage, savePackageToFolder } from "dpkit" +import { helpConfiguration } from "../../helpers/help.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const copyPackageCommand = new Command("copy") + .configureHelp(helpConfiguration) + .description("Copy a local or remote Data Package to a local folder") + + .addArgument(params.positionalDescriptorPath) + .addOption(params.toFolder.makeOptionMandatory()) + .addOption(params.withRemote) + + .action(async (path, options) => { + const session = Session.create({ + title: "Validate package", + }) + + const dp = await session.task("Loading package", loadPackage(path)) + + await session.task( + "Copying package", + savePackageToFolder(dp, { + folderPath: options.toFolder, + withRemote: options.withRemote, + }), + ) + + session.success(`Package from "${path}" copied to "${options.toFolder}"`) + }) diff --git a/cli/commands/package/errors.tsx b/cli/commands/package/errors.tsx new file mode 100644 index 00000000..9eedb797 --- /dev/null +++ b/cli/commands/package/errors.tsx @@ -0,0 +1,29 @@ +import { Command } from "commander" +import { validatePackage } from "dpkit" +import React from "react" +import { ReportGrid } from "../../components/ReportGrid.tsx" +import { helpConfiguration } from "../../helpers/help.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const errorsPackageCommand = new Command("errors") + .configureHelp(helpConfiguration) + .description("Show errors for a data package from a local or remote path") + + .addArgument(params.positionalDescriptorPath) + .addOption(params.json) + + .action(async (path, options) => { + const session = Session.create({ + title: "Package errors", + json: options.json, + }) + + const report = await session.task("Finding errors", validatePackage(path)) + + if (report.valid) { + session.success("Package is valid") + } + + session.render(report, ) + }) diff --git a/cli/commands/package/index.ts b/cli/commands/package/index.ts new file mode 100644 index 00000000..ec040217 --- /dev/null +++ b/cli/commands/package/index.ts @@ -0,0 +1,21 @@ +import { Command } from "commander" +import { helpConfiguration } from "../../helpers/help.ts" +import { archivePackageCommand } from "./archive.ts" +import { copyPackageCommand } from "./copy.ts" +import { errorsPackageCommand } from "./errors.tsx" +import { inferPackageCommand } from "./infer.tsx" +import { publishPackageCommand } from "./publish/index.ts" +import { showPackageCommand } from "./show.tsx" +import { validatePackageCommand } from "./validate.tsx" + +export const packageCommand = new Command("package") + .configureHelp(helpConfiguration) + .description("Data Package related commands") + + .addCommand(archivePackageCommand) + .addCommand(copyPackageCommand) + .addCommand(errorsPackageCommand) + .addCommand(inferPackageCommand) + .addCommand(showPackageCommand) + .addCommand(publishPackageCommand) + .addCommand(validatePackageCommand) diff --git a/cli/commands/package/infer.tsx b/cli/commands/package/infer.tsx new file mode 100644 index 00000000..9a1d21a0 --- /dev/null +++ b/cli/commands/package/infer.tsx @@ -0,0 +1,35 @@ +import { Command } from "commander" +import { inferPackage } from "dpkit" +import React from "react" +import { PackageGrid } from "../../components/PackageGrid.tsx" +import { helpConfiguration } from "../../helpers/help.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const inferPackageCommand = new Command("infer") + .configureHelp(helpConfiguration) + .description("Infer a data package from local or remote file paths") + + .addArgument(params.positionalFilePaths) + .addOption(params.json) + + .action(async (paths, options) => { + const session = Session.create({ + title: "Infer resource", + json: options.json, + }) + + const sourcePackage = { + resources: paths.map(path => ({ path })), + } + + const targetPackage = await session.task( + "Inferring package", + inferPackage(sourcePackage), + ) + + await session.render( + targetPackage, + , + ) + }) diff --git a/cli/commands/package/load.ts b/cli/commands/package/load.ts deleted file mode 100644 index ce6c21b0..00000000 --- a/cli/commands/package/load.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { Command } from "@oclif/core" -import { loadPackage } from "dpkit" -import * as options from "../../options/index.ts" -import * as params from "../../params/index.ts" - -export default class LoadPackage extends Command { - static override description = "Load a Data Package descriptor" - - static override args = { - path: params.requriedDescriptorPath, - } - - static override flags = { - json: options.json, - } - - public async run() { - const { args, flags } = await this.parse(LoadPackage) - - const dp = await loadPackage(args.path) - - if (flags.json) { - this.logJson(dp) - return - } - - console.log(dp) - } -} diff --git a/cli/commands/package/publish/ckan.ts b/cli/commands/package/publish/ckan.ts new file mode 100644 index 00000000..19c14ddc --- /dev/null +++ b/cli/commands/package/publish/ckan.ts @@ -0,0 +1,59 @@ +import { Option } from "commander" +import { Command } from "commander" +import { loadPackage, savePackageToCkan } from "dpkit" +import { helpConfiguration } from "../../../helpers/help.ts" +import { Session } from "../../../helpers/session.ts" +import * as params from "../../../params/index.ts" + +export const toCkanApiKey = new Option( + "--to-api-key ", + "API key for CKAN API", +) + +export const toCkanUrl = new Option( + "--to-ckan-url ", + "Base CKAN url to publish to", +) + +export const toCkanOwnerOrg = new Option( + "--to-owner-org ", + "Owner organization for the CKAN dataset", +) + +export const toCkanDatasetName = new Option( + "--to-dataset-name ", + "Name for the CKAN dataset", +) + +export const ckanPublishPackageCommand = new Command("ckan") + .configureHelp(helpConfiguration) + .description("Publish a data package from a local or remote path to CKAN") + + .addArgument(params.positionalDescriptorPath) + .addOption(params.withRemote) + + .optionsGroup("CKAN") + .addOption(toCkanApiKey.makeOptionMandatory()) + .addOption(toCkanUrl.makeOptionMandatory()) + .addOption(toCkanOwnerOrg.makeOptionMandatory()) + .addOption(toCkanDatasetName.makeOptionMandatory()) + + .action(async (path, options) => { + const session = Session.create({ + title: "Publish package", + }) + + const dp = await session.task("Loading package", loadPackage(path)) + + const result = await session.task( + "Publishing package", + savePackageToCkan(dp, { + ckanUrl: options.toCkanUrl, + apiKey: options.toApiKey, + ownerOrg: options.toOwnerOrg, + datasetName: options.toDatasetName, + }), + ) + + session.success(`Package from "${path}" published to "${result.path}"`) + }) diff --git a/cli/commands/package/publish/github.ts b/cli/commands/package/publish/github.ts new file mode 100644 index 00000000..3117109a --- /dev/null +++ b/cli/commands/package/publish/github.ts @@ -0,0 +1,52 @@ +import { Option } from "commander" +import { Command } from "commander" +import { loadPackage, savePackageToGithub } from "dpkit" +import { helpConfiguration } from "../../../helpers/help.ts" +import { Session } from "../../../helpers/session.ts" +import * as params from "../../../params/index.ts" + +export const toGithubApiKey = new Option( + "--to-api-key ", + "API key for GitHub API", +) + +export const toGithubRepo = new Option( + "--to-repo ", + "GitHub repository name", +) + +export const toGithubOrg = new Option( + "--to-org ", + "GitHub organization (optional, defaults to user repositories)", +) + +export const githubPublishPackageCommand = new Command("github") + .configureHelp(helpConfiguration) + .description("Publish a data package from a local or remote path to GitHub") + + .addArgument(params.positionalDescriptorPath) + .addOption(params.withRemote) + + .optionsGroup("GitHub") + .addOption(toGithubApiKey.makeOptionMandatory()) + .addOption(toGithubRepo.makeOptionMandatory()) + .addOption(toGithubOrg) + + .action(async (path, options) => { + const session = Session.create({ + title: "Publish package", + }) + + const dp = await session.task("Loading package", loadPackage(path)) + + const result = await session.task( + "Publishing package", + savePackageToGithub(dp, { + apiKey: options.toApiKey, + repo: options.toRepo, + org: options.toOrg, + }), + ) + + session.success(`Package from "${path}" published to "${result.path}"`) + }) diff --git a/cli/commands/package/publish/index.ts b/cli/commands/package/publish/index.ts new file mode 100644 index 00000000..4a3311e7 --- /dev/null +++ b/cli/commands/package/publish/index.ts @@ -0,0 +1,13 @@ +import { Command } from "commander" +import { helpConfiguration } from "../../../helpers/help.ts" +import { ckanPublishPackageCommand } from "./ckan.ts" +import { githubPublishPackageCommand } from "./github.ts" +import { zenodoPublishPackageCommand } from "./zenodo.ts" + +export const publishPackageCommand = new Command("publish") + .configureHelp(helpConfiguration) + .description("Publish data packages to various platforms") + + .addCommand(ckanPublishPackageCommand) + .addCommand(githubPublishPackageCommand) + .addCommand(zenodoPublishPackageCommand) diff --git a/cli/commands/package/publish/zenodo.ts b/cli/commands/package/publish/zenodo.ts new file mode 100644 index 00000000..674f1a3c --- /dev/null +++ b/cli/commands/package/publish/zenodo.ts @@ -0,0 +1,45 @@ +import { Option } from "commander" +import { Command } from "commander" +import { loadPackage, savePackageToZenodo } from "dpkit" +import { helpConfiguration } from "../../../helpers/help.ts" +import { Session } from "../../../helpers/session.ts" +import * as params from "../../../params/index.ts" + +export const toZenodoApiKey = new Option( + "--to-api-key ", + "API key for Zenodo API", +) + +export const toZenodoSandbox = new Option( + "--to-sandbox", + "Use Zenodo sandbox environment", +).default(false) + +export const zenodoPublishPackageCommand = new Command("zenodo") + .configureHelp(helpConfiguration) + .description("Publish a data package from a local or remote path to Zenodo") + + .addArgument(params.positionalDescriptorPath) + .addOption(params.withRemote) + + .optionsGroup("Zenodo") + .addOption(toZenodoApiKey.makeOptionMandatory()) + .addOption(toZenodoSandbox) + + .action(async (path, options) => { + const session = Session.create({ + title: "Publish package", + }) + + const dp = await session.task("Loading package", loadPackage(path)) + + const result = await session.task( + "Publishing package", + savePackageToZenodo(dp, { + apiKey: options.toApiKey, + sandbox: options.toSandbox, + }), + ) + + session.success(`Package from "${path}" published to "${result.path}"`) + }) diff --git a/cli/commands/package/show.tsx b/cli/commands/package/show.tsx new file mode 100644 index 00000000..8bedc204 --- /dev/null +++ b/cli/commands/package/show.tsx @@ -0,0 +1,25 @@ +import { Command } from "commander" +import { loadPackage } from "dpkit" +import React from "react" +import { PackageGrid } from "../../components/PackageGrid.tsx" +import { helpConfiguration } from "../../helpers/help.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const showPackageCommand = new Command("show") + .configureHelp(helpConfiguration) + .description("Show a Data Package descriptor") + + .addArgument(params.positionalDescriptorPath) + .addOption(params.json) + + .action(async (path, options) => { + const session = Session.create({ + title: "Show package", + json: options.json, + }) + + const dp = await session.task("Loading package", loadPackage(path)) + + await session.render(dp, ) + }) diff --git a/cli/commands/package/validate.tsx b/cli/commands/package/validate.tsx new file mode 100644 index 00000000..9f0bf413 --- /dev/null +++ b/cli/commands/package/validate.tsx @@ -0,0 +1,35 @@ +import { Command } from "commander" +import { validatePackage } from "dpkit" +import React from "react" +import { ReportGrid } from "../../components/ReportGrid.tsx" +import { helpConfiguration } from "../../helpers/help.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const validatePackageCommand = new Command("validate") + .configureHelp(helpConfiguration) + .description("Validate a data package from a local or remote path") + + .addArgument(params.positionalDescriptorPath) + .addOption(params.json) + + .action(async (path, options) => { + const session = Session.create({ + title: "Validate package", + json: options.json, + }) + + const report = await session.task( + "Validating package", + validatePackage(path), + ) + + if (report.valid) { + session.success("Package is valid") + } + + session.render( + report, + , + ) + }) diff --git a/cli/commands/resource/.keep b/cli/commands/resource/.keep deleted file mode 100644 index e69de29b..00000000 diff --git a/cli/commands/resource/errors.tsx b/cli/commands/resource/errors.tsx new file mode 100644 index 00000000..5ad7da42 --- /dev/null +++ b/cli/commands/resource/errors.tsx @@ -0,0 +1,37 @@ +import { Command } from "commander" +import { validateResource } from "dpkit" +import React from "react" +import { ReportGrid } from "../../components/ReportGrid.tsx" +import { helpConfiguration } from "../../helpers/help.ts" +import { selectResource } from "../../helpers/resource.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const errorsResourceCommand = new Command("errors") + .configureHelp(helpConfiguration) + .description("Show errors for a data resource from a local or remote path") + + .addArgument(params.optionalPositionalDescriptorPath) + .addOption(params.fromPackage) + .addOption(params.fromResource) + .addOption(params.json) + + .action(async (path, options) => { + const session = Session.create({ + title: "Resource errors", + json: options.json, + }) + + const descriptor = path ? path : await selectResource(session, options) + + const report = await session.task( + "Finding errors", + validateResource(descriptor), + ) + + if (report.valid) { + session.success("Resource is valid") + } + + session.render(report, ) + }) diff --git a/cli/commands/resource/index.ts b/cli/commands/resource/index.ts new file mode 100644 index 00000000..c2e66d69 --- /dev/null +++ b/cli/commands/resource/index.ts @@ -0,0 +1,15 @@ +import { Command } from "commander" +import { helpConfiguration } from "../../helpers/help.ts" +import { errorsResourceCommand } from "./errors.tsx" +import { inferResourceCommand } from "./infer.tsx" +import { showResourceCommand } from "./show.tsx" +import { validateResourceCommand } from "./validate.tsx" + +export const resourceCommand = new Command("resource") + .configureHelp(helpConfiguration) + .description("Data Resource related commands") + + .addCommand(errorsResourceCommand) + .addCommand(inferResourceCommand) + .addCommand(showResourceCommand) + .addCommand(validateResourceCommand) diff --git a/cli/commands/resource/infer.tsx b/cli/commands/resource/infer.tsx new file mode 100644 index 00000000..e2f35cd9 --- /dev/null +++ b/cli/commands/resource/infer.tsx @@ -0,0 +1,42 @@ +import { Command } from "commander" +import { inferResource } from "dpkit" +import React from "react" +import { ResourceGrid } from "../../components/ResourceGrid.tsx" +import { helpConfiguration } from "../../helpers/help.ts" +import { isEmptyObject } from "../../helpers/object.ts" +import { selectResource } from "../../helpers/resource.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const inferResourceCommand = new Command("infer") + .configureHelp(helpConfiguration) + .description("Infer a data resource from a table") + + .addArgument(params.positionalTablePath) + .addOption(params.fromPackage) + .addOption(params.fromResource) + .addOption(params.json) + + .action(async (path, options) => { + const session = Session.create({ + title: "Infer resource", + json: options.json, + }) + + const resource = path ? { path } : await selectResource(session, options) + + const result = await session.task( + "Inferring resource", + inferResource(resource), + ) + + if (isEmptyObject(result)) { + Session.terminate("Could not infer resource") + } + + await session.render( + result, + // @ts-ignore + , + ) + }) diff --git a/cli/commands/resource/show.tsx b/cli/commands/resource/show.tsx new file mode 100644 index 00000000..43b336c4 --- /dev/null +++ b/cli/commands/resource/show.tsx @@ -0,0 +1,44 @@ +import { Command } from "commander" +import { loadResourceDescriptor } from "dpkit" +import type { Resource } from "dpkit" +import React from "react" +import { ResourceGrid } from "../../components/ResourceGrid.tsx" +import { helpConfiguration } from "../../helpers/help.ts" +import { isEmptyObject } from "../../helpers/object.ts" +import { selectResource } from "../../helpers/resource.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const showResourceCommand = new Command("show") + .configureHelp(helpConfiguration) + .description("Show a data resource from a local or remote path") + + .addArgument(params.positionalTablePath) + .addOption(params.fromPackage) + .addOption(params.fromResource) + .addOption(params.json) + + .action(async (path, options) => { + const session = Session.create({ + title: "Show resource", + json: options.json, + }) + + let resource: Resource | undefined + + if (!path) { + resource = await selectResource(session, options) + } else { + // @ts-ignore + resource = await session.task( + "Loading resource", + loadResourceDescriptor(path), + ) + } + + if (isEmptyObject(resource)) { + Session.terminate("Resource is not available") + } + + await session.render(resource, ) + }) diff --git a/cli/commands/resource/validate.tsx b/cli/commands/resource/validate.tsx new file mode 100644 index 00000000..b3ae7795 --- /dev/null +++ b/cli/commands/resource/validate.tsx @@ -0,0 +1,37 @@ +import { Command } from "commander" +import { validateResource } from "dpkit" +import React from "react" +import { ReportGrid } from "../../components/ReportGrid.tsx" +import { helpConfiguration } from "../../helpers/help.ts" +import { selectResource } from "../../helpers/resource.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const validateResourceCommand = new Command("validate") + .configureHelp(helpConfiguration) + .description("Validate a data resource from a local or remote path") + + .addArgument(params.optionalPositionalDescriptorPath) + .addOption(params.fromPackage) + .addOption(params.fromResource) + .addOption(params.json) + + .action(async (path, options) => { + const session = Session.create({ + title: "Validate resource", + json: options.json, + }) + + const descriptor = path ? path : await selectResource(session, options) + + const report = await session.task( + "Validating resource", + validateResource(descriptor), + ) + + if (report.valid) { + session.success("Resource is valid") + } + + session.render(report, ) + }) diff --git a/cli/commands/schema/.keep b/cli/commands/schema/.keep deleted file mode 100644 index e69de29b..00000000 diff --git a/cli/commands/schema/errors.tsx b/cli/commands/schema/errors.tsx new file mode 100644 index 00000000..5955e83c --- /dev/null +++ b/cli/commands/schema/errors.tsx @@ -0,0 +1,62 @@ +import { Command } from "commander" +import { loadDescriptor, validateSchema } from "dpkit" +import type { Descriptor } from "dpkit" +import React from "react" +import { ReportGrid } from "../../components/ReportGrid.tsx" +import { helpConfiguration } from "../../helpers/help.ts" +import { selectResource } from "../../helpers/resource.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const errorsSchemaCommand = new Command("errors") + .configureHelp(helpConfiguration) + .description("Show errors for a table schema from a local or remote path") + + .addArgument(params.positionalTablePath) + .addOption(params.fromPackage) + .addOption(params.fromResource) + .addOption(params.json) + + .action(async (path, options) => { + const session = Session.create({ + title: "Schema errors", + json: options.json, + }) + + let descriptor: Descriptor | undefined + + if (!path) { + const resource = await selectResource(session, options) + + if (!resource.schema) { + Session.terminate("Schema is not available") + } + + if (typeof resource.schema !== "string") { + descriptor = resource.schema as unknown as Descriptor + } else { + path = resource.schema + } + } + + if (!descriptor) { + const result = await session.task( + "Loading schema", + // @ts-ignore + loadDescriptor(path), + ) + + descriptor = result.descriptor + } + + const report = await session.task( + "Validating schema", + validateSchema(descriptor), + ) + + if (report.valid) { + session.success("Schema is valid") + } + + session.render(report, ) + }) diff --git a/cli/commands/schema/index.ts b/cli/commands/schema/index.ts new file mode 100644 index 00000000..200f757c --- /dev/null +++ b/cli/commands/schema/index.ts @@ -0,0 +1,15 @@ +import { Command } from "commander" +import { helpConfiguration } from "../../helpers/help.ts" +import { errorsSchemaCommand } from "./errors.tsx" +import { inferSchemaCommand } from "./infer.tsx" +import { showSchemaCommand } from "./show.tsx" +import { validateSchemaCommand } from "./validate.tsx" + +export const schemaCommand = new Command("schema") + .configureHelp(helpConfiguration) + .description("Table Schema related commands") + + .addCommand(errorsSchemaCommand) + .addCommand(inferSchemaCommand) + .addCommand(showSchemaCommand) + .addCommand(validateSchemaCommand) diff --git a/cli/commands/schema/infer.tsx b/cli/commands/schema/infer.tsx new file mode 100644 index 00000000..a6680aaf --- /dev/null +++ b/cli/commands/schema/infer.tsx @@ -0,0 +1,44 @@ +import { Command } from "commander" +import { inferSchema, loadTable } from "dpkit" +import React from "react" +import { SchemaGrid } from "../../components/SchemaGrid.tsx" +import { helpConfiguration } from "../../helpers/help.ts" +import { isEmptyObject } from "../../helpers/object.ts" +import { selectResource } from "../../helpers/resource.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const inferSchemaCommand = new Command("infer") + .configureHelp(helpConfiguration) + .description("Infer a table schema from a table") + + .addArgument(params.positionalTablePath) + .addOption(params.fromPackage) + .addOption(params.fromResource) + .addOption(params.json) + + .action(async (path, options) => { + const session = Session.create({ + title: "Infer schema", + json: options.json, + }) + + const resource = path ? { path } : await selectResource(session, options) + + const table = await session.task("Loading table", loadTable(resource)) + + const inferredSchema = await session.task( + "Inferring schema", + inferSchema(table), + ) + + if (isEmptyObject(inferredSchema)) { + Session.terminate("Could not infer schema") + } + + await session.render( + inferredSchema, + // @ts-ignore + , + ) + }) diff --git a/cli/commands/schema/show.tsx b/cli/commands/schema/show.tsx new file mode 100644 index 00000000..50963339 --- /dev/null +++ b/cli/commands/schema/show.tsx @@ -0,0 +1,52 @@ +import { Command } from "commander" +import { loadSchema } from "dpkit" +import type { Schema } from "dpkit" +import React from "react" +import { SchemaGrid } from "../../components/SchemaGrid.tsx" +import { helpConfiguration } from "../../helpers/help.ts" +import { isEmptyObject } from "../../helpers/object.ts" +import { selectResource } from "../../helpers/resource.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const showSchemaCommand = new Command("show") + .configureHelp(helpConfiguration) + .description("Show a table schema from a local or remote path") + + .addArgument(params.positionalTablePath) + .addOption(params.fromPackage) + .addOption(params.fromResource) + .addOption(params.json) + + .action(async (path, options) => { + const session = Session.create({ + title: "Show schema", + json: options.json, + }) + + let schema: Schema | undefined + + if (!path) { + const resource = await selectResource(session, options) + if (!resource.schema) { + Session.terminate("Schema is not available") + } + + if (typeof resource.schema !== "string") { + schema = resource.schema + } else { + path = resource.schema + } + } + + if (!schema) { + // @ts-ignore + schema = await session.task("Loading schema", loadSchema(path)) + } + + if (isEmptyObject(schema)) { + Session.terminate("Schema is not available") + } + + await session.render(schema, ) + }) diff --git a/cli/commands/schema/validate.tsx b/cli/commands/schema/validate.tsx new file mode 100644 index 00000000..e8d773c2 --- /dev/null +++ b/cli/commands/schema/validate.tsx @@ -0,0 +1,63 @@ +import { Command } from "commander" +import { loadDescriptor, validateSchema } from "dpkit" +import type { Descriptor } from "dpkit" +import React from "react" +import { ReportGrid } from "../../components/ReportGrid.tsx" +import { helpConfiguration } from "../../helpers/help.ts" +import { selectResource } from "../../helpers/resource.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const validateSchemaCommand = new Command("validate") + .configureHelp(helpConfiguration) + .description("Validate a table schema from a local or remote path") + + .addArgument(params.positionalTablePath) + .addOption(params.fromPackage) + .addOption(params.fromResource) + .addOption(params.json) + + .action(async (path, options) => { + const session = Session.create({ + title: "Validate schema", + json: options.json, + }) + + let descriptor: Descriptor | undefined + + if (!path) { + const resource = await selectResource(session, options) + + if (!resource.schema) { + Session.terminate("Schema is not available") + } + + if (typeof resource.schema !== "string") { + descriptor = resource.schema as unknown as Descriptor + } else { + path = resource.schema + } + } + + if (!descriptor) { + const result = await session.task( + "Loading descriptor", + // @ts-ignore + loadDescriptor(path), + ) + + descriptor = result.descriptor + } + + const report = await session.task( + "Validating descriptor", + // @ts-ignore + validateSchema(descriptor), + ) + + if (report.valid) { + session.success("Schema is valid") + } + + session.render(report, ) + }) diff --git a/cli/commands/table/convert.tsx b/cli/commands/table/convert.tsx index cfc29fe2..6a6a4e7d 100644 --- a/cli/commands/table/convert.tsx +++ b/cli/commands/table/convert.tsx @@ -1,46 +1,91 @@ -import { Command } from "@oclif/core" +import { Command } from "commander" import { getTempFilePath, loadFile } from "dpkit" import { readTable, saveTable } from "dpkit" -import * as options from "../../options/index.ts" +import { createDialectFromOptions } from "../../helpers/dialect.ts" +import { createToDialectFromOptions } from "../../helpers/dialect.ts" +import { helpConfiguration } from "../../helpers/help.ts" +import { selectResource } from "../../helpers/resource.ts" +import { Session } from "../../helpers/session.ts" import * as params from "../../params/index.ts" -export default class ConvertTable extends Command { - static override description = - "Convert a table from a local or remote source path to a target path" - - static override args = { - path: params.requriedTablePath, - } - - // TODO: support toDialectOptions - static override flags = { - toPath: options.toPath, - toFormat: options.toFormat, - ...options.dialectOptions, - ...options.toDialectOptions, - } - - public async run() { - const { args, flags } = await this.parse(ConvertTable) - - const dialect = options.createDialectFromFlags(flags) - const table = await readTable({ path: args.path, dialect }) - - const toPath = flags.toPath ?? getTempFilePath() - const toDialect = options.createToDialectFromFlags(flags) - await saveTable(table, { - path: toPath, - format: flags.toFormat, - dialect: toDialect, +export const convertTableCommand = new Command("convert") + .configureHelp(helpConfiguration) + .description( + "Convert a table from a local or remote source path to a target path", + ) + + .addArgument(params.positionalTablePath) + .addOption(params.fromPackage) + .addOption(params.fromResource) + .addOption(params.toPath) + .addOption(params.toFormat) + + .optionsGroup("Table Dialect") + .addOption(params.delimiter) + .addOption(params.header) + .addOption(params.headerRows) + .addOption(params.headerJoin) + .addOption(params.commentRows) + .addOption(params.commentChar) + .addOption(params.quoteChar) + .addOption(params.doubleQuote) + .addOption(params.escapeChar) + .addOption(params.nullSequence) + .addOption(params.skipInitialSpace) + .addOption(params.property) + .addOption(params.itemType) + .addOption(params.itemKeys) + .addOption(params.sheetNumber) + .addOption(params.sheetName) + .addOption(params.table) + + .optionsGroup("Table Dialect (output)") + .addOption(params.toDelimiter) + .addOption(params.toHeader) + .addOption(params.toHeaderRows) + .addOption(params.toHeaderJoin) + .addOption(params.toCommentRows) + .addOption(params.toCommentChar) + .addOption(params.toQuoteChar) + .addOption(params.toDoubleQuote) + .addOption(params.toEscapeChar) + .addOption(params.toNullSequence) + .addOption(params.toSkipInitialSpace) + .addOption(params.toProperty) + .addOption(params.toItemType) + .addOption(params.toItemKeys) + .addOption(params.toSheetNumber) + .addOption(params.toSheetName) + .addOption(params.toTable) + + .action(async (path, options) => { + const session = Session.create({ + title: "Table errors", }) - if (!flags.toPath) { - // TODO: stream to stdout + const resource = path + ? { path, dialect: createDialectFromOptions(options) } + : await selectResource(session, options) + + const table = await session.task("Loading table", readTable(resource)) + + const toPath = options.toPath ?? getTempFilePath() + const toDialect = createToDialectFromOptions(options) + + await session.task( + "Saving table", + saveTable(table, { + path: toPath, + format: options.toFormat, + dialect: toDialect, + }), + ) + + if (!options.toPath) { const buffer = await loadFile(toPath) - this.log(buffer.toString().trim()) + console.log(buffer.toString().trim()) return } - this.log(`Converted table from ${args.path} to ${flags.toPath}`) - } -} + console.log(`Converted table from ${path} to ${options.toPath}`) + }) diff --git a/cli/commands/table/describe.tsx b/cli/commands/table/describe.tsx deleted file mode 100644 index 4e7bc690..00000000 --- a/cli/commands/table/describe.tsx +++ /dev/null @@ -1,37 +0,0 @@ -import { Command } from "@oclif/core" -import { readTable } from "dpkit" -import { render } from "ink" -import React from "react" -import { TableGrid } from "../../components/TableGrid.tsx" -import * as options from "../../options/index.ts" -import * as params from "../../params/index.ts" - -export default class DescribeTable extends Command { - static override description = "Describe a table from a local or remote path" - - static override args = { - path: params.requriedTablePath, - } - - static override flags = { - ...options.dialectOptions, - json: options.json, - } - - public async run() { - const { args, flags } = await this.parse(DescribeTable) - - const dialect = options.createDialectFromFlags(flags) - const table = await readTable({ path: args.path, dialect }) - - const df = await table.collect() - const stats = df.describe() - - if (flags.json) { - this.logJson(stats) - return - } - - render() - } -} diff --git a/cli/commands/table/errors.tsx b/cli/commands/table/errors.tsx new file mode 100644 index 00000000..abdefeed --- /dev/null +++ b/cli/commands/table/errors.tsx @@ -0,0 +1,56 @@ +import { Command } from "commander" +import { validateTable } from "dpkit" +import React from "react" +import { ReportGrid } from "../../components/ReportGrid.tsx" +import { createDialectFromOptions } from "../../helpers/dialect.ts" +import { helpConfiguration } from "../../helpers/help.ts" +import { selectResource } from "../../helpers/resource.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const errorsTableCommand = new Command("errors") + .configureHelp(helpConfiguration) + .description("Validate a table from a local or remote path") + + .addArgument(params.positionalTablePath) + .addOption(params.fromPackage) + .addOption(params.fromResource) + .addOption(params.json) + + .optionsGroup("Table Dialect") + .addOption(params.delimiter) + .addOption(params.header) + .addOption(params.headerRows) + .addOption(params.headerJoin) + .addOption(params.commentRows) + .addOption(params.commentChar) + .addOption(params.quoteChar) + .addOption(params.doubleQuote) + .addOption(params.escapeChar) + .addOption(params.nullSequence) + .addOption(params.skipInitialSpace) + .addOption(params.property) + .addOption(params.itemType) + .addOption(params.itemKeys) + .addOption(params.sheetNumber) + .addOption(params.sheetName) + .addOption(params.table) + + .action(async (path, options) => { + const session = Session.create({ + title: "Table errors", + json: options.json, + }) + + const resource = path + ? { path, dialect: createDialectFromOptions(options) } + : await selectResource(session, options) + + const report = await session.task("Finding errors", validateTable(resource)) + + if (report.valid) { + session.success("Table is valid") + } + + session.render(report, ) + }) diff --git a/cli/commands/table/explore.tsx b/cli/commands/table/explore.tsx index 76bc38ed..32dc70d8 100644 --- a/cli/commands/table/explore.tsx +++ b/cli/commands/table/explore.tsx @@ -1,35 +1,49 @@ -import { Command } from "@oclif/core" +import { Command } from "commander" import { readTable } from "dpkit" -import { render } from "ink" import React from "react" import { TableGrid } from "../../components/TableGrid.tsx" -import * as options from "../../options/index.ts" +import { createDialectFromOptions } from "../../helpers/dialect.ts" +import { helpConfiguration } from "../../helpers/help.ts" +import { selectResource } from "../../helpers/resource.ts" +import { Session } from "../../helpers/session.ts" import * as params from "../../params/index.ts" -export default class ExploreTable extends Command { - static override description = "Explore a table from a local or remote path" +export const exploreTableCommand = new Command("explore") + .configureHelp(helpConfiguration) + .description("Explore a table from a local or remote path") - static override args = { - path: params.requriedTablePath, - } + .addArgument(params.positionalTablePath) + .addOption(params.fromPackage) + .addOption(params.fromResource) - static override flags = { - ...options.dialectOptions, - } + .optionsGroup("Table Dialect") + .addOption(params.delimiter) + .addOption(params.header) + .addOption(params.headerRows) + .addOption(params.headerJoin) + .addOption(params.commentRows) + .addOption(params.commentChar) + .addOption(params.quoteChar) + .addOption(params.doubleQuote) + .addOption(params.escapeChar) + .addOption(params.nullSequence) + .addOption(params.skipInitialSpace) + .addOption(params.property) + .addOption(params.itemType) + .addOption(params.itemKeys) + .addOption(params.sheetNumber) + .addOption(params.sheetName) + .addOption(params.table) - public async run() { - const { args, flags } = await this.parse(ExploreTable) + .action(async (path, options) => { + const session = Session.create({ + title: "Explore table", + }) - const dialect = options.createDialectFromFlags(flags) - const table = await readTable({ path: args.path, dialect }) + const resource = path + ? { path, dialect: createDialectFromOptions(options) } + : await selectResource(session, options) - if (flags.json) { - const df = await table.slice(0, 10).collect() - const data = df.toRecords() - this.logJson(data) - return - } - - render() - } -} + const table = await session.task("Loading table", readTable(resource)) + await session.render(table, ) + }) diff --git a/cli/commands/table/index.ts b/cli/commands/table/index.ts new file mode 100644 index 00000000..a56294e3 --- /dev/null +++ b/cli/commands/table/index.ts @@ -0,0 +1,21 @@ +import { Command } from "commander" +import { helpConfiguration } from "../../helpers/help.ts" +import { convertTableCommand } from "./convert.tsx" +import { errorsTableCommand } from "./errors.tsx" +import { exploreTableCommand } from "./explore.tsx" +import { queryTableCommand } from "./query.tsx" +import { scriptTableCommand } from "./script.tsx" +import { statsTableCommand } from "./stats.tsx" +import { validateTableCommand } from "./validate.tsx" + +export const tableCommand = new Command("table") + .configureHelp(helpConfiguration) + .description("Table related commands") + + .addCommand(convertTableCommand) + .addCommand(errorsTableCommand) + .addCommand(exploreTableCommand) + .addCommand(queryTableCommand) + .addCommand(scriptTableCommand) + .addCommand(statsTableCommand) + .addCommand(validateTableCommand) diff --git a/cli/commands/table/query.tsx b/cli/commands/table/query.tsx index acd278a9..6886c660 100644 --- a/cli/commands/table/query.tsx +++ b/cli/commands/table/query.tsx @@ -1,22 +1,39 @@ -import { Command } from "@oclif/core" -//import { readTable } from "dpkit" -import * as options from "../../options/index.ts" +import { Command } from "commander" +import { helpConfiguration } from "../../helpers/help.ts" +import { Session } from "../../helpers/session.ts" import * as params from "../../params/index.ts" -export default class QueryTable extends Command { - static override description = - "Start a querying session for a table from a local or remote path" +export const queryTableCommand = new Command("query") + .configureHelp(helpConfiguration) + .description( + "Start a querying session for a table from a local or remote path", + ) + .addArgument(params.positionalTablePath) - static override args = { - path: params.requriedTablePath, - } + .optionsGroup("Table Dialect") + .addOption(params.delimiter) + .addOption(params.header) + .addOption(params.headerRows) + .addOption(params.headerJoin) + .addOption(params.commentRows) + .addOption(params.commentChar) + .addOption(params.quoteChar) + .addOption(params.doubleQuote) + .addOption(params.escapeChar) + .addOption(params.nullSequence) + .addOption(params.skipInitialSpace) + .addOption(params.property) + .addOption(params.itemType) + .addOption(params.itemKeys) + .addOption(params.sheetNumber) + .addOption(params.sheetName) + .addOption(params.table) - static override flags = { - ...options.dialectOptions, - } + .action(async (_path, _options) => { + // @ts-ignore + const session = Session.create({ + title: "Query table", + }) - public async run() { - //const { args, flags } = await this.parse(QueryTable) - throw new Error("Not implemented") - } -} + Session.terminate("Query command not implemented yet") + }) diff --git a/cli/commands/table/script.tsx b/cli/commands/table/script.tsx index 102dd237..d6df4b0f 100644 --- a/cli/commands/table/script.tsx +++ b/cli/commands/table/script.tsx @@ -1,28 +1,52 @@ import repl from "node:repl" -import { Command } from "@oclif/core" +import { Command } from "commander" import { readTable } from "dpkit" -import * as options from "../../options/index.ts" +import { createDialectFromOptions } from "../../helpers/dialect.ts" +import { helpConfiguration } from "../../helpers/help.ts" +import { selectResource } from "../../helpers/resource.ts" +import { Session } from "../../helpers/session.ts" import * as params from "../../params/index.ts" -export default class ScriptTable extends Command { - static override description = - "Start a scripting session for a table from a local or remote path" +export const scriptTableCommand = new Command("script") + .configureHelp(helpConfiguration) + .description( + "Start a scripting session for a table from a local or remote path", + ) - static override args = { - path: params.requriedTablePath, - } + .addArgument(params.positionalTablePath) + .addOption(params.fromPackage) + .addOption(params.fromResource) - static override flags = { - ...options.dialectOptions, - } + .optionsGroup("Table Dialect") + .addOption(params.delimiter) + .addOption(params.header) + .addOption(params.headerRows) + .addOption(params.headerJoin) + .addOption(params.commentRows) + .addOption(params.commentChar) + .addOption(params.quoteChar) + .addOption(params.doubleQuote) + .addOption(params.escapeChar) + .addOption(params.nullSequence) + .addOption(params.skipInitialSpace) + .addOption(params.property) + .addOption(params.itemType) + .addOption(params.itemKeys) + .addOption(params.sheetNumber) + .addOption(params.sheetName) + .addOption(params.table) - public async run() { - const { args, flags } = await this.parse(ScriptTable) + .action(async (path, options) => { + const session = Session.create({ + title: "Explore table", + }) - const dialect = options.createDialectFromFlags(flags) - const table = await readTable({ path: args.path, dialect }) + const resource = path + ? { path, dialect: createDialectFromOptions(options) } + : await selectResource(session, options) - const session = repl.start({ prompt: "dp> " }) - session.context.table = table - } -} + const table = await session.task("Loading table", readTable(resource)) + + const replSession = repl.start({ prompt: "dp> " }) + replSession.context.table = table + }) diff --git a/cli/commands/table/stats.tsx b/cli/commands/table/stats.tsx new file mode 100644 index 00000000..e85a5827 --- /dev/null +++ b/cli/commands/table/stats.tsx @@ -0,0 +1,54 @@ +import { Command } from "commander" +import { readTable } from "dpkit" +import React from "react" +import { DataGrid } from "../../components/DataGrid.tsx" +import { createDialectFromOptions } from "../../helpers/dialect.ts" +import { helpConfiguration } from "../../helpers/help.ts" +import { selectResource } from "../../helpers/resource.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +export const statsTableCommand = new Command("stats") + .configureHelp(helpConfiguration) + .description("Show stats for a table from a local or remote path") + + .addArgument(params.positionalTablePath) + .addOption(params.fromPackage) + .addOption(params.fromResource) + .addOption(params.json) + + .optionsGroup("Table Dialect") + .addOption(params.delimiter) + .addOption(params.header) + .addOption(params.headerRows) + .addOption(params.headerJoin) + .addOption(params.commentRows) + .addOption(params.commentChar) + .addOption(params.quoteChar) + .addOption(params.doubleQuote) + .addOption(params.escapeChar) + .addOption(params.nullSequence) + .addOption(params.skipInitialSpace) + .addOption(params.property) + .addOption(params.itemType) + .addOption(params.itemKeys) + .addOption(params.sheetNumber) + .addOption(params.sheetName) + .addOption(params.table) + .action(async (path, options) => { + const session = Session.create({ + title: "Table stats", + json: options.json, + }) + + const resource = path + ? { path, dialect: createDialectFromOptions(options) } + : await selectResource(session, options) + + const table = await session.task("Loading table", readTable(resource)) + const df = await session.task("Calculating stats", table.collect()) + + const stats = df.describe().toRecords() + + session.render(stats, ) + }) diff --git a/cli/commands/table/validate.tsx b/cli/commands/table/validate.tsx index d3571d8f..c2323cad 100644 --- a/cli/commands/table/validate.tsx +++ b/cli/commands/table/validate.tsx @@ -1,46 +1,40 @@ -import { Command } from "@oclif/core" +import { Command } from "commander" import { validateTable } from "dpkit" -import { render } from "ink" import React from "react" -import { ErrorGrid } from "../../components/ErrorGrid.tsx" -import * as options from "../../options/index.ts" +import { ReportGrid } from "../../components/ReportGrid.tsx" +import { createDialectFromOptions } from "../../helpers/dialect.ts" +import { helpConfiguration } from "../../helpers/help.ts" +import { selectResource } from "../../helpers/resource.ts" +import { Session } from "../../helpers/session.ts" import * as params from "../../params/index.ts" -export default class ExploreTable extends Command { - static override description = "Explore a table from a local or remote path" +export const validateTableCommand = new Command("validate") + .configureHelp(helpConfiguration) + .description("Validate a table from a local or remote path") - static override args = { - path: params.requriedTablePath, - } + .addArgument(params.positionalTablePath) + .addOption(params.fromPackage) + .addOption(params.fromResource) + .addOption(params.json) - static override flags = { - ...options.dialectOptions, - json: options.json, - } + .action(async (path, options) => { + const session = Session.create({ + title: "Validate Table", + json: options.json, + }) - public async run() { - const { args, flags } = await this.parse(ExploreTable) + const resource = path + ? { path, dialect: createDialectFromOptions(options) } + : await selectResource(session, options) - const dialect = options.createDialectFromFlags(flags) - const { errors } = await validateTable({ path: args.path, dialect }) + const report = await session.task( + "Validating table", + validateTable(resource), + ) - if (flags.json) { - this.logJson(errors) - return + if (report.valid) { + session.success("Table is valid") } - render( - , - ) - } -} + session.render(report, ) + }) diff --git a/cli/components/DataGrid.tsx b/cli/components/DataGrid.tsx index 398da95a..c504248f 100644 --- a/cli/components/DataGrid.tsx +++ b/cli/components/DataGrid.tsx @@ -1,6 +1,9 @@ import { Box, Text } from "ink" import React from "react" +// TODO: Accept column names as an option +// TODO: Autocalculate geometry (e.g. row height etc) + const MIN_COLUMN_WIDTH = 15 export type Order = { col: number; dir: "asc" | "desc" } @@ -8,10 +11,26 @@ export function DataGrid(props: { data: Record[] col?: number order?: Order + rowHeight?: number + borderColor?: "green" | "red" }) { - const { data, col, order } = props + const { data, col, order, rowHeight, borderColor = "green" } = props + + // TODO: fix $schema related cludge + const colNames = Object.keys(data[0] ?? {}).filter(name => name !== "$schema") + + // TODO: fix type related cludge + if (colNames.includes("type")) { + colNames.splice(colNames.indexOf("type"), 1) + colNames.unshift("type") + } + + // TODO: fix resource related cludge + if (colNames.includes("resource")) { + colNames.splice(colNames.indexOf("resource"), 1) + colNames.unshift("resource") + } - const colNames = Object.keys(data[0] ?? {}) const colWidth = Math.min( process.stdout.columns / colNames.length, MIN_COLUMN_WIDTH, @@ -27,7 +46,7 @@ export function DataGrid(props: { @@ -64,7 +83,7 @@ export function DataGrid(props: { ? "#444" : undefined } - height={2} + height={rowHeight} overflow="hidden" > {(row[name] ?? "").toString()} diff --git a/cli/components/DialectGrid.tsx b/cli/components/DialectGrid.tsx new file mode 100644 index 00000000..8995c4b7 --- /dev/null +++ b/cli/components/DialectGrid.tsx @@ -0,0 +1,11 @@ +import type { Dialect } from "dpkit" +import React from "react" +import { DataGrid } from "./DataGrid.tsx" + +// TODO: Support non-visible chars like TAB and CR + +export function DialectGrid(props: { dialect: Dialect }) { + const data = [props.dialect] + + return +} diff --git a/cli/components/ErrorGrid.tsx b/cli/components/ErrorGrid.tsx deleted file mode 100644 index 71818bfc..00000000 --- a/cli/components/ErrorGrid.tsx +++ /dev/null @@ -1,38 +0,0 @@ -import type { TableError } from "dpkit" -import { Box } from "ink" -import SelectInput from "ink-select-input" -import { DataFrame } from "nodejs-polars" -import { useState } from "react" -import React from "react" -import { objectKeys } from "ts-extras" -import { TableGrid } from "./TableGrid.tsx" - -export function ErrorGrid(props: { errors: TableError[] }) { - const { errors } = props - const [errorType, setErrorType] = useState(errors[0]?.type ?? "") - - const errorsByType = Object.groupBy(errors, error => error.type) - // @ts-ignore - const selectErrors = errorsByType[errorType] - - const table = DataFrame(selectErrors).lazy() - - const handleSelect = async (item: any) => { - setErrorType(item.value) - } - - return ( - - - ({ - label: type, - value: type, - }))} - /> - - - - ) -} diff --git a/cli/components/PackageGrid.tsx b/cli/components/PackageGrid.tsx new file mode 100644 index 00000000..7c52f552 --- /dev/null +++ b/cli/components/PackageGrid.tsx @@ -0,0 +1,18 @@ +import type { Package } from "dpkit" +import React from "react" +import { DataGrid } from "./DataGrid.tsx" + +// TODO: Support showing other package/resource properties + +export function PackageGrid(props: { dataPackage: Package }) { + const data = [ + Object.fromEntries( + props.dataPackage.resources.map(resource => [ + resource.name, + resource.path, + ]), + ), + ] + + return +} diff --git a/cli/components/ReportGrid.tsx b/cli/components/ReportGrid.tsx new file mode 100644 index 00000000..b897a668 --- /dev/null +++ b/cli/components/ReportGrid.tsx @@ -0,0 +1,46 @@ +//import type { TableError } from "dpkit" +import { countBy } from "es-toolkit" +import { DataFrame } from "nodejs-polars" +import React from "react" +import { DataGrid } from "./DataGrid.tsx" +import { TableGrid } from "./TableGrid.tsx" + +// TODO: Improve implementation (esp. typing) + +export function ReportGrid(props: { + report: { valid: boolean; errors: Record[] } + groupBy?: "type" | "resource/type" +}) { + const { report, groupBy } = props + + if (report.valid) { + return null + } + + if (groupBy === "resource/type") { + const groups = countBy(report.errors, error => + [error.resource ?? "", error.type].join("/"), + ) + + const data = Object.entries(groups).map(([resourceType, count]) => ({ + resource: resourceType.split("/")[0], + type: resourceType.split("/")[1], + count, + })) + + return + } + + if (groupBy === "type") { + const groups = countBy(report.errors, error => error.type) + const data = Object.entries(groups).map(([error, count]) => ({ + error, + count, + })) + + return + } + + const table = DataFrame(report.errors).lazy() + return +} diff --git a/cli/components/ResourceGrid.tsx b/cli/components/ResourceGrid.tsx new file mode 100644 index 00000000..cb218225 --- /dev/null +++ b/cli/components/ResourceGrid.tsx @@ -0,0 +1,11 @@ +import type { Resource } from "dpkit" +import React from "react" +import { DataGrid } from "./DataGrid.tsx" + +// TODO: Support better display of resource properties + +export function ResourceGrid(props: { resource: Resource }) { + const data = [props.resource] + + return +} diff --git a/cli/components/SchemaGrid.tsx b/cli/components/SchemaGrid.tsx new file mode 100644 index 00000000..ec5d302e --- /dev/null +++ b/cli/components/SchemaGrid.tsx @@ -0,0 +1,15 @@ +import type { Schema } from "dpkit" +import React from "react" +import { DataGrid } from "./DataGrid.tsx" + +// TODO: Support showing other schema/field properties + +export function SchemaGrid(props: { schema: Schema }) { + const data = [ + Object.fromEntries( + props.schema.fields.map(field => [field.name, field.type]), + ), + ] + + return +} diff --git a/cli/components/TableGrid.tsx b/cli/components/TableGrid.tsx index a261af21..954a6640 100644 --- a/cli/components/TableGrid.tsx +++ b/cli/components/TableGrid.tsx @@ -9,8 +9,12 @@ import { DataGrid } from "./DataGrid.tsx" const PAGE_SIZE = 10 type Data = Record[] -export function TableGrid(props: { table: Table }) { - const { table } = props +export function TableGrid(props: { + table: Table + borderColor?: "green" | "red" +}) { + const { table, borderColor } = props + const { exit } = useApp() const [col, setCol] = useState(0) const [page, setPage] = useState(1) @@ -92,7 +96,13 @@ export function TableGrid(props: { table: Table }) { return ( - + ) diff --git a/cli/entrypoints/dev.ts b/cli/entrypoints/dev.ts index cefc0f18..5a47c26e 100755 --- a/cli/entrypoints/dev.ts +++ b/cli/entrypoints/dev.ts @@ -1,4 +1,3 @@ #!cli/node_modules/.bin/tsx -import { execute } from "@oclif/core" -await execute({ dir: import.meta.url, development: true }) +await import("../main.ts") diff --git a/cli/entrypoints/run.ts b/cli/entrypoints/run.ts index c036c876..9dc8de9e 100755 --- a/cli/entrypoints/run.ts +++ b/cli/entrypoints/run.ts @@ -1,4 +1,3 @@ #!/usr/bin/env node -import { execute } from "@oclif/core" -await execute({ dir: import.meta.url }) +await import("../main.ts") diff --git a/cli/helpers/dialect.ts b/cli/helpers/dialect.ts new file mode 100644 index 00000000..47660411 --- /dev/null +++ b/cli/helpers/dialect.ts @@ -0,0 +1,163 @@ +import type { Dialect } from "dpkit" + +// TODO: Find a better way to construct dialects + +export function createDialectFromOptions(options: any) { + let dialect: Dialect | undefined + + if (options.delimiter) { + dialect = { ...dialect, delimiter: options.delimiter } + } + + if (options.header === false) { + dialect = { ...dialect, header: options.header } + } + + if (options.headerRows) { + dialect = { + ...dialect, + headerRows: options.headerRows.split(",").map(Number), + } + } + + if (options.headerJoin) { + dialect = { ...dialect, headerJoin: options.headerJoin } + } + + if (options.commentRows) { + dialect = { + ...dialect, + commentRows: options.commentRows.split(",").map(Number), + } + } + + if (options.commentChar) { + dialect = { ...dialect, commentChar: options.commentChar } + } + + if (options.quoteChar) { + dialect = { ...dialect, quoteChar: options.quoteChar } + } + + if (options.doubleQuote) { + dialect = { ...dialect, doubleQuote: options.doubleQuote } + } + + if (options.escapeChar) { + dialect = { ...dialect, escapeChar: options.escapeChar } + } + + if (options.nullSequence) { + dialect = { ...dialect, nullSequence: options.nullSequence } + } + + if (options.skipInitialSpace) { + dialect = { ...dialect, skipInitialSpace: options.skipInitialSpace } + } + + if (options.property) { + dialect = { ...dialect, property: options.property } + } + + if (options.itemType) { + dialect = { ...dialect, itemType: options.itemType } + } + + if (options.itemKeys) { + dialect = { ...dialect, itemKeys: options.itemKeys.split(",") } + } + + if (options.sheetNumber) { + dialect = { ...dialect, sheetNumber: options.sheetNumber } + } + + if (options.sheetName) { + dialect = { ...dialect, sheetName: options.sheetName } + } + + if (options.table) { + dialect = { ...dialect, table: options.table } + } + + return dialect +} + +export function createToDialectFromOptions(options: any) { + let dialect: Dialect | undefined + + if (options.toDelimiter) { + dialect = { ...dialect, delimiter: options.toDelimiter } + } + + if (options.toHeader === false) { + dialect = { ...dialect, header: options.toHeader } + } + + if (options.toHeaderRows) { + dialect = { + ...dialect, + headerRows: options.toHeaderRows.split(",").map(Number), + } + } + + if (options.toHeaderJoin) { + dialect = { ...dialect, headerJoin: options.toHeaderJoin } + } + + if (options.toCommentRows) { + dialect = { + ...dialect, + commentRows: options.toCommentRows.split(",").map(Number), + } + } + + if (options.toCommentChar) { + dialect = { ...dialect, commentChar: options.toCommentChar } + } + + if (options.toQuoteChar) { + dialect = { ...dialect, quoteChar: options.toQuoteChar } + } + + if (options.toDoubleQuote) { + dialect = { ...dialect, doubleQuote: options.toDoubleQuote } + } + + if (options.toEscapeChar) { + dialect = { ...dialect, escapeChar: options.toEscapeChar } + } + + if (options.toNullSequence) { + dialect = { ...dialect, nullSequence: options.toNullSequence } + } + + if (options.toSkipInitialSpace) { + dialect = { ...dialect, skipInitialSpace: options.toSkipInitialSpace } + } + + if (options.toProperty) { + dialect = { ...dialect, property: options.toProperty } + } + + if (options.toItemType) { + dialect = { ...dialect, itemType: options.toItemType } + } + + if (options.toItemKeys) { + dialect = { ...dialect, itemKeys: options.toItemKeys.split(",") } + } + + if (options.toSheetNumber) { + dialect = { ...dialect, sheetNumber: options.toSheetNumber } + } + + if (options.toSheetName) { + dialect = { ...dialect, sheetName: options.toSheetName } + } + + if (options.toTable) { + dialect = { ...dialect, table: options.toTable } + } + + return dialect +} diff --git a/cli/helpers/help.ts b/cli/helpers/help.ts new file mode 100644 index 00000000..68a955cf --- /dev/null +++ b/cli/helpers/help.ts @@ -0,0 +1,6 @@ +import type { HelpConfiguration } from "commander" +import pc from "picocolors" + +export const helpConfiguration: HelpConfiguration = { + styleTitle: str => pc.bold(str.toUpperCase().replace(":", "")), +} diff --git a/cli/helpers/object.ts b/cli/helpers/object.ts new file mode 100644 index 00000000..ea92287e --- /dev/null +++ b/cli/helpers/object.ts @@ -0,0 +1,3 @@ +export function isEmptyObject(obj: Record): boolean { + return Object.keys(obj).length === 0 +} diff --git a/cli/helpers/resource.ts b/cli/helpers/resource.ts new file mode 100644 index 00000000..3fd0714c --- /dev/null +++ b/cli/helpers/resource.ts @@ -0,0 +1,42 @@ +import { loadPackage } from "dpkit" +import { Session } from "./session.ts" + +export async function selectResource( + session: Session, + options: { package?: string; resource?: string }, +) { + if (!options.package) { + Session.terminate("Please provide a path argument or a package option") + } + + const dataPackage = await session.task( + "Loading package", + loadPackage(options.package), + ) + + const resourceName = + options.resource ?? + (await session.select({ + message: "Select resource", + options: dataPackage.resources.map(resource => ({ + label: resource.name, + value: resource.name, + })), + })) + + const resource = dataPackage.resources.find( + resource => resource.name === resourceName, + ) + + if (!resource) { + if (typeof resourceName !== "string") { + Session.terminate("Resource selection cancelled") + } + + Session.terminate( + `Resource "${resourceName}" is not found in the provided data package`, + ) + } + + return resource +} diff --git a/cli/helpers/session.ts b/cli/helpers/session.ts new file mode 100644 index 00000000..b8e6496f --- /dev/null +++ b/cli/helpers/session.ts @@ -0,0 +1,100 @@ +import { setImmediate } from "node:timers/promises" +import { spinner } from "@clack/prompts" +import { intro, log, outro, select } from "@clack/prompts" +import type { SelectOptions } from "@clack/prompts" +import exitHook from "exit-hook" +import { render } from "ink" +import pc from "picocolors" +import type React from "react" + +export class Session { + title: string + + static create(options: { title: string; json?: boolean }) { + const session = options.json + ? new JsonSession(options) + : new Session(options) + + session.start() + return session + } + + static terminate(message: string): never { + log.error(message) + process.exit(1) + } + + constructor(options: { title: string }) { + this.title = options.title + } + start() { + intro(pc.bold(this.title)) + this.#enableExitHook() + } + + success(message: string) { + log.success(message) + } + + error(message: string) { + log.error(message) + } + + async select(options: SelectOptions) { + return await select(options) + } + + async task(message: string, promise: Promise) { + // TODO: Consider spinner's onCancel or other solution when @clack/prompts@1.0 is released + // We disable/enable the exit hook to friend it with spinner's "Cancel" event + const loader = spinner() + + this.#disableExitHook?.() + loader.start(message) + + const result = await promise + + loader.stop(message) + this.#enableExitHook() + + return result + } + + async render(_object: any, node: React.ReactNode) { + // Without waiting for the next tick after clack prompts, + // ink render will be immidiately terminated + await setImmediate() + + const app = render(node) + await app.waitUntilExit() + } + + #disableExitHook?: ReturnType + #enableExitHook() { + this.#disableExitHook = exitHook(() => this.#handleExit()) + } + + #handleExit() { + outro( + `Problems? ${pc.underline(pc.cyan("https://github.com/datisthq/dpkit/issues"))}`, + ) + } +} + +export class JsonSession extends Session { + start = () => {} + success = () => {} + error = () => {} + + async select(_options: SelectOptions): Promise { + Session.terminate("Selection is not supported in JSON mode") + } + + async render(object: any, _node: React.ReactNode) { + console.log(JSON.stringify(object, null, 2)) + } + + async task(_message: string, promise: Promise) { + return await promise + } +} diff --git a/cli/main.ts b/cli/main.ts new file mode 100644 index 00000000..2cfaf5eb --- /dev/null +++ b/cli/main.ts @@ -0,0 +1,30 @@ +// TODO: Support tab completion when @bombsh/tab is released +//import tab from "@bombsh/tab/commander" +import { program } from "commander" +import { dialectCommand } from "./commands/dialect/index.ts" +import { fileCommand } from "./commands/file/index.ts" +import { packageCommand } from "./commands/package/index.ts" +import { resourceCommand } from "./commands/resource/index.ts" +import { schemaCommand } from "./commands/schema/index.ts" +import { tableCommand } from "./commands/table/index.ts" +import { helpConfiguration } from "./helpers/help.ts" +import metadata from "./package.json" with { type: "json" } + +const main = program + .name("dp") + .description( + "Fast data management CLI built on top of the Data Package standard and Polars DataFrames", + ) + + .version(metadata.version, "-v, --version") + .configureHelp(helpConfiguration) + + .addCommand(packageCommand) + .addCommand(resourceCommand) + .addCommand(dialectCommand) + .addCommand(schemaCommand) + .addCommand(tableCommand) + .addCommand(fileCommand) + +//tab(main) +main.parse() diff --git a/cli/options/dialect.ts b/cli/options/dialect.ts deleted file mode 100644 index d5db981e..00000000 --- a/cli/options/dialect.ts +++ /dev/null @@ -1,178 +0,0 @@ -import { Flags } from "@oclif/core" -import type { Dialect } from "dpkit" - -export const header = Flags.boolean({ - description: "whether the file includes a header row with field names", - default: true, -}) - -export const headerRows = Flags.string({ - description: - "comma-separated row numbers (zero-based) that are considered header rows", -}) - -export const headerJoin = Flags.string({ - description: "character used to join multi-line headers", -}) - -export const commentRows = Flags.string({ - description: "comma-separated rows to be excluded from the data (zero-based)", -}) - -export const commentChar = Flags.string({ - description: "character sequence denoting the start of a comment line", -}) - -export const delimiter = Flags.string({ - description: "character used to separate fields in the data", - char: "d", -}) - -export const lineTerminator = Flags.string({ - description: "character sequence used to terminate rows", -}) - -export const quoteChar = Flags.string({ - description: "character used to quote fields", -}) - -export const doubleQuote = Flags.boolean({ - description: - "whether a sequence of two quote characters represents a single quote", -}) - -export const escapeChar = Flags.string({ - description: "character used to escape the delimiter or quote characters", -}) - -export const nullSequence = Flags.string({ - description: - "character sequence representing null or missing values in the data", -}) - -export const skipInitialSpace = Flags.boolean({ - description: - "whether to ignore whitespace immediately following the delimiter", -}) - -export const property = Flags.string({ - description: "for JSON data, the property name containing the data array", -}) - -export const itemType = Flags.string({ - description: "the type of data item in the source", - options: ["array", "object"], -}) - -export const itemKeys = Flags.string({ - description: - "comma-separated object properties to extract as values (for object-based data items)", -}) - -export const sheetNumber = Flags.integer({ - description: "for spreadsheet data, the sheet number to read (zero-based)", -}) - -export const sheetName = Flags.string({ - description: "for spreadsheet data, the sheet name to read", -}) - -export const table = Flags.string({ - description: "for database sources, the table name to read", -}) - -export const dialectOptions = { - delimiter, - header, - headerRows, - headerJoin, - commentRows, - commentChar, - quoteChar, - doubleQuote, - escapeChar, - nullSequence, - skipInitialSpace, - property, - itemType, - itemKeys, - sheetNumber, - table, -} - -// TODO: rebase on types flags -export function createDialectFromFlags(flags: any) { - let dialect: Dialect | undefined - - if (flags.delimiter) { - dialect = { ...dialect, delimiter: flags.delimiter } - } - - if (flags.header === false) { - dialect = { ...dialect, header: flags.header } - } - - if (flags.headerRows) { - dialect = { - ...dialect, - headerRows: flags.headerRows.split(",").map(Number), - } - } - - if (flags.headerJoin) { - dialect = { ...dialect, headerJoin: flags.headerJoin } - } - - if (flags.commentRows) { - dialect = { - ...dialect, - commentRows: flags.commentRows.split(",").map(Number), - } - } - - if (flags.commentChar) { - dialect = { ...dialect, commentChar: flags.commentChar } - } - - if (flags.quoteChar) { - dialect = { ...dialect, quoteChar: flags.quoteChar } - } - - if (flags.doubleQuote) { - dialect = { ...dialect, doubleQuote: flags.doubleQuote } - } - - if (flags.escapeChar) { - dialect = { ...dialect, escapeChar: flags.escapeChar } - } - - if (flags.nullSequence) { - dialect = { ...dialect, nullSequence: flags.nullSequence } - } - - if (flags.skipInitialSpace) { - dialect = { ...dialect, skipInitialSpace: flags.skipInitialSpace } - } - - if (flags.property) { - dialect = { ...dialect, property: flags.property } - } - - if (flags.itemType) { - dialect = { ...dialect, itemType: flags.itemType } - } - - if (flags.itemKeys) { - dialect = { ...dialect, itemKeys: flags.itemKeys.split(",") } - } - - if (flags.sheetNumber) { - dialect = { ...dialect, sheetNumber: flags.sheetNumber } - } - - if (flags.table) { - dialect = { ...dialect, table: flags.table } - } - - return dialect -} diff --git a/cli/options/index.ts b/cli/options/index.ts deleted file mode 100644 index ea9f8d80..00000000 --- a/cli/options/index.ts +++ /dev/null @@ -1,5 +0,0 @@ -export * from "./json.ts" -export * from "./dialect.ts" -export * from "./path.ts" -export * from "./resource.ts" -export * from "./toDialect.ts" diff --git a/cli/options/json.ts b/cli/options/json.ts deleted file mode 100644 index f7d62b3f..00000000 --- a/cli/options/json.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { Flags } from "@oclif/core" - -export const json = Flags.boolean({ - description: "output as JSON", - char: "j", -}) diff --git a/cli/options/path.ts b/cli/options/path.ts deleted file mode 100644 index df4e97b5..00000000 --- a/cli/options/path.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { Flags } from "@oclif/core" - -export const toPath = Flags.string({ - description: "a local output path", -}) diff --git a/cli/options/resource.ts b/cli/options/resource.ts deleted file mode 100644 index f42643f1..00000000 --- a/cli/options/resource.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { Flags } from "@oclif/core" - -export const toFormat = Flags.string({ - description: "output resource format", -}) diff --git a/cli/options/toDialect.ts b/cli/options/toDialect.ts deleted file mode 100644 index bdb2b459..00000000 --- a/cli/options/toDialect.ts +++ /dev/null @@ -1,180 +0,0 @@ -import { Flags } from "@oclif/core" -import type { Dialect } from "dpkit" - -// TODO: merge with dialect.ts - -export const toHeader = Flags.boolean({ - description: "whether the file includes a header row with field names", - default: true, -}) - -export const toHeaderRows = Flags.string({ - description: - "comma-separated row numbers (zero-based) that are considered header rows", -}) - -export const toHeaderJoin = Flags.string({ - description: "character used to join multi-line headers", -}) - -export const toCommentRows = Flags.string({ - description: "comma-separated rows to be excluded from the data (zero-based)", -}) - -export const toCommentChar = Flags.string({ - description: "character sequence denoting the start of a comment line", -}) - -export const toDelimiter = Flags.string({ - description: "character used to separate fields in the data", - char: "d", -}) - -export const toLineTerminator = Flags.string({ - description: "character sequence used to terminate rows", -}) - -export const toQuoteChar = Flags.string({ - description: "character used to quote fields", -}) - -export const toDoubleQuote = Flags.boolean({ - description: - "whether a sequence of two quote characters represents a single quote", -}) - -export const toEscapeChar = Flags.string({ - description: "character used to escape the delimiter or quote characters", -}) - -export const toNullSequence = Flags.string({ - description: - "character sequence representing null or missing values in the data", -}) - -export const toSkipInitialSpace = Flags.boolean({ - description: - "whether to ignore whitespace immediately following the delimiter", -}) - -export const toProperty = Flags.string({ - description: "for JSON data, the property name containing the data array", -}) - -export const toItemType = Flags.string({ - description: "the type of data item in the source", - options: ["array", "object"], -}) - -export const toItemKeys = Flags.string({ - description: - "comma-separated object properties to extract as values (for object-based data items)", -}) - -export const toSheetNumber = Flags.integer({ - description: "for spreadsheet data, the sheet number to read (zero-based)", -}) - -export const toSheetName = Flags.string({ - description: "for spreadsheet data, the sheet name to read", -}) - -export const toTable = Flags.string({ - description: "for database sources, the table name to read", -}) - -export const toDialectOptions = { - toDelimiter, - toHeader, - toHeaderRows, - toHeaderJoin, - toCommentRows, - toCommentChar, - toQuoteChar, - toDoubleQuote, - toEscapeChar, - toNullSequence, - toSkipInitialSpace, - toProperty, - toItemType, - toItemKeys, - toSheetNumber, - toTable, -} - -// TODO: rebase on types flags -export function createToDialectFromFlags(flags: any) { - let dialect: Dialect | undefined - - if (flags.toDelimiter) { - dialect = { ...dialect, delimiter: flags.toDelimiter } - } - - if (flags.toHeader === false) { - dialect = { ...dialect, header: flags.toHeader } - } - - if (flags.toHeaderRows) { - dialect = { - ...dialect, - headerRows: flags.toHeaderRows.split(",").map(Number), - } - } - - if (flags.toHeaderJoin) { - dialect = { ...dialect, headerJoin: flags.toHeaderJoin } - } - - if (flags.toCommentRows) { - dialect = { - ...dialect, - commentRows: flags.toCommentRows.split(",").map(Number), - } - } - - if (flags.toCommentChar) { - dialect = { ...dialect, commentChar: flags.toCommentChar } - } - - if (flags.toQuoteChar) { - dialect = { ...dialect, quoteChar: flags.toQuoteChar } - } - - if (flags.toDoubleQuote) { - dialect = { ...dialect, doubleQuote: flags.toDoubleQuote } - } - - if (flags.toEscapeChar) { - dialect = { ...dialect, escapeChar: flags.toEscapeChar } - } - - if (flags.toNullSequence) { - dialect = { ...dialect, nullSequence: flags.toNullSequence } - } - - if (flags.toSkipInitialSpace) { - dialect = { ...dialect, skipInitialSpace: flags.toSkipInitialSpace } - } - - if (flags.toProperty) { - dialect = { ...dialect, property: flags.toProperty } - } - - if (flags.toItemType) { - dialect = { ...dialect, itemType: flags.toItemType } - } - - if (flags.toItemKeys) { - dialect = { ...dialect, itemKeys: flags.toItemKeys.split(",") } - } - - if (flags.toSheetNumber) { - dialect = { ...dialect, sheetNumber: flags.toSheetNumber } - } - - if (flags.toTable) { - dialect = { ...dialect, table: flags.toTable } - } - - return dialect -} diff --git a/cli/package.json b/cli/package.json index 9c4116a9..0cd687b9 100644 --- a/cli/package.json +++ b/cli/package.json @@ -23,23 +23,30 @@ ], "scripts": { "build": "tsc && pnpm build:copy && pnpm build:mode", - "build:copy": "cp entrypoints/*.cmd build/entrypoints && cp package.json build && cp .oclifrc.js build", + "build:copy": "cp entrypoints/*.cmd build/entrypoints && cp package.json build", "build:mode": "chmod +x ./build/entrypoints/*.js", "compile": "node ./scripts/compile.ts", "dev": "node ./scripts/dev.ts", "run": "node ./scripts/run.ts" }, "dependencies": { - "@oclif/core": "^4.5.2", - "@oclif/plugin-autocomplete": "^3.2.34", + "@clack/prompts": "^0.11.0", + "@commander-js/extra-typings": "^14.0.0", + "@inkjs/ui": "^2.0.0", + "commander": "^14.0.0", "dpkit": "workspace:*", + "es-toolkit": "^1.39.10", + "exit-hook": "^4.0.0", "ink": "^6.1.0", "ink-select-input": "^6.2.0", "nodejs-polars": "^0.21.0", + "picocolors": "^1.1.1", "react": "^19.1.1", + "tiny-invariant": "^1.3.3", "ts-extras": "^0.14.0" }, "devDependencies": { + "@types/node": "24.2.0", "@types/react": "19.1.9", "tsx": "4.20.3" } diff --git a/cli/params/ckan.ts b/cli/params/ckan.ts new file mode 100644 index 00000000..cb8ff119 --- /dev/null +++ b/cli/params/ckan.ts @@ -0,0 +1,18 @@ +import { Option } from "commander" + +export const toCkanApiKey = new Option( + "--to-ckan-api-key ", + "API key for CKAN API", +) + +export const toCkanUrl = new Option("--to-ckan-url ", "CKAN URL") + +export const toCkanOwnerOrg = new Option( + "--to-ckan-owner-org ", + "Owner organization for the CKAN dataset", +) + +export const toCkanDatasetName = new Option( + "--to-ckan-dataset-name ", + "Name for the CKAN dataset", +) diff --git a/cli/params/dialect.ts b/cli/params/dialect.ts new file mode 100644 index 00000000..919195ae --- /dev/null +++ b/cli/params/dialect.ts @@ -0,0 +1,181 @@ +import { Option } from "commander" + +export const header = new Option( + "--header", + "whether the file includes a header row with field names", +).default(true) + +export const headerRows = new Option( + "--header-rows ", + "comma-separated row numbers (zero-based) that are considered header rows", +) + +export const headerJoin = new Option( + "--header-join ", + "character used to join multi-line headers", +) + +export const commentRows = new Option( + "--comment-rows ", + "comma-separated rows to be excluded from the data (zero-based)", +) + +export const commentChar = new Option( + "--comment-char ", + "character sequence denoting the start of a comment line", +) + +export const delimiter = new Option( + "--delimiter ", + "character used to separate fields in the data", +) + +export const lineTerminator = new Option( + "--line-terminator ", + "character sequence used to terminate rows", +) + +export const quoteChar = new Option( + "--quote-char ", + "character used to quote fields", +) + +export const doubleQuote = new Option( + "--double-quote", + "whether a sequence of two quote characters represents a single quote", +) + +export const escapeChar = new Option( + "--escape-char ", + "character used to escape the delimiter or quote characters", +) + +export const nullSequence = new Option( + "--null-sequence ", + "character sequence representing null or missing values in the data", +) + +export const skipInitialSpace = new Option( + "--skip-initial-space", + "whether to ignore whitespace immediately following the delimiter", +) + +export const property = new Option( + "--property ", + "for JSON data, the property name containing the data array", +) + +export const itemType = new Option( + "--item-type ", + "the type of data item in the source", +).choices(["array", "object"]) + +export const itemKeys = new Option( + "--item-keys ", + "comma-separated object properties to extract as values (for object-based data items)", +) + +export const sheetNumber = new Option( + "--sheet-number ", + "for spreadsheet data, the sheet number to read (zero-based)", +).argParser(Number.parseInt) + +export const sheetName = new Option( + "--sheet-name ", + "for spreadsheet data, the sheet name to read", +) + +export const table = new Option( + "--table ", + "for database sources, the table name to read", +) + +export const toHeader = new Option( + "--to-header", + "whether the file includes a header row with field names", +).default(true) + +export const toHeaderRows = new Option( + "--to-header-rows ", + "comma-separated row numbers (zero-based) that are considered header rows", +) + +export const toHeaderJoin = new Option( + "--to-header-join ", + "character used to join multi-line headers", +) + +export const toCommentRows = new Option( + "--to-comment-rows ", + "comma-separated rows to be excluded from the data (zero-based)", +) + +export const toCommentChar = new Option( + "--to-comment-char ", + "character sequence denoting the start of a comment line", +) + +export const toDelimiter = new Option( + "--to-delimiter ", + "character used to separate fields in the data", +) + +export const toLineTerminator = new Option( + "--to-line-terminator ", + "character sequence used to terminate rows", +) + +export const toQuoteChar = new Option( + "--to-quote-char ", + "character used to quote fields", +) + +export const toDoubleQuote = new Option( + "--to-double-quote", + "whether a sequence of two quote characters represents a single quote", +) + +export const toEscapeChar = new Option( + "--to-escape-char ", + "character used to escape the delimiter or quote characters", +) + +export const toNullSequence = new Option( + "--to-null-sequence ", + "character sequence representing null or missing values in the data", +) + +export const toSkipInitialSpace = new Option( + "--to-skip-initial-space", + "whether to ignore whitespace immediately following the delimiter", +) + +export const toProperty = new Option( + "--to-property ", + "for JSON data, the property name containing the data array", +) + +export const toItemType = new Option( + "--to-item-type ", + "the type of data item in the source", +).choices(["array", "object"]) + +export const toItemKeys = new Option( + "--to-item-keys ", + "comma-separated object properties to extract as values (for object-based data items)", +) + +export const toSheetNumber = new Option( + "--to-sheet-number ", + "for spreadsheet data, the sheet number to read (zero-based)", +).argParser(Number.parseInt) + +export const toSheetName = new Option( + "--to-sheet-name ", + "for spreadsheet data, the sheet name to read", +) + +export const toTable = new Option( + "--to-table ", + "for database sources, the table name to read", +) diff --git a/cli/params/file.ts b/cli/params/file.ts new file mode 100644 index 00000000..4db3d8a6 --- /dev/null +++ b/cli/params/file.ts @@ -0,0 +1,12 @@ +import { Option } from "commander" + +export const hashType = new Option("--hash-type ", "hash type") + .choices(["md5", "sha1", "sha256", "sha512"]) + .default("sha256") + +export const bytes = new Option( + "--bytes ", + "expected file size in bytes", +) + +export const hash = new Option("--hash ", "expected file hash") diff --git a/cli/params/github.ts b/cli/params/github.ts new file mode 100644 index 00000000..63a81e3f --- /dev/null +++ b/cli/params/github.ts @@ -0,0 +1,16 @@ +import { Option } from "commander" + +export const toGithubApiKey = new Option( + "--to-github-api-key ", + "API key for GitHub API", +) + +export const toGithubRepo = new Option( + "--to-github-repo ", + "GitHub repository name", +) + +export const toGithubOrg = new Option( + "--to-github-org ", + "GitHub organization (optional, defaults to user repositories)", +) diff --git a/cli/params/index.ts b/cli/params/index.ts index 5bec7a0e..010f53cb 100644 --- a/cli/params/index.ts +++ b/cli/params/index.ts @@ -1 +1,7 @@ +export * from "./ckan.ts" +export * from "./dialect.ts" +export * from "./file.ts" +export * from "./json.ts" +export * from "./package.ts" export * from "./path.ts" +export * from "./resource.ts" diff --git a/cli/params/json.ts b/cli/params/json.ts new file mode 100644 index 00000000..7d770164 --- /dev/null +++ b/cli/params/json.ts @@ -0,0 +1,3 @@ +import { Option } from "commander" + +export const json = new Option("-j, --json", "output as JSON") diff --git a/cli/params/package.ts b/cli/params/package.ts new file mode 100644 index 00000000..17d88458 --- /dev/null +++ b/cli/params/package.ts @@ -0,0 +1,16 @@ +import { Option } from "commander" + +export const withRemote = new Option( + "--with-remote", + "include remote resources", +) + +export const fromPackage = new Option( + "-p --package ", + "package to select resource from", +) + +export const fromResource = new Option( + "-r --resource ", + "resource in provided package", +) diff --git a/cli/params/path.ts b/cli/params/path.ts index b94b40b8..132334b9 100644 --- a/cli/params/path.ts +++ b/cli/params/path.ts @@ -1,11 +1,43 @@ -import { Args } from "@oclif/core" +import { Argument, Option } from "commander" -export const requriedDescriptorPath = Args.string({ - description: "local or remote path to the descriptor", - required: true, -}) +export const optionalPositionalDescriptorPath = new Argument( + "[path]", + "local or remote path to the descriptor", +) -export const requriedTablePath = Args.string({ - description: "local or remote path to the table", - required: true, -}) +export const positionalDescriptorPath = new Argument( + "", + "local or remote path to the descriptor", +) + +export const positionalTablePath = new Argument( + "[path]", + "local or remote path to the table", +) + +export const positionalFilePath = new Argument( + "[path]", + "local or remote path to the file", +) + +export const requiredPositionalFilePath = new Argument( + "", + "local or remote path to the file", +) + +export const positionalFilePaths = new Argument( + "", + "local paths to files", +) + +export const toPath = new Option("--to-path ", "a local output path") + +export const toFolder = new Option( + "--to-folder ", + "a local output folder path", +) + +export const toArchive = new Option( + "--to-archive ", + "a local output zip file path", +) diff --git a/cli/params/resource.ts b/cli/params/resource.ts new file mode 100644 index 00000000..915f1292 --- /dev/null +++ b/cli/params/resource.ts @@ -0,0 +1,6 @@ +import { Option } from "commander" + +export const toFormat = new Option( + "--to-format ", + "output resource format", +) diff --git a/cli/params/zenodo.ts b/cli/params/zenodo.ts new file mode 100644 index 00000000..fb2f6a49 --- /dev/null +++ b/cli/params/zenodo.ts @@ -0,0 +1,11 @@ +import { Option } from "commander" + +export const toZenodoApiKey = new Option( + "--to-zenodo-api-key ", + "API key for Zenodo API", +) + +export const toZenodoSandbox = new Option( + "--to-zenodo-sandbox", + "Use Zenodo sandbox environment", +).default(false) diff --git a/cli/program.ts b/cli/program.ts new file mode 100644 index 00000000..8d0d0823 --- /dev/null +++ b/cli/program.ts @@ -0,0 +1,22 @@ +import { styleText } from "node:util" +import { program } from "commander" +import { dialectCommand } from "./commands/dialect/index.js" +import { packageCommand } from "./commands/package/index.js" +import { tableCommand } from "./commands/table/index.js" +import metadata from "./package.json" with { type: "json" } + +const root = program + .name("dp") + .description( + "Fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames", + ) + .version(metadata.version) + .configureHelp({ + styleTitle: str => styleText("bold", str.toUpperCase().slice(0, -1)), + }) + +root.addCommand(packageCommand) +root.addCommand(tableCommand) +root.addCommand(dialectCommand) + +root.parse() diff --git a/cli/scripts/compile.ts b/cli/scripts/compile.ts index 6b919e34..b7db5709 100644 --- a/cli/scripts/compile.ts +++ b/cli/scripts/compile.ts @@ -1,4 +1,5 @@ import { execa } from "execa" +import metadata from "../package.json" with { type: "json" } const $ = execa({ preferLocal: true, stdout: ["inherit"] }) // Cleanup @@ -18,4 +19,6 @@ pnpm deploy compile // Compile application -await $({ cwd: "compile" })`deno compile --allow-all entrypoints/run.ts` +await $({ + cwd: "compile", +})`deno compile --allow-all --no-check --output build/dp-${metadata.version}-x86_64-unknown-linux-gnu main.ts` diff --git a/core/dialect/assert.spec.ts b/core/dialect/assert.spec.ts index f0fc9eb5..86b46829 100644 --- a/core/dialect/assert.spec.ts +++ b/core/dialect/assert.spec.ts @@ -1,5 +1,5 @@ import { describe, expect, expectTypeOf, it } from "vitest" -import { AssertionError } from "../general/index.ts" +import { AssertionError } from "../error/index.ts" import type { Dialect } from "./Dialect.ts" import { assertDialect } from "./assert.ts" diff --git a/core/dialect/assert.ts b/core/dialect/assert.ts index b87140eb..b7446758 100644 --- a/core/dialect/assert.ts +++ b/core/dialect/assert.ts @@ -1,4 +1,5 @@ -import { AssertionError, type Descriptor } from "../general/index.ts" +import { AssertionError } from "../error/index.ts" +import type { Descriptor } from "../general/index.ts" import type { Dialect } from "./Dialect.ts" import { validateDialect } from "./validate.ts" diff --git a/core/general/Error.ts b/core/error/Assertion.ts similarity index 62% rename from core/general/Error.ts rename to core/error/Assertion.ts index 542186bc..4287e235 100644 --- a/core/general/Error.ts +++ b/core/error/Assertion.ts @@ -1,11 +1,4 @@ -import type { ErrorObject } from "ajv" - -/** - * A descriptor error - */ -export interface MetadataError extends ErrorObject { - type: "metadata" -} +import type { MetadataError } from "./Metadata.ts" /** * Thrown when a descriptor assertion fails diff --git a/core/error/Base.ts b/core/error/Base.ts new file mode 100644 index 00000000..cde68ce5 --- /dev/null +++ b/core/error/Base.ts @@ -0,0 +1,3 @@ +export interface BaseError { + type: string +} diff --git a/core/error/Metadata.ts b/core/error/Metadata.ts new file mode 100644 index 00000000..91512062 --- /dev/null +++ b/core/error/Metadata.ts @@ -0,0 +1,17 @@ +import type { BaseError } from "./Base.ts" + +/** + * A descriptor error + */ +export interface MetadataError extends BaseError { + type: "metadata" + keyword: string + instancePath: string + schemaPath: string + params: object + propertyName?: string + message?: string + schema?: any + parentSchema?: object + data?: any +} diff --git a/core/error/index.ts b/core/error/index.ts new file mode 100644 index 00000000..414f1fee --- /dev/null +++ b/core/error/index.ts @@ -0,0 +1,3 @@ +export { AssertionError } from "./Assertion.ts" +export type { BaseError } from "./Base.ts" +export type { MetadataError } from "./Metadata.ts" diff --git a/core/general/descriptor/validate.spec.ts b/core/general/descriptor/validate.spec.ts index accf7f2c..bc03fa9d 100644 --- a/core/general/descriptor/validate.spec.ts +++ b/core/general/descriptor/validate.spec.ts @@ -82,6 +82,7 @@ describe("validateDescriptor", () => { expect(error.keyword).toBe("required") expect(error.params).toBeDefined() if (error.params) { + // @ts-ignore expect(error.params.missingProperty).toBe("required_field") } } diff --git a/core/general/descriptor/validate.ts b/core/general/descriptor/validate.ts index b046c164..92c575c4 100644 --- a/core/general/descriptor/validate.ts +++ b/core/general/descriptor/validate.ts @@ -1,4 +1,4 @@ -import type { MetadataError } from "../Error.ts" +import type { MetadataError } from "../../error/index.ts" import { ajv } from "../profile/ajv.ts" import type { Descriptor } from "./Descriptor.ts" diff --git a/core/general/index.ts b/core/general/index.ts index 57c1b3e9..20eee670 100644 --- a/core/general/index.ts +++ b/core/general/index.ts @@ -8,7 +8,6 @@ export type { Descriptor } from "./descriptor/Descriptor.ts" export type { Metadata } from "./metadata/Metadata.ts" export { isDescriptor } from "./descriptor/Descriptor.ts" export { loadProfile } from "./profile/load.ts" -export { AssertionError, type MetadataError } from "./Error.ts" export { getName, getFormat, diff --git a/core/index.ts b/core/index.ts index 735a4049..807f6b4d 100644 --- a/core/index.ts +++ b/core/index.ts @@ -1,4 +1,5 @@ export * from "./general/index.ts" +export * from "./error/index.ts" export * from "./dialect/index.ts" export * from "./field/index.ts" export * from "./package/index.ts" diff --git a/core/package/assert.spec.ts b/core/package/assert.spec.ts index c167e8c6..e5253b77 100644 --- a/core/package/assert.spec.ts +++ b/core/package/assert.spec.ts @@ -1,5 +1,5 @@ import { describe, expect, expectTypeOf, it } from "vitest" -import { AssertionError } from "../general/index.ts" +import { AssertionError } from "../error/index.ts" import type { Package } from "./Package.ts" import { assertPackage } from "./assert.ts" diff --git a/core/package/assert.ts b/core/package/assert.ts index 92570015..1392feee 100644 --- a/core/package/assert.ts +++ b/core/package/assert.ts @@ -1,4 +1,5 @@ -import { AssertionError, type Descriptor } from "../general/index.ts" +import { AssertionError } from "../error/index.ts" +import type { Descriptor } from "../general/index.ts" import type { Package } from "./Package.ts" import { validatePackageDescriptor } from "./validate.ts" diff --git a/core/resource/assert.spec.ts b/core/resource/assert.spec.ts index 0299c8ad..d3a58358 100644 --- a/core/resource/assert.spec.ts +++ b/core/resource/assert.spec.ts @@ -1,5 +1,5 @@ import { describe, expect, expectTypeOf, it } from "vitest" -import { AssertionError } from "../general/index.ts" +import { AssertionError } from "../error/index.ts" import type { Resource } from "./Resource.ts" import { assertResource } from "./assert.ts" diff --git a/core/resource/assert.ts b/core/resource/assert.ts index 68375be2..dddd2954 100644 --- a/core/resource/assert.ts +++ b/core/resource/assert.ts @@ -1,4 +1,5 @@ -import { AssertionError, type Descriptor } from "../general/index.ts" +import { AssertionError } from "../error/index.ts" +import type { Descriptor } from "../general/index.ts" import type { Resource } from "./Resource.ts" import { validateResourceDescriptor } from "./validate.ts" diff --git a/core/resource/index.ts b/core/resource/index.ts index c17827f1..531c1774 100644 --- a/core/resource/index.ts +++ b/core/resource/index.ts @@ -1,5 +1,5 @@ export type { Resource } from "./Resource.ts" -export { inferFormat } from "./infer.ts" +export { inferResourceName, inferResourceFormat } from "./infer.ts" export { assertResource } from "./assert.ts" export { loadResourceDescriptor } from "./load.ts" export { saveResourceDescriptor } from "./save.ts" diff --git a/core/resource/infer.ts b/core/resource/infer.ts index 64416d1f..ceecbf8a 100644 --- a/core/resource/infer.ts +++ b/core/resource/infer.ts @@ -1,7 +1,21 @@ -import { getFilename, getFormat } from "../general/index.ts" +import { getFilename, getFormat, getName } from "../general/index.ts" import type { Resource } from "./Resource.ts" -export function inferFormat(resource: Partial) { +export function inferResourceName(resource: Partial) { + let name = resource.name + + if (!name) { + const path = Array.isArray(resource.path) ? resource.path[0] : resource.path + if (path) { + const filename = getFilename(path) + name = getName(filename) + } + } + + return name ?? "resource" +} + +export function inferResourceFormat(resource: Partial) { let format = resource.format if (!format) { diff --git a/core/resource/validate.ts b/core/resource/validate.ts index f5a80da5..06abe1db 100644 --- a/core/resource/validate.ts +++ b/core/resource/validate.ts @@ -1,6 +1,6 @@ import { loadDialect } from "../dialect/index.ts" +import { AssertionError } from "../error/index.ts" import { type Descriptor, validateDescriptor } from "../general/index.ts" -import { AssertionError } from "../general/index.ts" import { loadProfile } from "../general/index.ts" import { loadSchema } from "../schema/index.ts" import type { Resource } from "./Resource.ts" diff --git a/core/schema/assert.spec.ts b/core/schema/assert.spec.ts index db0a8cb0..3ba795f2 100644 --- a/core/schema/assert.spec.ts +++ b/core/schema/assert.spec.ts @@ -1,5 +1,5 @@ import { describe, expect, expectTypeOf, it } from "vitest" -import { AssertionError } from "../general/index.ts" +import { AssertionError } from "../error/index.ts" import type { Schema } from "./Schema.ts" import { assertSchema } from "./assert.ts" diff --git a/core/schema/assert.ts b/core/schema/assert.ts index 0a078986..230ac22b 100644 --- a/core/schema/assert.ts +++ b/core/schema/assert.ts @@ -1,4 +1,5 @@ -import { AssertionError, type Descriptor } from "../general/index.ts" +import { AssertionError } from "../error/index.ts" +import type { Descriptor } from "../general/index.ts" import type { Schema } from "./Schema.ts" import { validateSchema } from "./validate.ts" diff --git a/csv/plugin.ts b/csv/plugin.ts index d34bc16f..a8be7407 100644 --- a/csv/plugin.ts +++ b/csv/plugin.ts @@ -1,5 +1,5 @@ import type { Resource } from "@dpkit/core" -import { inferFormat } from "@dpkit/core" +import { inferResourceFormat } from "@dpkit/core" import type { InferDialectOptions, TablePlugin } from "@dpkit/table" import type { SaveTableOptions, Table } from "@dpkit/table" import { inferCsvDialect } from "./dialect/index.ts" @@ -32,6 +32,6 @@ export class CsvPlugin implements TablePlugin { } function getIsCsv(resource: Partial) { - const format = inferFormat(resource) + const format = inferResourceFormat(resource) return ["csv", "tsv"].includes(format ?? "") } diff --git a/docs/content/docs/guides/inline.md b/docs/content/docs/guides/inline.md index 0ed632e2..30ffceb5 100644 --- a/docs/content/docs/guides/inline.md +++ b/docs/content/docs/guides/inline.md @@ -1,7 +1,7 @@ --- title: Working with inline data sidebar: - label: Inline data + label: Inline Data order: 5 --- diff --git a/docs/content/docs/guides/table.md b/docs/content/docs/guides/table.md index a996fc20..af9f83e2 100644 --- a/docs/content/docs/guides/table.md +++ b/docs/content/docs/guides/table.md @@ -1,7 +1,7 @@ --- title: Working with tabular data sidebar: - label: Tabular data + label: Tabular Data order: 6 --- diff --git a/docs/content/docs/guides/visidata.md b/docs/content/docs/guides/visidata.md new file mode 100644 index 00000000..0d5a8f92 --- /dev/null +++ b/docs/content/docs/guides/visidata.md @@ -0,0 +1,30 @@ +--- +title: Using dpkit with VisiData +sidebar: + label: VisiData + order: 11 +--- + +[VisiData](https://www.visidata.org/) provide a powerful and flexible environment for data exploration, visualization, and analysis. dpkit can be used as a tool that prepared data for following usage in VisiData. + +## Installation + +- [Install dpkit CLI](../../cli/installation) +- [Install VisiData](https://www.visidata.org/install/) + +## Usage + +For example, we can use dpkit to copy a remote Data Package to a local folder and then load it in VisiData. + +```bash +dpkit copy https://zenodo.org/records/7559361 --toFolder dataset --withRemote +vd dataset/*.csv +``` + +:::tip +Functionality allowing to copy a Data Package to a SQLite database is under development. Once it is ready, it will be possible to prepare data to VisiData in more type-safe manner. +::: + +## References + +- [Visidata](https://www.visidata.org/) diff --git a/dpkit/dialect/infer.ts b/dpkit/dialect/infer.ts index c37b8087..fdf55d27 100644 --- a/dpkit/dialect/infer.ts +++ b/dpkit/dialect/infer.ts @@ -2,7 +2,7 @@ import type { Dialect, Resource } from "@dpkit/core" import type { InferDialectOptions } from "@dpkit/table" import { dpkit } from "../plugin.ts" -// TODO: implement inferDialect +// TODO: review default values being {} vs undefined export async function inferDialect( resource: Partial, diff --git a/dpkit/index.ts b/dpkit/index.ts index a28b8c61..d1e59557 100644 --- a/dpkit/index.ts +++ b/dpkit/index.ts @@ -13,5 +13,6 @@ export * from "@dpkit/zip" export * from "./dialect/index.ts" export * from "./package/index.ts" +export * from "./resource/index.ts" export * from "./table/index.ts" export * from "./plugin.ts" diff --git a/dpkit/package.json b/dpkit/package.json index 362e8b87..b2af2b4b 100644 --- a/dpkit/package.json +++ b/dpkit/package.json @@ -23,8 +23,8 @@ "@dpkit/arrow": "workspace:*", "@dpkit/camtrap": "workspace:*", "@dpkit/ckan": "workspace:*", - "@dpkit/csv": "workspace:*", "@dpkit/core": "workspace:*", + "@dpkit/csv": "workspace:*", "@dpkit/datahub": "workspace:*", "@dpkit/file": "workspace:*", "@dpkit/folder": "workspace:*", diff --git a/dpkit/package/index.ts b/dpkit/package/index.ts index 3a5cbd54..39f8f5db 100644 --- a/dpkit/package/index.ts +++ b/dpkit/package/index.ts @@ -1,2 +1,4 @@ export { loadPackage } from "./load.ts" export { savePackage } from "./save.ts" +export { inferPackage } from "./infer.ts" +export { validatePackage } from "./validate.ts" diff --git a/dpkit/package/infer.ts b/dpkit/package/infer.ts new file mode 100644 index 00000000..e942c879 --- /dev/null +++ b/dpkit/package/infer.ts @@ -0,0 +1,21 @@ +import type { Package, Resource } from "@dpkit/core" +import { inferResource } from "../resource/index.ts" + +// TODO: Move PartialPackage/Resource to @dpkit/core? + +interface PartialPackage extends Omit { + resources: Partial[] +} + +export async function inferPackage(dataPackage: PartialPackage) { + const result = { + ...dataPackage, + resources: await Promise.all( + dataPackage.resources.map(async resource => { + return await inferResource(resource) + }), + ), + } + + return result +} diff --git a/dpkit/package/validate.ts b/dpkit/package/validate.ts new file mode 100644 index 00000000..56941400 --- /dev/null +++ b/dpkit/package/validate.ts @@ -0,0 +1,57 @@ +import type { Descriptor, Package } from "@dpkit/core" +import { loadDescriptor, validatePackageDescriptor } from "@dpkit/core" +import { dpkit } from "../plugin.ts" +import { validateResource } from "../resource/index.ts" + +// TODO: Improve implementation +// TODO: Support multipart resources? (clarify on the specs level) + +export async function validatePackage( + source: string | Descriptor | Partial, + options?: { basepath?: string }, +) { + let descriptor: Descriptor | undefined + let basepath = options?.basepath + + if (typeof source !== "string") { + descriptor = source + } else { + for (const plugin of dpkit.plugins) { + const result = await plugin.loadPackage?.(source) + if (result) { + descriptor = result as unknown as Descriptor + break + } + } + + if (!descriptor) { + const result = await loadDescriptor(source) + descriptor = result.descriptor + basepath = result.basepath + } + } + + const { valid, errors, dataPackage } = await validatePackageDescriptor( + descriptor, + { basepath }, + ) + + if (!dataPackage) { + return { + valid, + errors: errors.map(error => ({ ...error, resource: undefined })), + } + } + + const resourceErrors = ( + await Promise.all( + dataPackage.resources.map(async resource => { + const { errors } = await validateResource(resource) + return errors.map(error => ({ ...error, resource: resource.name })) + }), + ) + ).flat() + + const resourceValid = !resourceErrors.length + return { valid: resourceValid, errors: resourceErrors } +} diff --git a/dpkit/resource/index.ts b/dpkit/resource/index.ts new file mode 100644 index 00000000..ea494c50 --- /dev/null +++ b/dpkit/resource/index.ts @@ -0,0 +1,2 @@ +export { inferResource } from "./infer.ts" +export { validateResource } from "./validate.ts" diff --git a/dpkit/resource/infer.ts b/dpkit/resource/infer.ts new file mode 100644 index 00000000..6eccd130 --- /dev/null +++ b/dpkit/resource/infer.ts @@ -0,0 +1,55 @@ +import type { Resource } from "@dpkit/core" +import { inferResourceFormat, inferResourceName } from "@dpkit/core" +import { prefetchFile } from "@dpkit/file" +import { inferFileBytes, inferFileEncoding, inferFileHash } from "@dpkit/file" +import { inferTable } from "../table/index.ts" + +// TODO: Support multipart resources? (clarify on the specs level) + +export async function inferResource(resource: Partial) { + const result = { + ...resource, + name: resource.name ?? inferResourceName(resource), + } + + if (!result.format) { + result.format = inferResourceFormat(resource) + } + + if (typeof resource.path === "string") { + const localPath = await prefetchFile(resource.path) + + if (!result.encoding) { + const encoding = await inferFileEncoding(localPath) + if (encoding) { + result.encoding = encoding + } + } + + if (!result.bytes) { + result.bytes = await inferFileBytes(localPath) + } + + if (!result.hash) { + result.hash = await inferFileHash(localPath) + } + + if (!result.schema) { + try { + const localResource = { ...resource, path: localPath } + const { dialect, schema } = await inferTable(localResource) + + result.dialect = dialect + result.schema = schema + } catch {} + } + } + + if (!result.type) { + if (result.schema) { + result.type = "table" + } + } + + return result +} diff --git a/dpkit/resource/validate.ts b/dpkit/resource/validate.ts new file mode 100644 index 00000000..cffe661e --- /dev/null +++ b/dpkit/resource/validate.ts @@ -0,0 +1,46 @@ +import type { Descriptor, Resource } from "@dpkit/core" +import { loadDescriptor, validateResourceDescriptor } from "@dpkit/core" +import { validateFile } from "@dpkit/file" +import { validateTable } from "../table/index.ts" + +// TODO: Support multipart resources? (clarify on the specs level) + +export async function validateResource( + pathOrDescriptorOrResource: string | Descriptor | Partial, + options?: { basepath?: string }, +) { + let descriptor = pathOrDescriptorOrResource + let basepath = options?.basepath + + if (typeof descriptor === "string") { + const result = await loadDescriptor(descriptor) + descriptor = result.descriptor + basepath = result.basepath + } + + const { valid, errors, resource } = await validateResourceDescriptor( + descriptor, + { basepath }, + ) + + if (!resource) { + return { valid, errors } + } + + if (resource.bytes || resource.hash) { + if (typeof resource.path === "string") { + return await validateFile(resource.path, { + bytes: resource.bytes, + hash: resource.hash, + }) + } + } + + try { + // TODO: rebase on not-rasing? + // It will raise if the resource is not a table + return await validateTable(resource) + } catch {} + + return { valid: true, errors: [] } +} diff --git a/dpkit/table/index.ts b/dpkit/table/index.ts index 09df76ca..5fb5a1f7 100644 --- a/dpkit/table/index.ts +++ b/dpkit/table/index.ts @@ -1,4 +1,5 @@ export { loadTable } from "./load.ts" export { readTable } from "./read.ts" export { saveTable } from "./save.ts" +export { inferTable } from "./infer.ts" export { validateTable } from "./validate.ts" diff --git a/dpkit/table/infer.ts b/dpkit/table/infer.ts index 49213a03..e9ed01af 100644 --- a/dpkit/table/infer.ts +++ b/dpkit/table/infer.ts @@ -5,6 +5,8 @@ import { inferSchema } from "@dpkit/table" import { inferDialect } from "../dialect/index.ts" import { loadTable } from "./load.ts" +// TODO: Allow non-tabular resources returning undefineds? + export async function inferTable( resource: Partial, ): Promise<{ dialect: Dialect; schema: Schema; table: Table }> { diff --git a/extend/CHANGELOG.md b/extend/CHANGELOG.md deleted file mode 100644 index e20bdb21..00000000 --- a/extend/CHANGELOG.md +++ /dev/null @@ -1,13 +0,0 @@ -# @dpkit/extend - -## 0.3.0 - -### Minor Changes - -- 313a275: Added Json/Jsonl/Arrow/Parquet support; Bootstrapped CLI - -## 0.2.0 - -### Minor Changes - -- edfde49: General framework improvements diff --git a/extend/README.md b/extend/README.md deleted file mode 100644 index 757f15fc..00000000 --- a/extend/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# @dpkit/extend - -dpkit is a fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/extend/index.ts b/extend/index.ts deleted file mode 100644 index e69de29b..00000000 diff --git a/extend/package.json b/extend/package.json deleted file mode 100644 index 48787cd5..00000000 --- a/extend/package.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "name": "@dpkit/extend", - "type": "module", - "version": "0.3.0", - "exports": "./build/index.js", - "license": "MIT", - "author": "Evgeny Karev", - "repository": "https://github.com/datisthq/dpkit", - "description": "Fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames", - "keywords": [ - "data", - "polars", - "dataframe", - "datapackage", - "tableschema", - "typescript", - "validation", - "quality", - "fair", - "extend" - ], - "scripts": { - "build": "tsc" - } -} diff --git a/extend/tsconfig.json b/extend/tsconfig.json deleted file mode 100644 index 3c43903c..00000000 --- a/extend/tsconfig.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "extends": "../tsconfig.json" -} diff --git a/file/error/Base.ts b/file/error/Base.ts new file mode 100644 index 00000000..a517c9cf --- /dev/null +++ b/file/error/Base.ts @@ -0,0 +1,3 @@ +import type { BaseError } from "@dpkit/core" + +export interface BaseFileError extends BaseError {} diff --git a/file/error/Bytes.ts b/file/error/Bytes.ts new file mode 100644 index 00000000..fbf5cce0 --- /dev/null +++ b/file/error/Bytes.ts @@ -0,0 +1,7 @@ +import type { BaseFileError } from "./Base.ts" + +export interface BytesError extends BaseFileError { + type: "file/bytes" + bytes: number + actualBytes: number +} diff --git a/file/error/File.ts b/file/error/File.ts new file mode 100644 index 00000000..997380d7 --- /dev/null +++ b/file/error/File.ts @@ -0,0 +1,4 @@ +import type { BytesError } from "./Bytes.ts" +import type { HashError } from "./Hash.ts" + +export type FileError = BytesError | HashError diff --git a/file/error/Hash.ts b/file/error/Hash.ts new file mode 100644 index 00000000..ed8bc4ab --- /dev/null +++ b/file/error/Hash.ts @@ -0,0 +1,7 @@ +import type { BaseFileError } from "./Base.ts" + +export interface HashError extends BaseFileError { + type: "file/hash" + hash: string + actualHash: string +} diff --git a/file/error/index.ts b/file/error/index.ts new file mode 100644 index 00000000..822f4c00 --- /dev/null +++ b/file/error/index.ts @@ -0,0 +1 @@ +export type { FileError } from "./File.ts" diff --git a/file/file/describe.ts b/file/file/describe.ts new file mode 100644 index 00000000..4188a2b6 --- /dev/null +++ b/file/file/describe.ts @@ -0,0 +1,15 @@ +import { prefetchFile } from "./fetch.ts" +import { inferFileBytes, inferFileHash } from "./infer.ts" +import type { HashType } from "./infer.ts" + +export async function describeFile( + path: string, + options?: { hashType?: HashType }, +) { + const localPath = await prefetchFile(path) + + const bytes = await inferFileBytes(localPath) + const hash = await inferFileHash(localPath, { hashType: options?.hashType }) + + return { bytes, hash } +} diff --git a/file/file/index.ts b/file/file/index.ts index da2aea7a..ed15776b 100644 --- a/file/file/index.ts +++ b/file/file/index.ts @@ -4,3 +4,6 @@ export { saveFile } from "./save.ts" export { getTempFilePath, writeTempFile } from "./temp.ts" export { assertLocalPathVacant, isLocalPathExist } from "./path.ts" export { prefetchFile, prefetchFiles } from "./fetch.ts" +export { inferFileEncoding, inferFileBytes, inferFileHash } from "./infer.ts" +export { describeFile } from "./describe.ts" +export { validateFile } from "./validate.ts" diff --git a/file/file/infer.ts b/file/file/infer.ts new file mode 100644 index 00000000..6aaaf087 --- /dev/null +++ b/file/file/infer.ts @@ -0,0 +1,48 @@ +import { stat } from "node:fs/promises" +import chardet from "chardet" +import { hashFile } from "hasha" +import { isBinaryFile } from "isbinaryfile" +import { prefetchFile } from "./fetch.ts" +import { loadFile } from "./load.ts" + +export type HashType = "md5" | "sha1" | "sha256" | "sha512" + +export async function inferFileHash( + path: string, + options?: { hashType?: HashType }, +) { + const localPath = await prefetchFile(path) + const algorithm = options?.hashType ?? "sha256" + + const result = await hashFile(localPath, { algorithm }) + return `${algorithm}:${result}` +} + +export async function inferFileBytes(path: string) { + const localPath = await prefetchFile(path) + + const result = await stat(localPath) + return result.size +} + +export async function inferFileEncoding( + path: string, + options?: { sampleBytes?: number; confidencePercent?: number }, +) { + const maxBytes = options?.sampleBytes ?? 10_000 + const confidencePercent = options?.confidencePercent ?? 75 + + const buffer = await loadFile(path, { maxBytes }) + const isBinary = await isBinaryFile(buffer) + + if (!isBinary) { + const matches = chardet.analyse(buffer) + for (const match of matches) { + if (match.confidence >= confidencePercent) { + return match.name.toLowerCase() + } + } + } + + return undefined +} diff --git a/file/file/validate.ts b/file/file/validate.ts new file mode 100644 index 00000000..fc892dc6 --- /dev/null +++ b/file/file/validate.ts @@ -0,0 +1,39 @@ +import type { FileError } from "../error/index.ts" +import { prefetchFile } from "./fetch.ts" +import { inferFileBytes, inferFileHash } from "./infer.ts" + +export async function validateFile( + path: string, + options?: { bytes?: number; hash?: string }, +) { + const errors: FileError[] = [] + const localPath = await prefetchFile(path) + + if (options?.bytes) { + const bytes = await inferFileBytes(localPath) + if (bytes !== options.bytes) { + errors.push({ + type: "file/bytes", + bytes: options.bytes, + actualBytes: bytes, + }) + } + } + + if (options?.hash) { + const [_hashValue, hashType = "md5"] = options.hash.split(":").toReversed() + // TODO: figure out how we should handle other hash types + // @ts-ignore + const hash = await inferFileHash(localPath, { hashType }) + if (hash !== options.hash) { + errors.push({ + type: "file/hash", + hash: options.hash, + actualHash: hash, + }) + } + } + + const valid = errors.length === 0 + return { valid, errors } +} diff --git a/file/index.ts b/file/index.ts index d7427de1..00ec5705 100644 --- a/file/index.ts +++ b/file/index.ts @@ -1,3 +1,4 @@ +export * from "./error/index.ts" export * from "./file/index.ts" export * from "./package/index.ts" export * from "./resource/index.ts" diff --git a/file/package.json b/file/package.json index 62014daa..30e93ceb 100644 --- a/file/package.json +++ b/file/package.json @@ -24,7 +24,10 @@ }, "dependencies": { "@dpkit/core": "workspace:*", + "chardet": "^2.1.0", "exit-hook": "^4.0.0", + "hasha": "^6.0.0", + "isbinaryfile": "^5.0.4", "tempy": "3.1.0", "tiny-invariant": "^1.3.3" } diff --git a/folder/package/save.ts b/folder/package/save.ts index 2a81302e..84438028 100644 --- a/folder/package/save.ts +++ b/folder/package/save.ts @@ -28,13 +28,13 @@ export async function savePackageToFolder( await saveResourceFiles(resource, { basepath, withRemote, - saveFile: async props => { + saveFile: async options => { await copyFile({ - sourcePath: props.normalizedPath, - targetPath: props.denormalizedPath, + sourcePath: options.normalizedPath, + targetPath: join(folderPath, options.denormalizedPath), }) - return props.denormalizedPath + return options.denormalizedPath }, }), ) diff --git a/github/package/save.ts b/github/package/save.ts index acaa833d..9a6ad7ff 100644 --- a/github/package/save.ts +++ b/github/package/save.ts @@ -9,7 +9,7 @@ import type { GithubPackage } from "./Package.ts" /** * Save a package to a Github repository - * @param props Object containing the package to save and Github details + * @param options Object containing the package to save and Github details * @returns Object with the repository URL */ export async function savePackageToGithub( @@ -38,23 +38,23 @@ export async function savePackageToGithub( await saveResourceFiles(resource, { basepath, withRemote: false, - saveFile: async props => { - const stream = await loadFileStream(props.normalizedPath) + saveFile: async options => { + const stream = await loadFileStream(options.normalizedPath) const payload = { - path: props.denormalizedPath, + path: options.denormalizedPath, content: Buffer.from(await buffer(stream)).toString("base64"), - message: `Added file "${props.denormalizedPath}"`, + message: `Added file "${options.denormalizedPath}"`, } await makeGithubApiRequest({ - endpoint: `/repos/${githubPackage.owner.login}/${repo}/contents/${props.denormalizedPath}`, + endpoint: `/repos/${githubPackage.owner.login}/${repo}/contents/${options.denormalizedPath}`, method: "PUT", payload, apiKey, }) - return props.denormalizedPath + return options.denormalizedPath }, }), ) diff --git a/json/plugin.ts b/json/plugin.ts index eab7b74e..d57a9ead 100644 --- a/json/plugin.ts +++ b/json/plugin.ts @@ -1,5 +1,5 @@ import type { Resource } from "@dpkit/core" -import { inferFormat } from "@dpkit/core" +import { inferResourceFormat } from "@dpkit/core" import type { TablePlugin } from "@dpkit/table" import type { SaveTableOptions, Table } from "@dpkit/table" import { loadJsonTable, loadJsonlTable } from "./table/index.ts" @@ -36,7 +36,7 @@ export class JsonPlugin implements TablePlugin { } function getFormatInfo(resource: Partial) { - const format = inferFormat(resource) + const format = inferResourceFormat(resource) const isJson = format === "json" const isJsonl = format === "jsonl" || format === "ndjson" return { isJson, isJsonl } diff --git a/parquet/plugin.ts b/parquet/plugin.ts index 4aacf435..d978d341 100644 --- a/parquet/plugin.ts +++ b/parquet/plugin.ts @@ -1,5 +1,5 @@ import type { Resource } from "@dpkit/core" -import { inferFormat } from "@dpkit/core" +import { inferResourceFormat } from "@dpkit/core" import type { TablePlugin } from "@dpkit/table" import type { SaveTableOptions, Table } from "@dpkit/table" import { loadParquetTable, saveParquetTable } from "./table/index.ts" @@ -21,6 +21,6 @@ export class ParquetPlugin implements TablePlugin { } function getIsParquet(resource: Partial) { - const format = inferFormat(resource) + const format = inferResourceFormat(resource) return format === "parquet" } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8607a30f..49aa62f1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -88,15 +88,27 @@ importers: cli: dependencies: - '@oclif/core': - specifier: ^4.5.2 - version: 4.5.2 - '@oclif/plugin-autocomplete': - specifier: ^3.2.34 - version: 3.2.34 + '@clack/prompts': + specifier: ^0.11.0 + version: 0.11.0 + '@commander-js/extra-typings': + specifier: ^14.0.0 + version: 14.0.0(commander@14.0.0) + '@inkjs/ui': + specifier: ^2.0.0 + version: 2.0.0(ink@6.1.0(@types/react@19.1.9)(react@19.1.1)) + commander: + specifier: ^14.0.0 + version: 14.0.0 dpkit: specifier: workspace:* version: link:../dpkit + es-toolkit: + specifier: ^1.39.10 + version: 1.39.10 + exit-hook: + specifier: ^4.0.0 + version: 4.0.0 ink: specifier: ^6.1.0 version: 6.1.0(@types/react@19.1.9)(react@19.1.1) @@ -106,13 +118,22 @@ importers: nodejs-polars: specifier: ^0.21.0 version: 0.21.0 + picocolors: + specifier: ^1.1.1 + version: 1.1.1 react: specifier: ^19.1.1 version: 19.1.1 + tiny-invariant: + specifier: ^1.3.3 + version: 1.3.3 ts-extras: specifier: ^0.14.0 version: 0.14.0 devDependencies: + '@types/node': + specifier: 24.2.0 + version: 24.2.0 '@types/react': specifier: 19.1.9 version: 19.1.9 @@ -255,16 +276,23 @@ importers: excel: {} - extend: {} - file: dependencies: '@dpkit/core': specifier: workspace:* version: link:../core + chardet: + specifier: ^2.1.0 + version: 2.1.0 exit-hook: specifier: ^4.0.0 version: 4.0.0 + hasha: + specifier: ^6.0.0 + version: 6.0.0 + isbinaryfile: + specifier: ^5.0.4 + version: 5.0.4 tempy: specifier: 3.1.0 version: 3.1.0 @@ -376,8 +404,6 @@ importers: specifier: ^6.0.6 version: 6.0.6 - ui: {} - zenodo: dependencies: '@dpkit/core': @@ -598,6 +624,17 @@ packages: '@changesets/write@0.4.0': resolution: {integrity: sha512-CdTLvIOPiCNuH71pyDu3rA+Q0n65cmAbXnwWH84rKGiFumFzkmHNT8KHTMEchcxN+Kl8I54xGUhJ7l3E7X396Q==} + '@clack/core@0.5.0': + resolution: {integrity: sha512-p3y0FIOwaYRUPRcMO7+dlmLh8PSRcrjuTndsiA0WAFbWES0mLZlrjVoBRZ9DzkPFJZG6KGkJmoEAY0ZcVWTkow==} + + '@clack/prompts@0.11.0': + resolution: {integrity: sha512-pMN5FcrEw9hUkZA4f+zLlzivQSeQf5dRGJjSUbvVYDLvpKCdQx5OaknvKzgbtXOizhP+SJJJjqEbOe55uKKfAw==} + + '@commander-js/extra-typings@14.0.0': + resolution: {integrity: sha512-hIn0ncNaJRLkZrxBIp5AsW/eXEHNKYQBh0aPdoUqNgD+Io3NIykQqpKFyKcuasZhicGaEZJX/JBSIkZ4e5x8Dg==} + peerDependencies: + commander: ~14.0.0 + '@ctrl/tinycolor@4.1.0': resolution: {integrity: sha512-WyOx8cJQ+FQus4Mm4uPIZA64gbk3Wxh0so5Lcii0aJifqwoVOlfFtorjLE0Hen4OYyHZMXDWqMmaQemBhgxFRQ==} engines: {node: '>=14'} @@ -1003,6 +1040,12 @@ packages: cpu: [x64] os: [win32] + '@inkjs/ui@2.0.0': + resolution: {integrity: sha512-5+8fJmwtF9UvikzLfph9sA+LS+l37Ij/szQltkuXLOAXwNkBX9innfzh4pLGXIB59vKEQUtc6D4qGvhD7h3pAg==} + engines: {node: '>=18'} + peerDependencies: + ink: '>=5' + '@isaacs/cliui@8.0.2': resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} engines: {node: '>=12'} @@ -1135,14 +1178,6 @@ packages: resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==} engines: {node: '>= 8'} - '@oclif/core@4.5.2': - resolution: {integrity: sha512-eQcKyrEcDYeZJKu4vUWiu0ii/1Gfev6GF4FsLSgNez5/+aQyAUCjg3ZWlurf491WiYZTXCWyKAxyPWk8DKv2MA==} - engines: {node: '>=18.0.0'} - - '@oclif/plugin-autocomplete@3.2.34': - resolution: {integrity: sha512-KhbPcNjitAU7jUojMXJ3l7duWVub0L0pEr3r3bLrpJBNuIJhoIJ7p56Ropcb7OMH2xcaz5B8HGq56cTOe1FHEg==} - engines: {node: '>=18.0.0'} - '@oslojs/encoding@1.1.0': resolution: {integrity: sha512-70wQhgYmndg4GCPxPPxPGevRKqTIJ2Nh4OkiMWmDAVYsTQ+Ta7Sq+rPevXyXGdzr30/qZBnyOalCszoMxlyldQ==} @@ -1486,10 +1521,6 @@ packages: resolution: {integrity: sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==} engines: {node: '>=6'} - ansi-escapes@4.3.2: - resolution: {integrity: sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==} - engines: {node: '>=8'} - ansi-escapes@7.0.0: resolution: {integrity: sha512-GdYO7a61mR0fOlAsvC9/rIHf7L96sBc6dEWzeOu+KAea5bZyQRPIpojrVoI4AXGJS/ycu/fBTdLrUkA4ODrvjw==} engines: {node: '>=18'} @@ -1510,10 +1541,6 @@ packages: resolution: {integrity: sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==} engines: {node: '>=12'} - ansis@3.17.0: - resolution: {integrity: sha512-0qWUglt9JEqLFr3w1I1pbrChn1grhaiAR2ocX1PP/flRmxgtwTzPFFFnfIlD6aMOLQZgSuCRlidD70lvx8yhzg==} - engines: {node: '>=14'} - anymatch@3.1.3: resolution: {integrity: sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==} engines: {node: '>= 8'} @@ -1559,9 +1586,6 @@ packages: engines: {node: 18.20.8 || ^20.3.0 || >=22.0.0, npm: '>=9.6.5', pnpm: '>=7.1.0'} hasBin: true - async@3.2.6: - resolution: {integrity: sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==} - auto-bind@5.0.1: resolution: {integrity: sha512-ooviqdwwgfIfNmDwo94wlshcdzfO64XV0Cg6oDsDYBJfITDz1EngD2z7DkbvCWn+XIMsIqW27sEVF6qcpJrRcg==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} @@ -1676,6 +1700,9 @@ packages: chardet@0.7.0: resolution: {integrity: sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==} + chardet@2.1.0: + resolution: {integrity: sha512-bNFETTG/pM5ryzQ9Ad0lJOTa6HWD/YsScAR3EnCPZRPlQh77JocYktSHOUHelyhm8IARL+o4c4F1bP5KVOjiRA==} + check-error@2.1.1: resolution: {integrity: sha512-OAlb+T7V4Op9OwdkjmguYRqncdlx5JiofwOAUkmTF+jNdHwzTaTs4sRAGpzLF3oOz5xAyDGrPgeIDFQmDOTiJw==} engines: {node: '>= 16'} @@ -1692,10 +1719,6 @@ packages: resolution: {integrity: sha512-l+2bNRMiQgcfILUi33labAZYIWlH1kWDp+ecNo5iisRKrbm0xcRyCww71/YU0Fkw0mAFpz9bJayXPjey6vkmaQ==} engines: {node: '>=8'} - clean-stack@3.0.1: - resolution: {integrity: sha512-lR9wNiMRcVQjSB3a7xXGLuz4cr4wJuuXlaAEbRutGowQTmlp7R72/DOgN21e8jdwblMWl9UOJMJXarX94pzKdg==} - engines: {node: '>=10'} - cli-boxes@3.0.0: resolution: {integrity: sha512-/lzGpEWL/8PfI0BmBOPRwp0c/wFNX1RdUML3jK/RcSBA9T8mZDdQpqYBKtCFTOfQbwPqWEOpjqW+Fnayc0969g==} engines: {node: '>=10'} @@ -1704,9 +1727,9 @@ packages: resolution: {integrity: sha512-VGtlMu3x/4DOtIUwEkRezxUZ2lBacNJCHash0N0WeZDBS+7Ux1dm3XWAgWYxLJFMMdOeXMHXorshEFhbMSGelg==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - cli-spinners@2.9.2: - resolution: {integrity: sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==} - engines: {node: '>=6'} + cli-spinners@3.2.0: + resolution: {integrity: sha512-pXftdQloMZzjCr3pCTIRniDcys6dDzgpgVhAHHk6TKBDbRuP1MkuetTF5KSv4YUutbOPa7+7ZrAJ2kVtbMqyXA==} + engines: {node: '>=18.20'} cli-truncate@4.0.0: resolution: {integrity: sha512-nPdaFdQ0h/GEigbPClz11D0v/ZJEwxmeVZGeMo3Z5StPtUTkA9o1lD6QwoirYiSDzbcwn2XcjwmCp68W1IS4TA==} @@ -1744,6 +1767,10 @@ packages: comma-separated-tokens@2.0.3: resolution: {integrity: sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==} + commander@14.0.0: + resolution: {integrity: sha512-2uM9rYjPvyq39NwLRqaiLtWHyDC1FvryJDa2ATTVims5YAS4PupsEQsDvP14FqhFr0P49CYDugi59xaxJlTXRA==} + engines: {node: '>=20'} + common-ancestor-path@1.0.1: resolution: {integrity: sha512-L3sHRo1pXXEqX8VU28kfgUY+YGsk09hPqZiZmLacNib6XNTCM8ubYeT7ryXQw8asB1sKgcU5lkB7ONug08aB8w==} @@ -1833,6 +1860,10 @@ packages: resolution: {integrity: sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==} engines: {node: '>=6'} + deepmerge@4.3.1: + resolution: {integrity: sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==} + engines: {node: '>=0.10.0'} + define-data-property@1.1.4: resolution: {integrity: sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==} engines: {node: '>= 0.4'} @@ -1909,11 +1940,6 @@ packages: ee-first@1.1.1: resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} - ejs@3.1.10: - resolution: {integrity: sha512-UeJmFfOrAQS8OJWPZ4qtgHyWExa088/MtK5UEyoJGFH67cDEXkZSviOiKRCZ4Xij0zxI3JECgYs3oKx+AizQBA==} - engines: {node: '>=0.10.0'} - hasBin: true - emoji-regex@10.4.0: resolution: {integrity: sha512-EC+0oUMY1Rqm4O6LLrgjtYDvcVYTy7chDnM4Q7030tP4Kwj3u/pR6gP9ygnp2CJMK5Gq+9Q2oqmrFJAz01DXjw==} @@ -1962,8 +1988,8 @@ packages: resolution: {integrity: sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==} engines: {node: '>= 0.4'} - es-toolkit@1.39.8: - resolution: {integrity: sha512-A8QO9TfF+rltS8BXpdu8OS+rpGgEdnRhqIVxO/ZmNvnXBYgOdSsxukT55ELyP94gZIntWJ+Li9QRrT2u1Kitpg==} + es-toolkit@1.39.10: + resolution: {integrity: sha512-E0iGnTtbDhkeczB0T+mxmoVlT4YNweEKBLq7oaU4p11mecdsZpNWOglI4895Vh4usbQ+LsJiuLuI2L0Vdmfm2w==} esast-util-from-estree@2.0.0: resolution: {integrity: sha512-4CyanoAudUSBAn5K13H4JhsMH6L9ZP7XbLVe/dKybkxMO7eDyLsT8UHl9TRNrU2Gr9nz+FovfSIjuXWJ81uVwQ==} @@ -1987,10 +2013,6 @@ packages: resolution: {integrity: sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==} engines: {node: '>=8'} - escape-string-regexp@4.0.0: - resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} - engines: {node: '>=10'} - escape-string-regexp@5.0.0: resolution: {integrity: sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==} engines: {node: '>=12'} @@ -2091,9 +2113,6 @@ packages: resolution: {integrity: sha512-d+l3qxjSesT4V7v2fh+QnmFnUWv9lSpjarhShNTgBOfA0ttejbQUAlHLitbjkoRiDulW0OPoQPYIGhIC8ohejg==} engines: {node: '>=18'} - filelist@1.0.4: - resolution: {integrity: sha512-w1cEuf3S+DrLCQL7ET6kz+gmlJdbq9J7yXCSjK/OZCPA+qEN1WyF4ZAf0YYJa4/shHJra2t/d/r8SV4Ji+x+8Q==} - fill-range@7.1.1: resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} engines: {node: '>=8'} @@ -2159,10 +2178,6 @@ packages: resolution: {integrity: sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==} engines: {node: '>= 0.4'} - get-package-type@0.1.0: - resolution: {integrity: sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==} - engines: {node: '>=8.0.0'} - get-proto@1.0.1: resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} engines: {node: '>= 0.4'} @@ -2214,6 +2229,10 @@ packages: resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==} engines: {node: '>= 0.4'} + hasha@6.0.0: + resolution: {integrity: sha512-MLydoyGp9QJcjlhE5lsLHXYpWayjjWqkavzju2ZWD2tYa1CgmML1K1gWAu22BLFa2eZ0OfvJ/DlfoVjaD54U2Q==} + engines: {node: '>=18'} + hasown@2.0.2: resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} engines: {node: '>= 0.4'} @@ -2328,10 +2347,6 @@ packages: import-meta-resolve@4.1.0: resolution: {integrity: sha512-I6fiaX09Xivtk+THaMfAwnA3MVA5Big1WHF1Dfx9hFuvNIWpXnorlkzhcQf6ehrqQiiZECRt1poOAkPmer3ruw==} - indent-string@4.0.0: - resolution: {integrity: sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==} - engines: {node: '>=8'} - indent-string@5.0.0: resolution: {integrity: sha512-m6FAo/spmsW2Ab2fU35JTYwtOKa2yAwXSwgjSv1TJzh4Mh7mC3lzAOVLBprb72XsTrgkEIsl7YrFNAiDiRhIGg==} engines: {node: '>=12'} @@ -2385,11 +2400,6 @@ packages: is-decimal@2.0.1: resolution: {integrity: sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A==} - is-docker@2.2.1: - resolution: {integrity: sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==} - engines: {node: '>=8'} - hasBin: true - is-docker@3.0.0: resolution: {integrity: sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} @@ -2460,14 +2470,14 @@ packages: resolution: {integrity: sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==} engines: {node: '>=0.10.0'} - is-wsl@2.2.0: - resolution: {integrity: sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==} - engines: {node: '>=8'} - is-wsl@3.1.0: resolution: {integrity: sha512-UcVfVfaK4Sc4m7X3dUSoHoozQGBEFeDC+zVo06t98xe8CzHSZZBekNXH+tu0NalHolcJ/QAGqS46Hef7QXBIMw==} engines: {node: '>=16'} + isbinaryfile@5.0.4: + resolution: {integrity: sha512-YKBKVkKhty7s8rxddb40oOkuP0NbaeXrQvLin6QMHL7Ypiy2RW9LwOVrVgZRyOrhQlayMd9t+D8yDy8MKFTSDQ==} + engines: {node: '>= 18.0.0'} + isexe@2.0.0: resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} @@ -2490,11 +2500,6 @@ packages: jackspeak@3.4.3: resolution: {integrity: sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==} - jake@10.9.4: - resolution: {integrity: sha512-wpHYzhxiVQL+IV05BLE2Xn34zW1S223hvjtqk0+gsPrwd/8JNLXJgZZM/iPFsYc1xyphF+6M6EvdE5E9MBGkDA==} - engines: {node: '>=10'} - hasBin: true - js-tokens@4.0.0: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} @@ -2527,10 +2532,6 @@ packages: resolution: {integrity: sha512-dhG34DXATL5hSxJbIexCft8FChFXtmskoZYnoPWjXQuebWYCNkVeV3KkGegCK9CP1oswI/vQibS2GY7Em/sJJA==} engines: {node: '>= 8'} - lilconfig@3.1.3: - resolution: {integrity: sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==} - engines: {node: '>=14'} - linkify-it@5.0.0: resolution: {integrity: sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ==} @@ -2796,10 +2797,6 @@ packages: resolution: {integrity: sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==} engines: {node: '>=6'} - minimatch@5.1.6: - resolution: {integrity: sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==} - engines: {node: '>=10'} - minimatch@9.0.5: resolution: {integrity: sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==} engines: {node: '>=16 || 14 >=14.17'} @@ -3515,10 +3512,6 @@ packages: resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} engines: {node: '>=8'} - supports-color@8.1.1: - resolution: {integrity: sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==} - engines: {node: '>=10'} - temp-dir@3.0.0: resolution: {integrity: sha512-nHc6S/bwIilKHNRgK/3jlhDoIHcp45YgyiwcAk46Tr0LfEqGBVpmiAyuiuxeVE44m3mXnEeVhaipLOEWmH+Njw==} engines: {node: '>=14.16'} @@ -3617,10 +3610,6 @@ packages: engines: {node: '>=18.0.0'} hasBin: true - type-fest@0.21.3: - resolution: {integrity: sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==} - engines: {node: '>=10'} - type-fest@1.4.0: resolution: {integrity: sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==} engines: {node: '>=10'} @@ -3921,17 +3910,10 @@ packages: engines: {node: '>=8'} hasBin: true - widest-line@3.1.0: - resolution: {integrity: sha512-NsmoXalsWVDMGupxZ5R08ka9flZjjiLvHVAWYOKtiKM8ujtZWr9cRffak+uSE48+Ob8ObalXpwyeUiyDD6QFgg==} - engines: {node: '>=8'} - widest-line@5.0.0: resolution: {integrity: sha512-c9bZp7b5YtRj2wOe6dlj32MK+Bx/M/d+9VB2SHM1OtsUHR0aV0tdP6DWh/iMt0kWi1t5g1Iudu6hQRNd1A4PVA==} engines: {node: '>=18'} - wordwrap@1.0.0: - resolution: {integrity: sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==} - wrap-ansi@7.0.0: resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==} engines: {node: '>=10'} @@ -4118,7 +4100,7 @@ snapshots: '@astrojs/telemetry@3.3.0': dependencies: ci-info: 4.3.0 - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 dlv: 1.1.3 dset: 3.1.4 is-docker: 3.0.0 @@ -4329,6 +4311,21 @@ snapshots: human-id: 4.1.1 prettier: 2.8.8 + '@clack/core@0.5.0': + dependencies: + picocolors: 1.1.1 + sisteransi: 1.0.5 + + '@clack/prompts@0.11.0': + dependencies: + '@clack/core': 0.5.0 + picocolors: 1.1.1 + sisteransi: 1.0.5 + + '@commander-js/extra-typings@14.0.0(commander@14.0.0)': + dependencies: + commander: 14.0.0 + '@ctrl/tinycolor@4.1.0': {} '@emnapi/core@1.4.5': @@ -4614,6 +4611,14 @@ snapshots: '@img/sharp-win32-x64@0.34.2': optional: true + '@inkjs/ui@2.0.0(ink@6.1.0(@types/react@19.1.9)(react@19.1.1))': + dependencies: + chalk: 5.5.0 + cli-spinners: 3.2.0 + deepmerge: 4.3.1 + figures: 6.1.0 + ink: 6.1.0(@types/react@19.1.9)(react@19.1.1) + '@isaacs/cliui@8.0.2': dependencies: string-width: 5.1.2 @@ -4765,36 +4770,6 @@ snapshots: '@nodelib/fs.scandir': 2.1.5 fastq: 1.19.1 - '@oclif/core@4.5.2': - dependencies: - ansi-escapes: 4.3.2 - ansis: 3.17.0 - clean-stack: 3.0.1 - cli-spinners: 2.9.2 - debug: 4.4.1(supports-color@8.1.1) - ejs: 3.1.10 - get-package-type: 0.1.0 - indent-string: 4.0.0 - is-wsl: 2.2.0 - lilconfig: 3.1.3 - minimatch: 9.0.5 - semver: 7.7.2 - string-width: 4.2.3 - supports-color: 8.1.1 - tinyglobby: 0.2.14 - widest-line: 3.1.0 - wordwrap: 1.0.0 - wrap-ansi: 7.0.0 - - '@oclif/plugin-autocomplete@3.2.34': - dependencies: - '@oclif/core': 4.5.2 - ansis: 3.17.0 - debug: 4.4.1(supports-color@8.1.1) - ejs: 3.1.10 - transitivePeerDependencies: - - supports-color - '@oslojs/encoding@1.1.0': {} '@pagefind/darwin-arm64@1.3.0': @@ -5067,7 +5042,7 @@ snapshots: dependencies: '@ampproject/remapping': 2.3.0 '@bcoe/v8-coverage': 1.0.2 - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 istanbul-lib-coverage: 3.2.2 istanbul-lib-report: 3.0.1 istanbul-lib-source-maps: 5.0.6 @@ -5160,10 +5135,6 @@ snapshots: ansi-colors@4.1.3: {} - ansi-escapes@4.3.2: - dependencies: - type-fest: 0.21.3 - ansi-escapes@7.0.0: dependencies: environment: 1.1.0 @@ -5178,8 +5149,6 @@ snapshots: ansi-styles@6.2.1: {} - ansis@3.17.0: {} - anymatch@3.1.3: dependencies: normalize-path: 3.0.0 @@ -5228,7 +5197,7 @@ snapshots: common-ancestor-path: 1.0.1 cookie: 1.0.2 cssesc: 3.0.0 - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 deterministic-object-hash: 2.0.2 devalue: 5.1.1 diff: 5.2.0 @@ -5311,8 +5280,6 @@ snapshots: - uploadthing - yaml - async@3.2.6: {} - auto-bind@5.0.1: {} axobject-query@4.1.0: {} @@ -5429,6 +5396,8 @@ snapshots: chardet@0.7.0: {} + chardet@2.1.0: {} + check-error@2.1.1: {} chokidar@4.0.3: @@ -5439,17 +5408,13 @@ snapshots: ci-info@4.3.0: {} - clean-stack@3.0.1: - dependencies: - escape-string-regexp: 4.0.0 - cli-boxes@3.0.0: {} cli-cursor@4.0.0: dependencies: restore-cursor: 4.0.0 - cli-spinners@2.9.2: {} + cli-spinners@3.2.0: {} cli-truncate@4.0.0: dependencies: @@ -5484,6 +5449,8 @@ snapshots: comma-separated-tokens@2.0.3: {} + commander@14.0.0: {} + common-ancestor-path@1.0.1: {} content-disposition@0.5.4: @@ -5544,11 +5511,9 @@ snapshots: dependencies: ms: 2.0.0 - debug@4.4.1(supports-color@8.1.1): + debug@4.4.1: dependencies: ms: 2.1.3 - optionalDependencies: - supports-color: 8.1.1 decode-named-character-reference@1.2.0: dependencies: @@ -5556,6 +5521,8 @@ snapshots: deep-eql@5.0.2: {} + deepmerge@4.3.1: {} + define-data-property@1.1.4: dependencies: es-define-property: 1.0.1 @@ -5616,10 +5583,6 @@ snapshots: ee-first@1.1.1: {} - ejs@3.1.10: - dependencies: - jake: 10.9.4 - emoji-regex@10.4.0: {} emoji-regex@8.0.0: {} @@ -5651,7 +5614,7 @@ snapshots: dependencies: es-errors: 1.3.0 - es-toolkit@1.39.8: {} + es-toolkit@1.39.10: {} esast-util-from-estree@2.0.0: dependencies: @@ -5702,8 +5665,6 @@ snapshots: escape-string-regexp@2.0.0: {} - escape-string-regexp@4.0.0: {} - escape-string-regexp@5.0.0: {} esprima@4.0.1: {} @@ -5847,10 +5808,6 @@ snapshots: dependencies: is-unicode-supported: 2.1.0 - filelist@1.0.4: - dependencies: - minimatch: 5.1.6 - fill-range@7.1.1: dependencies: to-regex-range: 5.0.1 @@ -5940,8 +5897,6 @@ snapshots: hasown: 2.0.2 math-intrinsics: 1.1.0 - get-package-type@0.1.0: {} - get-proto@1.0.1: dependencies: dunder-proto: 1.0.1 @@ -6009,6 +5964,11 @@ snapshots: has-symbols@1.1.0: {} + hasha@6.0.0: + dependencies: + is-stream: 3.0.0 + type-fest: 4.41.0 + hasown@2.0.2: dependencies: function-bind: 1.1.2 @@ -6222,7 +6182,7 @@ snapshots: http-graceful-shutdown@3.1.14: dependencies: - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 transitivePeerDependencies: - supports-color @@ -6244,8 +6204,6 @@ snapshots: import-meta-resolve@4.1.0: {} - indent-string@4.0.0: {} - indent-string@5.0.0: {} inherits@2.0.4: {} @@ -6268,7 +6226,7 @@ snapshots: cli-cursor: 4.0.0 cli-truncate: 4.0.0 code-excerpt: 4.0.0 - es-toolkit: 1.39.8 + es-toolkit: 1.39.10 indent-string: 5.0.0 is-in-ci: 1.0.0 patch-console: 2.0.0 @@ -6309,8 +6267,6 @@ snapshots: is-decimal@2.0.1: {} - is-docker@2.2.1: {} - is-docker@3.0.0: {} is-extglob@2.1.1: {} @@ -6355,14 +6311,12 @@ snapshots: is-windows@1.0.2: {} - is-wsl@2.2.0: - dependencies: - is-docker: 2.2.1 - is-wsl@3.1.0: dependencies: is-inside-container: 1.0.0 + isbinaryfile@5.0.4: {} + isexe@2.0.0: {} istanbul-lib-coverage@3.2.2: {} @@ -6376,7 +6330,7 @@ snapshots: istanbul-lib-source-maps@5.0.6: dependencies: '@jridgewell/trace-mapping': 0.3.29 - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 istanbul-lib-coverage: 3.2.2 transitivePeerDependencies: - supports-color @@ -6392,12 +6346,6 @@ snapshots: optionalDependencies: '@pkgjs/parseargs': 0.11.0 - jake@10.9.4: - dependencies: - async: 3.2.6 - filelist: 1.0.4 - picocolors: 1.1.1 - js-tokens@4.0.0: {} js-yaml@3.14.1: @@ -6427,8 +6375,6 @@ snapshots: klona@2.0.6: {} - lilconfig@3.1.3: {} - linkify-it@5.0.0: dependencies: uc.micro: 2.1.0 @@ -6936,7 +6882,7 @@ snapshots: micromark@4.0.2: dependencies: '@types/debug': 4.1.12 - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 decode-named-character-reference: 1.2.0 devlop: 1.1.0 micromark-core-commonmark: 2.0.3 @@ -6970,10 +6916,6 @@ snapshots: mimic-fn@2.1.0: {} - minimatch@5.1.6: - dependencies: - brace-expansion: 2.0.2 - minimatch@9.0.5: dependencies: brace-expansion: 2.0.2 @@ -7788,10 +7730,6 @@ snapshots: dependencies: has-flag: 4.0.0 - supports-color@8.1.1: - dependencies: - has-flag: 4.0.0 - temp-dir@3.0.0: {} tempy@3.1.0: @@ -7867,8 +7805,6 @@ snapshots: optionalDependencies: fsevents: 2.3.3 - type-fest@0.21.3: {} - type-fest@1.4.0: {} type-fest@2.19.0: {} @@ -8031,7 +7967,7 @@ snapshots: vite-node@3.1.4(@types/node@24.2.0)(tsx@4.20.3)(yaml@2.8.1): dependencies: cac: 6.7.14 - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 es-module-lexer: 1.7.0 pathe: 2.0.3 vite: 6.3.5(@types/node@24.2.0)(tsx@4.20.3)(yaml@2.8.1) @@ -8077,7 +8013,7 @@ snapshots: '@vitest/spy': 3.1.4 '@vitest/utils': 3.1.4 chai: 5.2.1 - debug: 4.4.1(supports-color@8.1.1) + debug: 4.4.1 expect-type: 1.2.2 magic-string: 0.30.17 pathe: 2.0.3 @@ -8128,16 +8064,10 @@ snapshots: siginfo: 2.0.0 stackback: 0.0.2 - widest-line@3.1.0: - dependencies: - string-width: 4.2.3 - widest-line@5.0.0: dependencies: string-width: 7.2.0 - wordwrap@1.0.0: {} - wrap-ansi@7.0.0: dependencies: ansi-styles: 4.3.0 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 717b560a..bfa5aa97 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -11,7 +11,6 @@ packages: - docs - dpkit - excel - - extend - file - folder - github @@ -21,6 +20,5 @@ packages: - parquet - table - test - - ui - zenodo - zip diff --git a/table/error/Base.ts b/table/error/Base.ts index 51539707..e0d2fcb6 100644 --- a/table/error/Base.ts +++ b/table/error/Base.ts @@ -1,3 +1,3 @@ -export interface BaseTableError { - type: string -} +import type { BaseError } from "@dpkit/core" + +export interface BaseTableError extends BaseError {} diff --git a/table/schema/infer.ts b/table/schema/infer.ts index 7354fcce..e7611819 100644 --- a/table/schema/infer.ts +++ b/table/schema/infer.ts @@ -3,6 +3,8 @@ import { col } from "nodejs-polars" import { getPolarsSchema } from "../schema/index.ts" import type { Table } from "../table/index.ts" +// TODO: Review default values being {fields: []} vs undefined + export type InferSchemaOptions = { sampleRows?: number confidence?: number diff --git a/tsconfig.json b/tsconfig.json index 8b1e1a39..b076abc1 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,7 +1,12 @@ { "$schema": "https://json.schemastore.org/tsconfig", + "include": ["${configDir}/**/*.ts", "${configDir}/**/*.tsx"], - "exclude": ["${configDir}/**/build/", "${configDir}/**/docs/"], + "exclude": [ + "${configDir}/**/docs/", + "${configDir}/**/build/", + "${configDir}/**/compile/" + ], "compilerOptions": { "target": "ES2022", diff --git a/ui/CHANGELOG.md b/ui/CHANGELOG.md deleted file mode 100644 index 67b728ad..00000000 --- a/ui/CHANGELOG.md +++ /dev/null @@ -1,13 +0,0 @@ -# @dpkit/ui - -## 0.3.0 - -### Minor Changes - -- 313a275: Added Json/Jsonl/Arrow/Parquet support; Bootstrapped CLI - -## 0.2.0 - -### Minor Changes - -- edfde49: General framework improvements diff --git a/ui/README.md b/ui/README.md deleted file mode 100644 index 5b3240e3..00000000 --- a/ui/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# @dpkit/ui - -dpkit is a fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/ui/index.ts b/ui/index.ts deleted file mode 100644 index e69de29b..00000000 diff --git a/ui/package.json b/ui/package.json deleted file mode 100644 index 9c0764bc..00000000 --- a/ui/package.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "name": "@dpkit/ui", - "type": "module", - "version": "0.3.0", - "exports": "./build/index.js", - "license": "MIT", - "author": "Evgeny Karev", - "repository": "https://github.com/datisthq/dpkit", - "description": "Fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames", - "keywords": [ - "data", - "polars", - "dataframe", - "datapackage", - "tableschema", - "typescript", - "validation", - "quality", - "fair", - "ui" - ], - "scripts": { - "build": "tsc" - } -} diff --git a/ui/tsconfig.json b/ui/tsconfig.json deleted file mode 100644 index 3c43903c..00000000 --- a/ui/tsconfig.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "extends": "../tsconfig.json" -} diff --git a/zenodo/package/save.ts b/zenodo/package/save.ts index c889f6b0..041c7e1f 100644 --- a/zenodo/package/save.ts +++ b/zenodo/package/save.ts @@ -12,7 +12,7 @@ import { denormalizeZenodoPackage } from "./process/denormalize.ts" /** * Save a package to Zenodo - * @param props Object containing the package to save and Zenodo API details + * @param options Object containing the package to save and Zenodo API details * @returns Object with the deposit URL and DOI */ export async function savePackageToZenodo( @@ -43,10 +43,10 @@ export async function savePackageToZenodo( basepath, withRemote: false, withoutFolders: true, - saveFile: async props => { + saveFile: async options => { const upload = { - name: props.denormalizedPath, - data: await blob(await loadFileStream(props.normalizedPath)), + name: options.denormalizedPath, + data: await blob(await loadFileStream(options.normalizedPath)), } // It seems that record and deposition files have different metadata @@ -59,7 +59,7 @@ export async function savePackageToZenodo( sandbox, }) - return props.denormalizedPath + return options.denormalizedPath }, }), ) diff --git a/zip/package/save.ts b/zip/package/save.ts index 59f6e071..ed6c930f 100644 --- a/zip/package/save.ts +++ b/zip/package/save.ts @@ -30,13 +30,13 @@ export async function savePackageToZip( await saveResourceFiles(resource, { basepath, withRemote, - saveFile: async props => { + saveFile: async options => { zipfile.addReadStream( - await loadFileStream(props.normalizedPath), - props.denormalizedPath, + await loadFileStream(options.normalizedPath), + options.denormalizedPath, ) - return props.denormalizedPath + return options.denormalizedPath }, }), )