diff --git a/core/package.json b/core/package.json index cc0bfbf1..e0ac37d7 100644 --- a/core/package.json +++ b/core/package.json @@ -27,5 +27,8 @@ "ajv": "^8.17.1", "quick-lru": "^7.0.1", "tiny-invariant": "^1.3.3" + }, + "devDependencies": { + "@dpkit/test": "workspace:*" } } diff --git a/core/package/fixtures/generated/validatePackageDescriptor-should-validate-camtrap-dp-144_965007081/recording.har b/core/package/fixtures/generated/validatePackageDescriptor-should-validate-camtrap-dp-144_965007081/recording.har new file mode 100644 index 00000000..6c78c77a --- /dev/null +++ b/core/package/fixtures/generated/validatePackageDescriptor-should-validate-camtrap-dp-144_965007081/recording.har @@ -0,0 +1,156 @@ +{ + "log": { + "_recordingName": "validatePackageDescriptor-should validate camtrap dp (#144)", + "creator": { + "comment": "persister:fs", + "name": "Polly.JS", + "version": "6.0.6" + }, + "entries": [ + { + "_id": "4306f83bec3be19c1183804f9d081277", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [], + "headersSize": 109, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://raw.githubusercontent.com/tdwg/camtrap-dp/refs/tags/1.0.2/example/datapackage.json" + }, + "response": { + "bodySize": 2603, + "content": { + "mimeType": "text/plain; charset=utf-8", + "size": 2603, + "text": "{\n \"resources\": [\n {\n \"name\": \"deployments\",\n \"path\": \"deployments.csv\",\n \"profile\": \"tabular-data-resource\",\n \"format\": \"csv\",\n \"mediatype\": \"text/csv\",\n \"encoding\": \"utf-8\",\n \"schema\": \"https://raw.githubusercontent.com/tdwg/camtrap-dp/1.0.2/deployments-table-schema.json\"\n },\n {\n \"name\": \"media\",\n \"path\": \"media.csv\",\n \"profile\": \"tabular-data-resource\",\n \"format\": \"csv\",\n \"mediatype\": \"text/csv\",\n \"encoding\": \"utf-8\",\n \"schema\": \"https://raw.githubusercontent.com/tdwg/camtrap-dp/1.0.2/media-table-schema.json\"\n },\n {\n \"name\": \"observations\",\n \"path\": \"observations.csv\",\n \"profile\": \"tabular-data-resource\",\n \"format\": \"csv\",\n \"mediatype\": \"text/csv\",\n \"encoding\": \"utf-8\",\n \"schema\": \"https://raw.githubusercontent.com/tdwg/camtrap-dp/1.0.2/observations-table-schema.json\"\n },\n {\n \"name\": \"individuals\",\n \"description\": \"Custom table/resource not part of the Camtrap DP model. Included to showcase that extending with more resources is possible.\",\n \"data\": [\n {\n \"id\": 1,\n \"individualName\": \"Reinaert\",\n \"scientificName\": \"Vulpes vulpes\"\n }\n ]\n }\n ],\n \"profile\": \"https://raw.githubusercontent.com/tdwg/camtrap-dp/1.0.2/camtrap-dp-profile.json\",\n \"name\": \"camtrap-dp-example-dataset\",\n \"id\": \"7cca70f5-ef8c-4f86-85fb-8f070937d7ab\",\n \"created\": \"2023-02-06T11:23:03Z\",\n \"title\": \"Sample from: MICA - Muskrat and coypu camera trap observations in Belgium, the Netherlands and Germany\",\n \"contributors\": [\n {\n \"title\": \"Axel Neukermans\",\n \"email\": \"axel.neukermans@inbo.be\",\n \"path\": \"https://orcid.org/0000-0003-0272-9180\",\n \"role\": \"contributor\",\n \"organization\": \"Research Institute for Nature and Forest (INBO)\"\n },\n {\n \"title\": \"Danny Van der beeck\",\n \"email\": \"daniel.vanderbeeck@gmail.com\"\n },\n {\n \"title\": \"Emma Cartuyvels\",\n \"email\": \"emma.cartuyvels@inbo.be\",\n \"role\": \"principalInvestigator\",\n \"organization\": \"Research Institute for Nature and Forest (INBO)\"\n },\n {\n \"title\": \"Peter Desmet\",\n \"email\": \"peter.desmet@inbo.be\",\n \"path\": \"https://orcid.org/0000-0002-8442-8025\",\n \"role\": \"contact\",\n \"organization\": \"Research Institute for Nature and Forest (INBO)\"\n },\n {\n \"title\": \"Research Institute for Nature and Forest (INBO)\",\n \"path\": \"https://inbo.be\",\n \"role\": \"rightsHolder\"\n },\n {\n \"title\": \"Research Institute for Nature and Forest (INBO)\",\n \"path\": \"https://inbo.be\",\n \"role\": \"publisher\"\n }\n ],\n \"description\": \"MICA - Muskrat and coypu camera trap observations in Belgium, the Netherlands and Germany is an occurrence dataset published by the Research Institute of Nature and Forest (INBO). It is part of the LIFE project MICA, in which innovative techniques are tested for a more efficient control of muskrat and coypu populations, both invasive species. This dataset is a sample of the original dataset and serves as an example of a Camera Trap Data Package (Camtrap DP).\",\n \"version\": \"1.0.2\",\n \"keywords\": [\n \"camera traps\",\n \"public awareness campaign\",\n \"flood protection\",\n \"flood control\",\n \"damage prevention\",\n \"animal damage\",\n \"pest control\",\n \"invasive alien species\",\n \"muskrat\",\n \"coypu\"\n ],\n \"image\": \"\",\n \"homepage\": \"https://camtrap-dp.tdwg.org/example/\",\n \"sources\": [\n {\n \"title\": \"Agouti\",\n \"path\": \"https://www.agouti.eu\",\n \"email\": \"agouti@wur.nl\",\n \"version\": \"v3.21\"\n }\n ],\n \"licenses\": [\n {\n \"name\": \"CC0-1.0\",\n \"scope\": \"data\"\n },\n {\n \"path\": \"http://creativecommons.org/licenses/by/4.0/\",\n \"scope\": \"media\"\n }\n ],\n \"bibliographicCitation\": \"Desmet P, Neukermans A, Van der beeck D, Cartuyvels E (2022). Sample from: MICA - Muskrat and coypu camera trap observations in Belgium, the Netherlands and Germany. Version 1.0.2. Research Institute for Nature and Forest (INBO). Dataset. https://camtrap-dp.tdwg.org/example/\",\n \"project\": {\n \"id\": \"86cabc14-d475-4439-98a7-e7b590bed60e\",\n \"title\": \"Management of Invasive Coypu and muskrAt in Europe\",\n \"acronym\": \"MICA\",\n \"description\": \"Invasive alien species such as the coypu and muskrat pose a major threat to biodiversity and cost millions of euros annually. By feeding on rushes and reeds, these animals cause serious damage to the environment in which they live and endangered species suffer from habitat loss. The disappearance of reeds and digging in dikes represents a safety risk for humans in the lowland areas. With the LIFE project MICA (), the partners from the participating countries want to develop a transnational plan for the management of coypu and muskrat populations in Europe and aim to reduce their population. The objective of an effective population control of coypu and muskrat is to protect lowlands from flooding, to prevent crop damage and loss of biodiversity. The objective of the project is to serve as a pilot and demonstration project in which ‘best practices’ are tested and new techniques are developed for a more efficient control of muskrat and coypu populations. By involving organisations from Belgium, Germany and the Netherlands, the project also promotes international cooperation and knowledge exchange in the field of muskrat and coypu management.\",\n \"samplingDesign\": \"targeted\",\n \"path\": \"https://lifemica.eu\",\n \"captureMethod\": [\n \"activityDetection\",\n \"timeLapse\"\n ],\n \"individualAnimals\": false,\n \"observationLevel\": [\n \"media\",\n \"event\"\n ]\n },\n \"coordinatePrecision\": 0.001,\n \"spatial\": {\n \"type\": \"Polygon\",\n \"coordinates\": [\n [\n [\n 4.013,\n 50.699\n ],\n [\n 5.659,\n 50.699\n ],\n [\n 5.659,\n 51.496\n ],\n [\n 4.013,\n 51.496\n ],\n [\n 4.013,\n 50.699\n ]\n ]\n ]\n },\n \"temporal\": {\n \"start\": \"2020-05-30\",\n \"end\": \"2021-04-18\"\n },\n \"taxonomic\": [\n {\n \"scientificName\": \"Anas platyrhynchos\",\n \"taxonID\": \"https://www.checklistbank.org/dataset/COL2023/taxon/DGP6\",\n \"taxonRank\": \"species\",\n \"vernacularNames\": {\n \"eng\": \"mallard\",\n \"nld\": \"wilde eend\"\n }\n },\n {\n \"scientificName\": \"Anas strepera\",\n \"taxonID\": \"https://www.checklistbank.org/dataset/COL2023/taxon/DGPL\",\n \"taxonRank\": \"species\",\n \"vernacularNames\": {\n \"eng\": \"gadwall\",\n \"nld\": \"krakeend\"\n }\n },\n {\n \"scientificName\": \"Ardea\",\n \"taxonID\": \"https://www.checklistbank.org/dataset/COL2023/taxon/32FH\",\n \"taxonRank\": \"genus\",\n \"vernacularNames\": {\n \"eng\": \"great herons\",\n \"nld\": \"reigers\"\n }\n },\n {\n \"scientificName\": \"Ardea cinerea\",\n \"taxonID\": \"https://www.checklistbank.org/dataset/COL2023/taxon/GCHS\",\n \"taxonRank\": \"species\",\n \"vernacularNames\": {\n \"eng\": \"grey heron\",\n \"nld\": \"blauwe reiger\"\n }\n },\n {\n \"scientificName\": \"Aves\",\n \"taxonID\": \"https://www.checklistbank.org/dataset/COL2023/taxon/V2\",\n \"taxonRank\": \"class\",\n \"vernacularNames\": {\n \"eng\": \"bird sp.\",\n \"nld\": \"vogel\"\n }\n },\n {\n \"scientificName\": \"Homo sapiens\",\n \"taxonID\": \"https://www.checklistbank.org/dataset/COL2023/taxon/6MB3T\",\n \"taxonRank\": \"species\",\n \"vernacularNames\": {\n \"eng\": \"human\",\n \"nld\": \"mens\"\n }\n },\n {\n \"scientificName\": \"Martes foina\",\n \"taxonID\": \"https://www.checklistbank.org/dataset/COL2023/taxon/3Y9VW\",\n \"taxonRank\": \"species\",\n \"vernacularNames\": {\n \"eng\": \"beech marten\",\n \"nld\": \"steenmarter\"\n }\n },\n {\n \"scientificName\": \"Mustela putorius\",\n \"taxonID\": \"https://www.checklistbank.org/dataset/COL2023/taxon/44QYC\",\n \"taxonRank\": \"species\",\n \"vernacularNames\": {\n \"eng\": \"European polecat\",\n \"nld\": \"bunzing\"\n }\n },\n {\n \"scientificName\": \"Rattus norvegicus\",\n \"taxonID\": \"https://www.checklistbank.org/dataset/COL2023/taxon/4RM67\",\n \"taxonRank\": \"species\",\n \"vernacularNames\": {\n \"eng\": \"brown rat\",\n \"nld\": \"bruine rat\"\n }\n },\n {\n \"scientificName\": \"Vulpes vulpes\",\n \"taxonID\": \"https://www.checklistbank.org/dataset/COL2023/taxon/5BSG3\",\n \"taxonRank\": \"species\",\n \"vernacularNames\": {\n \"eng\": \"red fox\",\n \"nld\": \"vos\"\n }\n }\n ],\n \"relatedIdentifiers\": [\n {\n \"relationType\": \"IsDerivedFrom\",\n \"relatedIdentifier\": \"https://doi.org/10.15468/5tb6ze\",\n \"resourceTypeGeneral\": \"Dataset\",\n \"relatedIdentifierType\": \"DOI\"\n },\n {\n \"relationType\": \"IsSupplementTo\",\n \"relatedIdentifier\": \"https://inbo.github.io/camtrapdp/\",\n \"resourceTypeGeneral\": \"Software\",\n \"relatedIdentifierType\": \"URL\"\n }\n ],\n \"references\": [ ]\n}\n" + }, + "cookies": [], + "headers": [ + { + "name": "accept-ranges", + "value": "bytes" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "cache-control", + "value": "max-age=300" + }, + { + "name": "connection", + "value": "keep-alive" + }, + { + "name": "content-encoding", + "value": "gzip" + }, + { + "name": "content-length", + "value": "2603" + }, + { + "name": "content-security-policy", + "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" + }, + { + "name": "content-type", + "value": "text/plain; charset=utf-8" + }, + { + "name": "cross-origin-resource-policy", + "value": "cross-origin" + }, + { + "name": "date", + "value": "Sat, 27 Sep 2025 09:29:45 GMT" + }, + { + "name": "etag", + "value": "W/\"568d81aa4b4f1148ac5358387a8367b5853f76cb76dfb282d001809dbe43e173\"" + }, + { + "name": "expires", + "value": "Sat, 27 Sep 2025 09:34:45 GMT" + }, + { + "name": "source-age", + "value": "223" + }, + { + "name": "strict-transport-security", + "value": "max-age=31536000" + }, + { + "name": "vary", + "value": "Authorization,Accept-Encoding" + }, + { + "name": "via", + "value": "1.1 varnish" + }, + { + "name": "x-cache", + "value": "HIT" + }, + { + "name": "x-cache-hits", + "value": "1" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "x-fastly-request-id", + "value": "992bc82874a7e7d37bbff9f4838c35e697fc63de" + }, + { + "name": "x-frame-options", + "value": "deny" + }, + { + "name": "x-github-request-id", + "value": "5F1A:36A173:B08AEF:CF6618:68D79B2B" + }, + { + "name": "x-served-by", + "value": "cache-lis1490053-LIS" + }, + { + "name": "x-timer", + "value": "S1758965386.503800,VS0,VE1" + }, + { + "name": "x-xss-protection", + "value": "1; mode=block" + } + ], + "headersSize": 902, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-09-27T09:29:45.355Z", + "time": 184, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 184 + } + } + ], + "pages": [], + "version": "1.2" + } +} diff --git a/core/package/validate.spec.ts b/core/package/validate.spec.ts index 1357f90c..efbf54a7 100644 --- a/core/package/validate.spec.ts +++ b/core/package/validate.spec.ts @@ -1,6 +1,10 @@ +import { useRecording } from "@dpkit/test" import { describe, expect, it } from "vitest" +import { loadDescriptor } from "../general/index.ts" import { validatePackageDescriptor } from "./validate.ts" +useRecording() + describe("validatePackageDescriptor", () => { it("returns valid result for valid package", async () => { const descriptor = { @@ -13,10 +17,10 @@ describe("validatePackageDescriptor", () => { ], } - const result = await validatePackageDescriptor(descriptor) + const { valid, errors } = await validatePackageDescriptor(descriptor) - expect(result.valid).toBe(true) - expect(result.errors).toEqual([]) + expect(valid).toBe(true) + expect(errors).toEqual([]) }) it("returns validation errors for invalid package", async () => { @@ -25,15 +29,24 @@ describe("validatePackageDescriptor", () => { resources: "not-an-array", // Should be an array } - const result = await validatePackageDescriptor(descriptor) + const { valid, errors } = await validatePackageDescriptor(descriptor) - expect(result.valid).toBe(false) - expect(result.errors.length).toBeGreaterThan(0) + expect(valid).toBe(false) + expect(errors.length).toBeGreaterThan(0) - const error = result.errors[0] + const error = errors[0] expect(error).toBeDefined() if (error) { expect(error.keyword).toBe("type") } }) + + it("should validate camtrap dp (#144)", async () => { + const { descriptor } = await loadDescriptor( + "https://raw.githubusercontent.com/tdwg/camtrap-dp/refs/tags/1.0.2/example/datapackage.json", + ) + + const { valid } = await validatePackageDescriptor(descriptor) + expect(valid).toBe(true) + }) }) diff --git a/core/resource/normalize.ts b/core/resource/normalize.ts index 4fd83fe4..1fef6998 100644 --- a/core/resource/normalize.ts +++ b/core/resource/normalize.ts @@ -1,6 +1,7 @@ import { normalizeDialect } from "../dialect/index.ts" import { isDescriptor, normalizePath } from "../general/index.ts" import type { Descriptor } from "../general/index.ts" +import { isRemotePath } from "../general/index.ts" import { normalizeSchema } from "../schema/index.ts" export function normalizeResource( @@ -23,7 +24,15 @@ export function normalizeResource( } function normalizeProfile(descriptor: Descriptor) { - descriptor.$schema = descriptor.$schema ?? descriptor.profile + // We can ignore data-package/data-resource/tabular-data-package/tabular-data-resource + // special value profiles because they are already incorporated into the dpkit's built-in profiles + // https://specs.frictionlessdata.io/profiles/#profile-property + const remoteProfile = + typeof descriptor.profile === "string" && isRemotePath(descriptor.profile) + ? descriptor.profile + : undefined + + descriptor.$schema = descriptor.$schema ?? remoteProfile } function normalizeUrl(descriptor: Descriptor) { diff --git a/dpkit/commands/dialect/explore.tsx b/dpkit/commands/dialect/explore.tsx index 5d362857..c31c7456 100644 --- a/dpkit/commands/dialect/explore.tsx +++ b/dpkit/commands/dialect/explore.tsx @@ -37,7 +37,8 @@ export const exploreDialectCommand = new Command("explore") ) if (!dialect || isEmptyObject(dialect)) { - Session.terminate("Dialect is not available") + session.terminate("Dialect is not available") + process.exit(1) // typescript ignore never return type above } await session.render(dialect, ) diff --git a/dpkit/commands/dialect/infer.tsx b/dpkit/commands/dialect/infer.tsx index ceff601b..2358c860 100644 --- a/dpkit/commands/dialect/infer.tsx +++ b/dpkit/commands/dialect/infer.tsx @@ -37,7 +37,8 @@ export const inferDialectCommand = new Command("infer") ) if (isEmptyObject(dialect)) { - Session.terminate("Could not infer dialect") + session.terminate("Could not infer dialect") + process.exit(1) // typescript ignore never return type above } await session.render(dialect, ) diff --git a/dpkit/commands/dialect/script.tsx b/dpkit/commands/dialect/script.tsx index 8ddbd12a..885e2a0d 100644 --- a/dpkit/commands/dialect/script.tsx +++ b/dpkit/commands/dialect/script.tsx @@ -38,7 +38,8 @@ export const scriptDialectCommand = new Command("script") ) if (!dialect || isEmptyObject(dialect)) { - Session.terminate("Dialect is not available") + session.terminate("Dialect is not available") + process.exit(1) // typescript ignore never return type above } console.log( diff --git a/dpkit/commands/dialect/validate.tsx b/dpkit/commands/dialect/validate.tsx index b308137b..6b1b179f 100644 --- a/dpkit/commands/dialect/validate.tsx +++ b/dpkit/commands/dialect/validate.tsx @@ -20,12 +20,15 @@ export const validateDialectCommand = new Command("validate") .addOption(params.json) .addOption(params.debug) .addOption(params.quit) + .addOption(params.all) .action(async (path, options) => { const session = Session.create({ title: "Validate dialect", json: options.json, debug: options.debug, + quit: options.quit, + all: options.all, }) const resource: Partial | undefined = !path @@ -46,7 +49,8 @@ export const validateDialectCommand = new Command("validate") : { descriptor: undefined } if (!descriptor) { - Session.terminate("Dialect is not available") + session.terminate("Dialect is not available") + process.exit(1) } const report = await session.task( @@ -54,7 +58,7 @@ export const validateDialectCommand = new Command("validate") validateDialect(descriptor), ) - if (report.errors.length && !options.quit) { + if (report.errors.length) { const type = await selectErrorType(session, report.errors) if (type) report.errors = report.errors.filter(e => e.type === type) } diff --git a/dpkit/commands/file/copy.ts b/dpkit/commands/file/copy.ts index 78cc941c..e63dca47 100644 --- a/dpkit/commands/file/copy.ts +++ b/dpkit/commands/file/copy.ts @@ -25,7 +25,8 @@ export const copyFileCommand = new Command("copy") const resource = await selectResource(session, options) if (typeof resource.path !== "string") { - Session.terminate("Only single file resources are supported") + session.terminate("Only single file resources are supported") + process.exit(1) // typescript ignore never return type above } path = resource.path diff --git a/dpkit/commands/file/describe.tsx b/dpkit/commands/file/describe.tsx index affaa974..959da92e 100644 --- a/dpkit/commands/file/describe.tsx +++ b/dpkit/commands/file/describe.tsx @@ -29,7 +29,8 @@ export const describeFileCommand = new Command("describe") const resource = await selectResource(session, options) if (typeof resource.path !== "string") { - Session.terminate("Only single file resources are supported") + session.terminate("Only single file resources are supported") + process.exit(1) // typescript ignore never return type above } path = resource.path diff --git a/dpkit/commands/file/validate.tsx b/dpkit/commands/file/validate.tsx index 51008fe8..9fc4f211 100644 --- a/dpkit/commands/file/validate.tsx +++ b/dpkit/commands/file/validate.tsx @@ -17,16 +17,20 @@ export const validateFileCommand = new Command("validate") .addOption(params.json) .addOption(params.debug) .addOption(params.quit) + .addOption(params.all) .action(async (path, options) => { const session = Session.create({ title: "Validate file", json: options.json, debug: options.debug, + quit: options.quit, + all: options.all, }) if (!options.bytes && !options.hash) { - Session.terminate("You must specify either --bytes or --hash") + session.terminate("You must specify either --bytes or --hash") + process.exit(1) // typescript ignore never return type above } const report = await session.task( @@ -37,7 +41,7 @@ export const validateFileCommand = new Command("validate") }), ) - if (report.errors.length && !options.quit) { + if (report.errors.length) { const type = await selectErrorType(session, report.errors) if (type) report.errors = report.errors.filter(e => e.type === type) } diff --git a/dpkit/commands/package/validate.tsx b/dpkit/commands/package/validate.tsx index 9cdd0ddb..7b59570a 100644 --- a/dpkit/commands/package/validate.tsx +++ b/dpkit/commands/package/validate.tsx @@ -15,12 +15,15 @@ export const validatePackageCommand = new Command("validate") .addOption(params.json) .addOption(params.debug) .addOption(params.quit) + .addOption(params.all) .action(async (path, options) => { const session = Session.create({ title: "Validate package", json: options.json, debug: options.debug, + quit: options.quit, + all: options.all, }) const report = await session.task( @@ -28,7 +31,7 @@ export const validatePackageCommand = new Command("validate") validatePackage(path), ) - if (report.errors.length && !options.quit) { + if (report.errors.length) { // @ts-ignore const name = await selectErrorResource(session, report.errors) // @ts-ignore diff --git a/dpkit/commands/resource/explore.tsx b/dpkit/commands/resource/explore.tsx index 2616a1c8..fd70c2d9 100644 --- a/dpkit/commands/resource/explore.tsx +++ b/dpkit/commands/resource/explore.tsx @@ -29,7 +29,8 @@ export const exploreResourceCommand = new Command("explore") : await selectResource(session, options) if (isEmptyObject(resource)) { - Session.terminate("Resource is not available") + session.terminate("Resource is not available") + process.exit(1) // typescript ignore never return type above } await session.render(resource, ) diff --git a/dpkit/commands/resource/infer.tsx b/dpkit/commands/resource/infer.tsx index 8eb7b3d6..0f3049b1 100644 --- a/dpkit/commands/resource/infer.tsx +++ b/dpkit/commands/resource/infer.tsx @@ -77,7 +77,8 @@ export const inferResourceCommand = new Command("infer") ) if (isEmptyObject(result)) { - Session.terminate("Could not infer resource") + session.terminate("Could not infer resource") + process.exit(1) // typescript ignore never return type above } await session.render(result, ) diff --git a/dpkit/commands/resource/script.tsx b/dpkit/commands/resource/script.tsx index 6efc7ed2..1006e5c6 100644 --- a/dpkit/commands/resource/script.tsx +++ b/dpkit/commands/resource/script.tsx @@ -30,7 +30,8 @@ export const scriptResourceCommand = new Command("script") : await selectResource(session, options) if (isEmptyObject(resource)) { - Session.terminate("Resource is not available") + session.terminate("Resource is not available") + process.exit(1) // typescript ignore never return type above } console.log( diff --git a/dpkit/commands/resource/validate.tsx b/dpkit/commands/resource/validate.tsx index e804ff40..442c40fe 100644 --- a/dpkit/commands/resource/validate.tsx +++ b/dpkit/commands/resource/validate.tsx @@ -18,12 +18,15 @@ export const validateResourceCommand = new Command("validate") .addOption(params.json) .addOption(params.debug) .addOption(params.quit) + .addOption(params.all) .action(async (path, options) => { const session = Session.create({ title: "Validate resource", json: options.json, debug: options.debug, + quit: options.quit, + all: options.all, }) const descriptor = path ? path : await selectResource(session, options) @@ -33,7 +36,7 @@ export const validateResourceCommand = new Command("validate") validateResource(descriptor), ) - if (report.errors.length && !options.quit) { + if (report.errors.length) { const type = await selectErrorType(session, report.errors) // @ts-ignore if (type) report.errors = report.errors.filter(e => e.type === type) diff --git a/dpkit/commands/schema/explore.tsx b/dpkit/commands/schema/explore.tsx index 760a9743..26edd3b2 100644 --- a/dpkit/commands/schema/explore.tsx +++ b/dpkit/commands/schema/explore.tsx @@ -37,7 +37,8 @@ export const exploreSchemaCommand = new Command("explore") ) if (!schema || isEmptyObject(schema)) { - Session.terminate("Schema is not available") + session.terminate("Schema is not available") + process.exit(1) // typescript ignore never return type above } await session.render(schema, ) diff --git a/dpkit/commands/schema/infer.tsx b/dpkit/commands/schema/infer.tsx index 0f6d59f9..44ebf7b7 100644 --- a/dpkit/commands/schema/infer.tsx +++ b/dpkit/commands/schema/infer.tsx @@ -85,7 +85,8 @@ export const inferSchemaCommand = new Command("infer") ) if (isEmptyObject(inferredSchema)) { - Session.terminate("Could not infer schema") + session.terminate("Could not infer schema") + process.exit(1) // typescript ignore never return type above } await session.render(inferredSchema, ) diff --git a/dpkit/commands/schema/validate.tsx b/dpkit/commands/schema/validate.tsx index 8eb2335e..961e20e1 100644 --- a/dpkit/commands/schema/validate.tsx +++ b/dpkit/commands/schema/validate.tsx @@ -20,12 +20,15 @@ export const validateSchemaCommand = new Command("validate") .addOption(params.json) .addOption(params.debug) .addOption(params.quit) + .addOption(params.all) .action(async (path, options) => { const session = Session.create({ title: "Validate schema", json: options.json, debug: options.debug, + quit: options.quit, + all: options.all, }) const resource: Partial | undefined = !path @@ -46,7 +49,8 @@ export const validateSchemaCommand = new Command("validate") : { descriptor: undefined } if (!descriptor) { - Session.terminate("Schema is not available") + session.terminate("Schema is not available") + process.exit(1) // typescript ignore never return type above } const report = await session.task( @@ -54,7 +58,7 @@ export const validateSchemaCommand = new Command("validate") validateSchema(descriptor), ) - if (report.errors.length && !options.quit) { + if (report.errors.length) { const type = await selectErrorType(session, report.errors) if (type) report.errors = report.errors.filter(e => e.type === type) } diff --git a/dpkit/commands/table/validate.tsx b/dpkit/commands/table/validate.tsx index 9af4c51b..f0fdc891 100644 --- a/dpkit/commands/table/validate.tsx +++ b/dpkit/commands/table/validate.tsx @@ -23,6 +23,7 @@ export const validateTableCommand = new Command("validate") .addOption(params.json) .addOption(params.debug) .addOption(params.quit) + .addOption(params.all) .optionsGroup("Table Dialect") .addOption(params.dialect) @@ -77,6 +78,8 @@ export const validateTableCommand = new Command("validate") title: "Validate Table", json: options.json, debug: options.debug, + quit: options.quit, + all: options.all, }) const dialect = options.dialect @@ -115,7 +118,7 @@ export const validateTableCommand = new Command("validate") validateTable(table, { schema }), ) - if (report.errors.length && !options.quit) { + if (report.errors.length) { const type = await selectErrorType(session, report.errors) if (type) report.errors = report.errors.filter(e => e.type === type) } diff --git a/dpkit/components/ErrorGrid.tsx b/dpkit/components/ErrorGrid.tsx index 5b876ac8..ee386362 100644 --- a/dpkit/components/ErrorGrid.tsx +++ b/dpkit/components/ErrorGrid.tsx @@ -7,7 +7,14 @@ export function ErrorGrid(props: { errors: (TableError | MetadataError | FileError)[] quit?: boolean }) { - const table = DataFrame(props.errors).lazy() + // TODO: Property process errors + const errors = props.errors.map(error => ({ + ...error, + // @ts-ignore + params: error.params ? error.params.toString() : undefined, + })) + + const table = DataFrame(errors).lazy() return } diff --git a/dpkit/helpers/error.ts b/dpkit/helpers/error.ts index f5b1de95..511bba69 100644 --- a/dpkit/helpers/error.ts +++ b/dpkit/helpers/error.ts @@ -10,6 +10,7 @@ export async function selectErrorResource( const name = await session.select({ message: "Select error resource", + skipable: true, options: [ { label: `all (${errors.length})`, value: undefined }, ...Object.entries(groups).map(([name, count]) => ({ @@ -30,6 +31,7 @@ export async function selectErrorType( const type = await session.select({ message: "Select error type", + skipable: true, options: [ { label: `all (${errors.length})`, value: undefined }, ...Object.entries(groups).map(([type, count]) => ({ diff --git a/dpkit/helpers/resource.ts b/dpkit/helpers/resource.ts index 6fab2d42..4b9eef3b 100644 --- a/dpkit/helpers/resource.ts +++ b/dpkit/helpers/resource.ts @@ -1,12 +1,12 @@ import { loadPackage } from "@dpkit/all" -import { Session } from "./session.ts" +import type { Session } from "./session.ts" export async function selectResource( session: Session, options: { package?: string; resource?: string }, ) { if (!options.package) { - Session.terminate("Please provide a path argument or a package option") + session.terminate("Please provide a path argument or a package option") } const dataPackage = await session.task( @@ -30,10 +30,10 @@ export async function selectResource( if (!resource) { if (typeof resourceName !== "string") { - Session.terminate("Resource selection cancelled") + session.terminate("Resource selection cancelled") } - Session.terminate( + session.terminate( `Resource "${resourceName}" is not found in the provided data package`, ) } diff --git a/dpkit/helpers/session.ts b/dpkit/helpers/session.ts index 68ed84d9..9518c807 100644 --- a/dpkit/helpers/session.ts +++ b/dpkit/helpers/session.ts @@ -7,15 +7,21 @@ import { render } from "ink" import pc from "picocolors" import type React from "react" +type SessionSelectOptions = SelectOptions & { skipable?: boolean } + export class Session { title: string debug: boolean + quit: boolean + all: boolean static create(options: { title: string json?: boolean text?: boolean debug?: boolean + quit?: boolean + all?: boolean }) { let session = new Session(options) @@ -26,14 +32,16 @@ export class Session { return session } - static terminate(message: string): never { - log.error(message) - process.exit(1) - } - - constructor(options: { title: string; debug?: boolean }) { + constructor(options: { + title: string + debug?: boolean + quit?: boolean + all?: boolean + }) { this.title = options.title this.debug = options.debug ?? false + this.quit = options.quit ?? false + this.all = options.all ?? false } start() { intro(pc.bold(this.title)) @@ -48,7 +56,16 @@ export class Session { log.error(message) } - async select(options: SelectOptions) { + terminate(message: string): never { + log.error(message) + process.exit(1) + } + + async select(options: SessionSelectOptions) { + if (options.skipable) { + if (this.quit || this.all) return undefined + } + return await select(options) } @@ -79,11 +96,7 @@ export class Session { } } - async render( - _object: any, - node?: React.ReactNode, - //options?: { quit?: boolean }, - ) { + async render(_object: any, node?: React.ReactNode) { // Without waiting for the next tick after clack prompts, // ink render will be immidiately terminated await setImmediate() @@ -109,8 +122,17 @@ class JsonSession extends Session { success = () => {} error = () => {} - async select(_options: SelectOptions): Promise { - Session.terminate("Selection is not supported in JSON mode") + terminate(message: string): never { + console.log(JSON.stringify({ error: message }, null, 2)) + process.exit(1) + } + + async select(options: SessionSelectOptions) { + if (options.skipable) { + return undefined + } + + this.terminate("Selection is not supported in JSON mode") } async render(object: any, _node: React.ReactNode) { @@ -132,8 +154,12 @@ class TextSession extends Session { success = () => {} error = () => {} - async select(_options: SelectOptions): Promise { - Session.terminate("Selection is not supported in TEXT mode") + async select(options: SessionSelectOptions) { + if (options.skipable) { + return undefined + } + + this.terminate("Selection is not supported in TEXT mode") } async render(object: any, _node: React.ReactNode) { diff --git a/dpkit/params/all.ts b/dpkit/params/all.ts new file mode 100644 index 00000000..8a4d9f8c --- /dev/null +++ b/dpkit/params/all.ts @@ -0,0 +1,6 @@ +import { Option } from "commander" + +export const all = new Option( + "-a, --all", + "skip selection prompts when all can be selected", +) diff --git a/dpkit/params/index.ts b/dpkit/params/index.ts index d0bc8e77..a174b0d2 100644 --- a/dpkit/params/index.ts +++ b/dpkit/params/index.ts @@ -1,3 +1,4 @@ +export * from "./all.ts" export * from "./ckan.ts" export * from "./debug.ts" export * from "./dialect.ts" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 815019b7..a949310b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -160,6 +160,10 @@ importers: tiny-invariant: specifier: ^1.3.3 version: 1.3.3 + devDependencies: + '@dpkit/test': + specifier: workspace:* + version: link:../test csv: dependencies: diff --git a/site/content/docs/guides/dialect.md b/site/content/docs/guides/dialect.md index 22ae0304..bfd520eb 100644 --- a/site/content/docs/guides/dialect.md +++ b/site/content/docs/guides/dialect.md @@ -84,6 +84,7 @@ dp dialect validate - `-j, --json`: Output validation results as JSON - `-d, --debug`: Enable debug mode - `-q, --quit`: Exit immediately after validation (don't prompt for error filtering) +- `-a, --all`: Skip selection prompts when all can be selected **Examples:** ```bash diff --git a/site/content/docs/guides/file.md b/site/content/docs/guides/file.md index 8ae27d38..65b00342 100644 --- a/site/content/docs/guides/file.md +++ b/site/content/docs/guides/file.md @@ -72,6 +72,7 @@ dp file validate - `--json`: Output validation results as JSON - `-d, --debug`: Enable debug mode - `-q, --quit`: Exit immediately after validation (don't prompt for error filtering) +- `-a, --all`: Skip selection prompts when all can be selected **Examples:** ```bash diff --git a/site/content/docs/guides/package.md b/site/content/docs/guides/package.md index d0870bf6..bdca6b49 100644 --- a/site/content/docs/guides/package.md +++ b/site/content/docs/guides/package.md @@ -133,6 +133,7 @@ dp package validate - `--json`: Output validation results as JSON - `-d, --debug`: Enable debug mode - `-q, --quit`: Exit immediately after validation (don't prompt for error filtering) +- `-a, --all`: Skip selection prompts when all can be selected **Examples:** ```bash diff --git a/site/content/docs/guides/resource.md b/site/content/docs/guides/resource.md index 537e0e1f..02a94af9 100644 --- a/site/content/docs/guides/resource.md +++ b/site/content/docs/guides/resource.md @@ -127,6 +127,7 @@ dp resource validate [descriptor-path] - `--json`: Output validation results as JSON - `--debug`: Enable debug mode - `-q, --quit`: Exit immediately after validation (don't prompt for error filtering) +- `-a, --all`: Skip selection prompts when all can be selected **Examples:** ```bash diff --git a/site/content/docs/guides/schema.md b/site/content/docs/guides/schema.md index 26e9186d..e7c43e04 100644 --- a/site/content/docs/guides/schema.md +++ b/site/content/docs/guides/schema.md @@ -127,6 +127,7 @@ dp schema validate - `--json`: Output validation results as JSON - `--debug`: Enable debug mode - `-q, --quit`: Exit immediately after validation (don't prompt for error filtering) +- `-a, --all`: Skip selection prompts when all can be selected **Examples:** ```bash diff --git a/site/content/docs/guides/table.md b/site/content/docs/guides/table.md index a493b3cf..8c87e37b 100644 --- a/site/content/docs/guides/table.md +++ b/site/content/docs/guides/table.md @@ -119,6 +119,7 @@ dp table validate - `--json`: Output validation results as JSON - `--debug`: Enable debug mode - `-q, --quit`: Exit immediately after validation (don't prompt for error filtering) +- `-a, --all`: Skip selection prompts when all can be selected **Examples:** ```bash diff --git a/zenodo/package/fixtures/generated/load.spec.ts.snap b/zenodo/package/fixtures/generated/load.spec.ts.snap index 1e3c5625..f6185b80 100644 --- a/zenodo/package/fixtures/generated/load.spec.ts.snap +++ b/zenodo/package/fixtures/generated/load.spec.ts.snap @@ -117,7 +117,7 @@ exports[`loadPackageFromZenodo > shoule merge datapackage.json if present 1`] = "profile": "tabular-data-package", "resources": [ { - "$schema": "tabular-data-resource", + "$schema": undefined, "encoding": "UTF-8", "format": "csv", "mediatype": "text/csv", @@ -353,7 +353,7 @@ exports[`loadPackageFromZenodo > shoule merge datapackage.json if present 1`] = "zenodo:url": undefined, }, { - "$schema": "tabular-data-resource", + "$schema": undefined, "encoding": "UTF-8", "format": "csv", "mediatype": "text/csv", @@ -566,7 +566,7 @@ exports[`loadPackageFromZenodo > shoule merge datapackage.json if present 1`] = "zenodo:url": undefined, }, { - "$schema": "tabular-data-resource", + "$schema": undefined, "encoding": "UTF-8", "format": "csv", "mediatype": "text/csv",