diff --git a/.changeset/heavy-mammals-battle.md b/.changeset/heavy-mammals-battle.md new file mode 100644 index 00000000..9be11eb6 --- /dev/null +++ b/.changeset/heavy-mammals-battle.md @@ -0,0 +1,11 @@ +--- +"@dpkit/camtrap": minor +"@dpkit/github": minor +"@dpkit/zenodo": minor +"@dpkit/ckan": minor +"@dpkit/core": minor +"@dpkit/file": minor +"@dpkit/zip": minor +--- + +Removed permissive generics diff --git a/README.md b/README.md index 92d7b2fa..71b588b5 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,11 @@ # dpkit + [![Build](https://img.shields.io/github/actions/workflow/status/datisthq/dpkit/general.yaml?branch=main)](https://github.com/datisthq/dpkit/actions) [![Coverage](https://img.shields.io/codecov/c/github/datisthq/dpkit/main)](https://codecov.io/gh/datisthq/dpkit) +> **Note:** This project is currently in active development. Feature and new plugin pull requests are not being accepted at this time due to the instability of the core APIs. Bug fixes are welcome! + dpkit is a fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). ## Funding diff --git a/camtrap/OVERVIEW.md b/camtrap/OVERVIEW.md index 5ad8a932..ee2ed4ad 100644 --- a/camtrap/OVERVIEW.md +++ b/camtrap/OVERVIEW.md @@ -1,2 +1,6 @@ # @dpkit/camtrap +:::note +This overview is under development. +::: + diff --git a/camtrap/package/assert.ts b/camtrap/package/assert.ts new file mode 100644 index 00000000..fb5bb8b9 --- /dev/null +++ b/camtrap/package/assert.ts @@ -0,0 +1,32 @@ +import { + AssertionError, + type Package, + validatePackageDescriptor, +} from "@dpkit/core" +import type { CamtrapPackage } from "./Package.js" + +const SUPPORTED_PROFILES = [ + "https://raw.githubusercontent.com/tdwg/camtrap-dp/1.0/camtrap-dp-profile.json", + "https://raw.githubusercontent.com/tdwg/camtrap-dp/1.0.1/camtrap-dp-profile.json", +] + +/** + * Assert a Package descriptor (JSON Object) against its profile + */ +export async function assertCamtrapPackage(props: { + datapackage: Package +}) { + if (!SUPPORTED_PROFILES.includes(props.datapackage.$schema ?? "")) { + throw new Error( + `Unsupported Camtrap DP profile: ${props.datapackage.$schema}`, + ) + } + + const { errors, datapackage } = await validatePackageDescriptor({ + // @ts-ignore (TODO: figure out descriptor/package boundary here) + descriptor: props.datapackage, + }) + + if (!datapackage) throw new AssertionError(errors) + return datapackage as CamtrapPackage +} diff --git a/camtrap/package/index.ts b/camtrap/package/index.ts index 221f2d67..9b8ce884 100644 --- a/camtrap/package/index.ts +++ b/camtrap/package/index.ts @@ -1 +1,2 @@ -export type { CamtrapPackage } from "./Package.ts" +export type { CamtrapPackage } from "./Package.js" +export { assertCamtrapPackage } from "./assert.js" diff --git a/ckan/OVERVIEW.md b/ckan/OVERVIEW.md index 8e30ce82..abec8174 100644 --- a/ckan/OVERVIEW.md +++ b/ckan/OVERVIEW.md @@ -1,2 +1,6 @@ # @dpkit/ckan +:::note +This overview is under development. +::: + diff --git a/ckan/package/load.ts b/ckan/package/load.ts index a76e3739..57992de6 100644 --- a/ckan/package/load.ts +++ b/ckan/package/load.ts @@ -8,7 +8,9 @@ import { normalizeCkanPackage } from "./process/normalize.js" * @param props Object containing the URL to the CKAN package * @returns Package object and cleanup function */ -export async function loadPackageFromCkan(props: { datasetUrl: string }) { +export async function loadPackageFromCkan(props: { + datasetUrl: string +}) { const { datasetUrl } = props const packageId = extractPackageId({ datasetUrl }) @@ -40,6 +42,7 @@ export async function loadPackageFromCkan(props: { datasetUrl: string }) { const datapackage = await mergePackages({ systemPackage, userPackagePath }) datapackage.resources = datapackage.resources.map(resource => { + // TODO: remove these keys completely return { ...resource, "ckan:key": undefined, "ckan:url": undefined } }) diff --git a/ckan/plugin.ts b/ckan/plugin.ts index 1c0da4ec..aed837a8 100644 --- a/ckan/plugin.ts +++ b/ckan/plugin.ts @@ -3,7 +3,10 @@ import { isRemotePath } from "@dpkit/core" import { loadPackageFromCkan } from "./package/load.js" export class CkanPlugin implements Plugin { - async loadPackage(props: { source: string; options?: Descriptor }) { + async loadPackage(props: { + source: string + options?: Descriptor + }) { const isRemote = isRemotePath({ path: props.source }) if (!isRemote) return undefined @@ -11,7 +14,9 @@ export class CkanPlugin implements Plugin { if (!withDataset) return undefined const cleanup = async () => {} - const datapackage = await loadPackageFromCkan({ datasetUrl: props.source }) + const datapackage = await loadPackageFromCkan({ + datasetUrl: props.source, + }) return { datapackage, cleanup } } diff --git a/core/OVERVIEW.md b/core/OVERVIEW.md index 49c0b908..1480d04e 100644 --- a/core/OVERVIEW.md +++ b/core/OVERVIEW.md @@ -1,2 +1,6 @@ # @dpkit/core +:::note +This overview is under development. +::: + diff --git a/core/dialect/process/normalize.ts b/core/dialect/process/normalize.ts index fa68890a..c4e2b58b 100644 --- a/core/dialect/process/normalize.ts +++ b/core/dialect/process/normalize.ts @@ -3,11 +3,17 @@ import type { Descriptor } from "../../general/index.js" export function normalizeDialect(props: { descriptor: Descriptor }) { const descriptor = globalThis.structuredClone(props.descriptor) + normalizeProfile({ descriptor }) normalizeTable({ descriptor }) return descriptor } +function normalizeProfile(props: { descriptor: Descriptor }) { + const { descriptor } = props + descriptor.$schema = descriptor.$schema ?? descriptor.profile +} + function normalizeTable(props: { descriptor: Descriptor }) { const { descriptor } = props diff --git a/core/dialect/save.ts b/core/dialect/save.ts index 87d70cf6..cd4cbcab 100644 --- a/core/dialect/save.ts +++ b/core/dialect/save.ts @@ -15,9 +15,7 @@ export async function saveDialect(props: { const { dialect, path } = props const descriptor = denormalizeDialect({ dialect }) - - descriptor.$schema = - descriptor.$schema ?? descriptor.profile ?? CURRENT_PROFILE + descriptor.$schema = descriptor.$schema ?? CURRENT_PROFILE await saveDescriptor({ descriptor, path }) } diff --git a/core/general/path.ts b/core/general/path.ts index 72f907fb..40f63350 100644 --- a/core/general/path.ts +++ b/core/general/path.ts @@ -66,14 +66,15 @@ export function normalizePath(props: { path: string basepath?: string }) { - const isBasepathRemote = isRemotePath({ path: props.basepath ?? "" }) const isPathRemote = isRemotePath({ path: props.path }) - const isRemote = isBasepathRemote || isPathRemote + const isBasepathRemote = isRemotePath({ path: props.basepath ?? "" }) - if (isRemote) { - const path = new URL( - !isPathRemote ? [props.basepath, props.path].join("/") : props.path, - ).toString() + if (isPathRemote) { + return new URL(props.path).toString() + } + + if (isBasepathRemote) { + const path = new URL([props.basepath, props.path].join("/")).toString() if (!path.startsWith(props.basepath ?? "")) { throw new Error( @@ -103,17 +104,17 @@ export function denormalizePath(props: { path: string basepath?: string }) { - const isBasepathRemote = isRemotePath({ path: props.basepath ?? "" }) const isPathRemote = isRemotePath({ path: props.path }) - const isRemote = isBasepathRemote || isPathRemote + const isBasepathRemote = isRemotePath({ path: props.basepath ?? "" }) - if (isRemote) { - const path = new URL(props.path).toString() - if (isPathRemote) { - return path - } + if (isPathRemote) { + return new URL(props.path).toString() + } + if (isBasepathRemote) { + const path = props.path const basepath = new URL(props.basepath ?? "").toString() + if (!path.startsWith(basepath)) { throw new Error( `Path ${props.path} is not a subpath of ${props.basepath}`, diff --git a/core/package/assert.ts b/core/package/assert.ts index 2b708577..ac269c93 100644 --- a/core/package/assert.ts +++ b/core/package/assert.ts @@ -1,15 +1,14 @@ import { AssertionError, type Descriptor } from "../general/index.js" -import type { Package } from "./Package.js" import { validatePackageDescriptor } from "./validate.js" /** * Assert a Package descriptor (JSON Object) against its profile */ -export async function assertPackage(props: { +export async function assertPackage(props: { descriptor: Descriptor basepath?: string }) { - const { errors, datapackage } = await validatePackageDescriptor(props) + const { errors, datapackage } = await validatePackageDescriptor(props) if (!datapackage) throw new AssertionError(errors) return datapackage } diff --git a/core/package/load.ts b/core/package/load.ts index 635f806b..f97713d4 100644 --- a/core/package/load.ts +++ b/core/package/load.ts @@ -1,15 +1,12 @@ import { loadDescriptor } from "../general/index.js" -import type { Package } from "./Package.js" import { assertPackage } from "./assert.js" /** * Load a Package descriptor (JSON Object) from a file or URL * Ensures the descriptor is valid against its profile */ -export async function loadPackageDescriptor< - T extends Package = Package, ->(props: { path: string }) { +export async function loadPackageDescriptor(props: { path: string }) { const { basepath, descriptor } = await loadDescriptor(props) - const datapackage = await assertPackage({ descriptor, basepath }) + const datapackage = await assertPackage({ descriptor, basepath }) return datapackage } diff --git a/core/package/merge.ts b/core/package/merge.ts index 5a253358..7a06910b 100644 --- a/core/package/merge.ts +++ b/core/package/merge.ts @@ -8,18 +8,10 @@ export async function mergePackages(props: { systemPackage: Package userPackagePath?: string }) { - const { systemPackage, userPackagePath } = props - - const userPackage = userPackagePath - ? await loadPackageDescriptor({ path: userPackagePath }) + const systemPackage = props.systemPackage + const userPackage = props.userPackagePath + ? await loadPackageDescriptor({ path: props.userPackagePath }) : undefined - if (!userPackage) { - return props.systemPackage - } - - // NOTE: - // Currently, it uses oversimplified logic till - // we get use feedback on the desired merging strategies - return { ...systemPackage, resources: userPackage.resources } + return { ...systemPackage, ...userPackage } } diff --git a/core/package/process/normalize.ts b/core/package/process/normalize.ts index 4eb6f35a..f17e71b4 100644 --- a/core/package/process/normalize.ts +++ b/core/package/process/normalize.ts @@ -8,12 +8,18 @@ export function normalizePackage(props: { const { basepath } = props const descriptor = globalThis.structuredClone(props.descriptor) + normalizeProfile({ descriptor }) normalizeResources({ descriptor, basepath }) normalizeContributors({ descriptor }) return descriptor } +function normalizeProfile(props: { descriptor: Descriptor }) { + const { descriptor } = props + descriptor.$schema = descriptor.$schema ?? descriptor.profile +} + function normalizeResources(props: { descriptor: Descriptor basepath?: string diff --git a/core/package/save.ts b/core/package/save.ts index 9857f065..49bc827c 100644 --- a/core/package/save.ts +++ b/core/package/save.ts @@ -13,12 +13,10 @@ export async function savePackageDescriptor(props: { path: string }) { const { datapackage, path } = props - const basepath = getBasepath({ path }) - const descriptor = denormalizePackage({ datapackage, basepath }) - descriptor.$schema = - descriptor.$schema ?? descriptor.profile ?? CURRENT_PROFILE + const descriptor = denormalizePackage({ datapackage, basepath }) + descriptor.$schema = descriptor.$schema ?? CURRENT_PROFILE await saveDescriptor({ descriptor, path: props.path }) } diff --git a/core/package/validate.ts b/core/package/validate.ts index be9d701f..808245e2 100644 --- a/core/package/validate.ts +++ b/core/package/validate.ts @@ -8,14 +8,12 @@ const DEFAULT_PROFILE = "https://datapackage.org/profiles/1.0/datapackage.json" /** * Validate a Package descriptor (JSON Object) against its profile */ -export async function validatePackageDescriptor< - T extends Package = Package, ->(props: { +export async function validatePackageDescriptor(props: { descriptor: Descriptor basepath?: string }) { const { descriptor, basepath } = props - let datapackage: T | undefined = undefined + let datapackage: Package | undefined = undefined const $schema = typeof props.descriptor.$schema === "string" @@ -27,7 +25,10 @@ export async function validatePackageDescriptor< if (valid) { // Validation + normalization = we can cast it - datapackage = normalizePackage({ descriptor, basepath }) as T + datapackage = normalizePackage({ + descriptor, + basepath, + }) as unknown as Package } return { valid, errors, datapackage } diff --git a/core/plugin.ts b/core/plugin.ts index 56836817..f0cd7ad1 100644 --- a/core/plugin.ts +++ b/core/plugin.ts @@ -2,16 +2,16 @@ import type { Descriptor } from "./general/index.js" import type { Package } from "./package/index.js" export interface Plugin { - loadPackage?: (props: { + loadPackage?(props: { source: string options?: Descriptor - }) => Promise< + }): Promise< undefined | { datapackage: Package; cleanup?: () => Promise } > - savePackage?: (props: { + savePackage?(props: { datapackage: Package target: string options?: Descriptor - }) => Promise + }): Promise } diff --git a/core/resource/process/normalize.ts b/core/resource/process/normalize.ts index 14892b55..ce573eae 100644 --- a/core/resource/process/normalize.ts +++ b/core/resource/process/normalize.ts @@ -10,6 +10,7 @@ export function normalizeResource(props: { const { basepath } = props const descriptor = globalThis.structuredClone(props.descriptor) + normalizeProfile({ descriptor }) normalizeUrl({ descriptor }) normalizeType({ descriptor }) normalizePaths({ descriptor, basepath }) @@ -20,6 +21,11 @@ export function normalizeResource(props: { return descriptor } +function normalizeProfile(props: { descriptor: Descriptor }) { + const { descriptor } = props + descriptor.$schema = descriptor.$schema ?? descriptor.profile +} + function normalizeUrl(props: { descriptor: Descriptor }) { const { descriptor } = props diff --git a/core/resource/save.ts b/core/resource/save.ts index 7b1858a6..6a535c80 100644 --- a/core/resource/save.ts +++ b/core/resource/save.ts @@ -13,12 +13,10 @@ export async function saveResourceDescriptor(props: { path: string }) { const { resource, path } = props - const basepath = getBasepath({ path }) - const descriptor = denormalizeResource({ resource, basepath }) - descriptor.$schema = - descriptor.$schema ?? descriptor.profile ?? CURRENT_PROFILE + const descriptor = denormalizeResource({ resource, basepath }) + descriptor.$schema = descriptor.$schema ?? CURRENT_PROFILE await saveDescriptor({ descriptor, path }) } diff --git a/core/schema/process/normalize.ts b/core/schema/process/normalize.ts index 85020f6e..925af44f 100644 --- a/core/schema/process/normalize.ts +++ b/core/schema/process/normalize.ts @@ -4,6 +4,7 @@ import type { Descriptor } from "../../general/index.js" export function normalizeSchema(props: { descriptor: Descriptor }) { const descriptor = globalThis.structuredClone(props.descriptor) + normalizeProfile({ descriptor }) normalizeFields({ descriptor }) normalizePrimaryKey({ descriptor }) normalizeForeignKeys({ descriptor }) @@ -12,6 +13,11 @@ export function normalizeSchema(props: { descriptor: Descriptor }) { return descriptor } +function normalizeProfile(props: { descriptor: Descriptor }) { + const { descriptor } = props + descriptor.$schema = descriptor.$schema ?? descriptor.profile +} + function normalizeFields(props: { descriptor: Descriptor }) { const { descriptor } = props diff --git a/core/schema/save.ts b/core/schema/save.ts index 3a8dc706..f5e62d21 100644 --- a/core/schema/save.ts +++ b/core/schema/save.ts @@ -15,9 +15,7 @@ export async function saveSchema(props: { const { schema, path } = props const descriptor = denormalizeSchema({ schema }) - - descriptor.$schema = - descriptor.$schema ?? descriptor.profile ?? CURRENT_PROFILE + descriptor.$schema = descriptor.$schema ?? CURRENT_PROFILE await saveDescriptor({ descriptor, path }) } diff --git a/docs/astro.config.ts b/docs/astro.config.ts index d3782856..0eb9b678 100644 --- a/docs/astro.config.ts +++ b/docs/astro.config.ts @@ -5,10 +5,11 @@ import starlightTypeDoc from "starlight-typedoc" const PACKAGES = { dpkit: "../dpkit", + "@dpkit/camtrap": "../camtrap", "@dpkit/ckan": "../ckan", "@dpkit/core": "../core", - "@dpkit/github": "../github", "@dpkit/file": "../file", + "@dpkit/github": "../github", "@dpkit/zenodo": "../zenodo", "@dpkit/zip": "../zip", } @@ -63,7 +64,7 @@ export default defineConfig({ { label: "Guides", autogenerate: { directory: "guides" } }, { label: "Packages", - collapsed: true, + //collapsed: true, items: generatePackageSidebars(), }, ], diff --git a/docs/content/docs/index.mdx b/docs/content/docs/index.mdx index b8ff5531..97744b54 100644 --- a/docs/content/docs/index.mdx +++ b/docs/content/docs/index.mdx @@ -11,7 +11,7 @@ hero: alt: Data Packages Illustration actions: - text: Get Started - link: /overview/installation/ + link: /overview/getting-started/ icon: right-arrow variant: primary - text: View on GitHub diff --git a/docs/content/docs/overview/getting-started.md b/docs/content/docs/overview/getting-started.md new file mode 100644 index 00000000..693c456d --- /dev/null +++ b/docs/content/docs/overview/getting-started.md @@ -0,0 +1,65 @@ +--- +title: Getting Started +sidebar: + order: 1 +--- + +This guide will help you get started with dpkit. If you are new to the core framework's tecnhologies, please take a look at the [Data Package standard](https://datapackage.org/) and [Polars DataFrames](https://pola-rs.github.io/polars) documentation. + + +## Installation + +:::note[Prerequisites] +- **Node.js v20+** +::: + +The framework can be installed as one package: + +```bash +npm install dpkit +``` + +Or cherry-picked from individual packages: + +```bash +npm install @dpkit/core @dpkit/zenodo +``` + +## TypeScript + +:::tip +Use **Node.js v24+** to be able to run TypeScript files directly with the `node` binary like `node my-data-script.ts` +::: + +dpkit is built with type safety in mind. It uses TypeScript to provide type definitions for all packages and to enforce type safety throughout the framework. It's highly reccomended to setup a TypeScript aware environment to work with the project. + +## Usage + +Here is an example of loading a Camtrap DP package from Zenodo merging system Zenodo metadata into a user data package and validating its metadata: + +```ts +import { loadPackage } from "dpkit" + +const { datapackage } = await loadPackage({ + source: "https://zenodo.org/records/10053903", +}) + +console.log(datapackage) +``` + +Example of using a Data Package extension in type-safe manner: + +```ts +import { loadPackage, assertCamtrapPackage } from "dpkit" + +const { datapackage } = await loadPackage({ + source: + "https://raw.githubusercontent.com/tdwg/camtrap-dp/refs/tags/1.0.1/example/datapackage.json", +}) + +const camtrapPackage = await assertCamtrapPackage({ datapackage }) + +console.log(camtrapPackage.taxonomic) +console.log(camtrapPackage.project.title) +console.log(camtrapPackage.bibliographicCitation) +``` diff --git a/docs/content/docs/overview/installation.md b/docs/content/docs/overview/installation.md deleted file mode 100644 index d16653dd..00000000 --- a/docs/content/docs/overview/installation.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -title: Installation -sidebar: - order: 1 ---- - -## Prerequisites - -- **Node.js**: v20.0.0 or higher - -## Installation - -```bash -npm install dpkit -``` diff --git a/docs/examples/getting-started-1.ts b/docs/examples/getting-started-1.ts new file mode 100644 index 00000000..530c57d9 --- /dev/null +++ b/docs/examples/getting-started-1.ts @@ -0,0 +1,7 @@ +import { loadPackage } from "dpkit" + +const { datapackage } = await loadPackage({ + source: "https://zenodo.org/records/10053903", +}) + +console.log(datapackage) diff --git a/docs/examples/getting-started-2.ts b/docs/examples/getting-started-2.ts new file mode 100644 index 00000000..12109f8a --- /dev/null +++ b/docs/examples/getting-started-2.ts @@ -0,0 +1,12 @@ +import { assertCamtrapPackage, loadPackage } from "dpkit" + +const { datapackage } = await loadPackage({ + source: + "https://raw.githubusercontent.com/tdwg/camtrap-dp/refs/tags/1.0.1/example/datapackage.json", +}) + +const camtrapPackage = await assertCamtrapPackage({ datapackage }) + +console.log(camtrapPackage.taxonomic) +console.log(camtrapPackage.project.title) +console.log(camtrapPackage.bibliographicCitation) diff --git a/docs/examples/savePackage.ts b/docs/examples/savePackage.ts deleted file mode 100644 index 4457d3d9..00000000 --- a/docs/examples/savePackage.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { loadPackage, savePackageToFolder } from "dpkit" - -const datapackage = await loadPackage({ - path: "https://raw.githubusercontent.com/datasets/natural-gas/refs/heads/main/datapackage.json", -}) - -await savePackageToFolder({ - datapackage, - folderPath: ".user/gas", - withRemote: true, -}) diff --git a/docs/examples/savePackage2.ts b/docs/examples/savePackage2.ts deleted file mode 100644 index 80eb2d8a..00000000 --- a/docs/examples/savePackage2.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { loadPackage, savePackageToFolder } from "dpkit" - -const datapackage = await loadPackage({ - path: "core/package/fixtures/package.json", -}) - -await savePackageToFolder({ datapackage, folderPath: ".user/test" }) diff --git a/docs/examples/savePackage3.ts b/docs/examples/savePackage3.ts deleted file mode 100644 index 0982ba8e..00000000 --- a/docs/examples/savePackage3.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { loadPackage, savePackageToZip } from "dpkit" - -const datapackage = await loadPackage({ - path: "https://raw.githubusercontent.com/datasets/natural-gas/refs/heads/main/datapackage.json", -}) - -await savePackageToZip({ - datapackage, - archivePath: ".user/gas.zip", - withRemote: true, -}) diff --git a/docs/examples/savePackage4.ts b/docs/examples/savePackage4.ts deleted file mode 100644 index c7248899..00000000 --- a/docs/examples/savePackage4.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { loadPackageFromZip } from "dpkit" - -const { datapackage } = await loadPackageFromZip({ - archivePath: ".user/gas.zip", -}) - -console.log(datapackage) diff --git a/dpkit/OVERVIEW.md b/dpkit/OVERVIEW.md index a2b1cdc9..7a052227 100644 --- a/dpkit/OVERVIEW.md +++ b/dpkit/OVERVIEW.md @@ -1,2 +1,6 @@ # dpkit +:::note +This overview is under development. +::: + diff --git a/file/OVERVIEW.md b/file/OVERVIEW.md index ebcdb89d..0f9cc6b6 100644 --- a/file/OVERVIEW.md +++ b/file/OVERVIEW.md @@ -1,2 +1,6 @@ # @dpkit/file +:::note +This overview is under development. +::: + diff --git a/file/plugin.ts b/file/plugin.ts index 0fe06e81..887e4f0e 100644 --- a/file/plugin.ts +++ b/file/plugin.ts @@ -4,7 +4,10 @@ import { isRemotePath } from "@dpkit/core" import { loadPackageFromFolder } from "./package/index.js" export class FilePlugin implements Plugin { - async loadPackage(props: { source: string; options?: Descriptor }) { + async loadPackage(props: { + source: string + options?: Descriptor + }) { const isRemote = isRemotePath({ path: props.source }) if (isRemote) return undefined diff --git a/github/OVERVIEW.md b/github/OVERVIEW.md index 82e6e6da..aabb7f73 100644 --- a/github/OVERVIEW.md +++ b/github/OVERVIEW.md @@ -1,2 +1,6 @@ # @dpkit/github +:::note +This overview is under development. +::: + diff --git a/github/package/fixtures/generated/load.spec.ts.snap b/github/package/fixtures/generated/load.spec.ts.snap index ddbb4566..fadcca50 100644 --- a/github/package/fixtures/generated/load.spec.ts.snap +++ b/github/package/fixtures/generated/load.spec.ts.snap @@ -109,25 +109,48 @@ exports[`loadPackageFromGithub > should load a package 1`] = ` exports[`loadPackageFromGithub > should merge datapackage.json if present 1`] = ` { + "$schema": undefined, "contributors": [ { - "path": "https://github.com/roll", - "role": "author", - "title": "roll", + "email": "rufus.pollock@okfn.org", + "role": undefined, + "roles": [ + "maintainer", + ], + "title": "Rufus Pollock", + }, + { + "email": "kdkusano@gmail.com", + "title": "Kristofer D. Kusano", }, ], "created": "2021-07-07T14:09:49Z", "description": "ISO 4217 List of Currencies and Currency Codes", - "homepage": "https://datahub.io/core/currency-codes", + "homepage": "http://www.iso.org/iso/currency_codes", + "keywords": [ + "iso", + "iso-4217", + "currency", + "codes", + ], + "licenses": [ + { + "name": "ODC-PDDL-1.0", + "path": "http://opendatacommons.org/licenses/pddl/", + "title": "Open Data Commons Public Domain Dedication and License v1.0", + }, + ], "name": "currency-codes", "resources": [ { + "$schema": undefined, "github:key": undefined, "github:url": undefined, "mimetype": "text/csv", "name": "codes-all", "path": "https://raw.githubusercontent.com/roll/currency-codes/refs/heads/master/data/codes-all.csv", "schema": { + "$schema": undefined, "fields": [ { "description": "Country or region name", @@ -168,6 +191,13 @@ exports[`loadPackageFromGithub > should merge datapackage.json if present 1`] = "size": "16863", }, ], - "title": "roll/currency-codes", + "sources": [ + { + "email": "office@currency-iso.org", + "name": "SIX Interbank Clearing Ltd (on behalf of ISO)", + "title": "SIX Interbank Clearing Ltd (on behalf of ISO)", + }, + ], + "title": "ISO 4217 Currency Codes", } `; diff --git a/github/package/load.ts b/github/package/load.ts index bec1878b..b55819a6 100644 --- a/github/package/load.ts +++ b/github/package/load.ts @@ -42,6 +42,7 @@ export async function loadPackageFromGithub(props: { const datapackage = await mergePackages({ systemPackage, userPackagePath }) datapackage.resources = datapackage.resources.map(resource => { + // TODO: remove these keys completely return { ...resource, "github:key": undefined, "github:url": undefined } }) diff --git a/github/plugin.ts b/github/plugin.ts index ade407c0..ddf6abe3 100644 --- a/github/plugin.ts +++ b/github/plugin.ts @@ -3,7 +3,10 @@ import { isRemotePath } from "@dpkit/core" import { loadPackageFromGithub } from "./package/load.js" export class GithubPlugin implements Plugin { - async loadPackage(props: { source: string; options?: Descriptor }) { + async loadPackage(props: { + source: string + options?: Descriptor + }) { const isRemote = isRemotePath({ path: props.source }) if (!isRemote) return undefined diff --git a/zenodo/OVERVIEW.md b/zenodo/OVERVIEW.md index c3db5d41..650c9200 100644 --- a/zenodo/OVERVIEW.md +++ b/zenodo/OVERVIEW.md @@ -1,2 +1,6 @@ # @dpkit/zenodo +:::note +This overview is under development. +::: + diff --git a/zenodo/package/fixtures/generated/load.spec.ts.snap b/zenodo/package/fixtures/generated/load.spec.ts.snap index 23092bbb..18f84c12 100644 --- a/zenodo/package/fixtures/generated/load.spec.ts.snap +++ b/zenodo/package/fixtures/generated/load.spec.ts.snap @@ -41,6 +41,7 @@ exports[`loadPackageFromZenodo > should load a package 1`] = ` exports[`loadPackageFromZenodo > shoule merge datapackage.json if present 1`] = ` { + "$schema": "tabular-data-package", "contributors": [ { "path": "Vogelwerkgroep Assen", @@ -91,6 +92,7 @@ exports[`loadPackageFromZenodo > shoule merge datapackage.json if present 1`] =

Acknowledgements

These data were collected by Bert Dijkstra and Rinus Dillerop from Vogelwerkgroep Assen, in collaboration with the Netherlands Institute of Ecology (NIOO-KNAW), Sovon, Radboud University and the University of Amsterdam (UvA). Funding was provided by the Prins Bernard Cultuurfonds Drenthe, municipality of Assen, IJsvogelfonds (from Birdlife Netherlands and Nationale Postcodeloterij) and the Waterleiding Maatschappij Drenthe. The dataset was published with funding from Stichting NLBIF - Netherlands Biodiversity Information Facility.

", + "id": "https://doi.org/10.5281/zenodo.10053903", "keywords": [ "animal movement", "animal tracking", @@ -112,8 +114,10 @@ exports[`loadPackageFromZenodo > shoule merge datapackage.json if present 1`] = }, ], "name": "record-10053903", + "profile": "tabular-data-package", "resources": [ { + "$schema": "tabular-data-resource", "encoding": "UTF-8", "format": "csv", "mediatype": "text/csv", @@ -121,6 +125,7 @@ exports[`loadPackageFromZenodo > shoule merge datapackage.json if present 1`] = "path": "https://zenodo.org/records/10053903/files/O_ASSEN-reference-data.csv", "profile": "tabular-data-resource", "schema": { + "$schema": undefined, "fields": [ { "description": "A unique identifier for the tag, provided by the data owner. If the data owner does not provide a tag ID, an internal Movebank tag identifier may sometimes be shown. Example: '2342'; Units: none; Entity described: tag", @@ -348,6 +353,7 @@ exports[`loadPackageFromZenodo > shoule merge datapackage.json if present 1`] = "zenodo:url": undefined, }, { + "$schema": "tabular-data-resource", "encoding": "UTF-8", "format": "csv", "mediatype": "text/csv", @@ -358,6 +364,7 @@ exports[`loadPackageFromZenodo > shoule merge datapackage.json if present 1`] = ], "profile": "tabular-data-resource", "schema": { + "$schema": undefined, "fields": [ { "description": "An identifier for the set of values associated with each event, i.e. sensor measurement. A unique event ID is assigned to every time-location or other time-measurement record in Movebank. If multiple measurements are included within a single row of a data file, they will share an event ID. If users import the same sensor measurement to Movebank multiple times, a separate event ID will be assigned to each. Example: '14328243575'; Units: none; Entity described: event", @@ -559,6 +566,7 @@ exports[`loadPackageFromZenodo > shoule merge datapackage.json if present 1`] = "zenodo:url": undefined, }, { + "$schema": "tabular-data-resource", "encoding": "UTF-8", "format": "csv", "mediatype": "text/csv", @@ -569,6 +577,7 @@ exports[`loadPackageFromZenodo > shoule merge datapackage.json if present 1`] = ], "profile": "tabular-data-resource", "schema": { + "$schema": undefined, "fields": [ { "description": "An identifier for the set of values associated with each event, i.e. sensor measurement. A unique event ID is assigned to every time-location or other time-measurement record in Movebank. If multiple measurements are included within a single row of a data file, they will share an event ID. If users import the same sensor measurement to Movebank multiple times, a separate event ID will be assigned to each. Example: '14328243575'; Units: none; Entity described: event", diff --git a/zenodo/package/load.ts b/zenodo/package/load.ts index a05eb59a..74aea0ef 100644 --- a/zenodo/package/load.ts +++ b/zenodo/package/load.ts @@ -34,6 +34,7 @@ export async function loadPackageFromZenodo(props: { const datapackage = await mergePackages({ systemPackage, userPackagePath }) datapackage.resources = datapackage.resources.map(resource => { + // TODO: remove these keys completely return { ...resource, "zenodo:key": undefined, "zenodo:url": undefined } }) diff --git a/zenodo/plugin.ts b/zenodo/plugin.ts index afec0898..7655c0fb 100644 --- a/zenodo/plugin.ts +++ b/zenodo/plugin.ts @@ -3,7 +3,10 @@ import { isRemotePath } from "@dpkit/core" import { loadPackageFromZenodo } from "./package/load.js" export class ZenodoPlugin implements Plugin { - async loadPackage(props: { source: string; options?: Descriptor }) { + async loadPackage(props: { + source: string + options?: Descriptor + }) { const isRemote = isRemotePath({ path: props.source }) if (!isRemote) return undefined diff --git a/zip/OVERVIEW.md b/zip/OVERVIEW.md index a704ff69..57f57cc9 100644 --- a/zip/OVERVIEW.md +++ b/zip/OVERVIEW.md @@ -1,2 +1,6 @@ # @dpkit/zip +:::note +This overview is under development. +::: + diff --git a/zip/package/load.ts b/zip/package/load.ts index ddefc9b3..7dc0ccdf 100644 --- a/zip/package/load.ts +++ b/zip/package/load.ts @@ -6,7 +6,9 @@ import { loadPackageDescriptor } from "@dpkit/core" import { temporaryDirectory } from "tempy" import yauzl from "yauzl-promise" -export async function loadPackageFromZip(props: { archivePath: string }) { +export async function loadPackageFromZip(props: { + archivePath: string +}) { const tempdir = temporaryDirectory() const zipfile = await yauzl.open(props.archivePath) diff --git a/zip/plugin.ts b/zip/plugin.ts index 1db5cabf..dd71d18d 100644 --- a/zip/plugin.ts +++ b/zip/plugin.ts @@ -2,7 +2,10 @@ import type { Descriptor, Package, Plugin } from "@dpkit/core" import { loadPackageFromZip, savePackageToZip } from "./package/index.js" export class ZipPlugin implements Plugin { - async loadPackage(props: { source: string; options?: Descriptor }) { + async loadPackage(props: { + source: string + options?: Descriptor + }) { const isZip = props.source.endsWith(".zip") if (!isZip) return undefined