From 8c6a94aa16d5c8f96f5b9f1d4bb2afa0394f9f00 Mon Sep 17 00:00:00 2001 From: roll Date: Wed, 28 May 2025 15:46:41 +0100 Subject: [PATCH 1/5] docs: added development notices --- README.md | 3 +++ camtrap/OVERVIEW.md | 4 ++++ ckan/OVERVIEW.md | 4 ++++ core/OVERVIEW.md | 4 ++++ docs/astro.config.ts | 3 ++- dpkit/OVERVIEW.md | 4 ++++ file/OVERVIEW.md | 4 ++++ github/OVERVIEW.md | 4 ++++ zenodo/OVERVIEW.md | 4 ++++ zip/OVERVIEW.md | 4 ++++ 10 files changed, 37 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 92d7b2fa..71b588b5 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,11 @@ # dpkit + [![Build](https://img.shields.io/github/actions/workflow/status/datisthq/dpkit/general.yaml?branch=main)](https://github.com/datisthq/dpkit/actions) [![Coverage](https://img.shields.io/codecov/c/github/datisthq/dpkit/main)](https://codecov.io/gh/datisthq/dpkit) +> **Note:** This project is currently in active development. Feature and new plugin pull requests are not being accepted at this time due to the instability of the core APIs. Bug fixes are welcome! + dpkit is a fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). ## Funding diff --git a/camtrap/OVERVIEW.md b/camtrap/OVERVIEW.md index 5ad8a932..ee2ed4ad 100644 --- a/camtrap/OVERVIEW.md +++ b/camtrap/OVERVIEW.md @@ -1,2 +1,6 @@ # @dpkit/camtrap +:::note +This overview is under development. +::: + diff --git a/ckan/OVERVIEW.md b/ckan/OVERVIEW.md index 8e30ce82..abec8174 100644 --- a/ckan/OVERVIEW.md +++ b/ckan/OVERVIEW.md @@ -1,2 +1,6 @@ # @dpkit/ckan +:::note +This overview is under development. +::: + diff --git a/core/OVERVIEW.md b/core/OVERVIEW.md index 49c0b908..1480d04e 100644 --- a/core/OVERVIEW.md +++ b/core/OVERVIEW.md @@ -1,2 +1,6 @@ # @dpkit/core +:::note +This overview is under development. +::: + diff --git a/docs/astro.config.ts b/docs/astro.config.ts index d3782856..aa7b11d0 100644 --- a/docs/astro.config.ts +++ b/docs/astro.config.ts @@ -5,10 +5,11 @@ import starlightTypeDoc from "starlight-typedoc" const PACKAGES = { dpkit: "../dpkit", + "@dpkit/camtrap": "../camtrap", "@dpkit/ckan": "../ckan", "@dpkit/core": "../core", - "@dpkit/github": "../github", "@dpkit/file": "../file", + "@dpkit/github": "../github", "@dpkit/zenodo": "../zenodo", "@dpkit/zip": "../zip", } diff --git a/dpkit/OVERVIEW.md b/dpkit/OVERVIEW.md index a2b1cdc9..7a052227 100644 --- a/dpkit/OVERVIEW.md +++ b/dpkit/OVERVIEW.md @@ -1,2 +1,6 @@ # dpkit +:::note +This overview is under development. +::: + diff --git a/file/OVERVIEW.md b/file/OVERVIEW.md index ebcdb89d..0f9cc6b6 100644 --- a/file/OVERVIEW.md +++ b/file/OVERVIEW.md @@ -1,2 +1,6 @@ # @dpkit/file +:::note +This overview is under development. +::: + diff --git a/github/OVERVIEW.md b/github/OVERVIEW.md index 82e6e6da..aabb7f73 100644 --- a/github/OVERVIEW.md +++ b/github/OVERVIEW.md @@ -1,2 +1,6 @@ # @dpkit/github +:::note +This overview is under development. +::: + diff --git a/zenodo/OVERVIEW.md b/zenodo/OVERVIEW.md index c3db5d41..650c9200 100644 --- a/zenodo/OVERVIEW.md +++ b/zenodo/OVERVIEW.md @@ -1,2 +1,6 @@ # @dpkit/zenodo +:::note +This overview is under development. +::: + diff --git a/zip/OVERVIEW.md b/zip/OVERVIEW.md index a704ff69..57f57cc9 100644 --- a/zip/OVERVIEW.md +++ b/zip/OVERVIEW.md @@ -1,2 +1,6 @@ # @dpkit/zip +:::note +This overview is under development. +::: + From c8fc5f202f62122ac9e6bb04ec6c73ca635dc1c1 Mon Sep 17 00:00:00 2001 From: roll Date: Wed, 28 May 2025 17:35:47 +0100 Subject: [PATCH 2/5] Updated getting started --- ckan/package/load.ts | 8 +++- ckan/plugin.ts | 11 +++-- core/package/merge.ts | 18 ++----- core/plugin.ts | 10 ++-- docs/astro.config.ts | 2 +- docs/content/docs/index.mdx | 2 +- docs/content/docs/overview/getting-started.md | 48 +++++++++++++++++++ docs/content/docs/overview/installation.md | 15 ------ docs/examples/getting-started.ts | 7 +++ docs/examples/savePackage.ts | 11 ----- docs/examples/savePackage2.ts | 7 --- docs/examples/savePackage3.ts | 11 ----- docs/examples/savePackage4.ts | 7 --- dpkit/package/load.ts | 8 ++-- file/package/load.ts | 7 ++- file/plugin.ts | 9 ++-- github/package/load.ts | 8 +++- github/plugin.ts | 9 ++-- zenodo/package/load.ts | 8 +++- zenodo/plugin.ts | 9 ++-- zip/package/load.ts | 7 ++- zip/plugin.ts | 7 ++- 22 files changed, 129 insertions(+), 100 deletions(-) create mode 100644 docs/content/docs/overview/getting-started.md delete mode 100644 docs/content/docs/overview/installation.md create mode 100644 docs/examples/getting-started.ts delete mode 100644 docs/examples/savePackage.ts delete mode 100644 docs/examples/savePackage2.ts delete mode 100644 docs/examples/savePackage3.ts delete mode 100644 docs/examples/savePackage4.ts diff --git a/ckan/package/load.ts b/ckan/package/load.ts index a76e3739..75a372d9 100644 --- a/ckan/package/load.ts +++ b/ckan/package/load.ts @@ -1,4 +1,5 @@ import { mergePackages } from "@dpkit/core" +import type { Package } from "@dpkit/core" import { makeCkanApiRequest } from "../general/index.js" import type { CkanPackage } from "./Package.js" import { normalizeCkanPackage } from "./process/normalize.js" @@ -8,7 +9,9 @@ import { normalizeCkanPackage } from "./process/normalize.js" * @param props Object containing the URL to the CKAN package * @returns Package object and cleanup function */ -export async function loadPackageFromCkan(props: { datasetUrl: string }) { +export async function loadPackageFromCkan(props: { + datasetUrl: string +}) { const { datasetUrl } = props const packageId = extractPackageId({ datasetUrl }) @@ -38,8 +41,9 @@ export async function loadPackageFromCkan(props: { datasetUrl: string }) { .map(resource => resource["ckan:url"]) .at(0) - const datapackage = await mergePackages({ systemPackage, userPackagePath }) + const datapackage = await mergePackages({ systemPackage, userPackagePath }) datapackage.resources = datapackage.resources.map(resource => { + // TODO: remove these keys completely return { ...resource, "ckan:key": undefined, "ckan:url": undefined } }) diff --git a/ckan/plugin.ts b/ckan/plugin.ts index 1c0da4ec..787b8cef 100644 --- a/ckan/plugin.ts +++ b/ckan/plugin.ts @@ -1,9 +1,12 @@ -import type { Descriptor, Plugin } from "@dpkit/core" +import type { Descriptor, Plugin, Package } from "@dpkit/core" import { isRemotePath } from "@dpkit/core" import { loadPackageFromCkan } from "./package/load.js" export class CkanPlugin implements Plugin { - async loadPackage(props: { source: string; options?: Descriptor }) { + async loadPackage(props: { + source: string + options?: Descriptor + }) { const isRemote = isRemotePath({ path: props.source }) if (!isRemote) return undefined @@ -11,7 +14,9 @@ export class CkanPlugin implements Plugin { if (!withDataset) return undefined const cleanup = async () => {} - const datapackage = await loadPackageFromCkan({ datasetUrl: props.source }) + const datapackage = await loadPackageFromCkan({ + datasetUrl: props.source, + }) return { datapackage, cleanup } } diff --git a/core/package/merge.ts b/core/package/merge.ts index 5a253358..b49de135 100644 --- a/core/package/merge.ts +++ b/core/package/merge.ts @@ -4,22 +4,14 @@ import { loadPackageDescriptor } from "./load.js" /** * Merges a system data package into a user data package if provided */ -export async function mergePackages(props: { +export async function mergePackages(props: { systemPackage: Package userPackagePath?: string }) { - const { systemPackage, userPackagePath } = props - - const userPackage = userPackagePath - ? await loadPackageDescriptor({ path: userPackagePath }) + const systemPackage = props.systemPackage + const userPackage = props.userPackagePath + ? await loadPackageDescriptor({ path: props.userPackagePath }) : undefined - if (!userPackage) { - return props.systemPackage - } - - // NOTE: - // Currently, it uses oversimplified logic till - // we get use feedback on the desired merging strategies - return { ...systemPackage, resources: userPackage.resources } + return { ...systemPackage, ...userPackage } as T } diff --git a/core/plugin.ts b/core/plugin.ts index 56836817..e85d2f97 100644 --- a/core/plugin.ts +++ b/core/plugin.ts @@ -2,16 +2,14 @@ import type { Descriptor } from "./general/index.js" import type { Package } from "./package/index.js" export interface Plugin { - loadPackage?: (props: { + loadPackage?(props: { source: string options?: Descriptor - }) => Promise< - undefined | { datapackage: Package; cleanup?: () => Promise } - > + }): Promise Promise }> - savePackage?: (props: { + savePackage?(props: { datapackage: Package target: string options?: Descriptor - }) => Promise + }): Promise } diff --git a/docs/astro.config.ts b/docs/astro.config.ts index aa7b11d0..0eb9b678 100644 --- a/docs/astro.config.ts +++ b/docs/astro.config.ts @@ -64,7 +64,7 @@ export default defineConfig({ { label: "Guides", autogenerate: { directory: "guides" } }, { label: "Packages", - collapsed: true, + //collapsed: true, items: generatePackageSidebars(), }, ], diff --git a/docs/content/docs/index.mdx b/docs/content/docs/index.mdx index b8ff5531..97744b54 100644 --- a/docs/content/docs/index.mdx +++ b/docs/content/docs/index.mdx @@ -11,7 +11,7 @@ hero: alt: Data Packages Illustration actions: - text: Get Started - link: /overview/installation/ + link: /overview/getting-started/ icon: right-arrow variant: primary - text: View on GitHub diff --git a/docs/content/docs/overview/getting-started.md b/docs/content/docs/overview/getting-started.md new file mode 100644 index 00000000..64a5b8b3 --- /dev/null +++ b/docs/content/docs/overview/getting-started.md @@ -0,0 +1,48 @@ +--- +title: Getting Started +sidebar: + order: 1 +--- + +This guide will help you get started with dpkit. If you are new to the core framework's tecnhologies, please take a look at the [Data Package standard](https://datapackage.org/) and [Polars DataFrames](https://pola-rs.github.io/polars) documentation. + + +## Installation + +:::note[Prerequisites] +- **Node.js v20+** +::: + +The framework can be installed as one package: + +```bash +npm install dpkit +``` + +Or cherry-picked from individual packages: + +```bash +npm install @dpkit/core @dpkit/zenodo +``` + +## TypeScript + +:::tip +Use **Node.js v24+** to be able to run TypeScript files directly with the `node` binary like `node my-data-script.ts` +::: + +dpkit is built with type safety in mind. It uses TypeScript to provide type definitions for all packages and to enforce type safety throughout the framework. It's highly reccomended to setup a TypeScript aware environment to work with the project. + +## Usage + +Here is an example of loading a Camtrap DP package from Zenodo preserving full type-safety of the extension and merging system Zenodo metadata into a user data package: + +```ts +import { loadPackage, type CamtrapPackage } from "dpkit" + +const { datapackage } = await loadPackage({ + source: "https://zenodo.org/records/10053903", +}) + +console.log(datapackage) +``` diff --git a/docs/content/docs/overview/installation.md b/docs/content/docs/overview/installation.md deleted file mode 100644 index d16653dd..00000000 --- a/docs/content/docs/overview/installation.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -title: Installation -sidebar: - order: 1 ---- - -## Prerequisites - -- **Node.js**: v20.0.0 or higher - -## Installation - -```bash -npm install dpkit -``` diff --git a/docs/examples/getting-started.ts b/docs/examples/getting-started.ts new file mode 100644 index 00000000..6d1dd799 --- /dev/null +++ b/docs/examples/getting-started.ts @@ -0,0 +1,7 @@ +import { loadPackage, type CamtrapPackage } from "dpkit" + +const { datapackage } = await loadPackage({ + source: "https://zenodo.org/records/10053903", +}) + +console.log(datapackage) diff --git a/docs/examples/savePackage.ts b/docs/examples/savePackage.ts deleted file mode 100644 index 4457d3d9..00000000 --- a/docs/examples/savePackage.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { loadPackage, savePackageToFolder } from "dpkit" - -const datapackage = await loadPackage({ - path: "https://raw.githubusercontent.com/datasets/natural-gas/refs/heads/main/datapackage.json", -}) - -await savePackageToFolder({ - datapackage, - folderPath: ".user/gas", - withRemote: true, -}) diff --git a/docs/examples/savePackage2.ts b/docs/examples/savePackage2.ts deleted file mode 100644 index 80eb2d8a..00000000 --- a/docs/examples/savePackage2.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { loadPackage, savePackageToFolder } from "dpkit" - -const datapackage = await loadPackage({ - path: "core/package/fixtures/package.json", -}) - -await savePackageToFolder({ datapackage, folderPath: ".user/test" }) diff --git a/docs/examples/savePackage3.ts b/docs/examples/savePackage3.ts deleted file mode 100644 index 0982ba8e..00000000 --- a/docs/examples/savePackage3.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { loadPackage, savePackageToZip } from "dpkit" - -const datapackage = await loadPackage({ - path: "https://raw.githubusercontent.com/datasets/natural-gas/refs/heads/main/datapackage.json", -}) - -await savePackageToZip({ - datapackage, - archivePath: ".user/gas.zip", - withRemote: true, -}) diff --git a/docs/examples/savePackage4.ts b/docs/examples/savePackage4.ts deleted file mode 100644 index c7248899..00000000 --- a/docs/examples/savePackage4.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { loadPackageFromZip } from "dpkit" - -const { datapackage } = await loadPackageFromZip({ - archivePath: ".user/gas.zip", -}) - -console.log(datapackage) diff --git a/dpkit/package/load.ts b/dpkit/package/load.ts index 614e248e..5a335c46 100644 --- a/dpkit/package/load.ts +++ b/dpkit/package/load.ts @@ -1,19 +1,19 @@ -import type { Descriptor } from "@dpkit/core" +import type { Descriptor, Package } from "@dpkit/core" import { loadPackageDescriptor } from "@dpkit/core" import { dpkit } from "../general/index.js" -export async function loadPackage(props: { +export async function loadPackage(props: { source: string options?: Descriptor }) { for (const plugin of dpkit.plugins) { - const result = await plugin.loadPackage?.(props) + const result = await plugin.loadPackage?.(props) if (result) return result } if (props.source.endsWith("datapackage.json")) { const cleanup = async () => {} - const datapackage = await loadPackageDescriptor({ path: props.source }) + const datapackage = await loadPackageDescriptor({ path: props.source }) return { datapackage, cleanup } } diff --git a/file/package/load.ts b/file/package/load.ts index 9d78e55f..a31f11f9 100644 --- a/file/package/load.ts +++ b/file/package/load.ts @@ -1,8 +1,11 @@ import { join } from "node:path" +import type { Package } from "@dpkit/core" import { loadPackageDescriptor } from "@dpkit/core" -export async function loadPackageFromFolder(props: { folderPath: string }) { - return loadPackageDescriptor({ +export async function loadPackageFromFolder< + T extends Package = Package, +>(props: { folderPath: string }) { + return loadPackageDescriptor({ path: join(props.folderPath, "datapackage.json"), }) } diff --git a/file/plugin.ts b/file/plugin.ts index 0fe06e81..a1ebd64e 100644 --- a/file/plugin.ts +++ b/file/plugin.ts @@ -1,10 +1,13 @@ import { stat } from "node:fs/promises" -import type { Descriptor, Plugin } from "@dpkit/core" +import type { Descriptor, Plugin, Package } from "@dpkit/core" import { isRemotePath } from "@dpkit/core" import { loadPackageFromFolder } from "./package/index.js" export class FilePlugin implements Plugin { - async loadPackage(props: { source: string; options?: Descriptor }) { + async loadPackage(props: { + source: string + options?: Descriptor + }) { const isRemote = isRemotePath({ path: props.source }) if (isRemote) return undefined @@ -12,7 +15,7 @@ export class FilePlugin implements Plugin { if (!isFolder) return undefined const cleanup = async () => {} - const datapackage = await loadPackageFromFolder({ + const datapackage = await loadPackageFromFolder({ folderPath: props.source, }) diff --git a/github/package/load.ts b/github/package/load.ts index bec1878b..ffb78f2c 100644 --- a/github/package/load.ts +++ b/github/package/load.ts @@ -1,4 +1,5 @@ import { mergePackages } from "@dpkit/core" +import type { Package } from "@dpkit/core" import { makeGithubApiRequest } from "../general/index.js" import type { GithubResource } from "../resource/index.js" import type { GithubPackage } from "./Package.js" @@ -9,7 +10,9 @@ import { normalizeGithubPackage } from "./process/normalize.js" * @param props Object containing the URL to the Github repository * @returns Package object */ -export async function loadPackageFromGithub(props: { +export async function loadPackageFromGithub< + T extends Package = Package, +>(props: { repoUrl: string apiKey?: string }) { @@ -40,8 +43,9 @@ export async function loadPackageFromGithub(props: { .map(resource => resource["github:url"]) .at(0) - const datapackage = await mergePackages({ systemPackage, userPackagePath }) + const datapackage = await mergePackages({ systemPackage, userPackagePath }) datapackage.resources = datapackage.resources.map(resource => { + // TODO: remove these keys completely return { ...resource, "github:key": undefined, "github:url": undefined } }) diff --git a/github/plugin.ts b/github/plugin.ts index ade407c0..bed8b62f 100644 --- a/github/plugin.ts +++ b/github/plugin.ts @@ -1,9 +1,12 @@ -import type { Descriptor, Plugin } from "@dpkit/core" +import type { Descriptor, Plugin, Package } from "@dpkit/core" import { isRemotePath } from "@dpkit/core" import { loadPackageFromGithub } from "./package/load.js" export class GithubPlugin implements Plugin { - async loadPackage(props: { source: string; options?: Descriptor }) { + async loadPackage(props: { + source: string + options?: Descriptor + }) { const isRemote = isRemotePath({ path: props.source }) if (!isRemote) return undefined @@ -11,7 +14,7 @@ export class GithubPlugin implements Plugin { if (!isGithub) return undefined const cleanup = async () => {} - const datapackage = await loadPackageFromGithub({ + const datapackage = await loadPackageFromGithub({ repoUrl: props.source, }) diff --git a/zenodo/package/load.ts b/zenodo/package/load.ts index a05eb59a..cf45b14c 100644 --- a/zenodo/package/load.ts +++ b/zenodo/package/load.ts @@ -1,6 +1,7 @@ import { mergePackages } from "@dpkit/core" import { makeZenodoApiRequest } from "../general/index.js" import type { ZenodoPackage } from "./Package.js" +import type { Package } from "@dpkit/core" import { normalizeZenodoPackage } from "./process/normalize.js" /** @@ -8,7 +9,9 @@ import { normalizeZenodoPackage } from "./process/normalize.js" * @param props Object containing the URL to the Zenodo deposit * @returns Package object */ -export async function loadPackageFromZenodo(props: { +export async function loadPackageFromZenodo< + T extends Package = Package, +>(props: { datasetUrl: string apiKey?: string }) { @@ -32,8 +35,9 @@ export async function loadPackageFromZenodo(props: { .map(resource => resource["zenodo:url"]) .at(0) - const datapackage = await mergePackages({ systemPackage, userPackagePath }) + const datapackage = await mergePackages({ systemPackage, userPackagePath }) datapackage.resources = datapackage.resources.map(resource => { + // TODO: remove these keys completely return { ...resource, "zenodo:key": undefined, "zenodo:url": undefined } }) diff --git a/zenodo/plugin.ts b/zenodo/plugin.ts index afec0898..e900b23c 100644 --- a/zenodo/plugin.ts +++ b/zenodo/plugin.ts @@ -1,9 +1,12 @@ -import type { Descriptor, Plugin } from "@dpkit/core" +import type { Descriptor, Plugin, Package } from "@dpkit/core" import { isRemotePath } from "@dpkit/core" import { loadPackageFromZenodo } from "./package/load.js" export class ZenodoPlugin implements Plugin { - async loadPackage(props: { source: string; options?: Descriptor }) { + async loadPackage(props: { + source: string + options?: Descriptor + }) { const isRemote = isRemotePath({ path: props.source }) if (!isRemote) return undefined @@ -11,7 +14,7 @@ export class ZenodoPlugin implements Plugin { if (!isZenodo) return undefined const cleanup = async () => {} - const datapackage = await loadPackageFromZenodo({ + const datapackage = await loadPackageFromZenodo({ datasetUrl: props.source, }) diff --git a/zip/package/load.ts b/zip/package/load.ts index ddefc9b3..31410916 100644 --- a/zip/package/load.ts +++ b/zip/package/load.ts @@ -1,4 +1,5 @@ import { createWriteStream } from "node:fs" +import type { Package } from "@dpkit/core" import { mkdir, rm } from "node:fs/promises" import { join } from "node:path" import { pipeline } from "node:stream/promises" @@ -6,7 +7,9 @@ import { loadPackageDescriptor } from "@dpkit/core" import { temporaryDirectory } from "tempy" import yauzl from "yauzl-promise" -export async function loadPackageFromZip(props: { archivePath: string }) { +export async function loadPackageFromZip(props: { + archivePath: string +}) { const tempdir = temporaryDirectory() const zipfile = await yauzl.open(props.archivePath) @@ -28,7 +31,7 @@ export async function loadPackageFromZip(props: { archivePath: string }) { await zipfile.close() } - const datapackage = await loadPackageDescriptor({ + const datapackage = await loadPackageDescriptor({ path: join(tempdir, "datapackage.json"), }) diff --git a/zip/plugin.ts b/zip/plugin.ts index 1db5cabf..aebdf3fc 100644 --- a/zip/plugin.ts +++ b/zip/plugin.ts @@ -2,11 +2,14 @@ import type { Descriptor, Package, Plugin } from "@dpkit/core" import { loadPackageFromZip, savePackageToZip } from "./package/index.js" export class ZipPlugin implements Plugin { - async loadPackage(props: { source: string; options?: Descriptor }) { + async loadPackage(props: { + source: string + options?: Descriptor + }) { const isZip = props.source.endsWith(".zip") if (!isZip) return undefined - const { datapackage, cleanup } = await loadPackageFromZip({ + const { datapackage, cleanup } = await loadPackageFromZip({ archivePath: props.source, }) From 8424b7e5ac5bc3ad9e387064d39b628b37b4741d Mon Sep 17 00:00:00 2001 From: roll Date: Wed, 28 May 2025 17:46:35 +0100 Subject: [PATCH 3/5] Removed wrong generics --- ckan/package/load.ts | 5 ++--- ckan/plugin.ts | 6 +++--- core/package/assert.ts | 5 ++--- core/package/load.ts | 7 ++----- core/package/merge.ts | 6 +++--- core/package/validate.ts | 11 ++++++----- core/plugin.ts | 6 ++++-- dpkit/package/load.ts | 8 ++++---- file/package/load.ts | 7 ++----- file/plugin.ts | 6 +++--- github/package/load.ts | 7 ++----- github/plugin.ts | 6 +++--- zenodo/package/load.ts | 7 ++----- zenodo/plugin.ts | 6 +++--- zip/package/load.ts | 5 ++--- zip/plugin.ts | 4 ++-- 16 files changed, 45 insertions(+), 57 deletions(-) diff --git a/ckan/package/load.ts b/ckan/package/load.ts index 75a372d9..57992de6 100644 --- a/ckan/package/load.ts +++ b/ckan/package/load.ts @@ -1,5 +1,4 @@ import { mergePackages } from "@dpkit/core" -import type { Package } from "@dpkit/core" import { makeCkanApiRequest } from "../general/index.js" import type { CkanPackage } from "./Package.js" import { normalizeCkanPackage } from "./process/normalize.js" @@ -9,7 +8,7 @@ import { normalizeCkanPackage } from "./process/normalize.js" * @param props Object containing the URL to the CKAN package * @returns Package object and cleanup function */ -export async function loadPackageFromCkan(props: { +export async function loadPackageFromCkan(props: { datasetUrl: string }) { const { datasetUrl } = props @@ -41,7 +40,7 @@ export async function loadPackageFromCkan(props: { .map(resource => resource["ckan:url"]) .at(0) - const datapackage = await mergePackages({ systemPackage, userPackagePath }) + const datapackage = await mergePackages({ systemPackage, userPackagePath }) datapackage.resources = datapackage.resources.map(resource => { // TODO: remove these keys completely return { ...resource, "ckan:key": undefined, "ckan:url": undefined } diff --git a/ckan/plugin.ts b/ckan/plugin.ts index 787b8cef..aed837a8 100644 --- a/ckan/plugin.ts +++ b/ckan/plugin.ts @@ -1,9 +1,9 @@ -import type { Descriptor, Plugin, Package } from "@dpkit/core" +import type { Descriptor, Plugin } from "@dpkit/core" import { isRemotePath } from "@dpkit/core" import { loadPackageFromCkan } from "./package/load.js" export class CkanPlugin implements Plugin { - async loadPackage(props: { + async loadPackage(props: { source: string options?: Descriptor }) { @@ -14,7 +14,7 @@ export class CkanPlugin implements Plugin { if (!withDataset) return undefined const cleanup = async () => {} - const datapackage = await loadPackageFromCkan({ + const datapackage = await loadPackageFromCkan({ datasetUrl: props.source, }) diff --git a/core/package/assert.ts b/core/package/assert.ts index 2b708577..ac269c93 100644 --- a/core/package/assert.ts +++ b/core/package/assert.ts @@ -1,15 +1,14 @@ import { AssertionError, type Descriptor } from "../general/index.js" -import type { Package } from "./Package.js" import { validatePackageDescriptor } from "./validate.js" /** * Assert a Package descriptor (JSON Object) against its profile */ -export async function assertPackage(props: { +export async function assertPackage(props: { descriptor: Descriptor basepath?: string }) { - const { errors, datapackage } = await validatePackageDescriptor(props) + const { errors, datapackage } = await validatePackageDescriptor(props) if (!datapackage) throw new AssertionError(errors) return datapackage } diff --git a/core/package/load.ts b/core/package/load.ts index 635f806b..f97713d4 100644 --- a/core/package/load.ts +++ b/core/package/load.ts @@ -1,15 +1,12 @@ import { loadDescriptor } from "../general/index.js" -import type { Package } from "./Package.js" import { assertPackage } from "./assert.js" /** * Load a Package descriptor (JSON Object) from a file or URL * Ensures the descriptor is valid against its profile */ -export async function loadPackageDescriptor< - T extends Package = Package, ->(props: { path: string }) { +export async function loadPackageDescriptor(props: { path: string }) { const { basepath, descriptor } = await loadDescriptor(props) - const datapackage = await assertPackage({ descriptor, basepath }) + const datapackage = await assertPackage({ descriptor, basepath }) return datapackage } diff --git a/core/package/merge.ts b/core/package/merge.ts index b49de135..7a06910b 100644 --- a/core/package/merge.ts +++ b/core/package/merge.ts @@ -4,14 +4,14 @@ import { loadPackageDescriptor } from "./load.js" /** * Merges a system data package into a user data package if provided */ -export async function mergePackages(props: { +export async function mergePackages(props: { systemPackage: Package userPackagePath?: string }) { const systemPackage = props.systemPackage const userPackage = props.userPackagePath - ? await loadPackageDescriptor({ path: props.userPackagePath }) + ? await loadPackageDescriptor({ path: props.userPackagePath }) : undefined - return { ...systemPackage, ...userPackage } as T + return { ...systemPackage, ...userPackage } } diff --git a/core/package/validate.ts b/core/package/validate.ts index be9d701f..808245e2 100644 --- a/core/package/validate.ts +++ b/core/package/validate.ts @@ -8,14 +8,12 @@ const DEFAULT_PROFILE = "https://datapackage.org/profiles/1.0/datapackage.json" /** * Validate a Package descriptor (JSON Object) against its profile */ -export async function validatePackageDescriptor< - T extends Package = Package, ->(props: { +export async function validatePackageDescriptor(props: { descriptor: Descriptor basepath?: string }) { const { descriptor, basepath } = props - let datapackage: T | undefined = undefined + let datapackage: Package | undefined = undefined const $schema = typeof props.descriptor.$schema === "string" @@ -27,7 +25,10 @@ export async function validatePackageDescriptor< if (valid) { // Validation + normalization = we can cast it - datapackage = normalizePackage({ descriptor, basepath }) as T + datapackage = normalizePackage({ + descriptor, + basepath, + }) as unknown as Package } return { valid, errors, datapackage } diff --git a/core/plugin.ts b/core/plugin.ts index e85d2f97..f0cd7ad1 100644 --- a/core/plugin.ts +++ b/core/plugin.ts @@ -2,10 +2,12 @@ import type { Descriptor } from "./general/index.js" import type { Package } from "./package/index.js" export interface Plugin { - loadPackage?(props: { + loadPackage?(props: { source: string options?: Descriptor - }): Promise Promise }> + }): Promise< + undefined | { datapackage: Package; cleanup?: () => Promise } + > savePackage?(props: { datapackage: Package diff --git a/dpkit/package/load.ts b/dpkit/package/load.ts index 5a335c46..614e248e 100644 --- a/dpkit/package/load.ts +++ b/dpkit/package/load.ts @@ -1,19 +1,19 @@ -import type { Descriptor, Package } from "@dpkit/core" +import type { Descriptor } from "@dpkit/core" import { loadPackageDescriptor } from "@dpkit/core" import { dpkit } from "../general/index.js" -export async function loadPackage(props: { +export async function loadPackage(props: { source: string options?: Descriptor }) { for (const plugin of dpkit.plugins) { - const result = await plugin.loadPackage?.(props) + const result = await plugin.loadPackage?.(props) if (result) return result } if (props.source.endsWith("datapackage.json")) { const cleanup = async () => {} - const datapackage = await loadPackageDescriptor({ path: props.source }) + const datapackage = await loadPackageDescriptor({ path: props.source }) return { datapackage, cleanup } } diff --git a/file/package/load.ts b/file/package/load.ts index a31f11f9..9d78e55f 100644 --- a/file/package/load.ts +++ b/file/package/load.ts @@ -1,11 +1,8 @@ import { join } from "node:path" -import type { Package } from "@dpkit/core" import { loadPackageDescriptor } from "@dpkit/core" -export async function loadPackageFromFolder< - T extends Package = Package, ->(props: { folderPath: string }) { - return loadPackageDescriptor({ +export async function loadPackageFromFolder(props: { folderPath: string }) { + return loadPackageDescriptor({ path: join(props.folderPath, "datapackage.json"), }) } diff --git a/file/plugin.ts b/file/plugin.ts index a1ebd64e..887e4f0e 100644 --- a/file/plugin.ts +++ b/file/plugin.ts @@ -1,10 +1,10 @@ import { stat } from "node:fs/promises" -import type { Descriptor, Plugin, Package } from "@dpkit/core" +import type { Descriptor, Plugin } from "@dpkit/core" import { isRemotePath } from "@dpkit/core" import { loadPackageFromFolder } from "./package/index.js" export class FilePlugin implements Plugin { - async loadPackage(props: { + async loadPackage(props: { source: string options?: Descriptor }) { @@ -15,7 +15,7 @@ export class FilePlugin implements Plugin { if (!isFolder) return undefined const cleanup = async () => {} - const datapackage = await loadPackageFromFolder({ + const datapackage = await loadPackageFromFolder({ folderPath: props.source, }) diff --git a/github/package/load.ts b/github/package/load.ts index ffb78f2c..b55819a6 100644 --- a/github/package/load.ts +++ b/github/package/load.ts @@ -1,5 +1,4 @@ import { mergePackages } from "@dpkit/core" -import type { Package } from "@dpkit/core" import { makeGithubApiRequest } from "../general/index.js" import type { GithubResource } from "../resource/index.js" import type { GithubPackage } from "./Package.js" @@ -10,9 +9,7 @@ import { normalizeGithubPackage } from "./process/normalize.js" * @param props Object containing the URL to the Github repository * @returns Package object */ -export async function loadPackageFromGithub< - T extends Package = Package, ->(props: { +export async function loadPackageFromGithub(props: { repoUrl: string apiKey?: string }) { @@ -43,7 +40,7 @@ export async function loadPackageFromGithub< .map(resource => resource["github:url"]) .at(0) - const datapackage = await mergePackages({ systemPackage, userPackagePath }) + const datapackage = await mergePackages({ systemPackage, userPackagePath }) datapackage.resources = datapackage.resources.map(resource => { // TODO: remove these keys completely return { ...resource, "github:key": undefined, "github:url": undefined } diff --git a/github/plugin.ts b/github/plugin.ts index bed8b62f..ddf6abe3 100644 --- a/github/plugin.ts +++ b/github/plugin.ts @@ -1,9 +1,9 @@ -import type { Descriptor, Plugin, Package } from "@dpkit/core" +import type { Descriptor, Plugin } from "@dpkit/core" import { isRemotePath } from "@dpkit/core" import { loadPackageFromGithub } from "./package/load.js" export class GithubPlugin implements Plugin { - async loadPackage(props: { + async loadPackage(props: { source: string options?: Descriptor }) { @@ -14,7 +14,7 @@ export class GithubPlugin implements Plugin { if (!isGithub) return undefined const cleanup = async () => {} - const datapackage = await loadPackageFromGithub({ + const datapackage = await loadPackageFromGithub({ repoUrl: props.source, }) diff --git a/zenodo/package/load.ts b/zenodo/package/load.ts index cf45b14c..74aea0ef 100644 --- a/zenodo/package/load.ts +++ b/zenodo/package/load.ts @@ -1,7 +1,6 @@ import { mergePackages } from "@dpkit/core" import { makeZenodoApiRequest } from "../general/index.js" import type { ZenodoPackage } from "./Package.js" -import type { Package } from "@dpkit/core" import { normalizeZenodoPackage } from "./process/normalize.js" /** @@ -9,9 +8,7 @@ import { normalizeZenodoPackage } from "./process/normalize.js" * @param props Object containing the URL to the Zenodo deposit * @returns Package object */ -export async function loadPackageFromZenodo< - T extends Package = Package, ->(props: { +export async function loadPackageFromZenodo(props: { datasetUrl: string apiKey?: string }) { @@ -35,7 +32,7 @@ export async function loadPackageFromZenodo< .map(resource => resource["zenodo:url"]) .at(0) - const datapackage = await mergePackages({ systemPackage, userPackagePath }) + const datapackage = await mergePackages({ systemPackage, userPackagePath }) datapackage.resources = datapackage.resources.map(resource => { // TODO: remove these keys completely return { ...resource, "zenodo:key": undefined, "zenodo:url": undefined } diff --git a/zenodo/plugin.ts b/zenodo/plugin.ts index e900b23c..7655c0fb 100644 --- a/zenodo/plugin.ts +++ b/zenodo/plugin.ts @@ -1,9 +1,9 @@ -import type { Descriptor, Plugin, Package } from "@dpkit/core" +import type { Descriptor, Plugin } from "@dpkit/core" import { isRemotePath } from "@dpkit/core" import { loadPackageFromZenodo } from "./package/load.js" export class ZenodoPlugin implements Plugin { - async loadPackage(props: { + async loadPackage(props: { source: string options?: Descriptor }) { @@ -14,7 +14,7 @@ export class ZenodoPlugin implements Plugin { if (!isZenodo) return undefined const cleanup = async () => {} - const datapackage = await loadPackageFromZenodo({ + const datapackage = await loadPackageFromZenodo({ datasetUrl: props.source, }) diff --git a/zip/package/load.ts b/zip/package/load.ts index 31410916..7dc0ccdf 100644 --- a/zip/package/load.ts +++ b/zip/package/load.ts @@ -1,5 +1,4 @@ import { createWriteStream } from "node:fs" -import type { Package } from "@dpkit/core" import { mkdir, rm } from "node:fs/promises" import { join } from "node:path" import { pipeline } from "node:stream/promises" @@ -7,7 +6,7 @@ import { loadPackageDescriptor } from "@dpkit/core" import { temporaryDirectory } from "tempy" import yauzl from "yauzl-promise" -export async function loadPackageFromZip(props: { +export async function loadPackageFromZip(props: { archivePath: string }) { const tempdir = temporaryDirectory() @@ -31,7 +30,7 @@ export async function loadPackageFromZip(props: { await zipfile.close() } - const datapackage = await loadPackageDescriptor({ + const datapackage = await loadPackageDescriptor({ path: join(tempdir, "datapackage.json"), }) diff --git a/zip/plugin.ts b/zip/plugin.ts index aebdf3fc..dd71d18d 100644 --- a/zip/plugin.ts +++ b/zip/plugin.ts @@ -2,14 +2,14 @@ import type { Descriptor, Package, Plugin } from "@dpkit/core" import { loadPackageFromZip, savePackageToZip } from "./package/index.js" export class ZipPlugin implements Plugin { - async loadPackage(props: { + async loadPackage(props: { source: string options?: Descriptor }) { const isZip = props.source.endsWith(".zip") if (!isZip) return undefined - const { datapackage, cleanup } = await loadPackageFromZip({ + const { datapackage, cleanup } = await loadPackageFromZip({ archivePath: props.source, }) From 672f4ea3b1ddc0239caebe4804fac5cd13f4cd17 Mon Sep 17 00:00:00 2001 From: roll Date: Wed, 28 May 2025 18:23:13 +0100 Subject: [PATCH 4/5] Fixed camtrap example --- camtrap/package/assert.ts | 32 +++++++++++++++ camtrap/package/index.ts | 3 +- core/dialect/process/normalize.ts | 6 +++ core/dialect/save.ts | 4 +- core/general/path.ts | 27 +++++++------ core/package/process/normalize.ts | 6 +++ core/package/save.ts | 6 +-- core/resource/process/normalize.ts | 6 +++ core/resource/save.ts | 6 +-- core/schema/process/normalize.ts | 6 +++ core/schema/save.ts | 4 +- docs/content/docs/overview/getting-started.md | 23 +++++++++-- docs/examples/getting-started-1.ts | 7 ++++ docs/examples/getting-started-2.ts | 12 ++++++ docs/examples/getting-started.ts | 7 ---- .../fixtures/generated/load.spec.ts.snap | 40 ++++++++++++++++--- .../fixtures/generated/load.spec.ts.snap | 9 +++++ 17 files changed, 161 insertions(+), 43 deletions(-) create mode 100644 camtrap/package/assert.ts create mode 100644 docs/examples/getting-started-1.ts create mode 100644 docs/examples/getting-started-2.ts delete mode 100644 docs/examples/getting-started.ts diff --git a/camtrap/package/assert.ts b/camtrap/package/assert.ts new file mode 100644 index 00000000..fb5bb8b9 --- /dev/null +++ b/camtrap/package/assert.ts @@ -0,0 +1,32 @@ +import { + AssertionError, + type Package, + validatePackageDescriptor, +} from "@dpkit/core" +import type { CamtrapPackage } from "./Package.js" + +const SUPPORTED_PROFILES = [ + "https://raw.githubusercontent.com/tdwg/camtrap-dp/1.0/camtrap-dp-profile.json", + "https://raw.githubusercontent.com/tdwg/camtrap-dp/1.0.1/camtrap-dp-profile.json", +] + +/** + * Assert a Package descriptor (JSON Object) against its profile + */ +export async function assertCamtrapPackage(props: { + datapackage: Package +}) { + if (!SUPPORTED_PROFILES.includes(props.datapackage.$schema ?? "")) { + throw new Error( + `Unsupported Camtrap DP profile: ${props.datapackage.$schema}`, + ) + } + + const { errors, datapackage } = await validatePackageDescriptor({ + // @ts-ignore (TODO: figure out descriptor/package boundary here) + descriptor: props.datapackage, + }) + + if (!datapackage) throw new AssertionError(errors) + return datapackage as CamtrapPackage +} diff --git a/camtrap/package/index.ts b/camtrap/package/index.ts index 221f2d67..9b8ce884 100644 --- a/camtrap/package/index.ts +++ b/camtrap/package/index.ts @@ -1 +1,2 @@ -export type { CamtrapPackage } from "./Package.ts" +export type { CamtrapPackage } from "./Package.js" +export { assertCamtrapPackage } from "./assert.js" diff --git a/core/dialect/process/normalize.ts b/core/dialect/process/normalize.ts index fa68890a..c4e2b58b 100644 --- a/core/dialect/process/normalize.ts +++ b/core/dialect/process/normalize.ts @@ -3,11 +3,17 @@ import type { Descriptor } from "../../general/index.js" export function normalizeDialect(props: { descriptor: Descriptor }) { const descriptor = globalThis.structuredClone(props.descriptor) + normalizeProfile({ descriptor }) normalizeTable({ descriptor }) return descriptor } +function normalizeProfile(props: { descriptor: Descriptor }) { + const { descriptor } = props + descriptor.$schema = descriptor.$schema ?? descriptor.profile +} + function normalizeTable(props: { descriptor: Descriptor }) { const { descriptor } = props diff --git a/core/dialect/save.ts b/core/dialect/save.ts index 87d70cf6..cd4cbcab 100644 --- a/core/dialect/save.ts +++ b/core/dialect/save.ts @@ -15,9 +15,7 @@ export async function saveDialect(props: { const { dialect, path } = props const descriptor = denormalizeDialect({ dialect }) - - descriptor.$schema = - descriptor.$schema ?? descriptor.profile ?? CURRENT_PROFILE + descriptor.$schema = descriptor.$schema ?? CURRENT_PROFILE await saveDescriptor({ descriptor, path }) } diff --git a/core/general/path.ts b/core/general/path.ts index 72f907fb..40f63350 100644 --- a/core/general/path.ts +++ b/core/general/path.ts @@ -66,14 +66,15 @@ export function normalizePath(props: { path: string basepath?: string }) { - const isBasepathRemote = isRemotePath({ path: props.basepath ?? "" }) const isPathRemote = isRemotePath({ path: props.path }) - const isRemote = isBasepathRemote || isPathRemote + const isBasepathRemote = isRemotePath({ path: props.basepath ?? "" }) - if (isRemote) { - const path = new URL( - !isPathRemote ? [props.basepath, props.path].join("/") : props.path, - ).toString() + if (isPathRemote) { + return new URL(props.path).toString() + } + + if (isBasepathRemote) { + const path = new URL([props.basepath, props.path].join("/")).toString() if (!path.startsWith(props.basepath ?? "")) { throw new Error( @@ -103,17 +104,17 @@ export function denormalizePath(props: { path: string basepath?: string }) { - const isBasepathRemote = isRemotePath({ path: props.basepath ?? "" }) const isPathRemote = isRemotePath({ path: props.path }) - const isRemote = isBasepathRemote || isPathRemote + const isBasepathRemote = isRemotePath({ path: props.basepath ?? "" }) - if (isRemote) { - const path = new URL(props.path).toString() - if (isPathRemote) { - return path - } + if (isPathRemote) { + return new URL(props.path).toString() + } + if (isBasepathRemote) { + const path = props.path const basepath = new URL(props.basepath ?? "").toString() + if (!path.startsWith(basepath)) { throw new Error( `Path ${props.path} is not a subpath of ${props.basepath}`, diff --git a/core/package/process/normalize.ts b/core/package/process/normalize.ts index 4eb6f35a..f17e71b4 100644 --- a/core/package/process/normalize.ts +++ b/core/package/process/normalize.ts @@ -8,12 +8,18 @@ export function normalizePackage(props: { const { basepath } = props const descriptor = globalThis.structuredClone(props.descriptor) + normalizeProfile({ descriptor }) normalizeResources({ descriptor, basepath }) normalizeContributors({ descriptor }) return descriptor } +function normalizeProfile(props: { descriptor: Descriptor }) { + const { descriptor } = props + descriptor.$schema = descriptor.$schema ?? descriptor.profile +} + function normalizeResources(props: { descriptor: Descriptor basepath?: string diff --git a/core/package/save.ts b/core/package/save.ts index 9857f065..49bc827c 100644 --- a/core/package/save.ts +++ b/core/package/save.ts @@ -13,12 +13,10 @@ export async function savePackageDescriptor(props: { path: string }) { const { datapackage, path } = props - const basepath = getBasepath({ path }) - const descriptor = denormalizePackage({ datapackage, basepath }) - descriptor.$schema = - descriptor.$schema ?? descriptor.profile ?? CURRENT_PROFILE + const descriptor = denormalizePackage({ datapackage, basepath }) + descriptor.$schema = descriptor.$schema ?? CURRENT_PROFILE await saveDescriptor({ descriptor, path: props.path }) } diff --git a/core/resource/process/normalize.ts b/core/resource/process/normalize.ts index 14892b55..ce573eae 100644 --- a/core/resource/process/normalize.ts +++ b/core/resource/process/normalize.ts @@ -10,6 +10,7 @@ export function normalizeResource(props: { const { basepath } = props const descriptor = globalThis.structuredClone(props.descriptor) + normalizeProfile({ descriptor }) normalizeUrl({ descriptor }) normalizeType({ descriptor }) normalizePaths({ descriptor, basepath }) @@ -20,6 +21,11 @@ export function normalizeResource(props: { return descriptor } +function normalizeProfile(props: { descriptor: Descriptor }) { + const { descriptor } = props + descriptor.$schema = descriptor.$schema ?? descriptor.profile +} + function normalizeUrl(props: { descriptor: Descriptor }) { const { descriptor } = props diff --git a/core/resource/save.ts b/core/resource/save.ts index 7b1858a6..6a535c80 100644 --- a/core/resource/save.ts +++ b/core/resource/save.ts @@ -13,12 +13,10 @@ export async function saveResourceDescriptor(props: { path: string }) { const { resource, path } = props - const basepath = getBasepath({ path }) - const descriptor = denormalizeResource({ resource, basepath }) - descriptor.$schema = - descriptor.$schema ?? descriptor.profile ?? CURRENT_PROFILE + const descriptor = denormalizeResource({ resource, basepath }) + descriptor.$schema = descriptor.$schema ?? CURRENT_PROFILE await saveDescriptor({ descriptor, path }) } diff --git a/core/schema/process/normalize.ts b/core/schema/process/normalize.ts index 85020f6e..925af44f 100644 --- a/core/schema/process/normalize.ts +++ b/core/schema/process/normalize.ts @@ -4,6 +4,7 @@ import type { Descriptor } from "../../general/index.js" export function normalizeSchema(props: { descriptor: Descriptor }) { const descriptor = globalThis.structuredClone(props.descriptor) + normalizeProfile({ descriptor }) normalizeFields({ descriptor }) normalizePrimaryKey({ descriptor }) normalizeForeignKeys({ descriptor }) @@ -12,6 +13,11 @@ export function normalizeSchema(props: { descriptor: Descriptor }) { return descriptor } +function normalizeProfile(props: { descriptor: Descriptor }) { + const { descriptor } = props + descriptor.$schema = descriptor.$schema ?? descriptor.profile +} + function normalizeFields(props: { descriptor: Descriptor }) { const { descriptor } = props diff --git a/core/schema/save.ts b/core/schema/save.ts index 3a8dc706..f5e62d21 100644 --- a/core/schema/save.ts +++ b/core/schema/save.ts @@ -15,9 +15,7 @@ export async function saveSchema(props: { const { schema, path } = props const descriptor = denormalizeSchema({ schema }) - - descriptor.$schema = - descriptor.$schema ?? descriptor.profile ?? CURRENT_PROFILE + descriptor.$schema = descriptor.$schema ?? CURRENT_PROFILE await saveDescriptor({ descriptor, path }) } diff --git a/docs/content/docs/overview/getting-started.md b/docs/content/docs/overview/getting-started.md index 64a5b8b3..693c456d 100644 --- a/docs/content/docs/overview/getting-started.md +++ b/docs/content/docs/overview/getting-started.md @@ -35,14 +35,31 @@ dpkit is built with type safety in mind. It uses TypeScript to provide type defi ## Usage -Here is an example of loading a Camtrap DP package from Zenodo preserving full type-safety of the extension and merging system Zenodo metadata into a user data package: +Here is an example of loading a Camtrap DP package from Zenodo merging system Zenodo metadata into a user data package and validating its metadata: ```ts -import { loadPackage, type CamtrapPackage } from "dpkit" +import { loadPackage } from "dpkit" -const { datapackage } = await loadPackage({ +const { datapackage } = await loadPackage({ source: "https://zenodo.org/records/10053903", }) console.log(datapackage) ``` + +Example of using a Data Package extension in type-safe manner: + +```ts +import { loadPackage, assertCamtrapPackage } from "dpkit" + +const { datapackage } = await loadPackage({ + source: + "https://raw.githubusercontent.com/tdwg/camtrap-dp/refs/tags/1.0.1/example/datapackage.json", +}) + +const camtrapPackage = await assertCamtrapPackage({ datapackage }) + +console.log(camtrapPackage.taxonomic) +console.log(camtrapPackage.project.title) +console.log(camtrapPackage.bibliographicCitation) +``` diff --git a/docs/examples/getting-started-1.ts b/docs/examples/getting-started-1.ts new file mode 100644 index 00000000..530c57d9 --- /dev/null +++ b/docs/examples/getting-started-1.ts @@ -0,0 +1,7 @@ +import { loadPackage } from "dpkit" + +const { datapackage } = await loadPackage({ + source: "https://zenodo.org/records/10053903", +}) + +console.log(datapackage) diff --git a/docs/examples/getting-started-2.ts b/docs/examples/getting-started-2.ts new file mode 100644 index 00000000..12109f8a --- /dev/null +++ b/docs/examples/getting-started-2.ts @@ -0,0 +1,12 @@ +import { assertCamtrapPackage, loadPackage } from "dpkit" + +const { datapackage } = await loadPackage({ + source: + "https://raw.githubusercontent.com/tdwg/camtrap-dp/refs/tags/1.0.1/example/datapackage.json", +}) + +const camtrapPackage = await assertCamtrapPackage({ datapackage }) + +console.log(camtrapPackage.taxonomic) +console.log(camtrapPackage.project.title) +console.log(camtrapPackage.bibliographicCitation) diff --git a/docs/examples/getting-started.ts b/docs/examples/getting-started.ts deleted file mode 100644 index 6d1dd799..00000000 --- a/docs/examples/getting-started.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { loadPackage, type CamtrapPackage } from "dpkit" - -const { datapackage } = await loadPackage({ - source: "https://zenodo.org/records/10053903", -}) - -console.log(datapackage) diff --git a/github/package/fixtures/generated/load.spec.ts.snap b/github/package/fixtures/generated/load.spec.ts.snap index ddbb4566..fadcca50 100644 --- a/github/package/fixtures/generated/load.spec.ts.snap +++ b/github/package/fixtures/generated/load.spec.ts.snap @@ -109,25 +109,48 @@ exports[`loadPackageFromGithub > should load a package 1`] = ` exports[`loadPackageFromGithub > should merge datapackage.json if present 1`] = ` { + "$schema": undefined, "contributors": [ { - "path": "https://github.com/roll", - "role": "author", - "title": "roll", + "email": "rufus.pollock@okfn.org", + "role": undefined, + "roles": [ + "maintainer", + ], + "title": "Rufus Pollock", + }, + { + "email": "kdkusano@gmail.com", + "title": "Kristofer D. Kusano", }, ], "created": "2021-07-07T14:09:49Z", "description": "ISO 4217 List of Currencies and Currency Codes", - "homepage": "https://datahub.io/core/currency-codes", + "homepage": "http://www.iso.org/iso/currency_codes", + "keywords": [ + "iso", + "iso-4217", + "currency", + "codes", + ], + "licenses": [ + { + "name": "ODC-PDDL-1.0", + "path": "http://opendatacommons.org/licenses/pddl/", + "title": "Open Data Commons Public Domain Dedication and License v1.0", + }, + ], "name": "currency-codes", "resources": [ { + "$schema": undefined, "github:key": undefined, "github:url": undefined, "mimetype": "text/csv", "name": "codes-all", "path": "https://raw.githubusercontent.com/roll/currency-codes/refs/heads/master/data/codes-all.csv", "schema": { + "$schema": undefined, "fields": [ { "description": "Country or region name", @@ -168,6 +191,13 @@ exports[`loadPackageFromGithub > should merge datapackage.json if present 1`] = "size": "16863", }, ], - "title": "roll/currency-codes", + "sources": [ + { + "email": "office@currency-iso.org", + "name": "SIX Interbank Clearing Ltd (on behalf of ISO)", + "title": "SIX Interbank Clearing Ltd (on behalf of ISO)", + }, + ], + "title": "ISO 4217 Currency Codes", } `; diff --git a/zenodo/package/fixtures/generated/load.spec.ts.snap b/zenodo/package/fixtures/generated/load.spec.ts.snap index 23092bbb..18f84c12 100644 --- a/zenodo/package/fixtures/generated/load.spec.ts.snap +++ b/zenodo/package/fixtures/generated/load.spec.ts.snap @@ -41,6 +41,7 @@ exports[`loadPackageFromZenodo > should load a package 1`] = ` exports[`loadPackageFromZenodo > shoule merge datapackage.json if present 1`] = ` { + "$schema": "tabular-data-package", "contributors": [ { "path": "Vogelwerkgroep Assen", @@ -91,6 +92,7 @@ exports[`loadPackageFromZenodo > shoule merge datapackage.json if present 1`] =

Acknowledgements

These data were collected by Bert Dijkstra and Rinus Dillerop from Vogelwerkgroep Assen, in collaboration with the Netherlands Institute of Ecology (NIOO-KNAW), Sovon, Radboud University and the University of Amsterdam (UvA). Funding was provided by the Prins Bernard Cultuurfonds Drenthe, municipality of Assen, IJsvogelfonds (from Birdlife Netherlands and Nationale Postcodeloterij) and the Waterleiding Maatschappij Drenthe. The dataset was published with funding from Stichting NLBIF - Netherlands Biodiversity Information Facility.

", + "id": "https://doi.org/10.5281/zenodo.10053903", "keywords": [ "animal movement", "animal tracking", @@ -112,8 +114,10 @@ exports[`loadPackageFromZenodo > shoule merge datapackage.json if present 1`] = }, ], "name": "record-10053903", + "profile": "tabular-data-package", "resources": [ { + "$schema": "tabular-data-resource", "encoding": "UTF-8", "format": "csv", "mediatype": "text/csv", @@ -121,6 +125,7 @@ exports[`loadPackageFromZenodo > shoule merge datapackage.json if present 1`] = "path": "https://zenodo.org/records/10053903/files/O_ASSEN-reference-data.csv", "profile": "tabular-data-resource", "schema": { + "$schema": undefined, "fields": [ { "description": "A unique identifier for the tag, provided by the data owner. If the data owner does not provide a tag ID, an internal Movebank tag identifier may sometimes be shown. Example: '2342'; Units: none; Entity described: tag", @@ -348,6 +353,7 @@ exports[`loadPackageFromZenodo > shoule merge datapackage.json if present 1`] = "zenodo:url": undefined, }, { + "$schema": "tabular-data-resource", "encoding": "UTF-8", "format": "csv", "mediatype": "text/csv", @@ -358,6 +364,7 @@ exports[`loadPackageFromZenodo > shoule merge datapackage.json if present 1`] = ], "profile": "tabular-data-resource", "schema": { + "$schema": undefined, "fields": [ { "description": "An identifier for the set of values associated with each event, i.e. sensor measurement. A unique event ID is assigned to every time-location or other time-measurement record in Movebank. If multiple measurements are included within a single row of a data file, they will share an event ID. If users import the same sensor measurement to Movebank multiple times, a separate event ID will be assigned to each. Example: '14328243575'; Units: none; Entity described: event", @@ -559,6 +566,7 @@ exports[`loadPackageFromZenodo > shoule merge datapackage.json if present 1`] = "zenodo:url": undefined, }, { + "$schema": "tabular-data-resource", "encoding": "UTF-8", "format": "csv", "mediatype": "text/csv", @@ -569,6 +577,7 @@ exports[`loadPackageFromZenodo > shoule merge datapackage.json if present 1`] = ], "profile": "tabular-data-resource", "schema": { + "$schema": undefined, "fields": [ { "description": "An identifier for the set of values associated with each event, i.e. sensor measurement. A unique event ID is assigned to every time-location or other time-measurement record in Movebank. If multiple measurements are included within a single row of a data file, they will share an event ID. If users import the same sensor measurement to Movebank multiple times, a separate event ID will be assigned to each. Example: '14328243575'; Units: none; Entity described: event", From 85af47e5c1f7cea8e818b20cdf21375cb5b57500 Mon Sep 17 00:00:00 2001 From: roll Date: Wed, 28 May 2025 18:23:56 +0100 Subject: [PATCH 5/5] Added changesets --- .changeset/heavy-mammals-battle.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .changeset/heavy-mammals-battle.md diff --git a/.changeset/heavy-mammals-battle.md b/.changeset/heavy-mammals-battle.md new file mode 100644 index 00000000..9be11eb6 --- /dev/null +++ b/.changeset/heavy-mammals-battle.md @@ -0,0 +1,11 @@ +--- +"@dpkit/camtrap": minor +"@dpkit/github": minor +"@dpkit/zenodo": minor +"@dpkit/ckan": minor +"@dpkit/core": minor +"@dpkit/file": minor +"@dpkit/zip": minor +--- + +Removed permissive generics