feat: finalize initial metadata functionality

.changeset/heavy-mammals-battle.md

@@ -0,0 +1,11 @@
+---
+"@dpkit/camtrap": minor
+"@dpkit/github": minor
+"@dpkit/zenodo": minor
+"@dpkit/ckan": minor
+"@dpkit/core": minor
+"@dpkit/file": minor
+"@dpkit/zip": minor
+---
+
+Removed permissive generics

README.md

@@ -1,8 +1,11 @@
 # dpkit
 
+
 [![Build](https://img.shields.io/github/actions/workflow/status/datisthq/dpkit/general.yaml?branch=main)](https://github.com/datisthq/dpkit/actions)
 [![Coverage](https://img.shields.io/codecov/c/github/datisthq/dpkit/main)](https://codecov.io/gh/datisthq/dpkit)
 
+> **Note:** This project is currently in active development. Feature and new plugin pull requests are not being accepted at this time due to the instability of the core APIs. Bug fixes are welcome!
+
 dpkit is a fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io).
 
 ## Funding

camtrap/OVERVIEW.md

@@ -1,2 +1,6 @@
 # @dpkit/camtrap
 
+:::note
+This overview is under development.
+:::
+

camtrap/package/assert.ts

@@ -0,0 +1,32 @@
+import {
+  AssertionError,
+  type Package,
+  validatePackageDescriptor,
+} from "@dpkit/core"
+import type { CamtrapPackage } from "./Package.js"
+
+const SUPPORTED_PROFILES = [
+  "https://raw.githubusercontent.com/tdwg/camtrap-dp/1.0/camtrap-dp-profile.json",
+  "https://raw.githubusercontent.com/tdwg/camtrap-dp/1.0.1/camtrap-dp-profile.json",
+]
+
+/**
+ * Assert a Package descriptor (JSON Object) against its profile
+ */
+export async function assertCamtrapPackage(props: {
+  datapackage: Package
+}) {
+  if (!SUPPORTED_PROFILES.includes(props.datapackage.$schema ?? "")) {
+    throw new Error(
+      `Unsupported Camtrap DP profile: ${props.datapackage.$schema}`,
+    )
+  }
+
+  const { errors, datapackage } = await validatePackageDescriptor({
+    // @ts-ignore (TODO: figure out descriptor/package boundary here)
+    descriptor: props.datapackage,
+  })
+
+  if (!datapackage) throw new AssertionError(errors)
+  return datapackage as CamtrapPackage
+}

camtrap/package/index.ts

@@ -1 +1,2 @@
-export type { CamtrapPackage } from "./Package.ts"
+export type { CamtrapPackage } from "./Package.js"
+export { assertCamtrapPackage } from "./assert.js"

ckan/OVERVIEW.md

@@ -1,2 +1,6 @@
 # @dpkit/ckan
 
+:::note
+This overview is under development.
+:::
+

ckan/package/load.ts

@@ -8,7 +8,9 @@ import { normalizeCkanPackage } from "./process/normalize.js"
  * @param props Object containing the URL to the CKAN package
  * @returns Package object and cleanup function
  */
-export async function loadPackageFromCkan(props: { datasetUrl: string }) {
+export async function loadPackageFromCkan(props: {
+  datasetUrl: string
+}) {
   const { datasetUrl } = props
 
   const packageId = extractPackageId({ datasetUrl })
@@ -40,6 +42,7 @@ export async function loadPackageFromCkan(props: { datasetUrl: string }) {
 
   const datapackage = await mergePackages({ systemPackage, userPackagePath })
   datapackage.resources = datapackage.resources.map(resource => {
+    // TODO: remove these keys completely
     return { ...resource, "ckan:key": undefined, "ckan:url": undefined }
   })
 

ckan/plugin.ts

@@ -3,15 +3,20 @@ import { isRemotePath } from "@dpkit/core"
 import { loadPackageFromCkan } from "./package/load.js"
 
 export class CkanPlugin implements Plugin {
-  async loadPackage(props: { source: string; options?: Descriptor }) {
+  async loadPackage(props: {
+    source: string
+    options?: Descriptor
+  }) {
     const isRemote = isRemotePath({ path: props.source })
     if (!isRemote) return undefined
 
     const withDataset = props.source.includes("/dataset/")
     if (!withDataset) return undefined
 
     const cleanup = async () => {}
-    const datapackage = await loadPackageFromCkan({ datasetUrl: props.source })
+    const datapackage = await loadPackageFromCkan({
+      datasetUrl: props.source,
+    })
 
     return { datapackage, cleanup }
   }

core/OVERVIEW.md

@@ -1,2 +1,6 @@
 # @dpkit/core
 
+:::note
+This overview is under development.
+:::
+

core/dialect/process/normalize.ts

@@ -3,11 +3,17 @@ import type { Descriptor } from "../../general/index.js"
 export function normalizeDialect(props: { descriptor: Descriptor }) {
   const descriptor = globalThis.structuredClone(props.descriptor)
 
+  normalizeProfile({ descriptor })
   normalizeTable({ descriptor })
 
   return descriptor
 }
 
+function normalizeProfile(props: { descriptor: Descriptor }) {
+  const { descriptor } = props
+  descriptor.$schema = descriptor.$schema ?? descriptor.profile
+}
+
 function normalizeTable(props: { descriptor: Descriptor }) {
   const { descriptor } = props
 

core/dialect/save.ts

@@ -15,9 +15,7 @@ export async function saveDialect(props: {
   const { dialect, path } = props
 
   const descriptor = denormalizeDialect({ dialect })
-
-  descriptor.$schema =
-    descriptor.$schema ?? descriptor.profile ?? CURRENT_PROFILE
+  descriptor.$schema = descriptor.$schema ?? CURRENT_PROFILE
 
   await saveDescriptor({ descriptor, path })
 }

core/general/path.ts

@@ -66,14 +66,15 @@ export function normalizePath(props: {
   path: string
   basepath?: string
 }) {
-  const isBasepathRemote = isRemotePath({ path: props.basepath ?? "" })
   const isPathRemote = isRemotePath({ path: props.path })
-  const isRemote = isBasepathRemote || isPathRemote
+  const isBasepathRemote = isRemotePath({ path: props.basepath ?? "" })
 
-  if (isRemote) {
-    const path = new URL(
-      !isPathRemote ? [props.basepath, props.path].join("/") : props.path,
-    ).toString()
+  if (isPathRemote) {
+    return new URL(props.path).toString()
+  }
+
+  if (isBasepathRemote) {
+    const path = new URL([props.basepath, props.path].join("/")).toString()
 
     if (!path.startsWith(props.basepath ?? "")) {
       throw new Error(
@@ -103,17 +104,17 @@ export function denormalizePath(props: {
   path: string
   basepath?: string
 }) {
-  const isBasepathRemote = isRemotePath({ path: props.basepath ?? "" })
   const isPathRemote = isRemotePath({ path: props.path })
-  const isRemote = isBasepathRemote || isPathRemote
+  const isBasepathRemote = isRemotePath({ path: props.basepath ?? "" })
 
-  if (isRemote) {
-    const path = new URL(props.path).toString()
-    if (isPathRemote) {
-      return path
-    }
+  if (isPathRemote) {
+    return new URL(props.path).toString()
+  }
 
+  if (isBasepathRemote) {
+    const path = props.path
     const basepath = new URL(props.basepath ?? "").toString()
+
     if (!path.startsWith(basepath)) {
       throw new Error(
         `Path ${props.path} is not a subpath of ${props.basepath}`,

core/package/assert.ts

@@ -1,15 +1,14 @@
 import { AssertionError, type Descriptor } from "../general/index.js"
-import type { Package } from "./Package.js"
 import { validatePackageDescriptor } from "./validate.js"
 
 /**
  * Assert a Package descriptor (JSON Object) against its profile
  */
-export async function assertPackage<T extends Package = Package>(props: {
+export async function assertPackage(props: {
   descriptor: Descriptor
   basepath?: string
 }) {
-  const { errors, datapackage } = await validatePackageDescriptor<T>(props)
+  const { errors, datapackage } = await validatePackageDescriptor(props)
   if (!datapackage) throw new AssertionError(errors)
   return datapackage
 }

core/package/load.ts

@@ -1,15 +1,12 @@
 import { loadDescriptor } from "../general/index.js"
-import type { Package } from "./Package.js"
 import { assertPackage } from "./assert.js"
 
 /**
  * Load a Package descriptor (JSON Object) from a file or URL
  * Ensures the descriptor is valid against its profile
  */
-export async function loadPackageDescriptor<
-  T extends Package = Package,
->(props: { path: string }) {
+export async function loadPackageDescriptor(props: { path: string }) {
   const { basepath, descriptor } = await loadDescriptor(props)
-  const datapackage = await assertPackage<T>({ descriptor, basepath })
+  const datapackage = await assertPackage({ descriptor, basepath })
   return datapackage
 }

core/package/merge.ts

@@ -8,18 +8,10 @@ export async function mergePackages(props: {
   systemPackage: Package
   userPackagePath?: string
 }) {
-  const { systemPackage, userPackagePath } = props
-
-  const userPackage = userPackagePath
-    ? await loadPackageDescriptor({ path: userPackagePath })
+  const systemPackage = props.systemPackage
+  const userPackage = props.userPackagePath
+    ? await loadPackageDescriptor({ path: props.userPackagePath })
     : undefined
 
-  if (!userPackage) {
-    return props.systemPackage
-  }
-
-  // NOTE:
-  // Currently, it uses oversimplified logic till
-  // we get use feedback on the desired merging strategies
-  return { ...systemPackage, resources: userPackage.resources }
+  return { ...systemPackage, ...userPackage }
 }

core/package/process/normalize.ts

@@ -8,12 +8,18 @@ export function normalizePackage(props: {
   const { basepath } = props
   const descriptor = globalThis.structuredClone(props.descriptor)
 
+  normalizeProfile({ descriptor })
   normalizeResources({ descriptor, basepath })
   normalizeContributors({ descriptor })
 
   return descriptor
 }
 
+function normalizeProfile(props: { descriptor: Descriptor }) {
+  const { descriptor } = props
+  descriptor.$schema = descriptor.$schema ?? descriptor.profile
+}
+
 function normalizeResources(props: {
   descriptor: Descriptor
   basepath?: string

core/package/save.ts

@@ -13,12 +13,10 @@ export async function savePackageDescriptor(props: {
   path: string
 }) {
   const { datapackage, path } = props
-
   const basepath = getBasepath({ path })
-  const descriptor = denormalizePackage({ datapackage, basepath })
 
-  descriptor.$schema =
-    descriptor.$schema ?? descriptor.profile ?? CURRENT_PROFILE
+  const descriptor = denormalizePackage({ datapackage, basepath })
+  descriptor.$schema = descriptor.$schema ?? CURRENT_PROFILE
 
   await saveDescriptor({ descriptor, path: props.path })
 }

core/package/validate.ts

@@ -8,14 +8,12 @@ const DEFAULT_PROFILE = "https://datapackage.org/profiles/1.0/datapackage.json"
 /**
  * Validate a Package descriptor (JSON Object) against its profile
  */
-export async function validatePackageDescriptor<
-  T extends Package = Package,
->(props: {
+export async function validatePackageDescriptor(props: {
   descriptor: Descriptor
   basepath?: string
 }) {
   const { descriptor, basepath } = props
-  let datapackage: T | undefined = undefined
+  let datapackage: Package | undefined = undefined
 
   const $schema =
     typeof props.descriptor.$schema === "string"
@@ -27,7 +25,10 @@ export async function validatePackageDescriptor<
 
   if (valid) {
     // Validation + normalization = we can cast it
-    datapackage = normalizePackage({ descriptor, basepath }) as T
+    datapackage = normalizePackage({
+      descriptor,
+      basepath,
+    }) as unknown as Package
   }
 
   return { valid, errors, datapackage }

core/plugin.ts

@@ -2,16 +2,16 @@ import type { Descriptor } from "./general/index.js"
 import type { Package } from "./package/index.js"
 
 export interface Plugin {
-  loadPackage?: (props: {
+  loadPackage?(props: {
     source: string
     options?: Descriptor
-  }) => Promise<
+  }): Promise<
     undefined | { datapackage: Package; cleanup?: () => Promise<void> }
   >
 
-  savePackage?: (props: {
+  savePackage?(props: {
     datapackage: Package
     target: string
     options?: Descriptor
-  }) => Promise<undefined | { path?: string }>
+  }): Promise<undefined | { path?: string }>
 }

core/resource/process/normalize.ts

@@ -10,6 +10,7 @@ export function normalizeResource(props: {
   const { basepath } = props
   const descriptor = globalThis.structuredClone(props.descriptor)
 
+  normalizeProfile({ descriptor })
   normalizeUrl({ descriptor })
   normalizeType({ descriptor })
   normalizePaths({ descriptor, basepath })
@@ -20,6 +21,11 @@ export function normalizeResource(props: {
   return descriptor
 }
 
+function normalizeProfile(props: { descriptor: Descriptor }) {
+  const { descriptor } = props
+  descriptor.$schema = descriptor.$schema ?? descriptor.profile
+}
+
 function normalizeUrl(props: { descriptor: Descriptor }) {
   const { descriptor } = props
 

core/resource/save.ts

@@ -13,12 +13,10 @@ export async function saveResourceDescriptor(props: {
   path: string
 }) {
   const { resource, path } = props
-
   const basepath = getBasepath({ path })
-  const descriptor = denormalizeResource({ resource, basepath })
 
-  descriptor.$schema =
-    descriptor.$schema ?? descriptor.profile ?? CURRENT_PROFILE
+  const descriptor = denormalizeResource({ resource, basepath })
+  descriptor.$schema = descriptor.$schema ?? CURRENT_PROFILE
 
   await saveDescriptor({ descriptor, path })
 }

core/schema/process/normalize.ts

@@ -4,6 +4,7 @@ import type { Descriptor } from "../../general/index.js"
 export function normalizeSchema(props: { descriptor: Descriptor }) {
   const descriptor = globalThis.structuredClone(props.descriptor)
 
+  normalizeProfile({ descriptor })
   normalizeFields({ descriptor })
   normalizePrimaryKey({ descriptor })
   normalizeForeignKeys({ descriptor })
@@ -12,6 +13,11 @@ export function normalizeSchema(props: { descriptor: Descriptor }) {
   return descriptor
 }
 
+function normalizeProfile(props: { descriptor: Descriptor }) {
+  const { descriptor } = props
+  descriptor.$schema = descriptor.$schema ?? descriptor.profile
+}
+
 function normalizeFields(props: { descriptor: Descriptor }) {
   const { descriptor } = props