diff --git a/.changeset/twenty-kids-own.md b/.changeset/twenty-kids-own.md new file mode 100644 index 00000000..788cc068 --- /dev/null +++ b/.changeset/twenty-kids-own.md @@ -0,0 +1,23 @@ +--- +"@dpkit/camtrap": minor +"@dpkit/parquet": minor +"@dpkit/extend": minor +"@dpkit/github": minor +"@dpkit/zenodo": minor +"dpkit": minor +"@dpkit/excel": minor +"@dpkit/table": minor +"@dpkit/avro": minor +"@dpkit/ckan": minor +"@dpkit/core": minor +"@dpkit/file": minor +"@dpkit/json": minor +"@dpkit/cli": minor +"@dpkit/csv": minor +"@dpkit/ods": minor +"@dpkit/zip": minor +"@dpkit/db": minor +"@dpkit/ui": minor +--- + +General framework improvements diff --git a/.github/state.yaml b/.github/stale.yaml similarity index 100% rename from .github/state.yaml rename to .github/stale.yaml diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c00b6b83..67c6c799 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,8 +1,8 @@ --- title: Contributing +sidebar: + order: 2 --- -# Contributing to dpkit - Thank you for your interest in contributing to dpkit! This document provides guidelines and instructions for contributing to this project. ## Project Overview diff --git a/README.md b/README.md index ccad3163..fe916ce8 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ -# Data Package in TypeScript +# dpkit [![Build](https://img.shields.io/github/actions/workflow/status/datisthq/dpkit/general.yaml?branch=main)](https://github.com/datisthq/dpkit/actions) [![Coverage](https://img.shields.io/codecov/c/github/datisthq/dpkit/main)](https://codecov.io/gh/datisthq/dpkit) -Data Package implementation in TypeScript. Data Package is a standard consisting of a set of simple yet extensible specifications to describe datasets, data files and tabular data. It is a data definition language (DDL) and data API that facilitates findability, accessibility, interoperability, and reusability (FAIR) of data. For more information, please visit the [documentation portal](https://dpkit.datist.io). +dpkit is a fast TypeScript data management framework based on the Data Package standard. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). ## Funding diff --git a/avro/README.md b/avro/README.md index f4b62388..e99059a5 100644 --- a/avro/README.md +++ b/avro/README.md @@ -1,3 +1,3 @@ # @dpkit/avro -Data Package implementation in TypeScript. Data Package is a standard consisting of a set of simple yet extensible specifications to describe datasets, data files and tabular data. It is a data definition language (DDL) and data API that facilitates findability, accessibility, interoperability, and reusability (FAIR) of data. For more information, please visit the [documentation portal](https://dpkit.datist.io). +dpkit is a fast TypeScript data management framework based on the Data Package standard. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/avro/package.json b/avro/package.json index b213ac47..005768ab 100644 --- a/avro/package.json +++ b/avro/package.json @@ -6,7 +6,7 @@ "license": "MIT", "author": "Evgeny Karev", "repository": "https://github.com/datisthq/dpkit", - "description": "Data Package implementation in TypeScript.", + "description": "Fast TypeScript data management framework based on the Data Package standard", "keywords": [ "data", "package", diff --git a/bin/README.md b/bin/README.md index 87343160..39465a28 100644 --- a/bin/README.md +++ b/bin/README.md @@ -1,3 +1,3 @@ # @dpkit/bin -Data Package implementation in TypeScript. Data Package is a standard consisting of a set of simple yet extensible specifications to describe datasets, data files and tabular data. It is a data definition language (DDL) and data API that facilitates findability, accessibility, interoperability, and reusability (FAIR) of data. For more information, please visit the [documentation portal](https://dpkit.datist.io). +dpkit is a fast TypeScript data management framework based on the Data Package standard. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/camtrap/OVERVIEW.md b/camtrap/OVERVIEW.md new file mode 100644 index 00000000..5ad8a932 --- /dev/null +++ b/camtrap/OVERVIEW.md @@ -0,0 +1,2 @@ +# @dpkit/camtrap + diff --git a/camtrap/README.md b/camtrap/README.md new file mode 100644 index 00000000..41ece1ca --- /dev/null +++ b/camtrap/README.md @@ -0,0 +1,3 @@ +# @dpkit/camtrap + +dpkit is a fast TypeScript data management framework based on the Data Package standard. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/camtrap/index.ts b/camtrap/index.ts new file mode 100644 index 00000000..58d9fce6 --- /dev/null +++ b/camtrap/index.ts @@ -0,0 +1 @@ +export * from "./package/index.js" diff --git a/camtrap/package.json b/camtrap/package.json new file mode 100644 index 00000000..2430ae77 --- /dev/null +++ b/camtrap/package.json @@ -0,0 +1,25 @@ +{ + "name": "@dpkit/camtrap", + "type": "module", + "main": "build/index.js", + "version": "0.1.0", + "license": "MIT", + "author": "Evgeny Karev", + "repository": "https://github.com/datisthq/dpkit", + "description": "Fast TypeScript data management framework based on the Data Package standard", + "keywords": [ + "data", + "package", + "typescript", + "validation", + "quality", + "fair", + "camtrap" + ], + "scripts": { + "build": "tsc --build" + }, + "dependencies": { + "@dpkit/core": "workspace:*" + } +} diff --git a/camtrap/package/Package.ts b/camtrap/package/Package.ts new file mode 100644 index 00000000..d6d35132 --- /dev/null +++ b/camtrap/package/Package.ts @@ -0,0 +1,270 @@ +import type { Package } from "@dpkit/core" +import type { Contributor, License } from "@dpkit/core" + +/** + * Camera Trap Data Package interface based on the TDWG specification + * @see https://camtrap-dp.tdwg.org/metadata/ + */ +export interface CamtrapPackage extends Package { + /** + * Package profile identifier + * @required + */ + profile: string + + /** + * Creation date of the package + * @required + * @format ISO 8601 + */ + created: string + + /** + * Contributors to the package + * @required + */ + contributors: CamtrapContributor[] + + /** + * Project metadata + * @required + */ + project: { + /** + * Project identifier + */ + id?: string + + /** + * Project title + * @required + */ + title: string + + /** + * Project acronym + */ + acronym?: string + + /** + * Project description + */ + description?: string + + /** + * Project URL or path + */ + path?: string + + /** + * Sampling design methodology + * @required + */ + samplingDesign: + | "simpleRandom" + | "systematicRandom" + | "clusteredRandom" + | "experimental" + | "targeted" + | "opportunistic" + + /** + * Capture method used + * @required + */ + captureMethod: Array<"activityDetection" | "timeLapse"> + + /** + * Whether individual animals were identified + * @required + */ + individualAnimals: boolean + + /** + * Level at which observations are recorded + * @required + */ + observationLevel: Array<"media" | "event"> + } + + /** + * Spatial coverage of the data + * @required + */ + spatial: GeoJSON + + /** + * Temporal coverage of the data + * @required + */ + temporal: { + /** + * Start date of temporal coverage + * @required + * @format ISO 8601 + */ + start: string + + /** + * End date of temporal coverage + * @required + * @format ISO 8601 + */ + end: string + } + + /** + * Taxonomic coverage of the data + * @required + */ + taxonomic: TaxonomicCoverage[] + + /** + * Precision of geographic coordinates + */ + coordinatePrecision?: number + + /** + * Bibliographic citation for the dataset + */ + bibliographicCitation?: string + + /** + * Related identifiers for the dataset + */ + relatedIdentifiers?: RelatedIdentifier[] + + /** + * Licenses for the package + * Extended with scope property + */ + licenses?: CamtrapLicense[] +} + +/** + * GeoJSON specification for spatial data + */ +export interface GeoJSON { + type: string + coordinates?: number[] | number[][] | number[][][] | number[][][][] + geometries?: any[] + bbox?: number[] + properties?: Record +} + +/** + * Taxonomic Coverage specification + */ +export interface TaxonomicCoverage { + /** + * Scientific name of the taxon + */ + scientificName: string + + /** + * Taxonomic identifier + */ + taxonID?: string + + /** + * Taxonomic rank + */ + taxonRank?: + | "kingdom" + | "phylum" + | "class" + | "order" + | "family" + | "genus" + | "species" + | "subspecies" + + /** + * Kingdom classification + */ + kingdom?: string + + /** + * Phylum classification + */ + phylum?: string + + /** + * Class classification + */ + class?: string + + /** + * Order classification + */ + order?: string + + /** + * Family classification + */ + family?: string + + /** + * Genus classification + */ + genus?: string + + /** + * Vernacular (common) names by language code + */ + vernacularNames?: Record +} + +/** + * Related identifier specification + */ +export interface RelatedIdentifier { + /** + * Identifier value + */ + identifier: string + + /** + * Relationship type + */ + relation: + | "isPartOf" + | "hasPart" + | "isVersionOf" + | "isSourceOf" + | "isDerivedFrom" + | "isDocumentedBy" + | "documents" + | "isReferencedBy" + | "references" + + /** + * Identifier scheme + */ + scheme?: "DOI" | "ARK" | "URL" | "HANDLE" | "PURL" +} + +/** + * Extended Contributor interface specific to Camera Trap Data Package + */ +export interface CamtrapContributor extends Contributor { + /** + * Role of the contributor in the project + */ + role?: + | "contact" + | "principalInvestigator" + | "rightsHolder" + | "publisher" + | "contributor" +} + +/** + * Extended License interface specific to Camera Trap Data Package + */ +export interface CamtrapLicense extends License { + /** + * Scope of the license + */ + scope: "data" | "media" +} diff --git a/camtrap/package/index.ts b/camtrap/package/index.ts new file mode 100644 index 00000000..221f2d67 --- /dev/null +++ b/camtrap/package/index.ts @@ -0,0 +1 @@ +export type { CamtrapPackage } from "./Package.ts" diff --git a/site/tsconfig.json b/camtrap/tsconfig.json similarity index 100% rename from site/tsconfig.json rename to camtrap/tsconfig.json diff --git a/camtrap/typedoc.json b/camtrap/typedoc.json new file mode 100644 index 00000000..fc33b815 --- /dev/null +++ b/camtrap/typedoc.json @@ -0,0 +1,5 @@ +{ + "entryPoints": ["index.ts"], + "projectDocuments": ["CHANGELOG.md", "OVERVIEW.md"], + "skipErrorChecking": true +} diff --git a/ckan/OVERVIEW.md b/ckan/OVERVIEW.md new file mode 100644 index 00000000..8e30ce82 --- /dev/null +++ b/ckan/OVERVIEW.md @@ -0,0 +1,2 @@ +# @dpkit/ckan + diff --git a/ckan/README.md b/ckan/README.md index 97cc82ab..2ec48912 100644 --- a/ckan/README.md +++ b/ckan/README.md @@ -1,3 +1,3 @@ # @dpkit/ckan -Data Package implementation in TypeScript. Data Package is a standard consisting of a set of simple yet extensible specifications to describe datasets, data files and tabular data. It is a data definition language (DDL) and data API that facilitates findability, accessibility, interoperability, and reusability (FAIR) of data. For more information, please visit the [documentation portal](https://dpkit.datist.io). +dpkit is a fast TypeScript data management framework based on the Data Package standard. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/ckan/general/request.ts b/ckan/general/request.ts index f95f3055..d69ee387 100644 --- a/ckan/general/request.ts +++ b/ckan/general/request.ts @@ -1,6 +1,6 @@ import type { Descriptor } from "@dpkit/core" -export async function makeCkanApiRequest(props: { +export async function makeCkanApiRequest(props: { ckanUrl: string action: string payload: Descriptor @@ -8,7 +8,7 @@ export async function makeCkanApiRequest(props: { apiKey?: string }) { let body: string | FormData - const headers: Descriptor = {} + const headers: Record = {} const url = new URL(props.ckanUrl) url.pathname = `/api/3/action/${props.action}` @@ -47,5 +47,5 @@ export async function makeCkanApiRequest(props: { throw new Error(`CKAN API error: ${data.error}`) } - return data.result as Descriptor + return data.result as T } diff --git a/ckan/package.json b/ckan/package.json index ce0dc095..0d8f21ce 100644 --- a/ckan/package.json +++ b/ckan/package.json @@ -6,7 +6,7 @@ "license": "MIT", "author": "Evgeny Karev", "repository": "https://github.com/datisthq/dpkit", - "description": "Data Package implementation in TypeScript.", + "description": "Fast TypeScript data management framework based on the Data Package standard", "keywords": [ "data", "package", @@ -16,14 +16,14 @@ "fair", "ckan" ], + "scripts": { + "build": "tsc --build" + }, "dependencies": { "@dpkit/core": "workspace:*", "@dpkit/file": "workspace:*" }, "devDependencies": { "@dpkit/test": "workspace:*" - }, - "scripts": { - "build": "tsc --build" } } diff --git a/ckan/package/fixtures/generated/load.spec.ts.snap b/ckan/package/fixtures/generated/load.spec.ts.snap index f03250ad..e9a4ccb4 100644 --- a/ckan/package/fixtures/generated/load.spec.ts.snap +++ b/ckan/package/fixtures/generated/load.spec.ts.snap @@ -49,6 +49,8 @@ The criteria underpinning collection units and the methodology by which they are "resources": [ { "bytes": 896648, + "ckan:key": undefined, + "ckan:url": undefined, "description": "Collection-level records describing the holdings of the Natural History Museum as of April 2025. See the dataset description and the other resources in this repository for more detail on navigating and interpreting this data: - [Data dictionary](https://data.nhm.ac.uk/dataset/join-the-dots-collection-level-descriptions/resource/09b2f07b-9e77-44b7-8645-ddcba1266629) @@ -206,6 +208,8 @@ The criteria underpinning collection units and the methodology by which they are }, { "bytes": 5868, + "ckan:key": undefined, + "ckan:url": undefined, "description": "Description of the fields found in [JtD collection-level records](https://data.nhm.ac.uk/dataset/join-the-dots-collection-level-descriptions/resource/660de433-447b-40ad-9aaf-77c98f8c2ba9)", "format": "csv", "mediatype": "text/csv", @@ -243,6 +247,8 @@ The criteria underpinning collection units and the methodology by which they are }, { "bytes": 8108818, + "ckan:key": undefined, + "ckan:url": undefined, "description": "Documentation produced for NHM staff in 2018: a thorough overview of the Join the Dots data, processes and rationale. Some elements of the JtD framework have been amended slightly since this document was assembled, but it remains a useful reference for navigating the collection-level dataset presented in this package. ", "format": "pdf", "mediatype": "application/pdf", diff --git a/ckan/package/load.ts b/ckan/package/load.ts index 48bc4dd1..a76e3739 100644 --- a/ckan/package/load.ts +++ b/ckan/package/load.ts @@ -1,3 +1,4 @@ +import { mergePackages } from "@dpkit/core" import { makeCkanApiRequest } from "../general/index.js" import type { CkanPackage } from "./Package.js" import { normalizeCkanPackage } from "./process/normalize.js" @@ -10,7 +11,16 @@ import { normalizeCkanPackage } from "./process/normalize.js" export async function loadPackageFromCkan(props: { datasetUrl: string }) { const { datasetUrl } = props - const ckanPackage = await loadCkanPackage({ datasetUrl }) + const packageId = extractPackageId({ datasetUrl }) + if (!packageId) { + throw new Error(`Failed to extract package ID from URL: ${datasetUrl}`) + } + + const ckanPackage = await makeCkanApiRequest({ + ckanUrl: datasetUrl, + action: "package_show", + payload: { id: packageId }, + }) for (const resource of ckanPackage.resources) { const resourceId = resource.id @@ -22,28 +32,18 @@ export async function loadPackageFromCkan(props: { datasetUrl: string }) { } } - const datapackage = normalizeCkanPackage({ ckanPackage }) - return datapackage -} - -/** - * Fetch package data from CKAN API - */ -async function loadCkanPackage(props: { datasetUrl: string }) { - const { datasetUrl } = props - - const packageId = extractPackageId({ datasetUrl }) - if (!packageId) { - throw new Error(`Failed to extract package ID from URL: ${datasetUrl}`) - } + const systemPackage = normalizeCkanPackage({ ckanPackage }) + const userPackagePath = systemPackage.resources + .filter(resource => resource["ckan:key"] === "datapackage.json") + .map(resource => resource["ckan:url"]) + .at(0) - const result = await makeCkanApiRequest({ - ckanUrl: datasetUrl, - action: "package_show", - payload: { id: packageId }, + const datapackage = await mergePackages({ systemPackage, userPackagePath }) + datapackage.resources = datapackage.resources.map(resource => { + return { ...resource, "ckan:key": undefined, "ckan:url": undefined } }) - return result as CkanPackage + return datapackage } /** @@ -79,6 +79,7 @@ async function loadCkanSchema(props: { payload: { resource_id: resourceId, limit: 0 }, }) + // @ts-ignore const fields = result.fields.filter( (field: any) => field.id !== "_id" && field.id !== "_full_text", ) diff --git a/ckan/package/save.ts b/ckan/package/save.ts index 1a01bba6..f9c3c0b4 100644 --- a/ckan/package/save.ts +++ b/ckan/package/save.ts @@ -12,6 +12,7 @@ import { saveResourceFiles, } from "@dpkit/file" import { makeCkanApiRequest } from "../general/index.js" +import type { CkanResource } from "../resource/index.js" import { denormalizeCkanResource } from "../resource/index.js" import { denormalizeCkanPackage } from "./process/denormalize.js" @@ -70,7 +71,7 @@ export async function savePackageToCkan(props: { ), } - const result = await makeCkanApiRequest({ + const result = await makeCkanApiRequest({ action: "resource_create", payload, upload, diff --git a/ckan/resource/process/normalize.ts b/ckan/resource/process/normalize.ts index 73504033..1cbdb9f5 100644 --- a/ckan/resource/process/normalize.ts +++ b/ckan/resource/process/normalize.ts @@ -1,5 +1,5 @@ import type { Resource } from "@dpkit/core" -import { isTableResource } from "@dpkit/core" +import { getFilename } from "@dpkit/core" import { normalizeCkanSchema } from "../../schema/index.js" import type { CkanResource } from "../Resource.js" @@ -16,6 +16,8 @@ export function normalizeCkanResource(props: { const resource: Resource = { name: slugifyName(ckanResource.name), path: ckanResource.url, + "ckan:key": getFilename({ path: ckanResource.url }), + "ckan:url": ckanResource.url, } if (ckanResource.description) { @@ -41,9 +43,7 @@ export function normalizeCkanResource(props: { if (ckanResource.schema) { resource.type = "table" - if (isTableResource(resource)) { - resource.schema = normalizeCkanSchema({ ckanSchema: ckanResource.schema }) - } + resource.schema = normalizeCkanSchema({ ckanSchema: ckanResource.schema }) } return resource diff --git a/ckan/typedoc.json b/ckan/typedoc.json new file mode 100644 index 00000000..fc33b815 --- /dev/null +++ b/ckan/typedoc.json @@ -0,0 +1,5 @@ +{ + "entryPoints": ["index.ts"], + "projectDocuments": ["CHANGELOG.md", "OVERVIEW.md"], + "skipErrorChecking": true +} diff --git a/cli/README.md b/cli/README.md index 9d82da40..7ecde18a 100644 --- a/cli/README.md +++ b/cli/README.md @@ -1,3 +1,3 @@ # @dpkit/cli -Data Package implementation in TypeScript. Data Package is a standard consisting of a set of simple yet extensible specifications to describe datasets, data files and tabular data. It is a data definition language (DDL) and data API that facilitates findability, accessibility, interoperability, and reusability (FAIR) of data. For more information, please visit the [documentation portal](https://dpkit.datist.io). +dpkit is a fast TypeScript data management framework based on the Data Package standard. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/cli/package.json b/cli/package.json index e3fdc346..80b7a1c0 100644 --- a/cli/package.json +++ b/cli/package.json @@ -6,7 +6,7 @@ "license": "MIT", "author": "Evgeny Karev", "repository": "https://github.com/datisthq/dpkit", - "description": "Data Package implementation in TypeScript.", + "description": "Fast TypeScript data management framework based on the Data Package standard", "keywords": [ "data", "package", diff --git a/core/OVERVIEW.md b/core/OVERVIEW.md new file mode 100644 index 00000000..49c0b908 --- /dev/null +++ b/core/OVERVIEW.md @@ -0,0 +1,2 @@ +# @dpkit/core + diff --git a/core/README.md b/core/README.md index 45152184..ed649d51 100644 --- a/core/README.md +++ b/core/README.md @@ -1,3 +1,3 @@ # @dpkit/core -Data Package implementation in TypeScript. Data Package is a standard consisting of a set of simple yet extensible specifications to describe datasets, data files and tabular data. It is a data definition language (DDL) and data API that facilitates findability, accessibility, interoperability, and reusability (FAIR) of data. For more information, please visit the [documentation portal](https://dpkit.datist.io). +dpkit is a fast TypeScript data management framework based on the Data Package standard. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/core/dialect/Dialect.ts b/core/dialect/Dialect.ts index 9d9b5346..960bf4fa 100644 --- a/core/dialect/Dialect.ts +++ b/core/dialect/Dialect.ts @@ -1,9 +1,11 @@ +import type { Metadata } from "../general/index.js" + /** * Descriptor that describes the structure of tabular data, such as delimiters, * headers, and other features. Following the Data Package standard: * https://datapackage.org/standard/table-dialect/ */ -export interface Dialect { +export interface Dialect extends Metadata { /** * The name of this dialect */ diff --git a/core/dialect/assert.spec.ts b/core/dialect/assert.spec.ts index de17fa9f..0df44347 100644 --- a/core/dialect/assert.spec.ts +++ b/core/dialect/assert.spec.ts @@ -5,18 +5,18 @@ import { assertDialect } from "./assert.js" describe("assertDialect", () => { it("returns typed dialect when valid", async () => { - const validDialect: Dialect = { + const descriptor = { delimiter: ";", header: true, skipInitialSpace: true, } - const result = await assertDialect({ - descriptor: validDialect, + const dialect = await assertDialect({ + descriptor, }) - expectTypeOf(result).toEqualTypeOf() - expect(result).toEqual(validDialect) + expectTypeOf(dialect).toEqualTypeOf() + expect(dialect).toEqual(descriptor) }) it("throws ValidationError when dialect is invalid", async () => { diff --git a/core/dialect/assert.ts b/core/dialect/assert.ts index 138b8c4e..e26cbcb5 100644 --- a/core/dialect/assert.ts +++ b/core/dialect/assert.ts @@ -1,11 +1,14 @@ import { AssertionError, type Descriptor } from "../general/index.js" +import type { Dialect } from "./Dialect.js" import { validateDialect } from "./validate.js" /** * Assert a Dialect descriptor (JSON Object) against its profile */ -export async function assertDialect(props: { descriptor: Descriptor }) { - const { dialect, errors } = await validateDialect(props) +export async function assertDialect(props: { + descriptor: Descriptor +}) { + const { dialect, errors } = await validateDialect(props) if (!dialect) throw new AssertionError(errors) return dialect } diff --git a/core/dialect/index.ts b/core/dialect/index.ts index 55994f80..fd850fa3 100644 --- a/core/dialect/index.ts +++ b/core/dialect/index.ts @@ -3,3 +3,5 @@ export { validateDialect } from "./validate.js" export type { Dialect } from "./Dialect.js" export { loadDialect } from "./load.js" export { saveDialect } from "./save.js" +export { normalizeDialect } from "./process/normalize.js" +export { denormalizeDialect } from "./process/denormalize.js" diff --git a/core/dialect/load.ts b/core/dialect/load.ts index f0648630..b1b3ef03 100644 --- a/core/dialect/load.ts +++ b/core/dialect/load.ts @@ -1,12 +1,15 @@ import { loadDescriptor } from "../general/index.js" +import type { Dialect } from "./Dialect.js" import { assertDialect } from "./assert.js" /** * Load a Dialect descriptor (JSON Object) from a file or URL * Ensures the descriptor is valid against its profile */ -export async function loadDialect(props: { path: string }) { +export async function loadDialect(props: { + path: string +}) { const { descriptor } = await loadDescriptor(props) - const dialect = await assertDialect({ descriptor }) + const dialect = await assertDialect({ descriptor }) return dialect } diff --git a/core/dialect/save.spec.ts b/core/dialect/save.spec.ts index f82bce78..ace4c99f 100644 --- a/core/dialect/save.spec.ts +++ b/core/dialect/save.spec.ts @@ -45,10 +45,21 @@ describe("saveDialect", () => { const content = await fs.readFile(testPath, "utf-8") const parsedContent = JSON.parse(content) - expect(parsedContent).toEqual(testDialect) + // Remove $schema property for test comparison + const { $schema, ...dialectWithoutSchema } = parsedContent + expect(dialectWithoutSchema).toEqual(testDialect) + expect($schema).toBe( + "https://datapackage.org/profiles/2.0/tabledialect.json", + ) + + // Create expected format with $schema for comparison + const expectedWithSchema = { + ...testDialect, + $schema: "https://datapackage.org/profiles/2.0/tabledialect.json", + } // Verify the pretty formatting - const expectedFormat = JSON.stringify(testDialect, null, 2) + const expectedFormat = JSON.stringify(expectedWithSchema, null, 2) expect(content).toEqual(expectedFormat) }) }) diff --git a/core/dialect/save.ts b/core/dialect/save.ts index 5f1f1e6d..87d70cf6 100644 --- a/core/dialect/save.ts +++ b/core/dialect/save.ts @@ -2,6 +2,8 @@ import { saveDescriptor } from "../general/index.js" import type { Dialect } from "./Dialect.js" import { denormalizeDialect } from "./process/denormalize.js" +const CURRENT_PROFILE = "https://datapackage.org/profiles/2.0/tabledialect.json" + /** * Save a Dialect to a file path * Works in Node.js environments @@ -13,5 +15,9 @@ export async function saveDialect(props: { const { dialect, path } = props const descriptor = denormalizeDialect({ dialect }) + + descriptor.$schema = + descriptor.$schema ?? descriptor.profile ?? CURRENT_PROFILE + await saveDescriptor({ descriptor, path }) } diff --git a/core/dialect/validate.spec.ts b/core/dialect/validate.spec.ts index 4c0aad1c..1a2e2b41 100644 --- a/core/dialect/validate.spec.ts +++ b/core/dialect/validate.spec.ts @@ -1,15 +1,14 @@ import { describe, expect, it } from "vitest" -import type { Dialect } from "./Dialect.js" import { validateDialect } from "./validate.js" describe("validateDialect", () => { it("returns valid result for valid dialect", async () => { - const validDialect: Dialect = { + const descriptor = { delimiter: ";", } const result = await validateDialect({ - descriptor: validDialect, + descriptor, }) expect(result.valid).toBe(true) diff --git a/core/dialect/validate.ts b/core/dialect/validate.ts index aac5fa93..0fde448b 100644 --- a/core/dialect/validate.ts +++ b/core/dialect/validate.ts @@ -3,26 +3,29 @@ import { loadProfile } from "../general/index.js" import type { Dialect } from "./Dialect.js" import { normalizeDialect } from "./process/normalize.js" +const DEFAULT_PROFILE = "https://datapackage.org/profiles/1.0/tabledialect.json" + /** * Validate a Dialect descriptor (JSON Object) against its profile */ -export async function validateDialect(props: { +export async function validateDialect(props: { descriptor: Descriptor }) { const { descriptor } = props - let dialect: Dialect | undefined = undefined + let dialect: T | undefined = undefined - const $dialect = props.descriptor.$dialect ?? DEFAULT_PROFILE - const profile = await loadProfile({ path: $dialect }) + const $schema = + typeof props.descriptor.$schema === "string" + ? props.descriptor.$schema + : DEFAULT_PROFILE + const profile = await loadProfile({ path: $schema }) const { valid, errors } = await validateDescriptor({ ...props, profile }) if (valid) { // Validation + normalization = we can cast it - dialect = normalizeDialect({ descriptor }) as Dialect + dialect = normalizeDialect({ descriptor }) as T } return { valid, errors, dialect } } - -const DEFAULT_PROFILE = "https://datapackage.org/profiles/1.0/tabledialect.json" diff --git a/core/general/descriptor/Descriptor.ts b/core/general/descriptor/Descriptor.ts index 762e1e4d..89cdd2af 100644 --- a/core/general/descriptor/Descriptor.ts +++ b/core/general/descriptor/Descriptor.ts @@ -1 +1,5 @@ -export type Descriptor = Record +export type Descriptor = Record + +export function isDescriptor(value: unknown): value is Descriptor { + return !!value && typeof value === "object" && !Array.isArray(value) +} diff --git a/core/general/index.ts b/core/general/index.ts index 39e90f34..6500bf29 100644 --- a/core/general/index.ts +++ b/core/general/index.ts @@ -4,6 +4,8 @@ export { loadDescriptor } from "./descriptor/load.js" export { saveDescriptor } from "./descriptor/save.js" export { validateDescriptor } from "./descriptor/validate.js" export type { Descriptor } from "./descriptor/Descriptor.js" +export type { Metadata } from "./metadata/Metadata.js" +export { isDescriptor } from "./descriptor/Descriptor.js" export { loadProfile } from "./profile/load.js" export { AssertionError, type DescriptorError } from "./Error.js" export { diff --git a/core/general/metadata/Metadata.ts b/core/general/metadata/Metadata.ts new file mode 100644 index 00000000..8434eb62 --- /dev/null +++ b/core/general/metadata/Metadata.ts @@ -0,0 +1,6 @@ +export type Metadata = { + /** + * Custom properties + */ + [key in `${string}:${string}`]: any +} diff --git a/core/general/path.spec.ts b/core/general/path.spec.ts index fa4de305..03e0dd12 100644 --- a/core/general/path.spec.ts +++ b/core/general/path.spec.ts @@ -13,56 +13,56 @@ describe("isRemotePath", () => { { description: "http URL", path: "http://example.com/path/to/file.txt", - expected: true, + isRemote: true, }, { description: "https URL", path: "https://example.com/path/to/file.txt", - expected: true, + isRemote: true, }, { description: "ftp URL", path: "ftp://example.com/path/to/file.txt", - expected: true, + isRemote: true, }, { description: "file URL", path: "file:///path/to/file.txt", - expected: true, + isRemote: true, }, { description: "absolute path", path: "/path/to/file.txt", - expected: false, + isRemote: false, }, { description: "relative path", path: "path/to/file.txt", - expected: false, + isRemote: false, }, { description: "current directory path", path: "./file.txt", - expected: false, + isRemote: false, }, { description: "parent directory path", path: "../file.txt", - expected: false, + isRemote: false, }, { description: "empty string", path: "", - expected: false, + isRemote: false, }, { // new URL considers this to be a valid URL description: "protocol without slashes", path: "http:example.com", - expected: true, + isRemote: true, }, - ])("$description", ({ path, expected }) => { - expect(isRemotePath({ path })).toBe(expected) + ])("$description", ({ path, isRemote }) => { + expect(isRemotePath({ path })).toBe(isRemote) }) }) @@ -202,12 +202,6 @@ describe("normalizePath", () => { basepath: "http://example.com", normalizedPath: "http://example.com/path/to/file.txt", }, - { - description: "absolute path", - path: "/absolute/path/to/file.txt", - basepath: undefined, - normalizedPath: relative(process.cwd(), "/absolute/path/to/file.txt"), - }, { description: "local path with absolute basepath", path: "file.txt", @@ -223,6 +217,26 @@ describe("normalizePath", () => { ])("$description", ({ path, basepath, normalizedPath }) => { expect(normalizePath({ path, basepath })).toEqual(normalizedPath) }) + + it.each([ + { + description: "absolute path", + path: "/absolute/path/to/file.txt", + basepath: undefined, + }, + { + description: "local traversed path", + path: "../file.txt", + basepath: "/folder", + }, + { + description: "remote traversed path", + path: "../file.txt", + basepath: "http://example.com/data", + }, + ])("$description -- throw", ({ path, basepath }) => { + expect(() => normalizePath({ path, basepath })).toThrow() + }) }) describe("denormalizePath", () => { diff --git a/core/general/path.ts b/core/general/path.ts index fd20b578..72f907fb 100644 --- a/core/general/path.ts +++ b/core/general/path.ts @@ -71,12 +71,17 @@ export function normalizePath(props: { const isRemote = isBasepathRemote || isPathRemote if (isRemote) { - const path = !isPathRemote - ? [props.basepath, props.path].join("/") - : props.path + const path = new URL( + !isPathRemote ? [props.basepath, props.path].join("/") : props.path, + ).toString() - // Returns normalized remote path - return new URL(path).toString() + if (!path.startsWith(props.basepath ?? "")) { + throw new Error( + `Path ${props.path} is not a subpath of ${props.basepath}`, + ) + } + + return path } if (!node) { @@ -87,7 +92,10 @@ export function normalizePath(props: { ? node.path.join(props.basepath, props.path) : props.path - // Returns normalized local path + if (node.path.relative(props.basepath ?? "", path).startsWith("..")) { + throw new Error(`Path ${props.path} is not a subpath of ${props.basepath}`) + } + return node.path.relative(process.cwd(), node.path.resolve(path)) } diff --git a/core/package.json b/core/package.json index ed605ece..c3d8440f 100644 --- a/core/package.json +++ b/core/package.json @@ -6,7 +6,7 @@ "license": "MIT", "author": "Evgeny Karev", "repository": "https://github.com/datisthq/dpkit", - "description": "Data Package implementation in TypeScript.", + "description": "Fast TypeScript data management framework based on the Data Package standard", "keywords": [ "data", "package", @@ -22,6 +22,7 @@ "dependencies": { "ajv": "^8.17.1", "github-slugger": "^2.0.0", - "quick-lru": "^7.0.1" + "quick-lru": "^7.0.1", + "tiny-invariant": "^1.3.3" } } diff --git a/core/package/Package.ts b/core/package/Package.ts index 895af0b8..3fa5dcd4 100644 --- a/core/package/Package.ts +++ b/core/package/Package.ts @@ -1,3 +1,4 @@ +import type { Metadata } from "../general/index.js" import type { License, Resource, Source } from "../resource/index.js" import type { Contributor } from "./Contributor.js" @@ -5,7 +6,7 @@ import type { Contributor } from "./Contributor.js" * Data Package interface based on the Frictionless Data specification * @see https://datapackage.org/standard/data-package/ */ -export interface Package { +export interface Package extends Metadata { /** * Unique package identifier * Should use lowercase alphanumeric characters, periods, hyphens, and underscores diff --git a/core/package/assert.spec.ts b/core/package/assert.spec.ts index 4f7cc852..cfdcce8f 100644 --- a/core/package/assert.spec.ts +++ b/core/package/assert.spec.ts @@ -5,7 +5,7 @@ import { assertPackage } from "./assert.js" describe("assertPackage", () => { it("returns typed package when valid", async () => { - const validPackage: Package = { + const descriptor = { name: "example-package", resources: [ { @@ -15,12 +15,12 @@ describe("assertPackage", () => { ], } - const result = await assertPackage({ - descriptor: validPackage, + const datapackage = await assertPackage({ + descriptor, }) - expectTypeOf(result).toEqualTypeOf() - expect(result).toEqual(validPackage) + expectTypeOf(datapackage).toEqualTypeOf() + expect(datapackage).toEqual(descriptor) }) it("throws AssertionError when package is invalid", async () => { diff --git a/core/package/assert.ts b/core/package/assert.ts index ac269c93..2b708577 100644 --- a/core/package/assert.ts +++ b/core/package/assert.ts @@ -1,14 +1,15 @@ import { AssertionError, type Descriptor } from "../general/index.js" +import type { Package } from "./Package.js" import { validatePackageDescriptor } from "./validate.js" /** * Assert a Package descriptor (JSON Object) against its profile */ -export async function assertPackage(props: { +export async function assertPackage(props: { descriptor: Descriptor basepath?: string }) { - const { errors, datapackage } = await validatePackageDescriptor(props) + const { errors, datapackage } = await validatePackageDescriptor(props) if (!datapackage) throw new AssertionError(errors) return datapackage } diff --git a/core/package/index.ts b/core/package/index.ts index 1c0d55ae..17958a22 100644 --- a/core/package/index.ts +++ b/core/package/index.ts @@ -6,3 +6,4 @@ export { validatePackageDescriptor } from "./validate.js" export { normalizePackage } from "./process/normalize.js" export { denormalizePackage } from "./process/denormalize.js" export type { Contributor } from "./Contributor.js" +export { mergePackages } from "./merge.js" diff --git a/core/package/load.ts b/core/package/load.ts index f97713d4..635f806b 100644 --- a/core/package/load.ts +++ b/core/package/load.ts @@ -1,12 +1,15 @@ import { loadDescriptor } from "../general/index.js" +import type { Package } from "./Package.js" import { assertPackage } from "./assert.js" /** * Load a Package descriptor (JSON Object) from a file or URL * Ensures the descriptor is valid against its profile */ -export async function loadPackageDescriptor(props: { path: string }) { +export async function loadPackageDescriptor< + T extends Package = Package, +>(props: { path: string }) { const { basepath, descriptor } = await loadDescriptor(props) - const datapackage = await assertPackage({ descriptor, basepath }) + const datapackage = await assertPackage({ descriptor, basepath }) return datapackage } diff --git a/core/package/merge.ts b/core/package/merge.ts new file mode 100644 index 00000000..5a253358 --- /dev/null +++ b/core/package/merge.ts @@ -0,0 +1,25 @@ +import type { Package } from "./Package.js" +import { loadPackageDescriptor } from "./load.js" + +/** + * Merges a system data package into a user data package if provided + */ +export async function mergePackages(props: { + systemPackage: Package + userPackagePath?: string +}) { + const { systemPackage, userPackagePath } = props + + const userPackage = userPackagePath + ? await loadPackageDescriptor({ path: userPackagePath }) + : undefined + + if (!userPackage) { + return props.systemPackage + } + + // NOTE: + // Currently, it uses oversimplified logic till + // we get use feedback on the desired merging strategies + return { ...systemPackage, resources: userPackage.resources } +} diff --git a/core/package/save.ts b/core/package/save.ts index 3fa8c388..9857f065 100644 --- a/core/package/save.ts +++ b/core/package/save.ts @@ -2,6 +2,8 @@ import { getBasepath, saveDescriptor } from "../general/index.js" import type { Package } from "./Package.js" import { denormalizePackage } from "./process/denormalize.js" +const CURRENT_PROFILE = "https://datapackage.org/profiles/2.0/datapackage.json" + /** * Save a Package to a file path * Works in Node.js environments @@ -11,8 +13,12 @@ export async function savePackageDescriptor(props: { path: string }) { const { datapackage, path } = props - const basepath = getBasepath({ path }) + const basepath = getBasepath({ path }) const descriptor = denormalizePackage({ datapackage, basepath }) + + descriptor.$schema = + descriptor.$schema ?? descriptor.profile ?? CURRENT_PROFILE + await saveDescriptor({ descriptor, path: props.path }) } diff --git a/core/package/validate.spec.ts b/core/package/validate.spec.ts index 4dcae852..569302fb 100644 --- a/core/package/validate.spec.ts +++ b/core/package/validate.spec.ts @@ -1,10 +1,9 @@ import { describe, expect, it } from "vitest" -import type { Package } from "./Package.js" import { validatePackageDescriptor } from "./validate.js" describe("validatePackageDescriptor", () => { it("returns valid result for valid package", async () => { - const validPackage: Package = { + const descriptor = { name: "example-package", resources: [ { @@ -15,7 +14,7 @@ describe("validatePackageDescriptor", () => { } const result = await validatePackageDescriptor({ - descriptor: validPackage, + descriptor, }) expect(result.valid).toBe(true) diff --git a/core/package/validate.ts b/core/package/validate.ts index 9096799a..be9d701f 100644 --- a/core/package/validate.ts +++ b/core/package/validate.ts @@ -3,27 +3,32 @@ import { loadProfile } from "../general/index.js" import type { Package } from "./Package.js" import { normalizePackage } from "./process/normalize.js" +const DEFAULT_PROFILE = "https://datapackage.org/profiles/1.0/datapackage.json" + /** * Validate a Package descriptor (JSON Object) against its profile */ -export async function validatePackageDescriptor(props: { +export async function validatePackageDescriptor< + T extends Package = Package, +>(props: { descriptor: Descriptor basepath?: string }) { const { descriptor, basepath } = props - let datapackage: Package | undefined = undefined + let datapackage: T | undefined = undefined - const $schema = descriptor.$schema ?? DEFAULT_PROFILE - const profile = await loadProfile({ path: $schema }) + const $schema = + typeof props.descriptor.$schema === "string" + ? props.descriptor.$schema + : DEFAULT_PROFILE + const profile = await loadProfile({ path: $schema }) const { valid, errors } = await validateDescriptor({ ...props, profile }) if (valid) { // Validation + normalization = we can cast it - datapackage = normalizePackage({ descriptor, basepath }) as Package + datapackage = normalizePackage({ descriptor, basepath }) as T } return { valid, errors, datapackage } } - -const DEFAULT_PROFILE = "https://datapackage.org/profiles/1.0/datapackage.json" diff --git a/core/resource/Resource.ts b/core/resource/Resource.ts index 488b4d3f..93f05809 100644 --- a/core/resource/Resource.ts +++ b/core/resource/Resource.ts @@ -1,11 +1,14 @@ +import type { Dialect } from "../dialect/Dialect.js" +import type { Metadata } from "../general/index.js" +import type { Schema } from "../schema/Schema.js" import type { License } from "./License.js" import type { Source } from "./Source.js" /** - * Data Resource interface based on the Frictionless Data specification + * Data Resource interface based on the Data Package standard * @see https://datapackage.org/standard/data-resource/ */ -export interface Resource { +export interface Resource extends Metadata { /** * Unique resource identifier * Should use lowercase alphanumeric characters, periods, hyphens, and underscores @@ -22,7 +25,7 @@ export interface Resource { * Inline data content instead of referencing an external file * Either path or data must be provided */ - data?: unknown + data?: unknown | unknown[][] | Record[] /** * The resource type @@ -77,4 +80,18 @@ export interface Resource { * License information */ licenses?: License[] + + /** + * Table dialect specification + * Describes delimiters, quote characters, etc. + * @see https://datapackage.org/standard/table-dialect/ + */ + dialect?: string | Dialect + + /** + * Schema for the tabular data + * Describes fields in the table, constraints, etc. + * @see https://datapackage.org/standard/table-schema/ + */ + schema?: string | Schema } diff --git a/core/resource/assert.spec.ts b/core/resource/assert.spec.ts index ad190791..93cef425 100644 --- a/core/resource/assert.spec.ts +++ b/core/resource/assert.spec.ts @@ -5,19 +5,19 @@ import { assertResource } from "./assert.js" describe("assertResource", () => { it("returns typed resource when valid", async () => { - const validResource: Resource = { + const descriptor = { name: "example-resource", path: "data.csv", format: "csv", encoding: "utf-8", } - const result = await assertResource({ - descriptor: validResource, + const resource = await assertResource({ + descriptor, }) - expectTypeOf(result).toEqualTypeOf() - expect(result).toEqual(validResource) + expectTypeOf(resource).toEqualTypeOf() + expect(resource).toEqual(descriptor) }) it("throws AssertionError when resource is invalid", async () => { diff --git a/core/resource/assert.ts b/core/resource/assert.ts index c28ae0cb..eb7ba2e0 100644 --- a/core/resource/assert.ts +++ b/core/resource/assert.ts @@ -1,14 +1,15 @@ import { AssertionError, type Descriptor } from "../general/index.js" +import type { Resource } from "./Resource.js" import { validateResourceDescriptor } from "./validate.js" /** * Assert a Resource descriptor (JSON Object) against its profile */ -export async function assertResource(props: { +export async function assertResource(props: { descriptor: Descriptor basepath?: string }) { - const { errors, resource } = await validateResourceDescriptor(props) + const { errors, resource } = await validateResourceDescriptor(props) if (!resource) throw new AssertionError(errors) return resource } diff --git a/core/resource/index.ts b/core/resource/index.ts index 187c0126..0a2b13e3 100644 --- a/core/resource/index.ts +++ b/core/resource/index.ts @@ -1,5 +1,4 @@ export type { Resource } from "./Resource.js" -export type * from "./types/index.js" export { assertResource } from "./assert.js" export { loadResourceDescriptor } from "./load.js" export { saveResourceDescriptor } from "./save.js" @@ -8,4 +7,3 @@ export { normalizeResource } from "./process/normalize.js" export { denormalizeResource } from "./process/denormalize.js" export type { Source } from "./Source.js" export type { License } from "./License.js" -export { isTableResource } from "./types/table.js" diff --git a/core/resource/load.ts b/core/resource/load.ts index d62783cf..7528a92f 100644 --- a/core/resource/load.ts +++ b/core/resource/load.ts @@ -1,12 +1,15 @@ import { loadDescriptor } from "../general/index.js" +import type { Resource } from "./Resource.js" import { assertResource } from "./assert.js" /** * Load a Resource descriptor (JSON Object) from a file or URL * Ensures the descriptor is valid against its profile */ -export async function loadResourceDescriptor(props: { path: string }) { +export async function loadResourceDescriptor< + T extends Resource = Resource, +>(props: { path: string }) { const { descriptor, basepath } = await loadDescriptor(props) - const resource = await assertResource({ descriptor, basepath }) + const resource = await assertResource({ descriptor, basepath }) return resource } diff --git a/core/resource/process/denormalize.ts b/core/resource/process/denormalize.ts index 20f1e1a5..c05fb77b 100644 --- a/core/resource/process/denormalize.ts +++ b/core/resource/process/denormalize.ts @@ -1,7 +1,8 @@ +import { denormalizeDialect } from "../../dialect/index.js" import { denormalizePath } from "../../general/index.js" import type { Descriptor } from "../../general/index.js" +import { denormalizeSchema } from "../../schema/index.js" import type { Resource } from "../Resource.js" -import { isTableResource } from "../types/table.js" export function denormalizeResource(props: { resource: Resource @@ -11,10 +12,11 @@ export function denormalizeResource(props: { const resource = globalThis.structuredClone(props.resource) denormalizePaths({ resource, basepath }) - // TODO: denormalizeDialect - // TODO: denormalizeSchema - return resource as Descriptor + const dialect = denormalizeResourceDialect({ resource }) + const schema = denormalizeResourceSchema({ resource }) + + return { ...resource, dialect, schema } as Descriptor } function denormalizePaths(props: { resource: Resource; basepath?: string }) { @@ -26,14 +28,32 @@ function denormalizePaths(props: { resource: Resource; basepath?: string }) { : denormalizePath({ path: resource.path, basepath }) } - if (isTableResource(resource)) { - for (const name of ["dialect", "schema"] as const) { - if (typeof resource[name] === "string") { - resource[name] = denormalizePath({ - path: resource[name], - basepath, - }) - } + for (const name of ["dialect", "schema"] as const) { + if (typeof resource[name] === "string") { + resource[name] = denormalizePath({ + path: resource[name], + basepath, + }) } } } + +function denormalizeResourceDialect(props: { resource: Resource }) { + const { resource } = props + + if (!resource.dialect || typeof resource.dialect === "string") { + return resource.dialect + } + + return denormalizeDialect({ dialect: resource.dialect }) +} + +function denormalizeResourceSchema(props: { resource: Resource }) { + const { resource } = props + + if (!resource.schema || typeof resource.schema === "string") { + return resource.schema + } + + return denormalizeSchema({ schema: resource.schema }) +} diff --git a/core/resource/process/normalize.ts b/core/resource/process/normalize.ts index 21d0b5e5..14892b55 100644 --- a/core/resource/process/normalize.ts +++ b/core/resource/process/normalize.ts @@ -1,5 +1,7 @@ -import { normalizePath } from "../../general/index.js" +import { normalizeDialect } from "../../dialect/index.js" +import { isDescriptor, normalizePath } from "../../general/index.js" import type { Descriptor } from "../../general/index.js" +import { normalizeSchema } from "../../schema/index.js" export function normalizeResource(props: { descriptor: Descriptor @@ -11,8 +13,9 @@ export function normalizeResource(props: { normalizeUrl({ descriptor }) normalizeType({ descriptor }) normalizePaths({ descriptor, basepath }) - // TODO: normalizeDialect - // TODO: normalizeSchema + + normalizeResourceDialect({ descriptor }) + normalizeResourceSchema({ descriptor }) return descriptor } @@ -67,3 +70,19 @@ function normalizePaths(props: { descriptor: Descriptor; basepath?: string }) { } } } + +function normalizeResourceDialect(props: { descriptor: Descriptor }) { + const { descriptor } = props + + if (isDescriptor(descriptor.dialect)) { + descriptor.dialect = normalizeDialect({ descriptor: descriptor.dialect }) + } +} + +function normalizeResourceSchema(props: { descriptor: Descriptor }) { + const { descriptor } = props + + if (isDescriptor(descriptor.schema)) { + descriptor.schema = normalizeSchema({ descriptor: descriptor.schema }) + } +} diff --git a/core/resource/save.ts b/core/resource/save.ts index bcf089a3..7b1858a6 100644 --- a/core/resource/save.ts +++ b/core/resource/save.ts @@ -2,6 +2,8 @@ import { getBasepath, saveDescriptor } from "../general/index.js" import type { Resource } from "./Resource.js" import { denormalizeResource } from "./process/denormalize.js" +const CURRENT_PROFILE = "https://datapackage.org/profiles/2.0/dataresource.json" + /** * Save a Resource to a file path * Works in Node.js environments @@ -11,8 +13,12 @@ export async function saveResourceDescriptor(props: { path: string }) { const { resource, path } = props - const basepath = getBasepath({ path }) + const basepath = getBasepath({ path }) const descriptor = denormalizeResource({ resource, basepath }) + + descriptor.$schema = + descriptor.$schema ?? descriptor.profile ?? CURRENT_PROFILE + await saveDescriptor({ descriptor, path }) } diff --git a/core/resource/types/index.ts b/core/resource/types/index.ts deleted file mode 100644 index 9cc30e3c..00000000 --- a/core/resource/types/index.ts +++ /dev/null @@ -1 +0,0 @@ -export type { TableResource } from "./table.js" diff --git a/core/resource/types/table.ts b/core/resource/types/table.ts deleted file mode 100644 index 6a68225a..00000000 --- a/core/resource/types/table.ts +++ /dev/null @@ -1,40 +0,0 @@ -import type { Dialect } from "../../dialect/Dialect.js" -import type { Resource } from "../../resource/Resource.js" -import type { Schema } from "../../schema/Schema.js" - -/** - * Check if a resource is a Table Resource and narrow the type - */ -export function isTableResource(resource: Resource): resource is TableResource { - return resource.type === "table" -} - -/** - * Table Resource interface extending the base Resource interface - * @see https://datapackage.org/standard/table-resource/ - */ -export interface TableResource extends Resource { - /** - * Resource type must be "table" - */ - type: "table" - - /** - * Inline tabular data - */ - data: any[][] | Record[] - - /** - * CSV dialect specification - * Describes delimiters, quote characters, etc. - * @see https://datapackage.org/standard/table-dialect/ - */ - dialect?: string | Dialect - - /** - * Schema for the tabular data - * Describes fields in the table, constraints, etc. - * @see https://datapackage.org/standard/table-schema/ - */ - schema: string | Schema -} diff --git a/core/resource/validate.spec.ts b/core/resource/validate.spec.ts index daeaf359..8cc7df3c 100644 --- a/core/resource/validate.spec.ts +++ b/core/resource/validate.spec.ts @@ -1,10 +1,9 @@ import { describe, expect, it } from "vitest" -import type { Resource } from "./Resource.js" import { validateResourceDescriptor } from "./validate.js" describe("validateResourceDescriptor", () => { it("returns valid result for valid resource", async () => { - const validResource: Resource = { + const descriptor = { name: "example-resource", path: "data.csv", format: "csv", @@ -12,7 +11,7 @@ describe("validateResourceDescriptor", () => { } const result = await validateResourceDescriptor({ - descriptor: validResource, + descriptor, }) expect(result.valid).toBe(true) diff --git a/core/resource/validate.ts b/core/resource/validate.ts index 6701c8c3..869665a3 100644 --- a/core/resource/validate.ts +++ b/core/resource/validate.ts @@ -1,29 +1,82 @@ +import { loadDialect } from "../dialect/index.js" import { type Descriptor, validateDescriptor } from "../general/index.js" +import { AssertionError } from "../general/index.js" import { loadProfile } from "../general/index.js" +import { loadSchema } from "../schema/index.js" import type { Resource } from "./Resource.js" import { normalizeResource } from "./process/normalize.js" +const DEFAULT_PROFILE = "https://datapackage.org/profiles/1.0/dataresource.json" + /** * Validate a Resource descriptor (JSON Object) against its profile */ -export async function validateResourceDescriptor(props: { +export async function validateResourceDescriptor< + T extends Resource = Resource, +>(props: { descriptor: Descriptor basepath?: string }) { const { descriptor, basepath } = props - let resource: Resource | undefined = undefined + let resource: T | undefined = undefined - const $schema = props.descriptor.$schema ?? DEFAULT_PROFILE - const profile = await loadProfile({ path: $schema }) + const $schema = + typeof props.descriptor.$schema === "string" + ? props.descriptor.$schema + : DEFAULT_PROFILE - const { valid, errors } = await validateDescriptor({ descriptor, profile }) + const profile = await loadProfile({ path: $schema }) + let { valid, errors } = await validateDescriptor({ descriptor, profile }) if (valid) { // Validation + normalization = we can cast it - resource = normalizeResource({ descriptor, basepath }) as Resource + resource = normalizeResource({ descriptor, basepath }) as T + } + + if (resource) { + const dialectErorrs = await validateDialectIfExternal({ resource }) + if (dialectErorrs) errors.push(...dialectErorrs) + + const schemaErorrs = await validateSchemaIfExternal({ resource }) + if (schemaErorrs) errors.push(...schemaErorrs) + + if (errors.length) { + resource = undefined + valid = false + } } return { valid, errors, resource } } -const DEFAULT_PROFILE = "https://datapackage.org/profiles/1.0/dataresource.json" +async function validateDialectIfExternal(props: { resource: Resource }) { + const { resource } = props + + if (typeof resource.dialect === "string") { + try { + await loadDialect({ path: resource.dialect }) + } catch (error) { + if (error instanceof AssertionError) { + return error.errors + } + } + } + + return undefined +} + +async function validateSchemaIfExternal(props: { resource: Resource }) { + const { resource } = props + + if (typeof resource.schema === "string") { + try { + await loadSchema({ path: resource.schema }) + } catch (error) { + if (error instanceof AssertionError) { + return error.errors + } + } + } + + return undefined +} diff --git a/core/schema/Field/types/base.ts b/core/schema/Field/types/base.ts index a50ed14a..236692b9 100644 --- a/core/schema/Field/types/base.ts +++ b/core/schema/Field/types/base.ts @@ -1,7 +1,9 @@ +import type { Metadata } from "../../../general/index.js" + /** * Base field properties common to all field types */ -export interface BaseField { +export interface BaseField extends Metadata { /** * Name of the field matching the column name */ diff --git a/core/schema/Schema.ts b/core/schema/Schema.ts index e63924d8..eada44a8 100644 --- a/core/schema/Schema.ts +++ b/core/schema/Schema.ts @@ -1,3 +1,4 @@ +import type { Metadata } from "../general/index.js" import type { Field } from "./Field/index.js" import type { ForeignKey } from "./ForeignKey.js" @@ -5,7 +6,7 @@ import type { ForeignKey } from "./ForeignKey.js" * Table Schema definition * Based on the specification at https://datapackage.org/standard/table-schema/ */ -export interface Schema { +export interface Schema extends Metadata { /** * Fields in this schema (required) */ diff --git a/core/schema/assert.spec.ts b/core/schema/assert.spec.ts index c91c813b..c7056f2e 100644 --- a/core/schema/assert.spec.ts +++ b/core/schema/assert.spec.ts @@ -5,7 +5,7 @@ import { assertSchema } from "./assert.js" describe("assertSchema", () => { it("returns typed schema when valid", async () => { - const validSchema: Schema = { + const descriptor = { fields: [ { name: "id", @@ -19,12 +19,12 @@ describe("assertSchema", () => { primaryKey: ["id"], } - const result = await assertSchema({ - descriptor: validSchema, + const schema = await assertSchema({ + descriptor, }) - expectTypeOf(result).toEqualTypeOf() - expect(result).toEqual(validSchema) + expectTypeOf(schema).toEqualTypeOf() + expect(schema).toEqual(descriptor) }) it("throws ValidationError when schema is invalid", async () => { diff --git a/core/schema/assert.ts b/core/schema/assert.ts index eb865a15..c9ab39de 100644 --- a/core/schema/assert.ts +++ b/core/schema/assert.ts @@ -1,11 +1,14 @@ import { AssertionError, type Descriptor } from "../general/index.js" +import type { Schema } from "./Schema.js" import { validateSchema } from "./validate.js" /** * Assert a Schema descriptor (JSON Object) against its profile */ -export async function assertSchema(props: { descriptor: Descriptor }) { - const { schema, errors } = await validateSchema(props) +export async function assertSchema(props: { + descriptor: Descriptor +}) { + const { schema, errors } = await validateSchema(props) if (!schema) throw new AssertionError(errors) return schema } diff --git a/core/schema/index.ts b/core/schema/index.ts index 688ee5aa..d863339b 100644 --- a/core/schema/index.ts +++ b/core/schema/index.ts @@ -1,8 +1,9 @@ export type { Schema } from "./Schema.js" export type { ForeignKey } from "./ForeignKey.js" - export { assertSchema } from "./assert.js" export { loadSchema } from "./load.js" export { saveSchema } from "./save.js" export { validateSchema } from "./validate.js" +export { normalizeSchema } from "./process/normalize.js" +export { denormalizeSchema } from "./process/denormalize.js" export * from "./Field/index.js" diff --git a/core/schema/load.ts b/core/schema/load.ts index 795651f9..7956f9bc 100644 --- a/core/schema/load.ts +++ b/core/schema/load.ts @@ -1,12 +1,15 @@ import { loadDescriptor } from "../general/index.js" +import type { Schema } from "./Schema.js" import { assertSchema } from "./assert.js" /** * Load a Schema descriptor (JSON Object) from a file or URL * Ensures the descriptor is valid against its profile */ -export async function loadSchema(props: { path: string }) { +export async function loadSchema(props: { + path: string +}) { const { descriptor } = await loadDescriptor(props) - const schema = await assertSchema({ descriptor }) + const schema = await assertSchema({ descriptor }) return schema } diff --git a/core/schema/process/denormalize.ts b/core/schema/process/denormalize.ts index c24b3f77..93b89c7f 100644 --- a/core/schema/process/denormalize.ts +++ b/core/schema/process/denormalize.ts @@ -3,5 +3,5 @@ import type { Schema } from "../Schema.js" export function denormalizeSchema(props: { schema: Schema }) { const schema = globalThis.structuredClone(props.schema) - return schema as Descriptor + return schema as unknown as Descriptor } diff --git a/core/schema/process/normalize.ts b/core/schema/process/normalize.ts index 66273b0c..85020f6e 100644 --- a/core/schema/process/normalize.ts +++ b/core/schema/process/normalize.ts @@ -1,3 +1,4 @@ +import invariant from "tiny-invariant" import type { Descriptor } from "../../general/index.js" export function normalizeSchema(props: { descriptor: Descriptor }) { @@ -19,6 +20,11 @@ function normalizeFields(props: { descriptor: Descriptor }) { return } + invariant( + Array.isArray(fields), + "Fields being an array is guaranteed by the validation", + ) + for (const field of fields) { normalizeFieldFormat({ field }) normalizeFieldMissingValues({ field }) @@ -28,7 +34,7 @@ function normalizeFields(props: { descriptor: Descriptor }) { } } -function normalizeFieldFormat(props: { field: any }) { +function normalizeFieldFormat(props: { field: Descriptor }) { const { field } = props const format = field.format @@ -43,7 +49,7 @@ function normalizeFieldFormat(props: { field: any }) { } } -function normalizeFieldMissingValues(props: { field: any }) { +function normalizeFieldMissingValues(props: { field: Descriptor }) { const { field } = props const missingValues = field.missingValues @@ -57,7 +63,7 @@ function normalizeFieldMissingValues(props: { field: any }) { } } -function normalizeFieldCategories(props: { field: any }) { +function normalizeFieldCategories(props: { field: Descriptor }) { const { field } = props const categories = field.categories @@ -71,7 +77,7 @@ function normalizeFieldCategories(props: { field: any }) { } } -function normalizeFieldCategoriesOrdered(props: { field: any }) { +function normalizeFieldCategoriesOrdered(props: { field: Descriptor }) { const { field } = props const categoriesOrdered = field.categoriesOrdered @@ -87,7 +93,7 @@ function normalizeFieldCategoriesOrdered(props: { field: any }) { } } -function normalizeFieldJsonschema(props: { field: any }) { +function normalizeFieldJsonschema(props: { field: Descriptor }) { const { field } = props const jsonschema = field.jsonschema @@ -148,6 +154,7 @@ function normalizeUniqueKeys(props: { descriptor: Descriptor }) { if (!Array.isArray(uniqueKeys)) { descriptor.uniqueKeys = undefined console.warn(`Ignoring v2.0 incompatible uniqueKeys: ${uniqueKeys}`) + return } for (const uniqueKey of uniqueKeys) { diff --git a/core/schema/save.spec.ts b/core/schema/save.spec.ts index d0839e70..bf54b9dc 100644 --- a/core/schema/save.spec.ts +++ b/core/schema/save.spec.ts @@ -52,9 +52,20 @@ describe("saveSchema", () => { const content = await fs.readFile(testPath, "utf-8") const parsedContent = JSON.parse(content) - expect(parsedContent).toEqual(testSchema) - const expectedFormat = JSON.stringify(testSchema, null, 2) + // Remove $schema property for test comparison + const { $schema, ...schemaWithoutSchema } = parsedContent + expect(schemaWithoutSchema).toEqual(testSchema) + expect($schema).toBe( + "https://datapackage.org/profiles/2.0/tableschema.json", + ) + + // Create expected format with $schema for comparison + const expectedWithSchema = { + ...testSchema, + $schema: "https://datapackage.org/profiles/2.0/tableschema.json", + } + const expectedFormat = JSON.stringify(expectedWithSchema, null, 2) expect(content).toEqual(expectedFormat) }) }) diff --git a/core/schema/save.ts b/core/schema/save.ts index 9ab7a90a..3a8dc706 100644 --- a/core/schema/save.ts +++ b/core/schema/save.ts @@ -2,6 +2,8 @@ import { saveDescriptor } from "../general/index.js" import type { Schema } from "./Schema.js" import { denormalizeSchema } from "./process/denormalize.js" +const CURRENT_PROFILE = "https://datapackage.org/profiles/2.0/tableschema.json" + /** * Save a Schema to a file path * Works in Node.js environments @@ -13,5 +15,9 @@ export async function saveSchema(props: { const { schema, path } = props const descriptor = denormalizeSchema({ schema }) + + descriptor.$schema = + descriptor.$schema ?? descriptor.profile ?? CURRENT_PROFILE + await saveDescriptor({ descriptor, path }) } diff --git a/core/schema/validate.spec.ts b/core/schema/validate.spec.ts index def49d74..2b1fa471 100644 --- a/core/schema/validate.spec.ts +++ b/core/schema/validate.spec.ts @@ -1,10 +1,9 @@ import { describe, expect, it } from "vitest" -import type { Schema } from "./Schema.js" import { validateSchema } from "./validate.js" describe("validateSchema", () => { it("returns empty array for valid schema", async () => { - const validSchema: Schema = { + const descriptor = { fields: [ { name: "id", @@ -18,7 +17,7 @@ describe("validateSchema", () => { } const result = await validateSchema({ - descriptor: validSchema, + descriptor, }) expect(result.valid).toBe(true) diff --git a/core/schema/validate.ts b/core/schema/validate.ts index 12210f88..d42ed77c 100644 --- a/core/schema/validate.ts +++ b/core/schema/validate.ts @@ -3,26 +3,29 @@ import { loadProfile } from "../general/index.js" import type { Schema } from "./Schema.js" import { normalizeSchema } from "./process/normalize.js" +const DEFAULT_PROFILE = "https://datapackage.org/profiles/1.0/tableschema.json" + /** * Validate a Schema descriptor (JSON Object) against its profile */ -export async function validateSchema(props: { +export async function validateSchema(props: { descriptor: Descriptor }) { const { descriptor } = props - let schema: Schema | undefined = undefined + let schema: T | undefined = undefined - const $schema = props.descriptor.$schema ?? DEFAULT_PROFILE - const profile = await loadProfile({ path: $schema }) + const $schema = + typeof props.descriptor.$schema === "string" + ? props.descriptor.$schema + : DEFAULT_PROFILE + const profile = await loadProfile({ path: $schema }) const { valid, errors } = await validateDescriptor({ ...props, profile }) if (valid) { // Validation + normalization = we can cast it - schema = normalizeSchema({ descriptor }) as Schema + schema = normalizeSchema({ descriptor }) as T } return { valid, errors, schema } } - -const DEFAULT_PROFILE = "https://datapackage.org/profiles/1.0/tableschema.json" diff --git a/core/typedoc.json b/core/typedoc.json index f8e49f3a..fc33b815 100644 --- a/core/typedoc.json +++ b/core/typedoc.json @@ -1,4 +1,5 @@ { "entryPoints": ["index.ts"], + "projectDocuments": ["CHANGELOG.md", "OVERVIEW.md"], "skipErrorChecking": true } diff --git a/csv/README.md b/csv/README.md index 6d26c028..001a7940 100644 --- a/csv/README.md +++ b/csv/README.md @@ -1,3 +1,3 @@ # @dpkit/csv -Data Package implementation in TypeScript. Data Package is a standard consisting of a set of simple yet extensible specifications to describe datasets, data files and tabular data. It is a data definition language (DDL) and data API that facilitates findability, accessibility, interoperability, and reusability (FAIR) of data. For more information, please visit the [documentation portal](https://dpkit.datist.io). +dpkit is a fast TypeScript data management framework based on the Data Package standard. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/csv/package.json b/csv/package.json index 9fbca842..023c358e 100644 --- a/csv/package.json +++ b/csv/package.json @@ -6,7 +6,7 @@ "license": "MIT", "author": "Evgeny Karev", "repository": "https://github.com/datisthq/dpkit", - "description": "Data Package implementation in TypeScript.", + "description": "Fast TypeScript data management framework based on the Data Package standard", "keywords": [ "data", "package", diff --git a/db/README.md b/db/README.md index f30d0749..1aac7bd7 100644 --- a/db/README.md +++ b/db/README.md @@ -1,3 +1,3 @@ # @dpkit/db -Data Package implementation in TypeScript. Data Package is a standard consisting of a set of simple yet extensible specifications to describe datasets, data files and tabular data. It is a data definition language (DDL) and data API that facilitates findability, accessibility, interoperability, and reusability (FAIR) of data. For more information, please visit the [documentation portal](https://dpkit.datist.io). +dpkit is a fast TypeScript data management framework based on the Data Package standard. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/db/package.json b/db/package.json index ced0697d..8cd9b530 100644 --- a/db/package.json +++ b/db/package.json @@ -6,7 +6,7 @@ "license": "MIT", "author": "Evgeny Karev", "repository": "https://github.com/datisthq/dpkit", - "description": "Data Package implementation in TypeScript.", + "description": "Fast TypeScript data management framework based on the Data Package standard", "keywords": [ "data", "package", diff --git a/docs/assets/about.png b/docs/assets/about.png new file mode 100644 index 00000000..7be21fc3 Binary files /dev/null and b/docs/assets/about.png differ diff --git a/docs/assets/hero.png b/docs/assets/hero.png index 0ab0a607..e37e6251 100644 Binary files a/docs/assets/hero.png and b/docs/assets/hero.png differ diff --git a/docs/assets/logo-dark.png b/docs/assets/logo-dark.png new file mode 100644 index 00000000..a34897a2 Binary files /dev/null and b/docs/assets/logo-dark.png differ diff --git a/docs/assets/logo-dark.svg b/docs/assets/logo-dark.svg deleted file mode 100644 index 975381a6..00000000 --- a/docs/assets/logo-dark.svg +++ /dev/null @@ -1,140 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/assets/logo-light.png b/docs/assets/logo-light.png new file mode 100644 index 00000000..0aee1841 Binary files /dev/null and b/docs/assets/logo-light.png differ diff --git a/docs/assets/logo-light.svg b/docs/assets/logo-light.svg deleted file mode 100644 index fe26ea2e..00000000 --- a/docs/assets/logo-light.svg +++ /dev/null @@ -1,26 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/assets/styles.css b/docs/assets/styles.css index 55a51fa4..754569b6 100644 --- a/docs/assets/styles.css +++ b/docs/assets/styles.css @@ -6,15 +6,15 @@ :root[data-theme="light"] { --purple-hsl: 209, 85%, 65%; - --sl-color-text-accent: #6fa7da; - --sl-color-banner-bg: #6fa7da; - /* --sl-color-bg-inline-code: #6fa7da; */ + --sl-color-text-accent: #228be6; + --sl-color-banner-bg: #228be6; + /* --sl-color-bg-inline-code: #228be6; */ } :root[data-theme="dark"] { - --sl-color-text-accent: #6fa7da; - --sl-color-banner-bg: #6fa7da; - /* --sl-color-bg-inline-code: #6fa7da; */ + --sl-color-text-accent: #228be6; + --sl-color-banner-bg: #228be6; + /* --sl-color-bg-inline-code: #228be6; */ } /* Scrollbar */ diff --git a/docs/astro.config.ts b/docs/astro.config.ts index 5b8aee34..69d8bd21 100644 --- a/docs/astro.config.ts +++ b/docs/astro.config.ts @@ -1,7 +1,17 @@ import starlight from "@astrojs/starlight" import { defineConfig } from "astro/config" import starlightScrollToTop from "starlight-scroll-to-top" -import starlightTypeDoc, { typeDocSidebarGroup } from "starlight-typedoc" +import starlightTypeDoc from "starlight-typedoc" + +const PACKAGES = { + dpkit: "../dpkit", + "@dpkit/ckan": "../ckan", + "@dpkit/core": "../core", + "@dpkit/github": "../github", + "@dpkit/file": "../file", + "@dpkit/zenodo": "../zenodo", + "@dpkit/zip": "../zip", +} export default defineConfig({ site: "https://dpkit.datist.io", @@ -9,14 +19,17 @@ export default defineConfig({ outDir: "build", integrations: [ starlight({ - title: "Data Package in TypeScript", + title: "dpkit", description: - "Data Package is a standard consisting of a set of simple yet extensible specifications to describe datasets, data files and tabular data. It is a data definition language (DDL) and data API that facilitates findability, accessibility, interoperability, and reusability (FAIR) of data.", + "dpkit is a fast TypeScript data management framework based on the Data Package standard. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub", customCss: ["/assets/styles.css"], + components: { + SocialIcons: "./components/Header/SocialIcons.astro", + }, logo: { - light: "/assets/logo-light.svg", - dark: "/assets/logo-dark.svg", - alt: "Data Package Logo", + light: "/assets/logo-light.png", + dark: "/assets/logo-dark.png", + alt: "DPkit Logo", replacesTitle: true, }, social: [ @@ -26,7 +39,7 @@ export default defineConfig({ href: "https://github.com/datisthq/dpkit", }, ], - favicon: "favicon.ico", + favicon: "favicon.png", editLink: { baseUrl: "https://github.com/datisthq/dpkit/edit/main/", }, @@ -35,9 +48,9 @@ export default defineConfig({ plugins: [ starlightScrollToTop(), starlightTypeDoc({ - entryPoints: ["../dpkit", "../core", "../file", "../zip"], + entryPoints: generatePackageEntrypoints(), tsconfig: "../tsconfig.json", - typeDoc: { entryPointStrategy: "packages" }, + typeDoc: { entryPointStrategy: "packages", router: "structure" }, output: "packages", sidebar: { label: "Packages", @@ -48,7 +61,11 @@ export default defineConfig({ sidebar: [ { label: "Overview", autogenerate: { directory: "overview" } }, { label: "Guides", autogenerate: { directory: "guides" } }, - typeDocSidebarGroup, + { + label: "Packages", + collapsed: true, + items: generatePackageSidebars(), + }, ], head: [ { @@ -63,3 +80,38 @@ export default defineConfig({ }), ], }) + +function generatePackageEntrypoints() { + return Object.values(PACKAGES) +} + +function generatePackageSidebars() { + return Object.entries(PACKAGES).map(([name, path]) => + generatePackageSidebar({ name }), + ) +} + +function generatePackageSidebar(props: { name: string }) { + const name = props.name + const slug = name.replace("@", "_") + + return { + label: name, + collapsed: true, + items: [ + { + label: "Overview", + slug: `packages/${slug}/overview`, + }, + { + label: "Changelog", + slug: `packages/${slug}/changelog`, + }, + { + label: "API Reference", + autogenerate: { directory: `packages/${slug}/index` }, + //collapsed: true, + }, + ], + } +} diff --git a/docs/components/About/About.astro b/docs/components/About/About.astro index bbb064f8..4981344e 100644 --- a/docs/components/About/About.astro +++ b/docs/components/About/About.astro @@ -1,15 +1,15 @@ --- import { Image } from "astro:assets" -import datist from "./datist.png" +import logo from "../../assets/about.png" ---
Brought to you by -
Datist
- We are bringing technological innovation and consultancy services to open data field. - Learn about Datist. +
Datist
+

We are bringing technological innovation and consultancy
services to open data field. + Learn about Datist

diff --git a/docs/content/docs/guides/validation.md b/docs/content/docs/guides/validation.md index 7997e7c8..beea8e8a 100644 --- a/docs/content/docs/guides/validation.md +++ b/docs/content/docs/guides/validation.md @@ -2,4 +2,8 @@ title: Validation --- +:::note +This guide is under development. +::: + diff --git a/docs/content/docs/index.mdx b/docs/content/docs/index.mdx index 8b73207b..8fbb203d 100644 --- a/docs/content/docs/index.mdx +++ b/docs/content/docs/index.mdx @@ -1,8 +1,61 @@ --- -title: Data Package in TypeScript -head: - - tag: title - content: Data Package in TypeScript +title: dpkit +template: splash +editUrl: false +lastUpdated: false +hero: + tagline: | + dpkit is a fast TypeScript data management framework based on the Data Package standard. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. + image: + file: ../../assets/hero.png + alt: Data Packages Illustration + actions: + - text: Get Started + link: /overview/installation/ + icon: right-arrow + variant: primary + - text: View on GitHub + link: https://github.com/datisthq/dpkit + icon: external + variant: secondary +banner: + content: | +

+ The project is currently in active development. + Please follow the Datist Blog for news and announcements. +

--- -Data Package implementation in TypeScript. Data Package is a standard consisting of a set of simple yet extensible specifications to describe datasets, data files and tabular data. It is a data definition language (DDL) and data API that facilitates findability, accessibility, interoperability, and reusability (FAIR) of data. +import About from "../../components/About/About.astro" +import ClickableCard from "../../components/Card/Clickable.astro" +import { CardGrid, LinkCard, Card } from "@astrojs/starlight/components" + +

Features

+ + + + Data operation is backed by Polars, an extremely fast DataFrame library. + + + The framework supports various data formats and data platform integrations. + + + The framework is 100% compatible with the Data Package (v2) standard. + + + Easy-to-use plugin system to extend the framework with custom data formats and data platforms. + + + +
+ +

Documentation

+ + + + + + + + + diff --git a/docs/content/docs/overview/contributing.md b/docs/content/docs/overview/contributing.md new file mode 120000 index 00000000..069558fa --- /dev/null +++ b/docs/content/docs/overview/contributing.md @@ -0,0 +1 @@ +../../../../CONTRIBUTING.md \ No newline at end of file diff --git a/docs/content/docs/overview/funding.md b/docs/content/docs/overview/funding.md new file mode 100644 index 00000000..fdea7321 --- /dev/null +++ b/docs/content/docs/overview/funding.md @@ -0,0 +1,14 @@ +--- +title: Funding +--- + +This project is funded through [NGI Zero Commons Fund](https://nlnet.nl/commonsfund/), a fund established by [NLnet](https://nlnet.nl) with financial support from the European Commission's [Next Generation Internet](https://ngi.eu) program. Learn more at the [NLnet project page](https://nlnet.nl/project/DataPackage-TS/). + + diff --git a/docs/content/docs/overview/installation.md b/docs/content/docs/overview/installation.md index 5ee638e3..d16653dd 100644 --- a/docs/content/docs/overview/installation.md +++ b/docs/content/docs/overview/installation.md @@ -1,11 +1,12 @@ --- title: Installation +sidebar: + order: 1 --- ## Prerequisites -- **Node.js**: v24.0.0 or higher -- **PNPM**: v10.0.0 or higher +- **Node.js**: v20.0.0 or higher ## Installation diff --git a/docs/public/favicon.ico b/docs/public/favicon.ico deleted file mode 100644 index 125945cd..00000000 Binary files a/docs/public/favicon.ico and /dev/null differ diff --git a/docs/public/favicon.png b/docs/public/favicon.png new file mode 100644 index 00000000..88a2c1f9 Binary files /dev/null and b/docs/public/favicon.png differ diff --git a/dpkit/OVERVIEW.md b/dpkit/OVERVIEW.md new file mode 100644 index 00000000..a2b1cdc9 --- /dev/null +++ b/dpkit/OVERVIEW.md @@ -0,0 +1,2 @@ +# dpkit + diff --git a/dpkit/README.md b/dpkit/README.md new file mode 100644 index 00000000..4c60b46b --- /dev/null +++ b/dpkit/README.md @@ -0,0 +1,3 @@ +# dpkit + +dpkit is a fast TypeScript data management framework based on the Data Package standard. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/dpkit/general/dpkit.ts b/dpkit/general/dpkit.ts index bc5bee88..28c096c7 100644 --- a/dpkit/general/dpkit.ts +++ b/dpkit/general/dpkit.ts @@ -1,6 +1,8 @@ import { CkanPlugin } from "@dpkit/ckan" import type { Plugin } from "@dpkit/core" import { FilePlugin } from "@dpkit/file" +import { GithubPlugin } from "@dpkit/github" +import { ZenodoPlugin } from "@dpkit/zenodo" import { ZipPlugin } from "@dpkit/zip" export class Dpkit { @@ -14,5 +16,7 @@ export class Dpkit { export const dpkit = new Dpkit() dpkit.register(CkanPlugin) +dpkit.register(GithubPlugin) +dpkit.register(ZenodoPlugin) dpkit.register(FilePlugin) dpkit.register(ZipPlugin) diff --git a/dpkit/index.ts b/dpkit/index.ts index 0554c5e8..77433a50 100644 --- a/dpkit/index.ts +++ b/dpkit/index.ts @@ -1,6 +1,8 @@ +export * from "@dpkit/camtrap" export * from "@dpkit/ckan" export * from "@dpkit/core" export * from "@dpkit/file" +export * from "@dpkit/github" export * from "@dpkit/zenodo" export * from "@dpkit/zip" diff --git a/dpkit/package.json b/dpkit/package.json index 09aa84ec..26b10a6d 100644 --- a/dpkit/package.json +++ b/dpkit/package.json @@ -6,7 +6,7 @@ "license": "MIT", "author": "Evgeny Karev", "repository": "https://github.com/datisthq/dpkit", - "description": "Data Package implementation in TypeScript.", + "description": "Fast TypeScript data management framework based on the Data Package standard", "keywords": [ "data", "package", @@ -17,9 +17,11 @@ "cli" ], "dependencies": { + "@dpkit/camtrap": "workspace:*", "@dpkit/ckan": "workspace:*", "@dpkit/core": "workspace:*", "@dpkit/file": "workspace:*", + "@dpkit/github": "workspace:*", "@dpkit/zenodo": "workspace:*", "@dpkit/zip": "workspace:*" }, diff --git a/dpkit/typedoc.json b/dpkit/typedoc.json index f8e49f3a..fc33b815 100644 --- a/dpkit/typedoc.json +++ b/dpkit/typedoc.json @@ -1,4 +1,5 @@ { "entryPoints": ["index.ts"], + "projectDocuments": ["CHANGELOG.md", "OVERVIEW.md"], "skipErrorChecking": true } diff --git a/excel/README.md b/excel/README.md index cc08a81f..b985c64e 100644 --- a/excel/README.md +++ b/excel/README.md @@ -1,3 +1,3 @@ # @dpkit/excel -Data Package implementation in TypeScript. Data Package is a standard consisting of a set of simple yet extensible specifications to describe datasets, data files and tabular data. It is a data definition language (DDL) and data API that facilitates findability, accessibility, interoperability, and reusability (FAIR) of data. For more information, please visit the [documentation portal](https://dpkit.datist.io). +dpkit is a fast TypeScript data management framework based on the Data Package standard. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/excel/package.json b/excel/package.json index abbd2884..f2895d2f 100644 --- a/excel/package.json +++ b/excel/package.json @@ -6,7 +6,7 @@ "license": "MIT", "author": "Evgeny Karev", "repository": "https://github.com/datisthq/dpkit", - "description": "Data Package implementation in TypeScript.", + "description": "Fast TypeScript data management framework based on the Data Package standard", "keywords": [ "data", "package", diff --git a/extend/README.md b/extend/README.md new file mode 100644 index 00000000..81b899e7 --- /dev/null +++ b/extend/README.md @@ -0,0 +1,3 @@ +# @dpkit/extend + +dpkit is a fast TypeScript data management framework based on the Data Package standard. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/site/index.ts b/extend/index.ts similarity index 100% rename from site/index.ts rename to extend/index.ts diff --git a/site/package.json b/extend/package.json similarity index 70% rename from site/package.json rename to extend/package.json index f05d99d4..3ae21fe7 100644 --- a/site/package.json +++ b/extend/package.json @@ -1,12 +1,12 @@ { - "name": "@dpkit/site", + "name": "@dpkit/extend", "type": "module", "main": "build/index.js", "version": "0.1.0", "license": "MIT", "author": "Evgeny Karev", "repository": "https://github.com/datisthq/dpkit", - "description": "Data Package implementation in TypeScript.", + "description": "Fast TypeScript data management framework based on the Data Package standard", "keywords": [ "data", "package", @@ -14,7 +14,7 @@ "validation", "quality", "fair", - "site" + "extend" ], "scripts": { "build": "tsc --build" diff --git a/extend/tsconfig.json b/extend/tsconfig.json new file mode 100644 index 00000000..7be40230 --- /dev/null +++ b/extend/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../tsconfig.json", + "include": ["**/*.ts"], + "exclude": ["**/build/"], + "compilerOptions": { + "outDir": "build", + "noEmit": false, + "declaration": true + } +} diff --git a/file/OVERVIEW.md b/file/OVERVIEW.md new file mode 100644 index 00000000..ebcdb89d --- /dev/null +++ b/file/OVERVIEW.md @@ -0,0 +1,2 @@ +# @dpkit/file + diff --git a/file/README.md b/file/README.md index 02ae7fa4..c0ce7de2 100644 --- a/file/README.md +++ b/file/README.md @@ -1,3 +1,3 @@ # @dpkit/data -Data Package implementation in TypeScript. Data Package is a standard consisting of a set of simple yet extensible specifications to describe datasets, data files and tabular data. It is a data definition language (DDL) and data API that facilitates findability, accessibility, interoperability, and reusability (FAIR) of data. For more information, please visit the [documentation portal](https://dpkit.datist.io). +dpkit is a fast TypeScript data management framework based on the Data Package standard. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/file/package.json b/file/package.json index 7bb8e072..a1aa3126 100644 --- a/file/package.json +++ b/file/package.json @@ -6,7 +6,7 @@ "license": "MIT", "author": "Evgeny Karev", "repository": "https://github.com/datisthq/dpkit", - "description": "Data Package implementation in TypeScript.", + "description": "Fast TypeScript data management framework based on the Data Package standard", "keywords": [ "data", "package", @@ -21,6 +21,7 @@ }, "dependencies": { "@dpkit/core": "workspace:*", - "tempy": "3.1.0" + "tempy": "3.1.0", + "tiny-invariant": "^1.3.3" } } diff --git a/file/resource/save.ts b/file/resource/save.ts index 687ef202..2960f534 100644 --- a/file/resource/save.ts +++ b/file/resource/save.ts @@ -4,8 +4,8 @@ import { denormalizeResource, getFilename, isRemotePath, - isTableResource, } from "@dpkit/core" +import invariant from "tiny-invariant" export type SaveFile = (props: { propertyName: string @@ -67,17 +67,16 @@ export async function saveResourceFiles(props: { } if (Array.isArray(resource.path)) { + invariant(Array.isArray(descriptor.path), "Multipart as resource.path") for (const [index, path] of resource.path.entries()) { descriptor.path[index] = await saveFile(path, "path", index) } } - if (isTableResource(resource)) { - for (const name of ["dialect", "schema"] as const) { - const path = resource[name] - if (typeof path === "string") { - descriptor[name] = await saveFile(path, name, 0) - } + for (const name of ["dialect", "schema"] as const) { + const path = resource[name] + if (typeof path === "string") { + descriptor[name] = await saveFile(path, name, 0) } } diff --git a/file/typedoc.json b/file/typedoc.json index f8e49f3a..fc33b815 100644 --- a/file/typedoc.json +++ b/file/typedoc.json @@ -1,4 +1,5 @@ { "entryPoints": ["index.ts"], + "projectDocuments": ["CHANGELOG.md", "OVERVIEW.md"], "skipErrorChecking": true } diff --git a/github/CHAT.md b/github/CHAT.md deleted file mode 100644 index 7c1f9a33..00000000 --- a/github/CHAT.md +++ /dev/null @@ -1,17 +0,0 @@ -# Github support - -We need to support Github similar to Zenodo and CKAN. - -## References - -- https://github.com/frictionlessdata/dplib-py/tree/main/dplib/plugins/github/models -- https://github.com/frictionlessdata/frictionless-py/blob/main/frictionless/portals/github/adapter.py - -## Prompts - -### 1. Initial implementation - -- Port `ckan` package as `github` package (also consult to `zenodo` package) -- Use dplib-py as a reference to adjust github types and process functions -- Use frictionless-py as a reference to adjust `loadPackageFromGithub` and `savePackageToGithub` -- Note that there is no Schema in github domain diff --git a/github/OVERVIEW.md b/github/OVERVIEW.md new file mode 100644 index 00000000..82e6e6da --- /dev/null +++ b/github/OVERVIEW.md @@ -0,0 +1,2 @@ +# @dpkit/github + diff --git a/github/README.md b/github/README.md index 8da6e72f..8d2b29b5 100644 --- a/github/README.md +++ b/github/README.md @@ -1,3 +1,3 @@ # @dpkit/github -Data Package implementation in TypeScript. Data Package is a standard consisting of a set of simple yet extensible specifications to describe datasets, data files and tabular data. It is a data definition language (DDL) and data API that facilitates findability, accessibility, interoperability, and reusability (FAIR) of data. For more information, please visit the [documentation portal](https://dpkit.datist.io). +dpkit is a fast TypeScript data management framework based on the Data Package standard. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/github/general/path.ts b/github/general/path.ts new file mode 100644 index 00000000..ba3ef247 --- /dev/null +++ b/github/general/path.ts @@ -0,0 +1,3 @@ +export function normalizeFileLink(props: { link: string }) { + return props.link.replace("/api/", "/").replace(/\/content$/, "") +} diff --git a/github/general/request.ts b/github/general/request.ts index e11d3e54..27eb8a39 100644 --- a/github/general/request.ts +++ b/github/general/request.ts @@ -34,11 +34,13 @@ export interface GithubApiRequestProps { /** * Makes a request to the Github API */ -export async function makeGithubApiRequest(props: GithubApiRequestProps) { +export async function makeGithubApiRequest( + props: GithubApiRequestProps, +) { const { endpoint, method = "GET", payload, upload, apiKey } = props let body: string | FormData | undefined - const headers: Descriptor = {} + const headers: Record = {} if (apiKey) { headers["Authorization"] = `Bearer ${apiKey}` @@ -72,17 +74,12 @@ export async function makeGithubApiRequest(props: GithubApiRequestProps) { body, }) - if (!response.ok) { + if (!response.ok || response.status === 204) { const errorText = await response.text() throw new Error( `Github API error: ${response.status} ${response.statusText}\n${errorText}`, ) } - // Check for no content responses - if (response.status === 204) { - return {} - } - - return (await response.json()) as Descriptor + return (await response.json()) as T } diff --git a/github/index.ts b/github/index.ts index 71449eaf..c3c86163 100644 --- a/github/index.ts +++ b/github/index.ts @@ -1,2 +1,3 @@ export * from "./package/index.js" export * from "./resource/index.js" +export * from "./plugin.js" diff --git a/github/package.json b/github/package.json index c2228148..63090731 100644 --- a/github/package.json +++ b/github/package.json @@ -6,7 +6,7 @@ "license": "MIT", "author": "Evgeny Karev", "repository": "https://github.com/datisthq/dpkit", - "description": "Data Package implementation in TypeScript.", + "description": "Fast TypeScript data management framework based on the Data Package standard", "keywords": [ "data", "package", @@ -16,14 +16,14 @@ "fair", "github" ], + "scripts": { + "build": "tsc --build" + }, "dependencies": { "@dpkit/core": "workspace:*", "@dpkit/file": "workspace:*" }, "devDependencies": { "@dpkit/test": "workspace:*" - }, - "scripts": { - "build": "tsc --build" } } diff --git a/github/package/fixtures/generated/load.spec.ts.snap b/github/package/fixtures/generated/load.spec.ts.snap index 6eb91729..ddbb4566 100644 --- a/github/package/fixtures/generated/load.spec.ts.snap +++ b/github/package/fixtures/generated/load.spec.ts.snap @@ -9,52 +9,163 @@ exports[`loadPackageFromGithub > should load a package 1`] = ` "title": "roll", }, ], - "created": "2021-07-07T14:09:49Z", - "description": "ISO 4217 List of Currencies and Currency Codes", - "homepage": "https://datahub.io/core/currency-codes", - "name": "currency-codes", + "created": "2019-11-11T10:48:02Z", + "name": "data", "resources": [ { - "bytes": 1752, + "bytes": 6, "format": "md", - "hash": "sha1:9c36869b88dfad9b9d610734966270b7e883b707", + "github:key": undefined, + "github:url": undefined, + "hash": "sha1:31cd35605bbee1d3b3f297d5b0462028a46a0380", "name": "readme", - "path": "README.md", + "path": "https://raw.githubusercontent.com/roll/data/refs/heads/master/README.md", + }, + { + "bytes": 67920590, + "format": "zip", + "github:key": undefined, + "github:url": undefined, + "hash": "sha1:7c32a3955b961c3fe7646fac499177ab5e5af524", + "name": "data", + "path": "https://raw.githubusercontent.com/roll/data/refs/heads/master/data/data.zip", }, { - "bytes": 47281, - "format": "xml", - "hash": "sha1:42609efee29fb5a7d98935b806ea135e3f455653", - "name": "table_a1", - "path": "archive/table_a1.xml", + "bytes": 192, + "format": "json", + "github:key": undefined, + "github:url": undefined, + "hash": "sha1:8905b138ed93ebb9e78945126ac2a423d31babc9", + "name": "meta", + "path": "https://raw.githubusercontent.com/roll/data/refs/heads/master/data/meta.json", }, { - "bytes": 29349, - "format": "xml", - "hash": "sha1:b2a569c403e6917aa466d0973d3b3f677821530f", - "name": "table_a2", - "path": "archive/table_a2.xml", + "bytes": 1000000, + "format": "csv", + "github:key": undefined, + "github:url": undefined, + "hash": "sha1:f79b42b814066ec87cc734d932c64733c47444df", + "name": "table1", + "path": "https://raw.githubusercontent.com/roll/data/refs/heads/master/data/table1.csv", }, { - "bytes": 17472, + "bytes": 10000000, "format": "csv", - "hash": "sha1:2c803df30c1d09e1c7f5de6661ca38f8e627d65b", - "name": "codes-all", - "path": "data/codes-all.csv", + "github:key": undefined, + "github:url": undefined, + "hash": "sha1:f352795df6534642cc0a9fe6d9b2b3f8fa80135a", + "name": "table10", + "path": "https://raw.githubusercontent.com/roll/data/refs/heads/master/data/table10.csv", }, { - "bytes": 2019, + "bytes": 100000000, + "format": "csv", + "github:key": undefined, + "github:url": undefined, + "hash": "sha1:734520644f4616825209c0c7a0750e5887b93a76", + "name": "table100", + "path": "https://raw.githubusercontent.com/roll/data/refs/heads/master/data/table100.csv", + }, + { + "bytes": 5000000, + "format": "csv", + "github:key": undefined, + "github:url": undefined, + "hash": "sha1:306b2975e0e469a2eec853b1a4856acb25a1dab6", + "name": "table5", + "path": "https://raw.githubusercontent.com/roll/data/refs/heads/master/data/table5.csv", + }, + { + "bytes": 50000000, + "format": "csv", + "github:key": undefined, + "github:url": undefined, + "hash": "sha1:eca005a30f75202d7ccdadbbb6789a9ae0389555", + "name": "table50", + "path": "https://raw.githubusercontent.com/roll/data/refs/heads/master/data/table50.csv", + }, + { + "bytes": 513, "format": "json", - "hash": "sha1:dfca466367e0b3d23294df83b72101cb5b084c09", - "name": "datapackage", - "path": "datapackage.json", + "github:key": undefined, + "github:url": undefined, + "hash": "sha1:4564795a02255050972182a43590b853dee8b8b5", + "name": "package", + "path": "https://raw.githubusercontent.com/roll/data/refs/heads/master/package.json", + }, + { + "bytes": 1952, + "format": "js", + "github:key": undefined, + "github:url": undefined, + "hash": "sha1:59f443ca5062f3f84937490509d6cc25571fb853", + "name": "generate", + "path": "https://raw.githubusercontent.com/roll/data/refs/heads/master/scripts/generate.js", }, + ], + "title": "roll/data", +} +`; + +exports[`loadPackageFromGithub > should merge datapackage.json if present 1`] = ` +{ + "contributors": [ { - "bytes": 3098, - "format": "sh", - "hash": "sha1:4632c9933cd6c76331301efa71dc5cd183c6dc89", - "name": "runall", - "path": "scripts/runall.sh", + "path": "https://github.com/roll", + "role": "author", + "title": "roll", + }, + ], + "created": "2021-07-07T14:09:49Z", + "description": "ISO 4217 List of Currencies and Currency Codes", + "homepage": "https://datahub.io/core/currency-codes", + "name": "currency-codes", + "resources": [ + { + "github:key": undefined, + "github:url": undefined, + "mimetype": "text/csv", + "name": "codes-all", + "path": "https://raw.githubusercontent.com/roll/currency-codes/refs/heads/master/data/codes-all.csv", + "schema": { + "fields": [ + { + "description": "Country or region name", + "name": "Entity", + "type": "string", + }, + { + "description": "Name of the currency", + "name": "Currency", + "type": "string", + }, + { + "description": "3 digit alphabetic code for the currency", + "name": "AlphabeticCode", + "title": "Alphabetic Code", + "type": "string", + }, + { + "description": "3 digit numeric code", + "name": "NumericCode", + "title": "Numeric Code", + "type": "number", + }, + { + "description": "", + "name": "MinorUnit", + "title": "Minor Unit", + "type": "string", + }, + { + "description": "Date currency withdrawn (values can be ranges or months", + "name": "WithdrawalDate", + "title": "Withdrawal Date", + "type": "string", + }, + ], + }, + "size": "16863", }, ], "title": "roll/currency-codes", diff --git a/github/package/fixtures/generated/should-load-a-package_2675098929/recording.har b/github/package/fixtures/generated/should-load-a-package_2675098929/recording.har index 869c5972..2017f36b 100644 --- a/github/package/fixtures/generated/should-load-a-package_2675098929/recording.har +++ b/github/package/fixtures/generated/should-load-a-package_2675098929/recording.har @@ -8,25 +8,25 @@ }, "entries": [ { - "_id": "1a1b6f9f2e0280a7960e568ae456fc69", + "_id": "af6e29ae2e06d9b1710553904135742d", "_order": 0, "cache": {}, "request": { "bodySize": 0, "cookies": [], "headers": [], - "headersSize": 67, + "headersSize": 57, "httpVersion": "HTTP/1.1", "method": "GET", "queryString": [], - "url": "https://api.github.com/repos/roll/currency-codes" + "url": "https://api.github.com/repos/roll/data" }, "response": { - "bodySize": 1990, + "bodySize": 1236, "content": { "mimeType": "application/json; charset=utf-8", - "size": 1990, - "text": "{\"id\":383820850,\"node_id\":\"MDEwOlJlcG9zaXRvcnkzODM4MjA4NTA=\",\"name\":\"currency-codes\",\"full_name\":\"roll/currency-codes\",\"private\":false,\"owner\":{\"login\":\"roll\",\"id\":557395,\"node_id\":\"MDQ6VXNlcjU1NzM5NQ==\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/557395?v=4\",\"gravatar_id\":\"\",\"url\":\"https://api.github.com/users/roll\",\"html_url\":\"https://github.com/roll\",\"followers_url\":\"https://api.github.com/users/roll/followers\",\"following_url\":\"https://api.github.com/users/roll/following{/other_user}\",\"gists_url\":\"https://api.github.com/users/roll/gists{/gist_id}\",\"starred_url\":\"https://api.github.com/users/roll/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/roll/subscriptions\",\"organizations_url\":\"https://api.github.com/users/roll/orgs\",\"repos_url\":\"https://api.github.com/users/roll/repos\",\"events_url\":\"https://api.github.com/users/roll/events{/privacy}\",\"received_events_url\":\"https://api.github.com/users/roll/received_events\",\"type\":\"User\",\"user_view_type\":\"public\",\"site_admin\":false},\"html_url\":\"https://github.com/roll/currency-codes\",\"description\":\"ISO 4217 List of Currencies and Currency Codes\",\"fork\":true,\"url\":\"https://api.github.com/repos/roll/currency-codes\",\"forks_url\":\"https://api.github.com/repos/roll/currency-codes/forks\",\"keys_url\":\"https://api.github.com/repos/roll/currency-codes/keys{/key_id}\",\"collaborators_url\":\"https://api.github.com/repos/roll/currency-codes/collaborators{/collaborator}\",\"teams_url\":\"https://api.github.com/repos/roll/currency-codes/teams\",\"hooks_url\":\"https://api.github.com/repos/roll/currency-codes/hooks\",\"issue_events_url\":\"https://api.github.com/repos/roll/currency-codes/issues/events{/number}\",\"events_url\":\"https://api.github.com/repos/roll/currency-codes/events\",\"assignees_url\":\"https://api.github.com/repos/roll/currency-codes/assignees{/user}\",\"branches_url\":\"https://api.github.com/repos/roll/currency-codes/branches{/branch}\",\"tags_url\":\"https://api.github.com/repos/roll/currency-codes/tags\",\"blobs_url\":\"https://api.github.com/repos/roll/currency-codes/git/blobs{/sha}\",\"git_tags_url\":\"https://api.github.com/repos/roll/currency-codes/git/tags{/sha}\",\"git_refs_url\":\"https://api.github.com/repos/roll/currency-codes/git/refs{/sha}\",\"trees_url\":\"https://api.github.com/repos/roll/currency-codes/git/trees{/sha}\",\"statuses_url\":\"https://api.github.com/repos/roll/currency-codes/statuses/{sha}\",\"languages_url\":\"https://api.github.com/repos/roll/currency-codes/languages\",\"stargazers_url\":\"https://api.github.com/repos/roll/currency-codes/stargazers\",\"contributors_url\":\"https://api.github.com/repos/roll/currency-codes/contributors\",\"subscribers_url\":\"https://api.github.com/repos/roll/currency-codes/subscribers\",\"subscription_url\":\"https://api.github.com/repos/roll/currency-codes/subscription\",\"commits_url\":\"https://api.github.com/repos/roll/currency-codes/commits{/sha}\",\"git_commits_url\":\"https://api.github.com/repos/roll/currency-codes/git/commits{/sha}\",\"comments_url\":\"https://api.github.com/repos/roll/currency-codes/comments{/number}\",\"issue_comment_url\":\"https://api.github.com/repos/roll/currency-codes/issues/comments{/number}\",\"contents_url\":\"https://api.github.com/repos/roll/currency-codes/contents/{+path}\",\"compare_url\":\"https://api.github.com/repos/roll/currency-codes/compare/{base}...{head}\",\"merges_url\":\"https://api.github.com/repos/roll/currency-codes/merges\",\"archive_url\":\"https://api.github.com/repos/roll/currency-codes/{archive_format}{/ref}\",\"downloads_url\":\"https://api.github.com/repos/roll/currency-codes/downloads\",\"issues_url\":\"https://api.github.com/repos/roll/currency-codes/issues{/number}\",\"pulls_url\":\"https://api.github.com/repos/roll/currency-codes/pulls{/number}\",\"milestones_url\":\"https://api.github.com/repos/roll/currency-codes/milestones{/number}\",\"notifications_url\":\"https://api.github.com/repos/roll/currency-codes/notifications{?since,all,participating}\",\"labels_url\":\"https://api.github.com/repos/roll/currency-codes/labels{/name}\",\"releases_url\":\"https://api.github.com/repos/roll/currency-codes/releases{/id}\",\"deployments_url\":\"https://api.github.com/repos/roll/currency-codes/deployments\",\"created_at\":\"2021-07-07T14:09:49Z\",\"updated_at\":\"2021-07-07T14:47:03Z\",\"pushed_at\":\"2021-07-07T14:47:01Z\",\"git_url\":\"git://github.com/roll/currency-codes.git\",\"ssh_url\":\"git@github.com:roll/currency-codes.git\",\"clone_url\":\"https://github.com/roll/currency-codes.git\",\"svn_url\":\"https://github.com/roll/currency-codes\",\"homepage\":\"https://datahub.io/core/currency-codes\",\"size\":87,\"stargazers_count\":0,\"watchers_count\":0,\"language\":\"Shell\",\"has_issues\":false,\"has_projects\":true,\"has_downloads\":true,\"has_wiki\":true,\"has_pages\":false,\"has_discussions\":false,\"forks_count\":0,\"mirror_url\":null,\"archived\":false,\"disabled\":false,\"open_issues_count\":0,\"license\":null,\"allow_forking\":true,\"is_template\":false,\"web_commit_signoff_required\":false,\"topics\":[],\"visibility\":\"public\",\"forks\":0,\"open_issues\":0,\"watchers\":0,\"default_branch\":\"master\",\"temp_clone_token\":null,\"parent\":{\"id\":6696644,\"node_id\":\"MDEwOlJlcG9zaXRvcnk2Njk2NjQ0\",\"name\":\"currency-codes\",\"full_name\":\"datasets/currency-codes\",\"private\":false,\"owner\":{\"login\":\"datasets\",\"id\":1643515,\"node_id\":\"MDEyOk9yZ2FuaXphdGlvbjE2NDM1MTU=\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/1643515?v=4\",\"gravatar_id\":\"\",\"url\":\"https://api.github.com/users/datasets\",\"html_url\":\"https://github.com/datasets\",\"followers_url\":\"https://api.github.com/users/datasets/followers\",\"following_url\":\"https://api.github.com/users/datasets/following{/other_user}\",\"gists_url\":\"https://api.github.com/users/datasets/gists{/gist_id}\",\"starred_url\":\"https://api.github.com/users/datasets/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/datasets/subscriptions\",\"organizations_url\":\"https://api.github.com/users/datasets/orgs\",\"repos_url\":\"https://api.github.com/users/datasets/repos\",\"events_url\":\"https://api.github.com/users/datasets/events{/privacy}\",\"received_events_url\":\"https://api.github.com/users/datasets/received_events\",\"type\":\"Organization\",\"user_view_type\":\"public\",\"site_admin\":false},\"html_url\":\"https://github.com/datasets/currency-codes\",\"description\":\"ISO 4217 List of Currencies and Currency Codes\",\"fork\":false,\"url\":\"https://api.github.com/repos/datasets/currency-codes\",\"forks_url\":\"https://api.github.com/repos/datasets/currency-codes/forks\",\"keys_url\":\"https://api.github.com/repos/datasets/currency-codes/keys{/key_id}\",\"collaborators_url\":\"https://api.github.com/repos/datasets/currency-codes/collaborators{/collaborator}\",\"teams_url\":\"https://api.github.com/repos/datasets/currency-codes/teams\",\"hooks_url\":\"https://api.github.com/repos/datasets/currency-codes/hooks\",\"issue_events_url\":\"https://api.github.com/repos/datasets/currency-codes/issues/events{/number}\",\"events_url\":\"https://api.github.com/repos/datasets/currency-codes/events\",\"assignees_url\":\"https://api.github.com/repos/datasets/currency-codes/assignees{/user}\",\"branches_url\":\"https://api.github.com/repos/datasets/currency-codes/branches{/branch}\",\"tags_url\":\"https://api.github.com/repos/datasets/currency-codes/tags\",\"blobs_url\":\"https://api.github.com/repos/datasets/currency-codes/git/blobs{/sha}\",\"git_tags_url\":\"https://api.github.com/repos/datasets/currency-codes/git/tags{/sha}\",\"git_refs_url\":\"https://api.github.com/repos/datasets/currency-codes/git/refs{/sha}\",\"trees_url\":\"https://api.github.com/repos/datasets/currency-codes/git/trees{/sha}\",\"statuses_url\":\"https://api.github.com/repos/datasets/currency-codes/statuses/{sha}\",\"languages_url\":\"https://api.github.com/repos/datasets/currency-codes/languages\",\"stargazers_url\":\"https://api.github.com/repos/datasets/currency-codes/stargazers\",\"contributors_url\":\"https://api.github.com/repos/datasets/currency-codes/contributors\",\"subscribers_url\":\"https://api.github.com/repos/datasets/currency-codes/subscribers\",\"subscription_url\":\"https://api.github.com/repos/datasets/currency-codes/subscription\",\"commits_url\":\"https://api.github.com/repos/datasets/currency-codes/commits{/sha}\",\"git_commits_url\":\"https://api.github.com/repos/datasets/currency-codes/git/commits{/sha}\",\"comments_url\":\"https://api.github.com/repos/datasets/currency-codes/comments{/number}\",\"issue_comment_url\":\"https://api.github.com/repos/datasets/currency-codes/issues/comments{/number}\",\"contents_url\":\"https://api.github.com/repos/datasets/currency-codes/contents/{+path}\",\"compare_url\":\"https://api.github.com/repos/datasets/currency-codes/compare/{base}...{head}\",\"merges_url\":\"https://api.github.com/repos/datasets/currency-codes/merges\",\"archive_url\":\"https://api.github.com/repos/datasets/currency-codes/{archive_format}{/ref}\",\"downloads_url\":\"https://api.github.com/repos/datasets/currency-codes/downloads\",\"issues_url\":\"https://api.github.com/repos/datasets/currency-codes/issues{/number}\",\"pulls_url\":\"https://api.github.com/repos/datasets/currency-codes/pulls{/number}\",\"milestones_url\":\"https://api.github.com/repos/datasets/currency-codes/milestones{/number}\",\"notifications_url\":\"https://api.github.com/repos/datasets/currency-codes/notifications{?since,all,participating}\",\"labels_url\":\"https://api.github.com/repos/datasets/currency-codes/labels{/name}\",\"releases_url\":\"https://api.github.com/repos/datasets/currency-codes/releases{/id}\",\"deployments_url\":\"https://api.github.com/repos/datasets/currency-codes/deployments\",\"created_at\":\"2012-11-14T23:19:47Z\",\"updated_at\":\"2025-04-15T19:49:30Z\",\"pushed_at\":\"2025-04-01T01:26:10Z\",\"git_url\":\"git://github.com/datasets/currency-codes.git\",\"ssh_url\":\"git@github.com:datasets/currency-codes.git\",\"clone_url\":\"https://github.com/datasets/currency-codes.git\",\"svn_url\":\"https://github.com/datasets/currency-codes\",\"homepage\":\"https://datahub.io/core/currency-codes\",\"size\":79,\"stargazers_count\":158,\"watchers_count\":158,\"language\":\"Shell\",\"has_issues\":true,\"has_projects\":true,\"has_downloads\":true,\"has_wiki\":true,\"has_pages\":false,\"has_discussions\":false,\"forks_count\":179,\"mirror_url\":null,\"archived\":false,\"disabled\":false,\"open_issues_count\":3,\"license\":null,\"allow_forking\":true,\"is_template\":false,\"web_commit_signoff_required\":false,\"topics\":[],\"visibility\":\"public\",\"forks\":179,\"open_issues\":3,\"watchers\":158,\"default_branch\":\"main\"},\"source\":{\"id\":6696644,\"node_id\":\"MDEwOlJlcG9zaXRvcnk2Njk2NjQ0\",\"name\":\"currency-codes\",\"full_name\":\"datasets/currency-codes\",\"private\":false,\"owner\":{\"login\":\"datasets\",\"id\":1643515,\"node_id\":\"MDEyOk9yZ2FuaXphdGlvbjE2NDM1MTU=\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/1643515?v=4\",\"gravatar_id\":\"\",\"url\":\"https://api.github.com/users/datasets\",\"html_url\":\"https://github.com/datasets\",\"followers_url\":\"https://api.github.com/users/datasets/followers\",\"following_url\":\"https://api.github.com/users/datasets/following{/other_user}\",\"gists_url\":\"https://api.github.com/users/datasets/gists{/gist_id}\",\"starred_url\":\"https://api.github.com/users/datasets/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/datasets/subscriptions\",\"organizations_url\":\"https://api.github.com/users/datasets/orgs\",\"repos_url\":\"https://api.github.com/users/datasets/repos\",\"events_url\":\"https://api.github.com/users/datasets/events{/privacy}\",\"received_events_url\":\"https://api.github.com/users/datasets/received_events\",\"type\":\"Organization\",\"user_view_type\":\"public\",\"site_admin\":false},\"html_url\":\"https://github.com/datasets/currency-codes\",\"description\":\"ISO 4217 List of Currencies and Currency Codes\",\"fork\":false,\"url\":\"https://api.github.com/repos/datasets/currency-codes\",\"forks_url\":\"https://api.github.com/repos/datasets/currency-codes/forks\",\"keys_url\":\"https://api.github.com/repos/datasets/currency-codes/keys{/key_id}\",\"collaborators_url\":\"https://api.github.com/repos/datasets/currency-codes/collaborators{/collaborator}\",\"teams_url\":\"https://api.github.com/repos/datasets/currency-codes/teams\",\"hooks_url\":\"https://api.github.com/repos/datasets/currency-codes/hooks\",\"issue_events_url\":\"https://api.github.com/repos/datasets/currency-codes/issues/events{/number}\",\"events_url\":\"https://api.github.com/repos/datasets/currency-codes/events\",\"assignees_url\":\"https://api.github.com/repos/datasets/currency-codes/assignees{/user}\",\"branches_url\":\"https://api.github.com/repos/datasets/currency-codes/branches{/branch}\",\"tags_url\":\"https://api.github.com/repos/datasets/currency-codes/tags\",\"blobs_url\":\"https://api.github.com/repos/datasets/currency-codes/git/blobs{/sha}\",\"git_tags_url\":\"https://api.github.com/repos/datasets/currency-codes/git/tags{/sha}\",\"git_refs_url\":\"https://api.github.com/repos/datasets/currency-codes/git/refs{/sha}\",\"trees_url\":\"https://api.github.com/repos/datasets/currency-codes/git/trees{/sha}\",\"statuses_url\":\"https://api.github.com/repos/datasets/currency-codes/statuses/{sha}\",\"languages_url\":\"https://api.github.com/repos/datasets/currency-codes/languages\",\"stargazers_url\":\"https://api.github.com/repos/datasets/currency-codes/stargazers\",\"contributors_url\":\"https://api.github.com/repos/datasets/currency-codes/contributors\",\"subscribers_url\":\"https://api.github.com/repos/datasets/currency-codes/subscribers\",\"subscription_url\":\"https://api.github.com/repos/datasets/currency-codes/subscription\",\"commits_url\":\"https://api.github.com/repos/datasets/currency-codes/commits{/sha}\",\"git_commits_url\":\"https://api.github.com/repos/datasets/currency-codes/git/commits{/sha}\",\"comments_url\":\"https://api.github.com/repos/datasets/currency-codes/comments{/number}\",\"issue_comment_url\":\"https://api.github.com/repos/datasets/currency-codes/issues/comments{/number}\",\"contents_url\":\"https://api.github.com/repos/datasets/currency-codes/contents/{+path}\",\"compare_url\":\"https://api.github.com/repos/datasets/currency-codes/compare/{base}...{head}\",\"merges_url\":\"https://api.github.com/repos/datasets/currency-codes/merges\",\"archive_url\":\"https://api.github.com/repos/datasets/currency-codes/{archive_format}{/ref}\",\"downloads_url\":\"https://api.github.com/repos/datasets/currency-codes/downloads\",\"issues_url\":\"https://api.github.com/repos/datasets/currency-codes/issues{/number}\",\"pulls_url\":\"https://api.github.com/repos/datasets/currency-codes/pulls{/number}\",\"milestones_url\":\"https://api.github.com/repos/datasets/currency-codes/milestones{/number}\",\"notifications_url\":\"https://api.github.com/repos/datasets/currency-codes/notifications{?since,all,participating}\",\"labels_url\":\"https://api.github.com/repos/datasets/currency-codes/labels{/name}\",\"releases_url\":\"https://api.github.com/repos/datasets/currency-codes/releases{/id}\",\"deployments_url\":\"https://api.github.com/repos/datasets/currency-codes/deployments\",\"created_at\":\"2012-11-14T23:19:47Z\",\"updated_at\":\"2025-04-15T19:49:30Z\",\"pushed_at\":\"2025-04-01T01:26:10Z\",\"git_url\":\"git://github.com/datasets/currency-codes.git\",\"ssh_url\":\"git@github.com:datasets/currency-codes.git\",\"clone_url\":\"https://github.com/datasets/currency-codes.git\",\"svn_url\":\"https://github.com/datasets/currency-codes\",\"homepage\":\"https://datahub.io/core/currency-codes\",\"size\":79,\"stargazers_count\":158,\"watchers_count\":158,\"language\":\"Shell\",\"has_issues\":true,\"has_projects\":true,\"has_downloads\":true,\"has_wiki\":true,\"has_pages\":false,\"has_discussions\":false,\"forks_count\":179,\"mirror_url\":null,\"archived\":false,\"disabled\":false,\"open_issues_count\":3,\"license\":null,\"allow_forking\":true,\"is_template\":false,\"web_commit_signoff_required\":false,\"topics\":[],\"visibility\":\"public\",\"forks\":179,\"open_issues\":3,\"watchers\":158,\"default_branch\":\"main\"},\"network_count\":179,\"subscribers_count\":0}" + "size": 1236, + "text": "{\"id\":220959516,\"node_id\":\"MDEwOlJlcG9zaXRvcnkyMjA5NTk1MTY=\",\"name\":\"data\",\"full_name\":\"roll/data\",\"private\":false,\"owner\":{\"login\":\"roll\",\"id\":557395,\"node_id\":\"MDQ6VXNlcjU1NzM5NQ==\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/557395?v=4\",\"gravatar_id\":\"\",\"url\":\"https://api.github.com/users/roll\",\"html_url\":\"https://github.com/roll\",\"followers_url\":\"https://api.github.com/users/roll/followers\",\"following_url\":\"https://api.github.com/users/roll/following{/other_user}\",\"gists_url\":\"https://api.github.com/users/roll/gists{/gist_id}\",\"starred_url\":\"https://api.github.com/users/roll/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/roll/subscriptions\",\"organizations_url\":\"https://api.github.com/users/roll/orgs\",\"repos_url\":\"https://api.github.com/users/roll/repos\",\"events_url\":\"https://api.github.com/users/roll/events{/privacy}\",\"received_events_url\":\"https://api.github.com/users/roll/received_events\",\"type\":\"User\",\"user_view_type\":\"public\",\"site_admin\":false},\"html_url\":\"https://github.com/roll/data\",\"description\":null,\"fork\":false,\"url\":\"https://api.github.com/repos/roll/data\",\"forks_url\":\"https://api.github.com/repos/roll/data/forks\",\"keys_url\":\"https://api.github.com/repos/roll/data/keys{/key_id}\",\"collaborators_url\":\"https://api.github.com/repos/roll/data/collaborators{/collaborator}\",\"teams_url\":\"https://api.github.com/repos/roll/data/teams\",\"hooks_url\":\"https://api.github.com/repos/roll/data/hooks\",\"issue_events_url\":\"https://api.github.com/repos/roll/data/issues/events{/number}\",\"events_url\":\"https://api.github.com/repos/roll/data/events\",\"assignees_url\":\"https://api.github.com/repos/roll/data/assignees{/user}\",\"branches_url\":\"https://api.github.com/repos/roll/data/branches{/branch}\",\"tags_url\":\"https://api.github.com/repos/roll/data/tags\",\"blobs_url\":\"https://api.github.com/repos/roll/data/git/blobs{/sha}\",\"git_tags_url\":\"https://api.github.com/repos/roll/data/git/tags{/sha}\",\"git_refs_url\":\"https://api.github.com/repos/roll/data/git/refs{/sha}\",\"trees_url\":\"https://api.github.com/repos/roll/data/git/trees{/sha}\",\"statuses_url\":\"https://api.github.com/repos/roll/data/statuses/{sha}\",\"languages_url\":\"https://api.github.com/repos/roll/data/languages\",\"stargazers_url\":\"https://api.github.com/repos/roll/data/stargazers\",\"contributors_url\":\"https://api.github.com/repos/roll/data/contributors\",\"subscribers_url\":\"https://api.github.com/repos/roll/data/subscribers\",\"subscription_url\":\"https://api.github.com/repos/roll/data/subscription\",\"commits_url\":\"https://api.github.com/repos/roll/data/commits{/sha}\",\"git_commits_url\":\"https://api.github.com/repos/roll/data/git/commits{/sha}\",\"comments_url\":\"https://api.github.com/repos/roll/data/comments{/number}\",\"issue_comment_url\":\"https://api.github.com/repos/roll/data/issues/comments{/number}\",\"contents_url\":\"https://api.github.com/repos/roll/data/contents/{+path}\",\"compare_url\":\"https://api.github.com/repos/roll/data/compare/{base}...{head}\",\"merges_url\":\"https://api.github.com/repos/roll/data/merges\",\"archive_url\":\"https://api.github.com/repos/roll/data/{archive_format}{/ref}\",\"downloads_url\":\"https://api.github.com/repos/roll/data/downloads\",\"issues_url\":\"https://api.github.com/repos/roll/data/issues{/number}\",\"pulls_url\":\"https://api.github.com/repos/roll/data/pulls{/number}\",\"milestones_url\":\"https://api.github.com/repos/roll/data/milestones{/number}\",\"notifications_url\":\"https://api.github.com/repos/roll/data/notifications{?since,all,participating}\",\"labels_url\":\"https://api.github.com/repos/roll/data/labels{/name}\",\"releases_url\":\"https://api.github.com/repos/roll/data/releases{/id}\",\"deployments_url\":\"https://api.github.com/repos/roll/data/deployments\",\"created_at\":\"2019-11-11T10:48:02Z\",\"updated_at\":\"2020-05-14T07:12:23Z\",\"pushed_at\":\"2020-05-14T07:12:13Z\",\"git_url\":\"git://github.com/roll/data.git\",\"ssh_url\":\"git@github.com:roll/data.git\",\"clone_url\":\"https://github.com/roll/data.git\",\"svn_url\":\"https://github.com/roll/data\",\"homepage\":null,\"size\":536228,\"stargazers_count\":1,\"watchers_count\":1,\"language\":\"JavaScript\",\"has_issues\":true,\"has_projects\":true,\"has_downloads\":true,\"has_wiki\":true,\"has_pages\":false,\"has_discussions\":false,\"forks_count\":0,\"mirror_url\":null,\"archived\":false,\"disabled\":false,\"open_issues_count\":0,\"license\":null,\"allow_forking\":true,\"is_template\":false,\"web_commit_signoff_required\":false,\"topics\":[],\"visibility\":\"public\",\"forks\":0,\"open_issues\":0,\"watchers\":1,\"default_branch\":\"master\",\"temp_clone_token\":null,\"network_count\":0,\"subscribers_count\":0}" }, "cookies": [], "headers": [ @@ -52,7 +52,7 @@ }, { "name": "content-length", - "value": "1990" + "value": "1236" }, { "name": "content-security-policy", @@ -64,15 +64,15 @@ }, { "name": "date", - "value": "Sat, 24 May 2025 11:31:08 GMT" + "value": "Wed, 28 May 2025 09:59:41 GMT" }, { "name": "etag", - "value": "W/\"93b52f692b954e6d1e8278d893c276f5a3c874a87588d75f27a83cd431427974\"" + "value": "W/\"cabe7e42ada08dc05ec06db40df679feb6e58d41f59407b819ccc4e49326c490\"" }, { "name": "last-modified", - "value": "Wed, 07 Jul 2021 14:47:03 GMT" + "value": "Thu, 14 May 2020 07:12:23 GMT" }, { "name": "referrer-policy", @@ -108,7 +108,7 @@ }, { "name": "x-github-request-id", - "value": "DA7A:A6F70:77F079:7BF99D:6831ADFC" + "value": "3878:1F0C59:13CFC:1456C:6836DE8D" }, { "name": "x-ratelimit-limit", @@ -116,11 +116,11 @@ }, { "name": "x-ratelimit-remaining", - "value": "41" + "value": "59" }, { "name": "x-ratelimit-reset", - "value": "1748088233" + "value": "1748429981" }, { "name": "x-ratelimit-resource", @@ -128,21 +128,21 @@ }, { "name": "x-ratelimit-used", - "value": "19" + "value": "1" }, { "name": "x-xss-protection", "value": "0" } ], - "headersSize": 1294, + "headersSize": 1292, "httpVersion": "HTTP/1.1", "redirectURL": "", "status": 200, "statusText": "OK" }, - "startedDateTime": "2025-05-24T11:31:08.101Z", - "time": 566, + "startedDateTime": "2025-05-28T09:59:40.738Z", + "time": 759, "timings": { "blocked": -1, "connect": -1, @@ -150,18 +150,18 @@ "receive": 0, "send": 0, "ssl": -1, - "wait": 566 + "wait": 759 } }, { - "_id": "392264ee4148eff756557a8163d81440", + "_id": "5c1796f31beae7f63ee90a22a252053d", "_order": 0, "cache": {}, "request": { "bodySize": 0, "cookies": [], "headers": [], - "headersSize": 96, + "headersSize": 86, "httpVersion": "HTTP/1.1", "method": "GET", "queryString": [ @@ -170,14 +170,14 @@ "value": "1" } ], - "url": "https://api.github.com/repos/roll/currency-codes/git/trees/master?recursive=1" + "url": "https://api.github.com/repos/roll/data/git/trees/master?recursive=1" }, "response": { - "bodySize": 874, + "bodySize": 798, "content": { "mimeType": "application/json; charset=utf-8", - "size": 874, - "text": "{\"sha\":\"e855efbcd7344e9ad8d89a8a1a31979678d67b17\",\"url\":\"https://api.github.com/repos/roll/currency-codes/git/trees/e855efbcd7344e9ad8d89a8a1a31979678d67b17\",\"tree\":[{\"path\":\".github\",\"mode\":\"040000\",\"type\":\"tree\",\"sha\":\"0c5ecb959cab3cebda3cf98e4bab0950a6242be3\",\"url\":\"https://api.github.com/repos/roll/currency-codes/git/trees/0c5ecb959cab3cebda3cf98e4bab0950a6242be3\"},{\"path\":\".github/frictionless.yaml\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"2bf0512efed4418e9264928a2bffa63af6ab1944\",\"size\":455,\"url\":\"https://api.github.com/repos/roll/currency-codes/git/blobs/2bf0512efed4418e9264928a2bffa63af6ab1944\"},{\"path\":\".github/workflows\",\"mode\":\"040000\",\"type\":\"tree\",\"sha\":\"842fcd20ad09704fbd17ccaf960e01a8b611cd7a\",\"url\":\"https://api.github.com/repos/roll/currency-codes/git/trees/842fcd20ad09704fbd17ccaf960e01a8b611cd7a\"},{\"path\":\".github/workflows/frictionless.yaml\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"aac9b28f30362e3e4c6e950a9bdd17e01280cd83\",\"size\":304,\"url\":\"https://api.github.com/repos/roll/currency-codes/git/blobs/aac9b28f30362e3e4c6e950a9bdd17e01280cd83\"},{\"path\":\".gitignore\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"7b14861d4f9b884288afd38fe4e7469789e9b34c\",\"size\":21,\"url\":\"https://api.github.com/repos/roll/currency-codes/git/blobs/7b14861d4f9b884288afd38fe4e7469789e9b34c\"},{\"path\":\".travis.yml\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"67c7e5588ed57eb0014cd8b3e6540000037dc7ba\",\"size\":294,\"url\":\"https://api.github.com/repos/roll/currency-codes/git/blobs/67c7e5588ed57eb0014cd8b3e6540000037dc7ba\"},{\"path\":\"README.md\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"9c36869b88dfad9b9d610734966270b7e883b707\",\"size\":1752,\"url\":\"https://api.github.com/repos/roll/currency-codes/git/blobs/9c36869b88dfad9b9d610734966270b7e883b707\"},{\"path\":\"archive\",\"mode\":\"040000\",\"type\":\"tree\",\"sha\":\"39b08adf12bb3f3adc170b3a77f338f8e02b63d1\",\"url\":\"https://api.github.com/repos/roll/currency-codes/git/trees/39b08adf12bb3f3adc170b3a77f338f8e02b63d1\"},{\"path\":\"archive/table_a1.xml\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"42609efee29fb5a7d98935b806ea135e3f455653\",\"size\":47281,\"url\":\"https://api.github.com/repos/roll/currency-codes/git/blobs/42609efee29fb5a7d98935b806ea135e3f455653\"},{\"path\":\"archive/table_a2.xml\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"b2a569c403e6917aa466d0973d3b3f677821530f\",\"size\":29349,\"url\":\"https://api.github.com/repos/roll/currency-codes/git/blobs/b2a569c403e6917aa466d0973d3b3f677821530f\"},{\"path\":\"data\",\"mode\":\"040000\",\"type\":\"tree\",\"sha\":\"0cf8433969ede15de089c1127cb6937d2ae13e3e\",\"url\":\"https://api.github.com/repos/roll/currency-codes/git/trees/0cf8433969ede15de089c1127cb6937d2ae13e3e\"},{\"path\":\"data/codes-all.csv\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"2c803df30c1d09e1c7f5de6661ca38f8e627d65b\",\"size\":17472,\"url\":\"https://api.github.com/repos/roll/currency-codes/git/blobs/2c803df30c1d09e1c7f5de6661ca38f8e627d65b\"},{\"path\":\"datapackage.json\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"dfca466367e0b3d23294df83b72101cb5b084c09\",\"size\":2019,\"url\":\"https://api.github.com/repos/roll/currency-codes/git/blobs/dfca466367e0b3d23294df83b72101cb5b084c09\"},{\"path\":\"scripts\",\"mode\":\"040000\",\"type\":\"tree\",\"sha\":\"ff0526e9a5e2d76f31d8038cbea148067203776a\",\"url\":\"https://api.github.com/repos/roll/currency-codes/git/trees/ff0526e9a5e2d76f31d8038cbea148067203776a\"},{\"path\":\"scripts/runall.sh\",\"mode\":\"100755\",\"type\":\"blob\",\"sha\":\"4632c9933cd6c76331301efa71dc5cd183c6dc89\",\"size\":3098,\"url\":\"https://api.github.com/repos/roll/currency-codes/git/blobs/4632c9933cd6c76331301efa71dc5cd183c6dc89\"}],\"truncated\":false}" + "size": 798, + "text": "{\"sha\":\"8094ea97a76e9435f90a9a73398754d7ec94cfac\",\"url\":\"https://api.github.com/repos/roll/data/git/trees/8094ea97a76e9435f90a9a73398754d7ec94cfac\",\"tree\":[{\"path\":\".gitignore\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"1f91d773a9b5e361a699bc10eff921c841831612\",\"size\":787,\"url\":\"https://api.github.com/repos/roll/data/git/blobs/1f91d773a9b5e361a699bc10eff921c841831612\"},{\"path\":\".prettierrc\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"36a4660e1d068962321affe174c604c8b5bfa6d5\",\"size\":90,\"url\":\"https://api.github.com/repos/roll/data/git/blobs/36a4660e1d068962321affe174c604c8b5bfa6d5\"},{\"path\":\"README.md\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"31cd35605bbee1d3b3f297d5b0462028a46a0380\",\"size\":6,\"url\":\"https://api.github.com/repos/roll/data/git/blobs/31cd35605bbee1d3b3f297d5b0462028a46a0380\"},{\"path\":\"data\",\"mode\":\"040000\",\"type\":\"tree\",\"sha\":\"6bec63ab526a9fb256f4681c48b9f760b5675eba\",\"url\":\"https://api.github.com/repos/roll/data/git/trees/6bec63ab526a9fb256f4681c48b9f760b5675eba\"},{\"path\":\"data/data.zip\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"7c32a3955b961c3fe7646fac499177ab5e5af524\",\"size\":67920590,\"url\":\"https://api.github.com/repos/roll/data/git/blobs/7c32a3955b961c3fe7646fac499177ab5e5af524\"},{\"path\":\"data/meta.json\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"8905b138ed93ebb9e78945126ac2a423d31babc9\",\"size\":192,\"url\":\"https://api.github.com/repos/roll/data/git/blobs/8905b138ed93ebb9e78945126ac2a423d31babc9\"},{\"path\":\"data/table1.csv\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"f79b42b814066ec87cc734d932c64733c47444df\",\"size\":1000000,\"url\":\"https://api.github.com/repos/roll/data/git/blobs/f79b42b814066ec87cc734d932c64733c47444df\"},{\"path\":\"data/table10.csv\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"f352795df6534642cc0a9fe6d9b2b3f8fa80135a\",\"size\":10000000,\"url\":\"https://api.github.com/repos/roll/data/git/blobs/f352795df6534642cc0a9fe6d9b2b3f8fa80135a\"},{\"path\":\"data/table100.csv\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"734520644f4616825209c0c7a0750e5887b93a76\",\"size\":100000000,\"url\":\"https://api.github.com/repos/roll/data/git/blobs/734520644f4616825209c0c7a0750e5887b93a76\"},{\"path\":\"data/table5.csv\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"306b2975e0e469a2eec853b1a4856acb25a1dab6\",\"size\":5000000,\"url\":\"https://api.github.com/repos/roll/data/git/blobs/306b2975e0e469a2eec853b1a4856acb25a1dab6\"},{\"path\":\"data/table50.csv\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"eca005a30f75202d7ccdadbbb6789a9ae0389555\",\"size\":50000000,\"url\":\"https://api.github.com/repos/roll/data/git/blobs/eca005a30f75202d7ccdadbbb6789a9ae0389555\"},{\"path\":\"package.json\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"4564795a02255050972182a43590b853dee8b8b5\",\"size\":513,\"url\":\"https://api.github.com/repos/roll/data/git/blobs/4564795a02255050972182a43590b853dee8b8b5\"},{\"path\":\"scripts\",\"mode\":\"040000\",\"type\":\"tree\",\"sha\":\"43a35f56f388fae0badbb2c5ce00489a982e796d\",\"url\":\"https://api.github.com/repos/roll/data/git/trees/43a35f56f388fae0badbb2c5ce00489a982e796d\"},{\"path\":\"scripts/generate.js\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"59f443ca5062f3f84937490509d6cc25571fb853\",\"size\":1952,\"url\":\"https://api.github.com/repos/roll/data/git/blobs/59f443ca5062f3f84937490509d6cc25571fb853\"}],\"truncated\":false}" }, "cookies": [], "headers": [ @@ -203,7 +203,7 @@ }, { "name": "content-length", - "value": "874" + "value": "798" }, { "name": "content-security-policy", @@ -215,15 +215,15 @@ }, { "name": "date", - "value": "Sat, 24 May 2025 11:31:08 GMT" + "value": "Wed, 28 May 2025 09:59:41 GMT" }, { "name": "etag", - "value": "W/\"4635290695f3b6591e9e38e1e37992e8a758b746f29fb6288f64ac5ea26b5d2d\"" + "value": "W/\"8b0b83664974e44f10a88c20b559e755b40b0569eb248c3d35681217d7acdd44\"" }, { "name": "last-modified", - "value": "Wed, 07 Jul 2021 14:47:03 GMT" + "value": "Thu, 14 May 2020 07:12:23 GMT" }, { "name": "referrer-policy", @@ -259,7 +259,7 @@ }, { "name": "x-github-request-id", - "value": "DA7A:A6F70:77F194:7BFABD:6831ADFC" + "value": "3878:1F0C59:13E18:146B0:6836DE8D" }, { "name": "x-ratelimit-limit", @@ -267,11 +267,11 @@ }, { "name": "x-ratelimit-remaining", - "value": "40" + "value": "58" }, { "name": "x-ratelimit-reset", - "value": "1748088233" + "value": "1748429981" }, { "name": "x-ratelimit-resource", @@ -279,21 +279,21 @@ }, { "name": "x-ratelimit-used", - "value": "20" + "value": "2" }, { "name": "x-xss-protection", "value": "0" } ], - "headersSize": 1293, + "headersSize": 1291, "httpVersion": "HTTP/1.1", "redirectURL": "", "status": 200, "statusText": "OK" }, - "startedDateTime": "2025-05-24T11:31:08.670Z", - "time": 238, + "startedDateTime": "2025-05-28T09:59:41.502Z", + "time": 288, "timings": { "blocked": -1, "connect": -1, @@ -301,7 +301,7 @@ "receive": 0, "send": 0, "ssl": -1, - "wait": 238 + "wait": 288 } } ], diff --git a/github/package/fixtures/generated/should-merge-datapackage-json-if-present_1884846678/recording.har b/github/package/fixtures/generated/should-merge-datapackage-json-if-present_1884846678/recording.har new file mode 100644 index 00000000..55aa6aec --- /dev/null +++ b/github/package/fixtures/generated/should-merge-datapackage-json-if-present_1884846678/recording.har @@ -0,0 +1,453 @@ +{ + "log": { + "_recordingName": "should merge datapackage.json if present", + "creator": { + "comment": "persister:fs", + "name": "Polly.JS", + "version": "6.0.6" + }, + "entries": [ + { + "_id": "1a1b6f9f2e0280a7960e568ae456fc69", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [], + "headersSize": 67, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://api.github.com/repos/roll/currency-codes" + }, + "response": { + "bodySize": 1990, + "content": { + "mimeType": "application/json; charset=utf-8", + "size": 1990, + "text": "{\"id\":383820850,\"node_id\":\"MDEwOlJlcG9zaXRvcnkzODM4MjA4NTA=\",\"name\":\"currency-codes\",\"full_name\":\"roll/currency-codes\",\"private\":false,\"owner\":{\"login\":\"roll\",\"id\":557395,\"node_id\":\"MDQ6VXNlcjU1NzM5NQ==\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/557395?v=4\",\"gravatar_id\":\"\",\"url\":\"https://api.github.com/users/roll\",\"html_url\":\"https://github.com/roll\",\"followers_url\":\"https://api.github.com/users/roll/followers\",\"following_url\":\"https://api.github.com/users/roll/following{/other_user}\",\"gists_url\":\"https://api.github.com/users/roll/gists{/gist_id}\",\"starred_url\":\"https://api.github.com/users/roll/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/roll/subscriptions\",\"organizations_url\":\"https://api.github.com/users/roll/orgs\",\"repos_url\":\"https://api.github.com/users/roll/repos\",\"events_url\":\"https://api.github.com/users/roll/events{/privacy}\",\"received_events_url\":\"https://api.github.com/users/roll/received_events\",\"type\":\"User\",\"user_view_type\":\"public\",\"site_admin\":false},\"html_url\":\"https://github.com/roll/currency-codes\",\"description\":\"ISO 4217 List of Currencies and Currency Codes\",\"fork\":true,\"url\":\"https://api.github.com/repos/roll/currency-codes\",\"forks_url\":\"https://api.github.com/repos/roll/currency-codes/forks\",\"keys_url\":\"https://api.github.com/repos/roll/currency-codes/keys{/key_id}\",\"collaborators_url\":\"https://api.github.com/repos/roll/currency-codes/collaborators{/collaborator}\",\"teams_url\":\"https://api.github.com/repos/roll/currency-codes/teams\",\"hooks_url\":\"https://api.github.com/repos/roll/currency-codes/hooks\",\"issue_events_url\":\"https://api.github.com/repos/roll/currency-codes/issues/events{/number}\",\"events_url\":\"https://api.github.com/repos/roll/currency-codes/events\",\"assignees_url\":\"https://api.github.com/repos/roll/currency-codes/assignees{/user}\",\"branches_url\":\"https://api.github.com/repos/roll/currency-codes/branches{/branch}\",\"tags_url\":\"https://api.github.com/repos/roll/currency-codes/tags\",\"blobs_url\":\"https://api.github.com/repos/roll/currency-codes/git/blobs{/sha}\",\"git_tags_url\":\"https://api.github.com/repos/roll/currency-codes/git/tags{/sha}\",\"git_refs_url\":\"https://api.github.com/repos/roll/currency-codes/git/refs{/sha}\",\"trees_url\":\"https://api.github.com/repos/roll/currency-codes/git/trees{/sha}\",\"statuses_url\":\"https://api.github.com/repos/roll/currency-codes/statuses/{sha}\",\"languages_url\":\"https://api.github.com/repos/roll/currency-codes/languages\",\"stargazers_url\":\"https://api.github.com/repos/roll/currency-codes/stargazers\",\"contributors_url\":\"https://api.github.com/repos/roll/currency-codes/contributors\",\"subscribers_url\":\"https://api.github.com/repos/roll/currency-codes/subscribers\",\"subscription_url\":\"https://api.github.com/repos/roll/currency-codes/subscription\",\"commits_url\":\"https://api.github.com/repos/roll/currency-codes/commits{/sha}\",\"git_commits_url\":\"https://api.github.com/repos/roll/currency-codes/git/commits{/sha}\",\"comments_url\":\"https://api.github.com/repos/roll/currency-codes/comments{/number}\",\"issue_comment_url\":\"https://api.github.com/repos/roll/currency-codes/issues/comments{/number}\",\"contents_url\":\"https://api.github.com/repos/roll/currency-codes/contents/{+path}\",\"compare_url\":\"https://api.github.com/repos/roll/currency-codes/compare/{base}...{head}\",\"merges_url\":\"https://api.github.com/repos/roll/currency-codes/merges\",\"archive_url\":\"https://api.github.com/repos/roll/currency-codes/{archive_format}{/ref}\",\"downloads_url\":\"https://api.github.com/repos/roll/currency-codes/downloads\",\"issues_url\":\"https://api.github.com/repos/roll/currency-codes/issues{/number}\",\"pulls_url\":\"https://api.github.com/repos/roll/currency-codes/pulls{/number}\",\"milestones_url\":\"https://api.github.com/repos/roll/currency-codes/milestones{/number}\",\"notifications_url\":\"https://api.github.com/repos/roll/currency-codes/notifications{?since,all,participating}\",\"labels_url\":\"https://api.github.com/repos/roll/currency-codes/labels{/name}\",\"releases_url\":\"https://api.github.com/repos/roll/currency-codes/releases{/id}\",\"deployments_url\":\"https://api.github.com/repos/roll/currency-codes/deployments\",\"created_at\":\"2021-07-07T14:09:49Z\",\"updated_at\":\"2021-07-07T14:47:03Z\",\"pushed_at\":\"2021-07-07T14:47:01Z\",\"git_url\":\"git://github.com/roll/currency-codes.git\",\"ssh_url\":\"git@github.com:roll/currency-codes.git\",\"clone_url\":\"https://github.com/roll/currency-codes.git\",\"svn_url\":\"https://github.com/roll/currency-codes\",\"homepage\":\"https://datahub.io/core/currency-codes\",\"size\":87,\"stargazers_count\":0,\"watchers_count\":0,\"language\":\"Shell\",\"has_issues\":false,\"has_projects\":true,\"has_downloads\":true,\"has_wiki\":true,\"has_pages\":false,\"has_discussions\":false,\"forks_count\":0,\"mirror_url\":null,\"archived\":false,\"disabled\":false,\"open_issues_count\":0,\"license\":null,\"allow_forking\":true,\"is_template\":false,\"web_commit_signoff_required\":false,\"topics\":[],\"visibility\":\"public\",\"forks\":0,\"open_issues\":0,\"watchers\":0,\"default_branch\":\"master\",\"temp_clone_token\":null,\"parent\":{\"id\":6696644,\"node_id\":\"MDEwOlJlcG9zaXRvcnk2Njk2NjQ0\",\"name\":\"currency-codes\",\"full_name\":\"datasets/currency-codes\",\"private\":false,\"owner\":{\"login\":\"datasets\",\"id\":1643515,\"node_id\":\"MDEyOk9yZ2FuaXphdGlvbjE2NDM1MTU=\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/1643515?v=4\",\"gravatar_id\":\"\",\"url\":\"https://api.github.com/users/datasets\",\"html_url\":\"https://github.com/datasets\",\"followers_url\":\"https://api.github.com/users/datasets/followers\",\"following_url\":\"https://api.github.com/users/datasets/following{/other_user}\",\"gists_url\":\"https://api.github.com/users/datasets/gists{/gist_id}\",\"starred_url\":\"https://api.github.com/users/datasets/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/datasets/subscriptions\",\"organizations_url\":\"https://api.github.com/users/datasets/orgs\",\"repos_url\":\"https://api.github.com/users/datasets/repos\",\"events_url\":\"https://api.github.com/users/datasets/events{/privacy}\",\"received_events_url\":\"https://api.github.com/users/datasets/received_events\",\"type\":\"Organization\",\"user_view_type\":\"public\",\"site_admin\":false},\"html_url\":\"https://github.com/datasets/currency-codes\",\"description\":\"ISO 4217 List of Currencies and Currency Codes\",\"fork\":false,\"url\":\"https://api.github.com/repos/datasets/currency-codes\",\"forks_url\":\"https://api.github.com/repos/datasets/currency-codes/forks\",\"keys_url\":\"https://api.github.com/repos/datasets/currency-codes/keys{/key_id}\",\"collaborators_url\":\"https://api.github.com/repos/datasets/currency-codes/collaborators{/collaborator}\",\"teams_url\":\"https://api.github.com/repos/datasets/currency-codes/teams\",\"hooks_url\":\"https://api.github.com/repos/datasets/currency-codes/hooks\",\"issue_events_url\":\"https://api.github.com/repos/datasets/currency-codes/issues/events{/number}\",\"events_url\":\"https://api.github.com/repos/datasets/currency-codes/events\",\"assignees_url\":\"https://api.github.com/repos/datasets/currency-codes/assignees{/user}\",\"branches_url\":\"https://api.github.com/repos/datasets/currency-codes/branches{/branch}\",\"tags_url\":\"https://api.github.com/repos/datasets/currency-codes/tags\",\"blobs_url\":\"https://api.github.com/repos/datasets/currency-codes/git/blobs{/sha}\",\"git_tags_url\":\"https://api.github.com/repos/datasets/currency-codes/git/tags{/sha}\",\"git_refs_url\":\"https://api.github.com/repos/datasets/currency-codes/git/refs{/sha}\",\"trees_url\":\"https://api.github.com/repos/datasets/currency-codes/git/trees{/sha}\",\"statuses_url\":\"https://api.github.com/repos/datasets/currency-codes/statuses/{sha}\",\"languages_url\":\"https://api.github.com/repos/datasets/currency-codes/languages\",\"stargazers_url\":\"https://api.github.com/repos/datasets/currency-codes/stargazers\",\"contributors_url\":\"https://api.github.com/repos/datasets/currency-codes/contributors\",\"subscribers_url\":\"https://api.github.com/repos/datasets/currency-codes/subscribers\",\"subscription_url\":\"https://api.github.com/repos/datasets/currency-codes/subscription\",\"commits_url\":\"https://api.github.com/repos/datasets/currency-codes/commits{/sha}\",\"git_commits_url\":\"https://api.github.com/repos/datasets/currency-codes/git/commits{/sha}\",\"comments_url\":\"https://api.github.com/repos/datasets/currency-codes/comments{/number}\",\"issue_comment_url\":\"https://api.github.com/repos/datasets/currency-codes/issues/comments{/number}\",\"contents_url\":\"https://api.github.com/repos/datasets/currency-codes/contents/{+path}\",\"compare_url\":\"https://api.github.com/repos/datasets/currency-codes/compare/{base}...{head}\",\"merges_url\":\"https://api.github.com/repos/datasets/currency-codes/merges\",\"archive_url\":\"https://api.github.com/repos/datasets/currency-codes/{archive_format}{/ref}\",\"downloads_url\":\"https://api.github.com/repos/datasets/currency-codes/downloads\",\"issues_url\":\"https://api.github.com/repos/datasets/currency-codes/issues{/number}\",\"pulls_url\":\"https://api.github.com/repos/datasets/currency-codes/pulls{/number}\",\"milestones_url\":\"https://api.github.com/repos/datasets/currency-codes/milestones{/number}\",\"notifications_url\":\"https://api.github.com/repos/datasets/currency-codes/notifications{?since,all,participating}\",\"labels_url\":\"https://api.github.com/repos/datasets/currency-codes/labels{/name}\",\"releases_url\":\"https://api.github.com/repos/datasets/currency-codes/releases{/id}\",\"deployments_url\":\"https://api.github.com/repos/datasets/currency-codes/deployments\",\"created_at\":\"2012-11-14T23:19:47Z\",\"updated_at\":\"2025-04-15T19:49:30Z\",\"pushed_at\":\"2025-04-01T01:26:10Z\",\"git_url\":\"git://github.com/datasets/currency-codes.git\",\"ssh_url\":\"git@github.com:datasets/currency-codes.git\",\"clone_url\":\"https://github.com/datasets/currency-codes.git\",\"svn_url\":\"https://github.com/datasets/currency-codes\",\"homepage\":\"https://datahub.io/core/currency-codes\",\"size\":79,\"stargazers_count\":158,\"watchers_count\":158,\"language\":\"Shell\",\"has_issues\":true,\"has_projects\":true,\"has_downloads\":true,\"has_wiki\":true,\"has_pages\":false,\"has_discussions\":false,\"forks_count\":179,\"mirror_url\":null,\"archived\":false,\"disabled\":false,\"open_issues_count\":3,\"license\":null,\"allow_forking\":true,\"is_template\":false,\"web_commit_signoff_required\":false,\"topics\":[],\"visibility\":\"public\",\"forks\":179,\"open_issues\":3,\"watchers\":158,\"default_branch\":\"main\"},\"source\":{\"id\":6696644,\"node_id\":\"MDEwOlJlcG9zaXRvcnk2Njk2NjQ0\",\"name\":\"currency-codes\",\"full_name\":\"datasets/currency-codes\",\"private\":false,\"owner\":{\"login\":\"datasets\",\"id\":1643515,\"node_id\":\"MDEyOk9yZ2FuaXphdGlvbjE2NDM1MTU=\",\"avatar_url\":\"https://avatars.githubusercontent.com/u/1643515?v=4\",\"gravatar_id\":\"\",\"url\":\"https://api.github.com/users/datasets\",\"html_url\":\"https://github.com/datasets\",\"followers_url\":\"https://api.github.com/users/datasets/followers\",\"following_url\":\"https://api.github.com/users/datasets/following{/other_user}\",\"gists_url\":\"https://api.github.com/users/datasets/gists{/gist_id}\",\"starred_url\":\"https://api.github.com/users/datasets/starred{/owner}{/repo}\",\"subscriptions_url\":\"https://api.github.com/users/datasets/subscriptions\",\"organizations_url\":\"https://api.github.com/users/datasets/orgs\",\"repos_url\":\"https://api.github.com/users/datasets/repos\",\"events_url\":\"https://api.github.com/users/datasets/events{/privacy}\",\"received_events_url\":\"https://api.github.com/users/datasets/received_events\",\"type\":\"Organization\",\"user_view_type\":\"public\",\"site_admin\":false},\"html_url\":\"https://github.com/datasets/currency-codes\",\"description\":\"ISO 4217 List of Currencies and Currency Codes\",\"fork\":false,\"url\":\"https://api.github.com/repos/datasets/currency-codes\",\"forks_url\":\"https://api.github.com/repos/datasets/currency-codes/forks\",\"keys_url\":\"https://api.github.com/repos/datasets/currency-codes/keys{/key_id}\",\"collaborators_url\":\"https://api.github.com/repos/datasets/currency-codes/collaborators{/collaborator}\",\"teams_url\":\"https://api.github.com/repos/datasets/currency-codes/teams\",\"hooks_url\":\"https://api.github.com/repos/datasets/currency-codes/hooks\",\"issue_events_url\":\"https://api.github.com/repos/datasets/currency-codes/issues/events{/number}\",\"events_url\":\"https://api.github.com/repos/datasets/currency-codes/events\",\"assignees_url\":\"https://api.github.com/repos/datasets/currency-codes/assignees{/user}\",\"branches_url\":\"https://api.github.com/repos/datasets/currency-codes/branches{/branch}\",\"tags_url\":\"https://api.github.com/repos/datasets/currency-codes/tags\",\"blobs_url\":\"https://api.github.com/repos/datasets/currency-codes/git/blobs{/sha}\",\"git_tags_url\":\"https://api.github.com/repos/datasets/currency-codes/git/tags{/sha}\",\"git_refs_url\":\"https://api.github.com/repos/datasets/currency-codes/git/refs{/sha}\",\"trees_url\":\"https://api.github.com/repos/datasets/currency-codes/git/trees{/sha}\",\"statuses_url\":\"https://api.github.com/repos/datasets/currency-codes/statuses/{sha}\",\"languages_url\":\"https://api.github.com/repos/datasets/currency-codes/languages\",\"stargazers_url\":\"https://api.github.com/repos/datasets/currency-codes/stargazers\",\"contributors_url\":\"https://api.github.com/repos/datasets/currency-codes/contributors\",\"subscribers_url\":\"https://api.github.com/repos/datasets/currency-codes/subscribers\",\"subscription_url\":\"https://api.github.com/repos/datasets/currency-codes/subscription\",\"commits_url\":\"https://api.github.com/repos/datasets/currency-codes/commits{/sha}\",\"git_commits_url\":\"https://api.github.com/repos/datasets/currency-codes/git/commits{/sha}\",\"comments_url\":\"https://api.github.com/repos/datasets/currency-codes/comments{/number}\",\"issue_comment_url\":\"https://api.github.com/repos/datasets/currency-codes/issues/comments{/number}\",\"contents_url\":\"https://api.github.com/repos/datasets/currency-codes/contents/{+path}\",\"compare_url\":\"https://api.github.com/repos/datasets/currency-codes/compare/{base}...{head}\",\"merges_url\":\"https://api.github.com/repos/datasets/currency-codes/merges\",\"archive_url\":\"https://api.github.com/repos/datasets/currency-codes/{archive_format}{/ref}\",\"downloads_url\":\"https://api.github.com/repos/datasets/currency-codes/downloads\",\"issues_url\":\"https://api.github.com/repos/datasets/currency-codes/issues{/number}\",\"pulls_url\":\"https://api.github.com/repos/datasets/currency-codes/pulls{/number}\",\"milestones_url\":\"https://api.github.com/repos/datasets/currency-codes/milestones{/number}\",\"notifications_url\":\"https://api.github.com/repos/datasets/currency-codes/notifications{?since,all,participating}\",\"labels_url\":\"https://api.github.com/repos/datasets/currency-codes/labels{/name}\",\"releases_url\":\"https://api.github.com/repos/datasets/currency-codes/releases{/id}\",\"deployments_url\":\"https://api.github.com/repos/datasets/currency-codes/deployments\",\"created_at\":\"2012-11-14T23:19:47Z\",\"updated_at\":\"2025-04-15T19:49:30Z\",\"pushed_at\":\"2025-04-01T01:26:10Z\",\"git_url\":\"git://github.com/datasets/currency-codes.git\",\"ssh_url\":\"git@github.com:datasets/currency-codes.git\",\"clone_url\":\"https://github.com/datasets/currency-codes.git\",\"svn_url\":\"https://github.com/datasets/currency-codes\",\"homepage\":\"https://datahub.io/core/currency-codes\",\"size\":79,\"stargazers_count\":158,\"watchers_count\":158,\"language\":\"Shell\",\"has_issues\":true,\"has_projects\":true,\"has_downloads\":true,\"has_wiki\":true,\"has_pages\":false,\"has_discussions\":false,\"forks_count\":179,\"mirror_url\":null,\"archived\":false,\"disabled\":false,\"open_issues_count\":3,\"license\":null,\"allow_forking\":true,\"is_template\":false,\"web_commit_signoff_required\":false,\"topics\":[],\"visibility\":\"public\",\"forks\":179,\"open_issues\":3,\"watchers\":158,\"default_branch\":\"main\"},\"network_count\":179,\"subscribers_count\":0}" + }, + "cookies": [], + "headers": [ + { + "name": "accept-ranges", + "value": "bytes" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "access-control-expose-headers", + "value": "ETag, Link, Location, Retry-After, X-GitHub-OTP, X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Used, X-RateLimit-Resource, X-RateLimit-Reset, X-OAuth-Scopes, X-Accepted-OAuth-Scopes, X-Poll-Interval, X-GitHub-Media-Type, X-GitHub-SSO, X-GitHub-Request-Id, Deprecation, Sunset" + }, + { + "name": "cache-control", + "value": "public, max-age=60, s-maxage=60" + }, + { + "name": "content-encoding", + "value": "gzip" + }, + { + "name": "content-length", + "value": "1990" + }, + { + "name": "content-security-policy", + "value": "default-src 'none'" + }, + { + "name": "content-type", + "value": "application/json; charset=utf-8" + }, + { + "name": "date", + "value": "Wed, 28 May 2025 09:59:41 GMT" + }, + { + "name": "etag", + "value": "W/\"93b52f692b954e6d1e8278d893c276f5a3c874a87588d75f27a83cd431427974\"" + }, + { + "name": "last-modified", + "value": "Wed, 07 Jul 2021 14:47:03 GMT" + }, + { + "name": "referrer-policy", + "value": "origin-when-cross-origin, strict-origin-when-cross-origin" + }, + { + "name": "server", + "value": "github.com" + }, + { + "name": "strict-transport-security", + "value": "max-age=31536000; includeSubdomains; preload" + }, + { + "name": "vary", + "value": "Accept,Accept-Encoding, Accept, X-Requested-With" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "x-frame-options", + "value": "deny" + }, + { + "name": "x-github-api-version-selected", + "value": "2022-11-28" + }, + { + "name": "x-github-media-type", + "value": "github.v3; format=json" + }, + { + "name": "x-github-request-id", + "value": "3878:1F0C59:13F54:147EE:6836DE8D" + }, + { + "name": "x-ratelimit-limit", + "value": "60" + }, + { + "name": "x-ratelimit-remaining", + "value": "57" + }, + { + "name": "x-ratelimit-reset", + "value": "1748429981" + }, + { + "name": "x-ratelimit-resource", + "value": "core" + }, + { + "name": "x-ratelimit-used", + "value": "3" + }, + { + "name": "x-xss-protection", + "value": "0" + } + ], + "headersSize": 1292, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-05-28T09:59:41.798Z", + "time": 277, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 277 + } + }, + { + "_id": "392264ee4148eff756557a8163d81440", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [], + "headersSize": 96, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [ + { + "name": "recursive", + "value": "1" + } + ], + "url": "https://api.github.com/repos/roll/currency-codes/git/trees/master?recursive=1" + }, + "response": { + "bodySize": 874, + "content": { + "mimeType": "application/json; charset=utf-8", + "size": 874, + "text": "{\"sha\":\"e855efbcd7344e9ad8d89a8a1a31979678d67b17\",\"url\":\"https://api.github.com/repos/roll/currency-codes/git/trees/e855efbcd7344e9ad8d89a8a1a31979678d67b17\",\"tree\":[{\"path\":\".github\",\"mode\":\"040000\",\"type\":\"tree\",\"sha\":\"0c5ecb959cab3cebda3cf98e4bab0950a6242be3\",\"url\":\"https://api.github.com/repos/roll/currency-codes/git/trees/0c5ecb959cab3cebda3cf98e4bab0950a6242be3\"},{\"path\":\".github/frictionless.yaml\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"2bf0512efed4418e9264928a2bffa63af6ab1944\",\"size\":455,\"url\":\"https://api.github.com/repos/roll/currency-codes/git/blobs/2bf0512efed4418e9264928a2bffa63af6ab1944\"},{\"path\":\".github/workflows\",\"mode\":\"040000\",\"type\":\"tree\",\"sha\":\"842fcd20ad09704fbd17ccaf960e01a8b611cd7a\",\"url\":\"https://api.github.com/repos/roll/currency-codes/git/trees/842fcd20ad09704fbd17ccaf960e01a8b611cd7a\"},{\"path\":\".github/workflows/frictionless.yaml\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"aac9b28f30362e3e4c6e950a9bdd17e01280cd83\",\"size\":304,\"url\":\"https://api.github.com/repos/roll/currency-codes/git/blobs/aac9b28f30362e3e4c6e950a9bdd17e01280cd83\"},{\"path\":\".gitignore\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"7b14861d4f9b884288afd38fe4e7469789e9b34c\",\"size\":21,\"url\":\"https://api.github.com/repos/roll/currency-codes/git/blobs/7b14861d4f9b884288afd38fe4e7469789e9b34c\"},{\"path\":\".travis.yml\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"67c7e5588ed57eb0014cd8b3e6540000037dc7ba\",\"size\":294,\"url\":\"https://api.github.com/repos/roll/currency-codes/git/blobs/67c7e5588ed57eb0014cd8b3e6540000037dc7ba\"},{\"path\":\"README.md\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"9c36869b88dfad9b9d610734966270b7e883b707\",\"size\":1752,\"url\":\"https://api.github.com/repos/roll/currency-codes/git/blobs/9c36869b88dfad9b9d610734966270b7e883b707\"},{\"path\":\"archive\",\"mode\":\"040000\",\"type\":\"tree\",\"sha\":\"39b08adf12bb3f3adc170b3a77f338f8e02b63d1\",\"url\":\"https://api.github.com/repos/roll/currency-codes/git/trees/39b08adf12bb3f3adc170b3a77f338f8e02b63d1\"},{\"path\":\"archive/table_a1.xml\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"42609efee29fb5a7d98935b806ea135e3f455653\",\"size\":47281,\"url\":\"https://api.github.com/repos/roll/currency-codes/git/blobs/42609efee29fb5a7d98935b806ea135e3f455653\"},{\"path\":\"archive/table_a2.xml\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"b2a569c403e6917aa466d0973d3b3f677821530f\",\"size\":29349,\"url\":\"https://api.github.com/repos/roll/currency-codes/git/blobs/b2a569c403e6917aa466d0973d3b3f677821530f\"},{\"path\":\"data\",\"mode\":\"040000\",\"type\":\"tree\",\"sha\":\"0cf8433969ede15de089c1127cb6937d2ae13e3e\",\"url\":\"https://api.github.com/repos/roll/currency-codes/git/trees/0cf8433969ede15de089c1127cb6937d2ae13e3e\"},{\"path\":\"data/codes-all.csv\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"2c803df30c1d09e1c7f5de6661ca38f8e627d65b\",\"size\":17472,\"url\":\"https://api.github.com/repos/roll/currency-codes/git/blobs/2c803df30c1d09e1c7f5de6661ca38f8e627d65b\"},{\"path\":\"datapackage.json\",\"mode\":\"100644\",\"type\":\"blob\",\"sha\":\"dfca466367e0b3d23294df83b72101cb5b084c09\",\"size\":2019,\"url\":\"https://api.github.com/repos/roll/currency-codes/git/blobs/dfca466367e0b3d23294df83b72101cb5b084c09\"},{\"path\":\"scripts\",\"mode\":\"040000\",\"type\":\"tree\",\"sha\":\"ff0526e9a5e2d76f31d8038cbea148067203776a\",\"url\":\"https://api.github.com/repos/roll/currency-codes/git/trees/ff0526e9a5e2d76f31d8038cbea148067203776a\"},{\"path\":\"scripts/runall.sh\",\"mode\":\"100755\",\"type\":\"blob\",\"sha\":\"4632c9933cd6c76331301efa71dc5cd183c6dc89\",\"size\":3098,\"url\":\"https://api.github.com/repos/roll/currency-codes/git/blobs/4632c9933cd6c76331301efa71dc5cd183c6dc89\"}],\"truncated\":false}" + }, + "cookies": [], + "headers": [ + { + "name": "accept-ranges", + "value": "bytes" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "access-control-expose-headers", + "value": "ETag, Link, Location, Retry-After, X-GitHub-OTP, X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Used, X-RateLimit-Resource, X-RateLimit-Reset, X-OAuth-Scopes, X-Accepted-OAuth-Scopes, X-Poll-Interval, X-GitHub-Media-Type, X-GitHub-SSO, X-GitHub-Request-Id, Deprecation, Sunset" + }, + { + "name": "cache-control", + "value": "public, max-age=60, s-maxage=60" + }, + { + "name": "content-encoding", + "value": "gzip" + }, + { + "name": "content-length", + "value": "874" + }, + { + "name": "content-security-policy", + "value": "default-src 'none'" + }, + { + "name": "content-type", + "value": "application/json; charset=utf-8" + }, + { + "name": "date", + "value": "Wed, 28 May 2025 09:59:42 GMT" + }, + { + "name": "etag", + "value": "W/\"4635290695f3b6591e9e38e1e37992e8a758b746f29fb6288f64ac5ea26b5d2d\"" + }, + { + "name": "last-modified", + "value": "Wed, 07 Jul 2021 14:47:03 GMT" + }, + { + "name": "referrer-policy", + "value": "origin-when-cross-origin, strict-origin-when-cross-origin" + }, + { + "name": "server", + "value": "github.com" + }, + { + "name": "strict-transport-security", + "value": "max-age=31536000; includeSubdomains; preload" + }, + { + "name": "vary", + "value": "Accept,Accept-Encoding, Accept, X-Requested-With" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "x-frame-options", + "value": "deny" + }, + { + "name": "x-github-api-version-selected", + "value": "2022-11-28" + }, + { + "name": "x-github-media-type", + "value": "github.v3; format=json" + }, + { + "name": "x-github-request-id", + "value": "3878:1F0C59:140AA:1493F:6836DE8E" + }, + { + "name": "x-ratelimit-limit", + "value": "60" + }, + { + "name": "x-ratelimit-remaining", + "value": "56" + }, + { + "name": "x-ratelimit-reset", + "value": "1748429981" + }, + { + "name": "x-ratelimit-resource", + "value": "core" + }, + { + "name": "x-ratelimit-used", + "value": "4" + }, + { + "name": "x-xss-protection", + "value": "0" + } + ], + "headersSize": 1291, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-05-28T09:59:42.077Z", + "time": 261, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 261 + } + }, + { + "_id": "9d2eca9b34eee81e983e0f36dc5e2abe", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [], + "headersSize": 107, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://raw.githubusercontent.com/roll/currency-codes/refs/heads/master/datapackage.json" + }, + "response": { + "bodySize": 679, + "content": { + "mimeType": "text/plain; charset=utf-8", + "size": 679, + "text": "{\n \"name\": \"currency-codes\",\n \"title\": \"ISO 4217 Currency Codes\",\n \"licenses\": [\n {\n \"name\": \"ODC-PDDL-1.0\",\n \"path\": \"http://opendatacommons.org/licenses/pddl/\",\n \"title\": \"Open Data Commons Public Domain Dedication and License v1.0\"\n }\n ],\n \"keywords\": [ \"iso\", \"iso-4217\", \"currency\", \"codes\" ],\n \"homepage\": \"http://www.iso.org/iso/currency_codes\",\n \"sources\": [{\n \"name\": \"SIX Interbank Clearing Ltd (on behalf of ISO)\",\n \"email\": \"office@currency-iso.org\",\n \"title\": \"SIX Interbank Clearing Ltd (on behalf of ISO)\"\n }],\n \"contributors\": [\n {\n \"title\": \"Rufus Pollock\",\n \"email\": \"rufus.pollock@okfn.org\",\n \"role\": \"maintainer\"\n },\n {\n \"title\": \"Kristofer D. Kusano\",\n \"email\": \"kdkusano@gmail.com\"\n }\n ],\n \"resources\": [\n {\n \"path\": \"data/codes-all.csv\",\n \"name\": \"codes-all\",\n \"mimetype\": \"text/csv\",\n \"size\": \"16863\",\n \"schema\": {\n \"fields\": [\n {\n \"name\": \"Entity\",\n \"type\": \"string\",\n \"description\": \"Country or region name\"\n },\n {\n \"name\": \"Currency\",\n \"type\": \"string\",\n \"description\": \"Name of the currency\"\n },\n {\n \"name\": \"AlphabeticCode\",\n \"title\": \"Alphabetic Code\",\n \"type\": \"string\",\n \"description\": \"3 digit alphabetic code for the currency\"\n },\n {\n \"name\": \"NumericCode\",\n \"title\": \"Numeric Code\",\n \"type\": \"number\",\n \"description\": \"3 digit numeric code\"\n },\n {\n \"name\": \"MinorUnit\",\n \"title\": \"Minor Unit\",\n \"type\": \"string\",\n \"description\": \"\"\n },\n {\n \"name\": \"WithdrawalDate\",\n \"title\": \"Withdrawal Date\",\n \"type\": \"string\",\n \"description\": \"Date currency withdrawn (values can be ranges or months\"\n }\n ]\n }\n }\n ]\n}\n" + }, + "cookies": [], + "headers": [ + { + "name": "accept-ranges", + "value": "bytes" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "cache-control", + "value": "max-age=300" + }, + { + "name": "connection", + "value": "keep-alive" + }, + { + "name": "content-encoding", + "value": "gzip" + }, + { + "name": "content-length", + "value": "679" + }, + { + "name": "content-security-policy", + "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" + }, + { + "name": "content-type", + "value": "text/plain; charset=utf-8" + }, + { + "name": "cross-origin-resource-policy", + "value": "cross-origin" + }, + { + "name": "date", + "value": "Wed, 28 May 2025 09:59:42 GMT" + }, + { + "name": "etag", + "value": "W/\"18535666ca34d8d5f92e12776d3673e8e31c05c2be5c50a439cee1f3213ab9cb\"" + }, + { + "name": "expires", + "value": "Wed, 28 May 2025 10:04:42 GMT" + }, + { + "name": "source-age", + "value": "177" + }, + { + "name": "strict-transport-security", + "value": "max-age=31536000" + }, + { + "name": "vary", + "value": "Authorization,Accept-Encoding" + }, + { + "name": "via", + "value": "1.1 varnish" + }, + { + "name": "x-cache", + "value": "HIT" + }, + { + "name": "x-cache-hits", + "value": "0" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "x-fastly-request-id", + "value": "b3139e6fb7d5d49cc6fc18b6ed9349583bc805bf" + }, + { + "name": "x-frame-options", + "value": "deny" + }, + { + "name": "x-github-request-id", + "value": "DE55:1500CA:D108E3:E7A375:6836DDDD" + }, + { + "name": "x-served-by", + "value": "cache-lis1490032-LIS" + }, + { + "name": "x-timer", + "value": "S1748426383.589923,VS0,VE1" + }, + { + "name": "x-xss-protection", + "value": "1; mode=block" + } + ], + "headersSize": 901, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-05-28T09:59:42.340Z", + "time": 436, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 436 + } + } + ], + "pages": [], + "version": "1.2" + } +} diff --git a/github/package/load.spec.ts b/github/package/load.spec.ts index 9afda93a..a77be2c5 100644 --- a/github/package/load.spec.ts +++ b/github/package/load.spec.ts @@ -6,6 +6,14 @@ describe("loadPackageFromGithub", () => { useRecording() it("should load a package", async () => { + const datapackage = await loadPackageFromGithub({ + repoUrl: "https://github.com/roll/data", + }) + + expect(datapackage).toMatchSnapshot() + }) + + it("should merge datapackage.json if present", async () => { const datapackage = await loadPackageFromGithub({ repoUrl: "https://github.com/roll/currency-codes", }) diff --git a/github/package/load.ts b/github/package/load.ts index d8e0ac83..bec1878b 100644 --- a/github/package/load.ts +++ b/github/package/load.ts @@ -1,4 +1,6 @@ +import { mergePackages } from "@dpkit/core" import { makeGithubApiRequest } from "../general/index.js" +import type { GithubResource } from "../resource/index.js" import type { GithubPackage } from "./Package.js" import { normalizeGithubPackage } from "./process/normalize.js" @@ -19,20 +21,30 @@ export async function loadPackageFromGithub(props: { throw new Error(`Failed to extract repository info from URL: ${repoUrl}`) } - const githubPackage = (await makeGithubApiRequest({ + const githubPackage = await makeGithubApiRequest({ endpoint: `/repos/${owner}/${repo}`, apiKey, - })) as GithubPackage + }) const ref = githubPackage.default_branch githubPackage.resources = ( - await makeGithubApiRequest({ + await makeGithubApiRequest<{ tree: GithubResource[] }>({ endpoint: `/repos/${owner}/${repo}/git/trees/${ref}?recursive=1`, apiKey, }) ).tree - const datapackage = normalizeGithubPackage({ githubPackage }) + const systemPackage = normalizeGithubPackage({ githubPackage }) + const userPackagePath = systemPackage.resources + .filter(resource => resource["github:key"] === "datapackage.json") + .map(resource => resource["github:url"]) + .at(0) + + const datapackage = await mergePackages({ systemPackage, userPackagePath }) + datapackage.resources = datapackage.resources.map(resource => { + return { ...resource, "github:key": undefined, "github:url": undefined } + }) + return datapackage } diff --git a/github/package/process/normalize.ts b/github/package/process/normalize.ts index 4c046df5..c662cf94 100644 --- a/github/package/process/normalize.ts +++ b/github/package/process/normalize.ts @@ -62,7 +62,12 @@ export function normalizeGithubPackage(props: { datapackage.resources = githubPackage.resources .filter(resource => !resource.path.startsWith(".")) .filter(resource => resource.type === "blob") - .map(resource => normalizeGithubResource({ githubResource: resource })) + .map(resource => + normalizeGithubResource({ + githubResource: resource, + defaultBranch: githubPackage.default_branch, + }), + ) } // Process topics as keywords diff --git a/github/package/save.ts b/github/package/save.ts index b449b37e..7ea0df0c 100644 --- a/github/package/save.ts +++ b/github/package/save.ts @@ -4,6 +4,7 @@ import { denormalizePackage, stringifyDescriptor } from "@dpkit/core" import { getPackageBasepath, readFileStream } from "@dpkit/file" import { saveResourceFiles } from "@dpkit/file" import { makeGithubApiRequest } from "../general/index.js" +import type { GithubPackage } from "./Package.js" /** * Save a package to a Github repository @@ -19,7 +20,7 @@ export async function savePackageToGithub(props: { const { datapackage, apiKey, org, repo } = props const basepath = getPackageBasepath({ datapackage }) - const githubPackage = await makeGithubApiRequest({ + const githubPackage = await makeGithubApiRequest({ endpoint: org ? `/orgs/${org}/repos` : "/user/repos", payload: { name: repo, auto_init: true }, method: "POST", diff --git a/github/plugin.ts b/github/plugin.ts new file mode 100644 index 00000000..ade407c0 --- /dev/null +++ b/github/plugin.ts @@ -0,0 +1,20 @@ +import type { Descriptor, Plugin } from "@dpkit/core" +import { isRemotePath } from "@dpkit/core" +import { loadPackageFromGithub } from "./package/load.js" + +export class GithubPlugin implements Plugin { + async loadPackage(props: { source: string; options?: Descriptor }) { + const isRemote = isRemotePath({ path: props.source }) + if (!isRemote) return undefined + + const isGithub = new URL(props.source).hostname === "github.com" + if (!isGithub) return undefined + + const cleanup = async () => {} + const datapackage = await loadPackageFromGithub({ + repoUrl: props.source, + }) + + return { datapackage, cleanup } + } +} diff --git a/github/resource/Resource.ts b/github/resource/Resource.ts index 18a09077..7498e17d 100644 --- a/github/resource/Resource.ts +++ b/github/resource/Resource.ts @@ -3,17 +3,17 @@ */ export interface GithubResource { /** - * File name + * File path within repository */ - name: string + path: string /** - * File path within repository + * File mode e.g. `100755` */ - path: string + mode: string /** - * File type + * File type e.g. `blob` */ type: string @@ -23,32 +23,12 @@ export interface GithubResource { size: number /** - * File SHA + * File SHA-1 */ sha: string /** - * File URL for viewing in browser - */ - html_url: string - - /** - * File download URL - */ - download_url: string - - /** - * File Git URL - */ - git_url: string - - /** - * File content (base64 encoded) - */ - content?: string - - /** - * File encoding + * File url on GitHub API */ - encoding?: string + url: string } diff --git a/github/resource/process/denormalize.ts b/github/resource/process/denormalize.ts index 005cdfe1..c7c61a04 100644 --- a/github/resource/process/denormalize.ts +++ b/github/resource/process/denormalize.ts @@ -11,29 +11,15 @@ export function denormalizeGithubResource(props: { resource: Resource content?: string }): Partial { - const { resource, content } = props + const { resource } = props if (!resource.path && !resource.name) { return {} } - // Get the filename from path or name - const name = - resource.name || - (typeof resource.path === "string" - ? resource.path.split("/").pop() || "unknown" - : "unknown") - const githubResource: Partial = { - name, path: resource.name, // Use resource name as path within repo } - // Include content if provided - if (content) { - githubResource.content = content - githubResource.encoding = "base64" - } - return githubResource } diff --git a/github/resource/process/normalize.ts b/github/resource/process/normalize.ts index b548e5c4..efca4596 100644 --- a/github/resource/process/normalize.ts +++ b/github/resource/process/normalize.ts @@ -9,18 +9,35 @@ import type { GithubResource } from "../Resource.js" */ export function normalizeGithubResource(props: { githubResource: GithubResource + defaultBranch: string }) { const { githubResource } = props - const path = githubResource.path - const filename = getFilename({ path }) - - const name = getName({ filename }) ?? githubResource.sha - const format = getFormat({ filename }) + const path = normalizeGithubPath({ + ...githubResource, + ref: props.defaultBranch, + }) - const bytes = githubResource.size - const hash = `sha1:${githubResource.sha}` + const filename = getFilename({ path }) + const resource: Resource = { + path, + name: getName({ filename }) ?? githubResource.sha, + bytes: githubResource.size, + hash: `sha1:${githubResource.sha}`, + format: getFormat({ filename }), + "github:key": githubResource.path, + "github:url": path, + } - const resource: Resource = { name, path, bytes, hash, format } return resource } + +function normalizeGithubPath(props: { + url: string + ref: string + path: string +}) { + const url = new URL(props.url) + const [owner, repo] = url.pathname.split("/").slice(2) + return `https://raw.githubusercontent.com/${owner}/${repo}/refs/heads/${props.ref}/${props.path}` +} diff --git a/github/typedoc.json b/github/typedoc.json new file mode 100644 index 00000000..fc33b815 --- /dev/null +++ b/github/typedoc.json @@ -0,0 +1,5 @@ +{ + "entryPoints": ["index.ts"], + "projectDocuments": ["CHANGELOG.md", "OVERVIEW.md"], + "skipErrorChecking": true +} diff --git a/json/README.md b/json/README.md index 8b88a70c..7ac3b9c3 100644 --- a/json/README.md +++ b/json/README.md @@ -1,3 +1,3 @@ # @dpkit/json -Data Package implementation in TypeScript. Data Package is a standard consisting of a set of simple yet extensible specifications to describe datasets, data files and tabular data. It is a data definition language (DDL) and data API that facilitates findability, accessibility, interoperability, and reusability (FAIR) of data. For more information, please visit the [documentation portal](https://dpkit.datist.io). +dpkit is a fast TypeScript data management framework based on the Data Package standard. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/json/package.json b/json/package.json index bb0d6235..380bf3b6 100644 --- a/json/package.json +++ b/json/package.json @@ -6,7 +6,7 @@ "license": "MIT", "author": "Evgeny Karev", "repository": "https://github.com/datisthq/dpkit", - "description": "Data Package implementation in TypeScript.", + "description": "Fast TypeScript data management framework based on the Data Package standard", "keywords": [ "data", "package", diff --git a/ods/README.md b/ods/README.md index 09889fa9..9dea240e 100644 --- a/ods/README.md +++ b/ods/README.md @@ -1,3 +1,3 @@ # @dpkit/ods -Data Package implementation in TypeScript. Data Package is a standard consisting of a set of simple yet extensible specifications to describe datasets, data files and tabular data. It is a data definition language (DDL) and data API that facilitates findability, accessibility, interoperability, and reusability (FAIR) of data. For more information, please visit the [documentation portal](https://dpkit.datist.io). +dpkit is a fast TypeScript data management framework based on the Data Package standard. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/ods/package.json b/ods/package.json index ba38a8fa..86c984a2 100644 --- a/ods/package.json +++ b/ods/package.json @@ -6,7 +6,7 @@ "license": "MIT", "author": "Evgeny Karev", "repository": "https://github.com/datisthq/dpkit", - "description": "Data Package implementation in TypeScript.", + "description": "Fast TypeScript data management framework based on the Data Package standard", "keywords": [ "data", "package", diff --git a/package.json b/package.json index 47983aaf..49cadecf 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "build": "pnpm -r build", "bump": "ncu -ws -u", "ci:install": "pnpm install --ignore-scripts", - "ci:publish": "pnpm -r publish --access public && changeset tag", + "ci:publish": "pnpm -r publish --access public --ignore-scripts && changeset tag", "check": "pnpm run lint && pnpm run type", "coverage": "vitest --ui", "format": "biome check --write", diff --git a/parquet/README.md b/parquet/README.md index e05220e0..93774ab1 100644 --- a/parquet/README.md +++ b/parquet/README.md @@ -1,3 +1,3 @@ # @dpkit/parquet -Data Package implementation in TypeScript. Data Package is a standard consisting of a set of simple yet extensible specifications to describe datasets, data files and tabular data. It is a data definition language (DDL) and data API that facilitates findability, accessibility, interoperability, and reusability (FAIR) of data. For more information, please visit the [documentation portal](https://dpkit.datist.io). +dpkit is a fast TypeScript data management framework based on the Data Package standard. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/parquet/package.json b/parquet/package.json index f4e393ec..b205d336 100644 --- a/parquet/package.json +++ b/parquet/package.json @@ -6,7 +6,7 @@ "license": "MIT", "author": "Evgeny Karev", "repository": "https://github.com/datisthq/dpkit", - "description": "Data Package implementation in TypeScript.", + "description": "Fast TypeScript data management framework based on the Data Package standard", "keywords": [ "data", "package", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 512955b8..f9ec4199 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -46,6 +46,12 @@ importers: bin: {} + camtrap: + dependencies: + '@dpkit/core': + specifier: workspace:* + version: link:../core + ckan: dependencies: '@dpkit/core': @@ -72,6 +78,9 @@ importers: quick-lru: specifier: ^7.0.1 version: 7.0.1 + tiny-invariant: + specifier: ^1.3.3 + version: 1.3.3 csv: {} @@ -106,6 +115,9 @@ importers: dpkit: dependencies: + '@dpkit/camtrap': + specifier: workspace:* + version: link:../camtrap '@dpkit/ckan': specifier: workspace:* version: link:../ckan @@ -115,6 +127,9 @@ importers: '@dpkit/file': specifier: workspace:* version: link:../file + '@dpkit/github': + specifier: workspace:* + version: link:../github '@dpkit/zenodo': specifier: workspace:* version: link:../zenodo @@ -124,6 +139,8 @@ importers: excel: {} + extend: {} + file: dependencies: '@dpkit/core': @@ -132,6 +149,9 @@ importers: tempy: specifier: 3.1.0 version: 3.1.0 + tiny-invariant: + specifier: ^1.3.3 + version: 1.3.3 github: dependencies: @@ -152,8 +172,6 @@ importers: parquet: {} - site: {} - table: {} test: @@ -2974,6 +2992,9 @@ packages: tiny-inflate@1.0.3: resolution: {integrity: sha512-pkY1fj1cKHb2seWDy0B16HeWyczlJA9/WW3u3c4z/NiWDsO3DOU5D7nhTLE9CF0yXv/QZFY7sEJmj24dK+Rrqw==} + tiny-invariant@1.3.3: + resolution: {integrity: sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==} + tinybench@2.9.0: resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} @@ -6861,6 +6882,8 @@ snapshots: tiny-inflate@1.0.3: {} + tiny-invariant@1.3.3: {} + tinybench@2.9.0: {} tinyexec@0.3.2: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 03b918fa..fc0c4941 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -2,6 +2,7 @@ dangerouslyAllowAllBuilds: true packages: - avro - bin + - camtrap - ckan - cli - core @@ -10,12 +11,12 @@ packages: - docs - dpkit - excel + - extend - file - github - json - ods - parquet - - site - table - test - ui diff --git a/renovate.json b/renovate.json index 5db72dd6..22a99432 100644 --- a/renovate.json +++ b/renovate.json @@ -1,6 +1,4 @@ { "$schema": "https://docs.renovatebot.com/renovate-schema.json", - "extends": [ - "config:recommended" - ] + "extends": ["config:recommended"] } diff --git a/site/README.md b/site/README.md deleted file mode 100644 index f4adff42..00000000 --- a/site/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# @dpkit/site - -Data Package implementation in TypeScript. Data Package is a standard consisting of a set of simple yet extensible specifications to describe datasets, data files and tabular data. It is a data definition language (DDL) and data API that facilitates findability, accessibility, interoperability, and reusability (FAIR) of data. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/table/README.md b/table/README.md index 71b3a99d..941ef5dd 100644 --- a/table/README.md +++ b/table/README.md @@ -1,3 +1,3 @@ # @dpkit/table -Data Package implementation in TypeScript. Data Package is a standard consisting of a set of simple yet extensible specifications to describe datasets, data files and tabular data. It is a data definition language (DDL) and data API that facilitates findability, accessibility, interoperability, and reusability (FAIR) of data. For more information, please visit the [documentation portal](https://dpkit.datist.io). +dpkit is a fast TypeScript data management framework based on the Data Package standard. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/test/README.md b/test/README.md index 57cf22c7..4b137e55 100644 --- a/test/README.md +++ b/test/README.md @@ -1,3 +1,3 @@ # @dpkit/test -Data Package implementation in TypeScript. Data Package is a standard consisting of a set of simple yet extensible specifications to describe datasets, data files and tabular data. It is a data definition language (DDL) and data API that facilitates findability, accessibility, interoperability, and reusability (FAIR) of data. For more information, please visit the [documentation portal](https://dpkit.datist.io). +dpkit is a fast TypeScript data management framework based on the Data Package standard. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/ui/README.md b/ui/README.md index 45c2fe5f..94a68cb1 100644 --- a/ui/README.md +++ b/ui/README.md @@ -1,3 +1,3 @@ # @dpkit/ui -Data Package implementation in TypeScript. Data Package is a standard consisting of a set of simple yet extensible specifications to describe datasets, data files and tabular data. It is a data definition language (DDL) and data API that facilitates findability, accessibility, interoperability, and reusability (FAIR) of data. For more information, please visit the [documentation portal](https://dpkit.datist.io). +dpkit is a fast TypeScript data management framework based on the Data Package standard. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/ui/package.json b/ui/package.json index 1881bca6..3040e331 100644 --- a/ui/package.json +++ b/ui/package.json @@ -6,7 +6,7 @@ "license": "MIT", "author": "Evgeny Karev", "repository": "https://github.com/datisthq/dpkit", - "description": "Data Package implementation in TypeScript.", + "description": "Fast TypeScript data management framework based on the Data Package standard", "keywords": [ "data", "package", diff --git a/zenodo/OVERVIEW.md b/zenodo/OVERVIEW.md new file mode 100644 index 00000000..c3db5d41 --- /dev/null +++ b/zenodo/OVERVIEW.md @@ -0,0 +1,2 @@ +# @dpkit/zenodo + diff --git a/zenodo/README.md b/zenodo/README.md index 79eaca49..01f5b42b 100644 --- a/zenodo/README.md +++ b/zenodo/README.md @@ -1,3 +1,3 @@ # @dpkit/zenodo -Data Package implementation in TypeScript. Data Package is a standard consisting of a set of simple yet extensible specifications to describe datasets, data files and tabular data. It is a data definition language (DDL) and data API that facilitates findability, accessibility, interoperability, and reusability (FAIR) of data. For more information, please visit the [documentation portal](https://dpkit.datist.io). +dpkit is a fast TypeScript data management framework based on the Data Package standard. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/zenodo/general/index.ts b/zenodo/general/index.ts index bd8002db..0e36dceb 100644 --- a/zenodo/general/index.ts +++ b/zenodo/general/index.ts @@ -1 +1 @@ -export * from "./request.js" +export { makeZenodoApiRequest } from "./request.js" diff --git a/zenodo/general/request.ts b/zenodo/general/request.ts index 572fc035..1b5239dc 100644 --- a/zenodo/general/request.ts +++ b/zenodo/general/request.ts @@ -1,6 +1,6 @@ import type { Descriptor } from "@dpkit/core" -export async function makeZenodoApiRequest(props: { +export async function makeZenodoApiRequest(props: { endpoint: string method?: "GET" | "POST" | "PUT" | "DELETE" payload?: Descriptor @@ -18,7 +18,7 @@ export async function makeZenodoApiRequest(props: { } = props let body: string | FormData | undefined - const headers: Descriptor = {} + const headers: Record = {} const baseUrl = sandbox ? "https://sandbox.zenodo.org/api" @@ -56,5 +56,5 @@ export async function makeZenodoApiRequest(props: { ) } - return (await response.json()) as Descriptor + return (await response.json()) as T } diff --git a/zenodo/index.ts b/zenodo/index.ts index 71449eaf..c3c86163 100644 --- a/zenodo/index.ts +++ b/zenodo/index.ts @@ -1,2 +1,3 @@ export * from "./package/index.js" export * from "./resource/index.js" +export * from "./plugin.js" diff --git a/zenodo/package.json b/zenodo/package.json index 1fb60493..6f185e5d 100644 --- a/zenodo/package.json +++ b/zenodo/package.json @@ -6,7 +6,7 @@ "license": "MIT", "author": "Evgeny Karev", "repository": "https://github.com/datisthq/dpkit", - "description": "Data Package implementation in TypeScript.", + "description": "Fast TypeScript data management framework based on the Data Package standard", "keywords": [ "data", "package", @@ -16,14 +16,14 @@ "fair", "zenodo" ], + "scripts": { + "build": "tsc --build" + }, "dependencies": { "@dpkit/core": "workspace:*", "@dpkit/file": "workspace:*" }, "devDependencies": { "@dpkit/test": "workspace:*" - }, - "scripts": { - "build": "tsc --build" } } diff --git a/zenodo/package/fixtures/generated/load.spec.ts.snap b/zenodo/package/fixtures/generated/load.spec.ts.snap index 2bcc2c49..23092bbb 100644 --- a/zenodo/package/fixtures/generated/load.spec.ts.snap +++ b/zenodo/package/fixtures/generated/load.spec.ts.snap @@ -4,56 +4,716 @@ exports[`loadPackageFromZenodo > should load a package 1`] = ` { "contributors": [ { - "path": "path", "role": "author", - "title": "title", + "title": "Sukhija, Vinay Jagdish", }, ], - "created": "2025-05-24", - "description": "description", + "created": "2025-05-27", + "description": "

A brief project to extract PM 10 concentration from Japan. Data set has been created using the OpenAQ data through a python script

", "keywords": [ - "keyword1", - "keyword2", + "PM10 concentrations", + "Air Quality", + "Japan", + "OpenAQ", ], "licenses": [ { "name": { - "id": "mit-license", + "id": "cc-by-4.0", }, }, ], - "name": "record-260528", + "name": "record-15525711", "resources": [ { - "bytes": 30, + "bytes": 272, "format": "csv", - "hash": "md5:6c2c61dd9b0e9c6876139a449ed87933", - "name": "table", - "path": "https://sandbox.zenodo.org/records/260528/files/table.csv", + "hash": "md5:5ac1b92a57ec809c546ea90334c845c9", + "name": "openaq_measurements_pm10_japan_20250527_095520", + "path": "https://zenodo.org/records/15525711/files/openaq_measurements_PM10_Japan_20250527_095520.csv", + "zenodo:key": undefined, + "zenodo:url": undefined, + }, + ], + "title": "PM10 Concentration in Japan", +} +`; + +exports[`loadPackageFromZenodo > shoule merge datapackage.json if present 1`] = ` +{ + "contributors": [ + { + "path": "Vogelwerkgroep Assen", + "role": "author", + "title": "Dijkstra, Bert", + }, + { + "path": "Vogelwerkgroep Assen", + "role": "author", + "title": "Dillerop, Rinus", + }, + { + "path": "Dutch Centre for Field Ornithology", + "role": "author", + "title": "Oosterbeek, Kees", }, { - "bytes": 23, - "format": "json", - "hash": "md5:0de7e07cbe2fc19dd077133951760eed", - "name": "dialect", - "path": "https://sandbox.zenodo.org/records/260528/files/dialect.json", + "path": "University of Amsterdam", + "role": "author", + "title": "Bouten, Willem", }, { - "bytes": 136, - "format": "json", - "hash": "md5:30208ec3060c83e82f626cdd19795c69", - "name": "schema", - "path": "https://sandbox.zenodo.org/records/260528/files/schema.json", + "path": "Research Institute for Nature and Forest", + "role": "author", + "title": "Desmet, Peter", }, { - "bytes": 824, - "format": "json", - "hash": "md5:028c7ddf791931e9738a3e7e8586ea34", - "name": "datapackage", - "path": "https://sandbox.zenodo.org/records/260528/files/datapackage.json", + "path": "Dutch Bryological and Lichenological Society", + "role": "author", + "title": "van der Kolk, Henk-Jan", + }, + { + "path": "Dutch Centre for Field Ornithology", + "role": "author", + "title": "Ens, Bruno J.", + }, + ], + "created": "2023-10-30", + "description": "

O_ASSEN - Eurasian oystercatchers (Haematopus ostralegus, Haematopodidae) breeding in Assen (the Netherlands) is a bird tracking dataset published by the Vogelwerkgroep Assen, Netherlands Institute of Ecology (NIOO-KNAW), Sovon, Radboud University, the University of Amsterdam and the Research Institute for Nature and Forest (INBO). It contains animal tracking data collected for the study O_ASSEN using trackers developed by the University of Amsterdam Bird Tracking System (UvA-BiTS, http://www.uva-bits.nl). The study was operational from 2018 to 2019. In total 6 individuals of Eurasian oystercatchers (Haematopus ostralegus) have been tagged as a breeding bird in the city of Assen (the Netherlands), mainly to study space use of oystercatchers breeding in urban areas. Data are uploaded from the UvA-BiTS database to Movebank and from there archived on Zenodo (see https://github.com/inbo/bird-tracking). No new data are expected.

+

See van der Kolk et al. (2022, https://doi.org/10.3897/zookeys.1123.90623) for a more detailed description of this dataset.

+

Files

+

Data in this package are exported from Movebank study 1605797471. Fields in the data follow the Movebank Attribute Dictionary and are described in datapackage.json. Files are structured as a Frictionless Data Package. You can access all data in R via https://zenodo.org/records/10053903/files/datapackage.json using frictionless.

+
    +
  • datapackage.json: technical description of the data files.
    • +
  • O_ASSEN-reference-data.csv: reference data about the animals, tags and deployments.
    • +
  • O_ASSEN-gps-yyyy.csv.gz: GPS data recorded by the tags, grouped by year.
    • +
  • O_ASSEN-acceleration-yyyy.csv.gz: acceleration data recorded by the tags, grouped by year.
    • +
+

Acknowledgements

+

These data were collected by Bert Dijkstra and Rinus Dillerop from Vogelwerkgroep Assen, in collaboration with the Netherlands Institute of Ecology (NIOO-KNAW), Sovon, Radboud University and the University of Amsterdam (UvA). Funding was provided by the Prins Bernard Cultuurfonds Drenthe, municipality of Assen, IJsvogelfonds (from Birdlife Netherlands and Nationale Postcodeloterij) and the Waterleiding Maatschappij Drenthe. The dataset was published with funding from Stichting NLBIF - Netherlands Biodiversity Information Facility.

", + "keywords": [ + "animal movement", + "animal tracking", + "gps tracking", + "accelerometer", + "altitude", + "temperature", + "biologging", + "birds", + "UvA-BiTS", + "Movebank", + "frictionlessdata", + ], + "licenses": [ + { + "name": { + "id": "cc-zero", + }, + }, + ], + "name": "record-10053903", + "resources": [ + { + "encoding": "UTF-8", + "format": "csv", + "mediatype": "text/csv", + "name": "reference-data", + "path": "https://zenodo.org/records/10053903/files/O_ASSEN-reference-data.csv", + "profile": "tabular-data-resource", + "schema": { + "fields": [ + { + "description": "A unique identifier for the tag, provided by the data owner. If the data owner does not provide a tag ID, an internal Movebank tag identifier may sometimes be shown. Example: '2342'; Units: none; Entity described: tag", + "format": "default", + "name": "tag-id", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000181/2/", + "title": "tag ID", + "type": "string", + }, + { + "description": "An individual identifier for the animal, provided by the data owner. Values are unique within the study. If the data owner does not provide an Animal ID, an internal Movebank animal identifier is sometimes shown. Example: 'TUSC_CV5'; Units: none; Entity described: individual", + "format": "default", + "name": "animal-id", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000016/3/", + "title": "animal ID", + "type": "string", + }, + { + "description": "The scientific name of the taxon on which the tag was deployed, as defined by the Integrated Taxonomic Information System www.itis.gov. If the species name can not be provided, this should be the lowest level taxonomic rank that can be determined and that is used in the ITIS taxonomy. Additional information can be provided using the term 'taxon detail'. The values 'test' and 'calibration' identify events relevant to animal tracking studies that should not be associated with a taxon. Format: controlled list; Entity described: individual", + "format": "default", + "name": "animal-taxon", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000024/4/", + "title": "animal taxon", + "type": "string", + }, + { + "description": "The timestamp when the tag deployment started. Data records recorded before this day and time are not associated with the animal related to the deployment. Values are typically defined by the data owner, and in some cases are created automatically during data import. Example: '2008-08-30 18:00:00.000'; Format: yyyy-MM-dd HH:mm:ss.SSS; Units: UTC or GPS time; Entity described: deployment", + "format": "%Y-%m-%d %H:%M:%S.%f", + "name": "deploy-on-date", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000081/3/", + "title": "deploy on timestamp", + "type": "datetime", + }, + { + "description": "The timestamp when the tag deployment ended. Data records recorded after this day and time are not associated with the animal related to the deployment. Values are typically defined by the data owner, and in some cases are created automatically during data import. Further information can be provided in 'deployment end type' and 'deployment end comments'. Example: '2009-10-01 12:00:00.000'; Format: yyyy-MM-dd HH:mm:ss.SSS; Units: UTC or GPS time; Entity described: deployment", + "format": "%Y-%m-%d %H:%M:%S.%f", + "name": "deploy-off-date", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000077/4/", + "title": "deploy off timestamp", + "type": "datetime", + }, + { + "description": "A name or unique identifier for a project associated with the deployment, for example a monitoring program or another data platform. Best practice is to include the name of the related database or organization followed by the project identifier. Example: 'MOTUS145'; Units: none; Entity described: deployment", + "format": "default", + "name": "alt-project-id", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000244/2/", + "title": "alt project ID", + "type": "string", + }, + { + "description": "Additional information about the animal. Example: 'first to fledge from nest'; Units: none; Entity described: individual", + "format": "default", + "name": "animal-comments", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000012/3/", + "title": "animal comments", + "type": "string", + }, + { + "description": "The age class or life stage of the animal at the beginning of the deployment. Can be years or months of age or terms such as 'adult', 'subadult' and 'juvenile'. Best practice is to define units in the values if needed (e.g. '2 years'). Example: 'juvenile, adult'; Units: none; Entity described: deployment", + "format": "default", + "name": "animal-life-stage", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000018/3/", + "title": "animal life stage", + "type": "string", + }, + { + "description": "The mass of the animal, typically at the beginning of the deployment. Example: '500'; Units: grams; Entity described: deployment", + "format": "default", + "name": "animal-mass", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000019/2/", + "title": "animal mass", + "type": "number", + }, + { + "description": "An alternate identifier for the animal. Used as the display name for animals shown in the Animal Tracker App. Example: 'Ali'; Units: none; Entity described: individual", + "format": "default", + "name": "animal-nickname", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000020/2/", + "title": "animal nickname", + "type": "string", + }, + { + "description": "A number or color scheme for a band or ring attached to the animal. Color bands and other markings can be stored in 'animal marker ID'. Example: '26225'; Units: none; Entity described: individual", + "format": "default", + "name": "animal-ring-id", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000022/3/", + "title": "animal ring ID", + "type": "string", + }, + { + "description": "The sex of the animal. Allowed values are m = male; f = female; u = unknown. Format: controlled list; Entity described: individual", + "format": "default", + "name": "animal-sex", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000023/3/", + "title": "animal sex", + "type": "string", + }, + { + "description": "The way a tag is attached to an animal. Details can be provided in 'attachment comments'. Values are chosen from a controlled list: backpack-harness = The tag is attached to the animal using a backpack-style harness; collar = The tag is attached by a collar around the animal's neck; ear-tag = The tag is attached to the animal's ear; fin mount = The tag is attached to the animal's fin; glue = The tag is attached to the animal using glue; harness = The tag is attached to the animal using a harness; implant = The tag is placed under the skin of the animal; leg-band = The tag is attached as a leg band or ring; leg-loop-harness = The tag is attached to the animal using a leg-loop-style harness; none = No tag was attached, e.g., for observations using natural markings; other = The tag is attached using another method; subcutaneous-anchor = The tag is attached using one or more anchors attached underneath the animal's skin; suction-cup = The tag is attached using one or more suction cups; sutures = The tag is attached by one or more sutures; tape = The tag is attached to the animal using tape. Format: controlled list; Entity described: deployment", + "format": "default", + "name": "attachment-type", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000052/5/", + "title": "attachment type", + "type": "string", + }, + { + "description": "The geographic latitude of the location where the animal was released. Intended primarily for cases in which the animal release location has higher accuracy than that derived from sensor data. Example: '27.3516'; Units: decimal degrees, WGS84 reference system; Entity described: deployment", + "format": "default", + "name": "deploy-on-latitude", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000078/3/", + "title": "deploy on latitude", + "type": "number", + }, + { + "description": "The geographic longitude of the location where the animal was released. Intended primarily for cases in which the animal release location has higher accuracy than that derived from sensor data. Example: '-97.3321'; Units: decimal degrees, WGS84 reference system; Entity described: deployment", + "format": "default", + "name": "deploy-on-longitude", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000079/3/", + "title": "deploy on longitude", + "type": "number", + }, + { + "description": "A list of additional measurements taken during capture of the animal at the start of the deployment. Recommended best practice is to define units and use a key:value encoding schema for a data interchange format such as JSON. Example: "{tarsusLengthInMillimeters:17.3, wingChordInMillimeters:125}"; Units: not defined; Entity described: deployment", + "format": "default", + "name": "deploy-on-measurements", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000356/2/", + "title": "deploy on measurements", + "type": "string", + }, + { + "description": "Additional information about the tag deployment that is not described by other reference data terms. Example: 'body length 154 cm; condition good'; Units: none; Entity described: deployment", + "format": "default", + "name": "deployment-comments", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000082/2/", + "title": "deployment comments", + "type": "string", + }, + { + "description": "A categorical classification describing the end of the tag deployment on the animal. Best practice is to clarify how the 'deploy-off timestamp', if present, was chosen. Values are chosen from a controlled list: analysis-end = the end time represents the end of the period of interest; captured = The tag remained on the animal but the animal was captured or confined; dead = The deployment ended with the death of the animal that was carrying the tag; dead/fall-off = The tag stopped moving, and it is not possible to determine whether it is due to death of the animal or unscheduled tag detachment; equipment-failure = The tag stopped working; fall-off = The attachment of the tag to the animal failed, and it fell of accidentally; other = other; released = The tag remained on the animal but the animal was released from captivity or confinement; removal = The tag was purposefully removed from the animal; scheduled-detachment = The tag was programmed to detach from the animal; transmission-end = The tag stopped transmitting usable data; unknown = The cause of the end of data availability or transmission is unknown. Format: controlled list; Entity described: deployment", + "format": "default", + "name": "deployment-end-type", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000084/5/", + "title": "deployment end type", + "type": "string", + }, + { + "description": "A unique identifier for the deployment of a tag on animal, provided by the data owner. If the data owner does not provide a Deployment ID, an internal Movebank deployment identifier may sometimes be shown. Example: 'Jane_42818'; Units: none; Entity described: deployment", + "format": "default", + "name": "deployment-id", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000085/3/", + "title": "deployment ID", + "type": "string", + }, + { + "description": "Comments about the location accuracy. This can further describe values provided in 'location error text', 'location error numerical', 'vertical error numerical', 'lat lower', 'lat upper', 'long lower' and/or 'long upper'. The percentile uncertainty can be provided using 'location error percentile'. Example: '1 standard deviation errors, assuming normal distribution, provided by the GPS unit'; Units: none; Entity described: deployment", + "format": "default", + "name": "location-accuracy-comments", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000141/3/", + "title": "location accuracy comments", + "type": "string", + }, + { + "description": "The way in which the animal was manipulated during the deployment. Additional information can be provided using 'manipulation comments'. Changes in manipulation status during deployment can be identified using 'manipulation status'. Values are chosen from a controlled list: confined = The animal's movement was restricted to within a defined area; domesticated = The animal is domesticated, for example, is a house pet or part of a managed herd; manipulated-other = The animal was manipulated in some other way, such as a physiological manipulation; none = The animal received no treatment other than tag attachment and related measurements and sampling (if applicable); reintroduction = The animal has been reintroduced as part of wildlife conservation or management efforts; relocated = The animal was released from a site other than the one at which it was captured. Format: controlled list; Entity described: deployment", + "format": "default", + "name": "manipulation-type", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000155/6/", + "title": "manipulation type", + "type": "string", + }, + { + "description": "A location such as the deployment site, study site, or colony name. Example: 'Pickerel Island North'; Units: none; Entity described: deployment", + "format": "default", + "name": "study-site", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000175/3/", + "title": "study site", + "type": "string", + }, + { + "description": "The tag firmware and version used during the deployment. If needed, identify the relevant sensors on the tag. Units: none; Entity described: deployment", + "format": "default", + "name": "tag-firmware", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000380/1/", + "title": "tag firmware", + "type": "string", + }, + { + "description": "The company or person that produced the tag. Example: 'Holohil'; Units: none; Entity described: tag", + "format": "default", + "name": "tag-manufacturer-name", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000183/3/", + "title": "tag manufacturer name", + "type": "string", + }, + { + "description": "The mass of the tag. Can be used with 'tag mass total' to define the mass of the tag separately from that of the tag with additional hardware. Example: '24'; Units: grams; Entity described: tag", + "format": "default", + "name": "tag-mass", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000184/4/", + "title": "tag mass", + "type": "number", + }, + { + "description": "The way the data are received from the tag. Values are chosen from a controlled list: ISS = Data are transferred via the International Space Station; LPWAN = Data are transferred through a low-power wide-area network, such as LoRa or Sigfox; multiple = Data are acquired using multiple methods; none = Data are obtained without use of an animal-borne tag, such as by observing a unique marking; other-wireless = Data are transferred via another form of wireless data transfer, such as a VHF transmitter/receiver; phone-network = Data are transferred via a phone network, such as GSM or AMPS; satellite = Data are transferred via satellite; tag-retrieval = The tag must be physically retrieved in order to obtain the data; telemetry-network = Data are obtained through a radio or acoustic telemetry network; Wi-Fi/Bluetooth = Data are transferred via a local Wi-Fi or Bluetooth system. Format: controlled list; Entity described: deployment", + "format": "default", + "name": "tag-readout-method", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000188/4/", + "title": "tag readout method", + "type": "string", + }, + { + "description": "The serial number of the tag. Example: 'MN93-33243'; Units: none; Entity described: tag", + "format": "default", + "name": "tag-serial-no", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000189/3/", + "title": "tag serial no", + "type": "string", + }, + ], + "primaryKey": [ + "animal-id", + "tag-id", + ], + }, + "zenodo:key": undefined, + "zenodo:url": undefined, + }, + { + "encoding": "UTF-8", + "format": "csv", + "mediatype": "text/csv", + "name": "gps", + "path": [ + "https://zenodo.org/records/10053903/files/O_ASSEN-gps-2018.csv.gz", + "https://zenodo.org/records/10053903/files/O_ASSEN-gps-2019.csv.gz", + ], + "profile": "tabular-data-resource", + "schema": { + "fields": [ + { + "description": "An identifier for the set of values associated with each event, i.e. sensor measurement. A unique event ID is assigned to every time-location or other time-measurement record in Movebank. If multiple measurements are included within a single row of a data file, they will share an event ID. If users import the same sensor measurement to Movebank multiple times, a separate event ID will be assigned to each. Example: '14328243575'; Units: none; Entity described: event", + "format": "default", + "name": "event-id", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000103/3/", + "title": "event ID", + "type": "integer", + }, + { + "description": "Determines whether an event is visible on the Movebank map. Allowed values are TRUE or FALSE. Values are calculated automatically, with TRUE indicating the event has not been flagged as an outlier by 'algorithm marked outlier', 'import marked outlier' or 'manually marked outlier', or that the user has overridden the results of these outlier attributes using 'manually marked valid' = TRUE. Units: none; Entity described: event", + "format": "default", + "name": "visible", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000209/3/", + "title": "visible", + "type": "boolean", + }, + { + "description": "The date and time corresponding to a sensor measurement or an estimate derived from sensor measurements. Example: '2008-08-14 18:31:00.000'; Format: yyyy-MM-dd HH:mm:ss.SSS; Units: UTC or GPS time; Entity described: event", + "format": "%Y-%m-%d %H:%M:%S.%f", + "name": "timestamp", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000200/2/", + "title": "timestamp", + "type": "datetime", + }, + { + "description": "The geographic longitude of the location as estimated by the sensor. Positive values are east of the Greenwich Meridian, negative values are west of it. Example: '-121.1761111'; Units: decimal degrees, WGS84 reference system; Entity described: event", + "format": "default", + "name": "location-long", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000146/2/", + "title": "location long", + "type": "number", + }, + { + "description": "The geographic latitude of the location as estimated by the sensor. Example: '-41.0982423'; Units: decimal degrees, WGS84 reference system; Entity described: event", + "format": "default", + "name": "location-lat", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000145/4/", + "title": "location lat", + "type": "number", + }, + { + "description": "The barometric air or water pressure. Example: '32536.0'; Units: mbar (hPa); Entity described: event", + "format": "default", + "name": "bar:barometric-pressure", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000055/3/", + "title": "barometric pressure", + "type": "number", + }, + { + "description": "The temperature measured by the tag (different from ambient temperature or internal body temperature of the animal). Example: '32.1'; Units: degrees Celsius; Entity described: event", + "format": "default", + "name": "external-temperature", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000104/2/", + "title": "external temperature", + "type": "number", + }, + { + "description": "Dilution of precision provided by the GPS. Example: '1.8'; Units: unitless; Entity described: event", + "format": "default", + "name": "gps:dop", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000115/2/", + "title": "GPS DOP", + "type": "number", + }, + { + "description": "The number of GPS satellites used to estimate the location. Example: '8'; Units: count; Entity described: event", + "format": "default", + "name": "gps:satellite-count", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000120/3/", + "title": "GPS satellite count", + "type": "integer", + }, + { + "description": "The time required to obtain the GPS location fix. Example: '36'; Units: seconds; Entity described: event", + "format": "default", + "name": "gps-time-to-fix", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000121/3/", + "title": "GPS time to fix", + "type": "number", + }, + { + "description": "The estimated ground speed provided by the sensor or calculated between consecutive locations. Example: '7.22'; Units: m/s; Entity described: event", + "format": "default", + "name": "ground-speed", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000124/2/", + "title": "ground speed", + "type": "number", + }, + { + "description": "The direction in which the tag is moving, in decimal degrees clockwise from north, as provided by the sensor or calculated between consecutive locations. Values range from 0-360: 0 = north, 90 = east, 180 = south, 270 = west. Example: '315.88'; Units: degrees clockwise from north; Entity described: event", + "format": "default", + "name": "heading", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000129/2/", + "title": "heading", + "type": "number", + }, + { + "description": "The estimated height of the tag above mean sea level, typically estimated by the tag. If altitudes are calculated as height above an ellipsoid, use 'height above ellipsoid'. Example: '34'; Units: meters; Entity described: event", + "format": "default", + "name": "height-above-msl", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000131/3/", + "title": "height above mean sea level", + "type": "number", + }, + { + "description": "Identifies events as outliers. Outliers have the value TRUE. Typically used to import a record of outliers that were identified by the data provider or owner with automated methods outside of Movebank. Information about how outliers were defined can be provided in 'outlier comments'. Units: none; Entity described: event", + "format": "default", + "name": "import-marked-outlier", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000133/3/", + "title": "import marked outlier", + "type": "boolean", + }, + { + "description": "An estimate of the horizontal error of the location including only numbers. (If the error estimates include non-numerical characters such as '>' use 'location error text'.) These values can be described using 'location error percentile' and 'location accuracy comments'. Example: '50'; Units: meters; Entity described: event", + "format": "default", + "name": "location-error-numerical", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000142/3/", + "title": "location error numerical", + "type": "number", + }, + { + "description": "Identifies events flagged manually as outliers, typically using the Event Editor in Movebank, and may also include outliers identified using other methods. Outliers have the value TRUE. Information about how outliers were defined can be provided in 'outlier comments'. Units: none; Entity described: event", + "format": "default", + "name": "manually-marked-outlier", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000156/3/", + "title": "manually marked outlier", + "type": "boolean", + }, + { + "description": "An estimate of the vertical error of the location. These values can be described using 'location error percentile' and 'location accuracy comments'. Example: '12'; Units: meters; Entity described: event", + "format": "default", + "name": "vertical-error-numerical", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000208/3/", + "title": "vertical error numerical", + "type": "number", + }, + { + "description": "The type of sensor with which data were collected. All sensors are associated with a tag id, and tags can contain multiple sensor types. Each event record in Movebank is assigned one sensor type. If values from multiple sensors are reported in a single event, the primary sensor is used. Values are chosen from a controlled list: acceleration = The sensor collects acceleration data; accessory-measurements = The sensor collects accessory measurements, such as battery voltage; acoustic-telemetry = The sensor transmits an acoustic signal that is detected by receivers to determine location; argos-doppler-shift = The sensor location is estimated by Argos using Doppler shift; barometer = The sensor records air or water pressure; bird-ring = The animal is identified by a band or ring that has a unique identifier; gps = The sensor uses GPS to determine location; gyroscope = The sensor records angular velocity; heart-rate = The sensor records or is used to calculate heart rate; magnetometer = The sensor records the magnetic field; natural-mark = The animal is identified by a unique natural marking; orientation = Quaternion components describing the orientation of the tag are derived from accelerometer and gyroscope measurements; proximity = The sensor identifies proximity to other tags; radio-transmitter = The sensor transmits a radio signal that is detected by receivers to determine location; sigfox-geolocation = The sensor location is determined by Sigfox using the received signal strength indicator; solar-geolocator = The sensor collects light levels, which are used to determine position (for processed locations); solar-geolocator-raw = The sensor collects light levels, which are used to determine position (for raw light-level measurements); solar-geolocator-twilight = The sensor collects light levels, which are used to determine position (for twilights calculated from light-level measurements). Format: controlled list; Entity described: event", + "format": "default", + "name": "sensor-type", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000170/6/", + "title": "sensor type", + "type": "string", + }, + { + "description": "This attribute has been merged with 'animal taxon'. The scientific name of the species on which the tag was deployed, as defined by the Integrated Taxonomic Information System (ITIS, www.itis.gov). If the species name can not be provided, this should be the lowest level taxonomic rank that can be determined and that is used in the ITIS taxonomy. Additional information can be provided using the term 'taxon detail'. Format: controlled list; Entity described: individual", + "format": "default", + "name": "individual-taxon-canonical-name", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000135/5/", + "title": "individual taxon canonical name", + "type": "string", + }, + { + "description": "This attribute has been merged with 'tag ID'. An identifier for the tag, provided by the data owner. Values are unique within the study. If the data owner does not provide a tag ID, an internal Movebank tag identifier may sometimes be shown. Example: '2342'; Units: none; Entity described: tag", + "format": "default", + "name": "tag-local-identifier", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000182/5/", + "title": "tag local identifier", + "type": "string", + }, + { + "description": "This attribute has been merged with 'animal ID'. An individual identifier for the animal, provided by the data owner. Values are unique within the study. If the data owner does not provide an Animal ID, an internal Movebank animal identifier is sometimes shown. Example: '91876A, Gary'; Units: none; Entity described: individual", + "format": "default", + "name": "individual-local-identifier", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000134/4/", + "title": "individual local identifier", + "type": "string", + }, + { + "description": "The name of the study in Movebank. Example: 'Coyotes, Kays and Bogan, Albany NY'; Units: none; Entity described: study", + "format": "default", + "name": "study-name", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000173/3/", + "title": "study name", + "type": "string", + }, + ], + "foreignKeys": [ + { + "fields": [ + "individual-local-identifier", + "tag-local-identifier", + ], + "reference": { + "fields": [ + "animal-id", + "tag-id", + ], + "resource": "reference-data", + }, + }, + ], + "primaryKey": [ + "event-id", + ], + }, + "zenodo:key": undefined, + "zenodo:url": undefined, + }, + { + "encoding": "UTF-8", + "format": "csv", + "mediatype": "text/csv", + "name": "acceleration", + "path": [ + "https://zenodo.org/records/10053903/files/O_ASSEN-acceleration-2018.csv.gz", + "https://zenodo.org/records/10053903/files/O_ASSEN-acceleration-2019.csv.gz", + ], + "profile": "tabular-data-resource", + "schema": { + "fields": [ + { + "description": "An identifier for the set of values associated with each event, i.e. sensor measurement. A unique event ID is assigned to every time-location or other time-measurement record in Movebank. If multiple measurements are included within a single row of a data file, they will share an event ID. If users import the same sensor measurement to Movebank multiple times, a separate event ID will be assigned to each. Example: '14328243575'; Units: none; Entity described: event", + "format": "default", + "name": "event-id", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000103/3/", + "title": "event ID", + "type": "integer", + }, + { + "description": "Determines whether an event is visible on the Movebank map. Allowed values are TRUE or FALSE. Values are calculated automatically, with TRUE indicating the event has not been flagged as an outlier by 'algorithm marked outlier', 'import marked outlier' or 'manually marked outlier', or that the user has overridden the results of these outlier attributes using 'manually marked valid' = TRUE. Units: none; Entity described: event", + "format": "default", + "name": "visible", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000209/3/", + "title": "visible", + "type": "boolean", + }, + { + "description": "The date and time corresponding to a sensor measurement or an estimate derived from sensor measurements. Example: '2008-08-14 18:31:00.000'; Format: yyyy-MM-dd HH:mm:ss.SSS; Units: UTC or GPS time; Entity described: event", + "format": "%Y-%m-%d %H:%M:%S.%f", + "name": "timestamp", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000200/2/", + "title": "timestamp", + "type": "datetime", + }, + { + "description": "Raw acceleration values provided by the tag for the X axis. Range and units may vary by provider, tag, and orientation of the sensor on the animal. Example: '0.556641'; Units: not defined; Entity described: event", + "format": "default", + "name": "acceleration-raw-x", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000002/2/", + "title": "acceleration raw x", + "type": "number", + }, + { + "description": "Raw acceleration values provided by the tag for the Y axis. Range and units may vary by provider, tag, and orientation of the sensor on the animal. Example: '0.09375'; Units: not defined; Entity described: event", + "format": "default", + "name": "acceleration-raw-y", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000003/2/", + "title": "acceleration raw y", + "type": "number", + }, + { + "description": "Raw acceleration values provided by the tag for the Z axis. Range and units may vary by provider, tag, and orientation of the sensor on the animal. Example: '-0.84375'; Units: not defined; Entity described: event", + "format": "default", + "name": "acceleration-raw-z", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000004/2/", + "title": "acceleration raw z", + "type": "number", + }, + { + "description": "The date and time when the sampling interval or burst began. Example: '2011-01-03 13:45:00.000'; Format: yyyy-MM-dd HH:mm:ss.SSS; Units: UTC or GPS time; Entity described: event", + "format": "%Y-%m-%d %H:%M:%S.%f", + "name": "start-timestamp", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000171/2/", + "title": "start timestamp", + "type": "datetime", + }, + { + "description": "Tilt provided by the accelerometer for the X axis. Example: '0'; Units: g forces (1 g = 9.8 m s^-2); Entity described: event", + "format": "default", + "name": "tilt-x", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000197/2/", + "title": "tilt x", + "type": "number", + }, + { + "description": "Tilt provided by the accelerometer for the Y axis. Example: '0'; Units: g forces (1 g = 9.8 m s^-2); Entity described: event", + "format": "default", + "name": "tilt-y", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000198/2/", + "title": "tilt y", + "type": "number", + }, + { + "description": "Tilt provided by the accelerometer for the Z axis. Example: '1'; Units: g forces (1 g = 9.8 m s^-2); Entity described: event", + "format": "default", + "name": "tilt-z", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000199/2/", + "title": "tilt z", + "type": "number", + }, + { + "description": "The type of sensor with which data were collected. All sensors are associated with a tag id, and tags can contain multiple sensor types. Each event record in Movebank is assigned one sensor type. If values from multiple sensors are reported in a single event, the primary sensor is used. Values are chosen from a controlled list: acceleration = The sensor collects acceleration data; accessory-measurements = The sensor collects accessory measurements, such as battery voltage; acoustic-telemetry = The sensor transmits an acoustic signal that is detected by receivers to determine location; argos-doppler-shift = The sensor location is estimated by Argos using Doppler shift; barometer = The sensor records air or water pressure; bird-ring = The animal is identified by a band or ring that has a unique identifier; gps = The sensor uses GPS to determine location; gyroscope = The sensor records angular velocity; heart-rate = The sensor records or is used to calculate heart rate; magnetometer = The sensor records the magnetic field; natural-mark = The animal is identified by a unique natural marking; orientation = Quaternion components describing the orientation of the tag are derived from accelerometer and gyroscope measurements; proximity = The sensor identifies proximity to other tags; radio-transmitter = The sensor transmits a radio signal that is detected by receivers to determine location; sigfox-geolocation = The sensor location is determined by Sigfox using the received signal strength indicator; solar-geolocator = The sensor collects light levels, which are used to determine position (for processed locations); solar-geolocator-raw = The sensor collects light levels, which are used to determine position (for raw light-level measurements); solar-geolocator-twilight = The sensor collects light levels, which are used to determine position (for twilights calculated from light-level measurements). Format: controlled list; Entity described: event", + "format": "default", + "name": "sensor-type", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000170/6/", + "title": "sensor type", + "type": "string", + }, + { + "description": "This attribute has been merged with 'animal taxon'. The scientific name of the species on which the tag was deployed, as defined by the Integrated Taxonomic Information System (ITIS, www.itis.gov). If the species name can not be provided, this should be the lowest level taxonomic rank that can be determined and that is used in the ITIS taxonomy. Additional information can be provided using the term 'taxon detail'. Format: controlled list; Entity described: individual", + "format": "default", + "name": "individual-taxon-canonical-name", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000135/5/", + "title": "individual taxon canonical name", + "type": "string", + }, + { + "description": "This attribute has been merged with 'tag ID'. An identifier for the tag, provided by the data owner. Values are unique within the study. If the data owner does not provide a tag ID, an internal Movebank tag identifier may sometimes be shown. Example: '2342'; Units: none; Entity described: tag", + "format": "default", + "name": "tag-local-identifier", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000182/5/", + "title": "tag local identifier", + "type": "string", + }, + { + "description": "This attribute has been merged with 'animal ID'. An individual identifier for the animal, provided by the data owner. Values are unique within the study. If the data owner does not provide an Animal ID, an internal Movebank animal identifier is sometimes shown. Example: '91876A, Gary'; Units: none; Entity described: individual", + "format": "default", + "name": "individual-local-identifier", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000134/4/", + "title": "individual local identifier", + "type": "string", + }, + { + "description": "The name of the study in Movebank. Example: 'Coyotes, Kays and Bogan, Albany NY'; Units: none; Entity described: study", + "format": "default", + "name": "study-name", + "skos:exactMatch": "http://vocab.nerc.ac.uk/collection/MVB/current/MVB000173/3/", + "title": "study name", + "type": "string", + }, + ], + "foreignKeys": [ + { + "fields": [ + "individual-local-identifier", + "tag-local-identifier", + ], + "reference": { + "fields": [ + "animal-id", + "tag-id", + ], + "resource": "reference-data", + }, + }, + ], + "primaryKey": [ + "event-id", + ], + }, + "zenodo:key": undefined, + "zenodo:url": undefined, }, ], - "title": "title", - "version": "1.0", + "title": "O_ASSEN - Eurasian oystercatchers (Haematopus ostralegus, Haematopodidae) breeding in Assen (the Netherlands)", } `; diff --git a/zenodo/package/fixtures/generated/should-load-a-package_2675098929/recording.har b/zenodo/package/fixtures/generated/should-load-a-package_2675098929/recording.har index 936b76e4..f8c4da0a 100644 --- a/zenodo/package/fixtures/generated/should-load-a-package_2675098929/recording.har +++ b/zenodo/package/fixtures/generated/should-load-a-package_2675098929/recording.har @@ -8,34 +8,34 @@ }, "entries": [ { - "_id": "c252115c2e378f6716d6b175112fdc7a", + "_id": "b91d0d032e9ab808b928e484fd3e9df0", "_order": 0, "cache": {}, "request": { "bodySize": 0, "cookies": [], "headers": [], - "headersSize": 64, + "headersSize": 58, "httpVersion": "HTTP/1.1", "method": "GET", "queryString": [], - "url": "https://sandbox.zenodo.org/api/records/260528" + "url": "https://zenodo.org/api/records/15525711" }, "response": { - "bodySize": 4196, + "bodySize": 3794, "content": { "mimeType": "application/json", - "size": 4196, - "text": "{\"created\": \"2025-05-24T09:01:09.188189+00:00\", \"modified\": \"2025-05-24T09:01:09.509867+00:00\", \"id\": 260528, \"conceptrecid\": \"260527\", \"doi\": \"10.5072/zenodo.260528\", \"conceptdoi\": \"10.5072/zenodo.260527\", \"doi_url\": \"https://handle.stage.datacite.org/10.5072/zenodo.260528\", \"metadata\": {\"title\": \"title\", \"doi\": \"10.5072/zenodo.260528\", \"publication_date\": \"2025-05-24\", \"description\": \"description\", \"access_right\": \"open\", \"creators\": [{\"name\": \"title\", \"affiliation\": \"path\"}], \"keywords\": [\"keyword1\", \"keyword2\"], \"version\": \"1.0\", \"resource_type\": {\"title\": \"Dataset\", \"type\": \"dataset\"}, \"license\": {\"id\": \"mit-license\"}, \"relations\": {\"version\": [{\"index\": 0, \"is_last\": true, \"parent\": {\"pid_type\": \"recid\", \"pid_value\": \"260527\"}}]}}, \"title\": \"title\", \"links\": {\"self\": \"https://sandbox.zenodo.org/api/records/260528\", \"self_html\": \"https://sandbox.zenodo.org/records/260528\", \"preview_html\": \"https://sandbox.zenodo.org/records/260528?preview=1\", \"doi\": \"https://handle.stage.datacite.org/10.5072/zenodo.260528\", \"self_doi\": \"https://handle.stage.datacite.org/10.5072/zenodo.260528\", \"self_doi_html\": \"https://sandbox.zenodo.org/doi/10.5072/zenodo.260528\", \"reserve_doi\": \"https://sandbox.zenodo.org/api/records/260528/draft/pids/doi\", \"parent\": \"https://sandbox.zenodo.org/api/records/260527\", \"parent_html\": \"https://sandbox.zenodo.org/records/260527\", \"parent_doi\": \"https://handle.stage.datacite.org/10.5072/zenodo.260527\", \"parent_doi_html\": \"https://sandbox.zenodo.org/doi/10.5072/zenodo.260527\", \"self_iiif_manifest\": \"https://sandbox.zenodo.org/api/iiif/record:260528/manifest\", \"self_iiif_sequence\": \"https://sandbox.zenodo.org/api/iiif/record:260528/sequence/default\", \"files\": \"https://sandbox.zenodo.org/api/records/260528/files\", \"media_files\": \"https://sandbox.zenodo.org/api/records/260528/media-files\", \"archive\": \"https://sandbox.zenodo.org/api/records/260528/files-archive\", \"archive_media\": \"https://sandbox.zenodo.org/api/records/260528/media-files-archive\", \"latest\": \"https://sandbox.zenodo.org/api/records/260528/versions/latest\", \"latest_html\": \"https://sandbox.zenodo.org/records/260528/latest\", \"versions\": \"https://sandbox.zenodo.org/api/records/260528/versions\", \"draft\": \"https://sandbox.zenodo.org/api/records/260528/draft\", \"access_links\": \"https://sandbox.zenodo.org/api/records/260528/access/links\", \"access_grants\": \"https://sandbox.zenodo.org/api/records/260528/access/grants\", \"access_users\": \"https://sandbox.zenodo.org/api/records/260528/access/users\", \"access_request\": \"https://sandbox.zenodo.org/api/records/260528/access/request\", \"access\": \"https://sandbox.zenodo.org/api/records/260528/access\", \"communities\": \"https://sandbox.zenodo.org/api/records/260528/communities\", \"communities-suggestions\": \"https://sandbox.zenodo.org/api/records/260528/communities-suggestions\", \"requests\": \"https://sandbox.zenodo.org/api/records/260528/requests\"}, \"updated\": \"2025-05-24T09:01:09.509867+00:00\", \"recid\": \"260528\", \"revision\": 3, \"files\": [{\"id\": \"09f0f6e5-aed0-45c7-b579-e000272be917\", \"key\": \"table.csv\", \"size\": 30, \"checksum\": \"md5:6c2c61dd9b0e9c6876139a449ed87933\", \"links\": {\"self\": \"https://sandbox.zenodo.org/api/records/260528/files/table.csv/content\"}}, {\"id\": \"17eba425-9e7e-4edc-abdb-d97514e5b7b4\", \"key\": \"dialect.json\", \"size\": 23, \"checksum\": \"md5:0de7e07cbe2fc19dd077133951760eed\", \"links\": {\"self\": \"https://sandbox.zenodo.org/api/records/260528/files/dialect.json/content\"}}, {\"id\": \"6a8affa9-f588-4fa3-a232-9b6c41be5469\", \"key\": \"schema.json\", \"size\": 136, \"checksum\": \"md5:30208ec3060c83e82f626cdd19795c69\", \"links\": {\"self\": \"https://sandbox.zenodo.org/api/records/260528/files/schema.json/content\"}}, {\"id\": \"677fffdd-9755-412f-b616-6b1ce9977c62\", \"key\": \"datapackage.json\", \"size\": 824, \"checksum\": \"md5:028c7ddf791931e9738a3e7e8586ea34\", \"links\": {\"self\": \"https://sandbox.zenodo.org/api/records/260528/files/datapackage.json/content\"}}], \"swh\": {}, \"owners\": [{\"id\": \"31914\"}], \"status\": \"published\", \"stats\": {\"downloads\": 0, \"unique_downloads\": 0, \"views\": 0, \"unique_views\": 0, \"version_downloads\": 0, \"version_unique_downloads\": 0, \"version_unique_views\": 0, \"version_views\": 0}, \"state\": \"done\", \"submitted\": true}" + "size": 3794, + "text": "{\"created\": \"2025-05-27T10:20:07.936985+00:00\", \"modified\": \"2025-05-27T10:20:08.174083+00:00\", \"id\": 15525711, \"conceptrecid\": \"15525710\", \"doi\": \"10.5281/zenodo.15525711\", \"conceptdoi\": \"10.5281/zenodo.15525710\", \"doi_url\": \"https://doi.org/10.5281/zenodo.15525711\", \"metadata\": {\"title\": \"PM10 Concentration in Japan\", \"doi\": \"10.5281/zenodo.15525711\", \"publication_date\": \"2025-05-27\", \"description\": \"

A brief project to extract PM 10 concentration from Japan. Data set has been created using the OpenAQ data through a python script

\", \"access_right\": \"open\", \"creators\": [{\"name\": \"Sukhija, Vinay Jagdish\", \"affiliation\": null, \"orcid\": \"0000-0003-2155-6940\"}], \"keywords\": [\"PM10 concentrations\", \"Air Quality\", \"Japan\", \"OpenAQ\"], \"custom\": {\"code:programmingLanguage\": [{\"id\": \"python\", \"title\": {\"en\": \"Python\"}}]}, \"resource_type\": {\"title\": \"Dataset\", \"type\": \"dataset\"}, \"meeting\": {\"title\": \"Open Data and Open Science Conference\", \"dates\": \"27 May 2025\", \"place\": \"Padova\"}, \"license\": {\"id\": \"cc-by-4.0\"}, \"relations\": {\"version\": [{\"index\": 0, \"is_last\": true, \"parent\": {\"pid_type\": \"recid\", \"pid_value\": \"15525710\"}}]}}, \"title\": \"PM10 Concentration in Japan\", \"links\": {\"self\": \"https://zenodo.org/api/records/15525711\", \"self_html\": \"https://zenodo.org/records/15525711\", \"preview_html\": \"https://zenodo.org/records/15525711?preview=1\", \"doi\": \"https://doi.org/10.5281/zenodo.15525711\", \"self_doi\": \"https://doi.org/10.5281/zenodo.15525711\", \"self_doi_html\": \"https://zenodo.org/doi/10.5281/zenodo.15525711\", \"reserve_doi\": \"https://zenodo.org/api/records/15525711/draft/pids/doi\", \"parent\": \"https://zenodo.org/api/records/15525710\", \"parent_html\": \"https://zenodo.org/records/15525710\", \"parent_doi\": \"https://doi.org/10.5281/zenodo.15525710\", \"parent_doi_html\": \"https://zenodo.org/doi/10.5281/zenodo.15525710\", \"self_iiif_manifest\": \"https://zenodo.org/api/iiif/record:15525711/manifest\", \"self_iiif_sequence\": \"https://zenodo.org/api/iiif/record:15525711/sequence/default\", \"files\": \"https://zenodo.org/api/records/15525711/files\", \"media_files\": \"https://zenodo.org/api/records/15525711/media-files\", \"archive\": \"https://zenodo.org/api/records/15525711/files-archive\", \"archive_media\": \"https://zenodo.org/api/records/15525711/media-files-archive\", \"latest\": \"https://zenodo.org/api/records/15525711/versions/latest\", \"latest_html\": \"https://zenodo.org/records/15525711/latest\", \"versions\": \"https://zenodo.org/api/records/15525711/versions\", \"draft\": \"https://zenodo.org/api/records/15525711/draft\", \"access_links\": \"https://zenodo.org/api/records/15525711/access/links\", \"access_grants\": \"https://zenodo.org/api/records/15525711/access/grants\", \"access_users\": \"https://zenodo.org/api/records/15525711/access/users\", \"access_request\": \"https://zenodo.org/api/records/15525711/access/request\", \"access\": \"https://zenodo.org/api/records/15525711/access\", \"communities\": \"https://zenodo.org/api/records/15525711/communities\", \"communities-suggestions\": \"https://zenodo.org/api/records/15525711/communities-suggestions\", \"requests\": \"https://zenodo.org/api/records/15525711/requests\"}, \"updated\": \"2025-05-27T10:20:08.174083+00:00\", \"recid\": \"15525711\", \"revision\": 4, \"files\": [{\"id\": \"c50a6efe-4946-4054-a3bb-797a35da03d0\", \"key\": \"openaq_measurements_PM10_Japan_20250527_095520.csv\", \"size\": 272, \"checksum\": \"md5:5ac1b92a57ec809c546ea90334c845c9\", \"links\": {\"self\": \"https://zenodo.org/api/records/15525711/files/openaq_measurements_PM10_Japan_20250527_095520.csv/content\"}}], \"swh\": {}, \"owners\": [{\"id\": \"1328821\"}], \"status\": \"published\", \"stats\": {\"downloads\": 0, \"unique_downloads\": 0, \"views\": 0, \"unique_views\": 0, \"version_downloads\": 0, \"version_unique_downloads\": 0, \"version_unique_views\": 0, \"version_views\": 0}, \"state\": \"done\", \"submitted\": true}" }, "cookies": [ { "httpOnly": true, - "name": "04f20c86f07421a9ec0f9d5ba4be544f", + "name": "5569e5a730cade8ff2b54f1e815f3670", "path": "/", "sameSite": "None", "secure": true, - "value": "10bd13aec24bd0e49d2374af0ac6de16" + "value": "1150a9296538db69f8d2c95afd8c3910" } ], "headers": [ @@ -65,15 +65,15 @@ }, { "name": "date", - "value": "Sat, 24 May 2025 09:07:34 GMT" + "value": "Tue, 27 May 2025 10:35:44 GMT" }, { "name": "etag", - "value": "W/\"3\"" + "value": "W/\"4\"" }, { "name": "link", - "value": " ; rel=\"linkset\" ; type=\"application/linkset+json\"" + "value": " ; rel=\"linkset\" ; type=\"application/linkset+json\"" }, { "name": "permissions-policy", @@ -85,7 +85,7 @@ }, { "name": "retry-after", - "value": "60" + "value": "59" }, { "name": "server", @@ -93,7 +93,7 @@ }, { "name": "set-cookie", - "value": "04f20c86f07421a9ec0f9d5ba4be544f=10bd13aec24bd0e49d2374af0ac6de16; path=/; HttpOnly; Secure; SameSite=None" + "value": "5569e5a730cade8ff2b54f1e815f3670=1150a9296538db69f8d2c95afd8c3910; path=/; HttpOnly; Secure; SameSite=None" }, { "name": "strict-transport-security", @@ -125,25 +125,25 @@ }, { "name": "x-ratelimit-reset", - "value": "1748077715" + "value": "1748342204" }, { "name": "x-request-id", - "value": "0e4f325b507d787a6f53d468865028e7" + "value": "0438503d83b8bf34f5ea707caee6734f" }, { "name": "x-xss-protection", "value": "1; mode=block" } ], - "headersSize": 1255, + "headersSize": 1249, "httpVersion": "HTTP/1.1", "redirectURL": "", "status": 200, "statusText": "OK" }, - "startedDateTime": "2025-05-24T09:07:33.850Z", - "time": 1102, + "startedDateTime": "2025-05-27T10:35:42.687Z", + "time": 597, "timings": { "blocked": -1, "connect": -1, @@ -151,7 +151,7 @@ "receive": 0, "send": 0, "ssl": -1, - "wait": 1102 + "wait": 597 } } ], diff --git a/zenodo/package/fixtures/generated/shoule-merge-datapackage-json-if-present_2284251141/recording.har b/zenodo/package/fixtures/generated/shoule-merge-datapackage-json-if-present_2284251141/recording.har new file mode 100644 index 00000000..a2d247db --- /dev/null +++ b/zenodo/package/fixtures/generated/shoule-merge-datapackage-json-if-present_2284251141/recording.har @@ -0,0 +1,300 @@ +{ + "log": { + "_recordingName": "shoule merge datapackage.json if present", + "creator": { + "comment": "persister:fs", + "name": "Polly.JS", + "version": "6.0.6" + }, + "entries": [ + { + "_id": "4c12784dd2adb5911ff7bf14f8b3d9b7", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [], + "headersSize": 58, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://zenodo.org/api/records/10053903" + }, + "response": { + "bodySize": 10436, + "content": { + "mimeType": "application/json", + "size": 10436, + "text": "{\"created\": \"2023-10-30T11:31:16.569518+00:00\", \"modified\": \"2025-05-15T09:13:47.716113+00:00\", \"id\": 10053903, \"conceptrecid\": \"5653310\", \"doi\": \"10.5281/zenodo.10053903\", \"conceptdoi\": \"10.5281/zenodo.5653310\", \"doi_url\": \"https://doi.org/10.5281/zenodo.10053903\", \"metadata\": {\"title\": \"O_ASSEN - Eurasian oystercatchers (Haematopus ostralegus, Haematopodidae) breeding in Assen (the Netherlands)\", \"doi\": \"10.5281/zenodo.10053903\", \"publication_date\": \"2023-10-30\", \"description\": \"

O_ASSEN - Eurasian oystercatchers (Haematopus ostralegus, Haematopodidae) breeding in Assen (the Netherlands) is a bird tracking dataset published by the Vogelwerkgroep Assen, Netherlands Institute of Ecology (NIOO-KNAW), Sovon, Radboud University, the University of Amsterdam and the Research Institute for Nature and Forest (INBO). It contains animal tracking data collected for the study O_ASSEN using trackers developed by the University of Amsterdam Bird Tracking System (UvA-BiTS, http://www.uva-bits.nl). The study was operational from 2018 to 2019. In total 6 individuals of Eurasian oystercatchers (Haematopus ostralegus) have been tagged as a breeding bird in the city of Assen (the Netherlands), mainly to study space use of oystercatchers breeding in urban areas. Data are uploaded from the UvA-BiTS database to Movebank and from there archived on Zenodo (see https://github.com/inbo/bird-tracking). No new data are expected.

\\n

See van der Kolk et al. (2022, https://doi.org/10.3897/zookeys.1123.90623) for a more detailed description of this dataset.

\\n

Files

\\n

Data in this package are exported from Movebank study 1605797471. Fields in the data follow the Movebank Attribute Dictionary and are described in datapackage.json. Files are structured as a Frictionless Data Package. You can access all data in R via https://zenodo.org/records/10053903/files/datapackage.json using frictionless.

\\n
    \\n
  • datapackage.json: technical description of the data files.
    • \\n
  • O_ASSEN-reference-data.csv: reference data about the animals, tags and deployments.
    • \\n
  • O_ASSEN-gps-yyyy.csv.gz: GPS data recorded by the tags, grouped by year.
    • \\n
  • O_ASSEN-acceleration-yyyy.csv.gz: acceleration data recorded by the tags, grouped by year.
    • \\n
\\n

Acknowledgements

\\n

These data were collected by Bert Dijkstra and Rinus Dillerop from Vogelwerkgroep Assen, in collaboration with the Netherlands Institute of Ecology (NIOO-KNAW), Sovon, Radboud University and the University of Amsterdam (UvA). Funding was provided by the Prins Bernard Cultuurfonds Drenthe, municipality of Assen, IJsvogelfonds (from Birdlife Netherlands and Nationale Postcodeloterij) and the Waterleiding Maatschappij Drenthe. The dataset was published with funding from Stichting NLBIF - Netherlands Biodiversity Information Facility.

\", \"access_right\": \"open\", \"creators\": [{\"name\": \"Dijkstra, Bert\", \"affiliation\": \"Vogelwerkgroep Assen\"}, {\"name\": \"Dillerop, Rinus\", \"affiliation\": \"Vogelwerkgroep Assen\"}, {\"name\": \"Oosterbeek, Kees\", \"affiliation\": \"Dutch Centre for Field Ornithology\"}, {\"name\": \"Bouten, Willem\", \"affiliation\": \"University of Amsterdam\", \"orcid\": \"0000-0002-5250-8872\"}, {\"name\": \"Desmet, Peter\", \"affiliation\": \"Research Institute for Nature and Forest\", \"orcid\": \"0000-0002-8442-8025\"}, {\"name\": \"van der Kolk, Henk-Jan\", \"affiliation\": \"Dutch Bryological and Lichenological Society\", \"orcid\": \"0000-0002-8023-379X\"}, {\"name\": \"Ens, Bruno J.\", \"affiliation\": \"Dutch Centre for Field Ornithology\", \"orcid\": \"0000-0002-4659-4807\"}], \"contributors\": [{\"name\": \"Desmet, Peter\", \"affiliation\": \"Research Institute for Nature and Forest\", \"type\": \"DataCurator\", \"orcid\": \"0000-0002-8442-8025\"}], \"keywords\": [\"animal movement\", \"animal tracking\", \"gps tracking\", \"accelerometer\", \"altitude\", \"temperature\", \"biologging\", \"birds\", \"UvA-BiTS\", \"Movebank\", \"frictionlessdata\"], \"related_identifiers\": [{\"identifier\": \"https://www.movebank.org/cms/webapp?gwt_fragment=page=studies,path=study1605797471\", \"relation\": \"isDerivedFrom\", \"resource_type\": \"dataset\", \"scheme\": \"url\"}, {\"identifier\": \"https://natuurtijdschriften.nl/pub/717840/DV2018032001001005.pdf\", \"relation\": \"isSourceOf\", \"resource_type\": \"publication-article\", \"scheme\": \"url\"}, {\"identifier\": \"https://natuurtijdschriften.nl/pub/1018631/DV2019032001001005.pdf\", \"relation\": \"isSourceOf\", \"resource_type\": \"publication-article\", \"scheme\": \"url\"}, {\"identifier\": \"https://www.gbif.org/dataset/226421f2-1d29-4950-901c-aba9d0e8f2bc\", \"relation\": \"isSourceOf\", \"resource_type\": \"dataset\", \"scheme\": \"url\"}, {\"identifier\": \"https://obis.org/dataset/550b4cc1-c40d-4070-a0cb-26e010eca9d4\", \"relation\": \"isSourceOf\", \"resource_type\": \"dataset\", \"scheme\": \"url\"}, {\"identifier\": \"10.3897/zookeys.1123.90623\", \"relation\": \"isDescribedBy\", \"resource_type\": \"publication-article\", \"scheme\": \"doi\"}, {\"identifier\": \"https://inbo.github.io/movepub/\", \"relation\": \"isCompiledBy\", \"resource_type\": \"software\", \"scheme\": \"url\"}], \"language\": \"eng\", \"resource_type\": {\"title\": \"Dataset\", \"type\": \"dataset\"}, \"license\": {\"id\": \"cc-zero\"}, \"grants\": [{\"code\": \"NLBIF\", \"internal_id\": \"0566bfb96::NLBIF\", \"funder\": {\"name\": \"Naturalis Biodiversity Center\"}, \"title\": \"Netherlands Biodiversity Information Facility\"}], \"communities\": [{\"id\": \"inbo\"}, {\"id\": \"netherlandsbiodiversityinformationfacility\"}, {\"id\": \"oscibio\"}], \"relations\": {\"version\": [{\"index\": 2, \"is_last\": true, \"parent\": {\"pid_type\": \"recid\", \"pid_value\": \"5653310\"}}]}, \"notes\": \"

This version adds alt-project-id to the reference-data and references the latest Movebank Attribute Dictionary.

\"}, \"title\": \"O_ASSEN - Eurasian oystercatchers (Haematopus ostralegus, Haematopodidae) breeding in Assen (the Netherlands)\", \"links\": {\"self\": \"https://zenodo.org/api/records/10053903\", \"self_html\": \"https://zenodo.org/records/10053903\", \"preview_html\": \"https://zenodo.org/records/10053903?preview=1\", \"doi\": \"https://doi.org/10.5281/zenodo.10053903\", \"self_doi\": \"https://doi.org/10.5281/zenodo.10053903\", \"self_doi_html\": \"https://zenodo.org/doi/10.5281/zenodo.10053903\", \"reserve_doi\": \"https://zenodo.org/api/records/10053903/draft/pids/doi\", \"parent\": \"https://zenodo.org/api/records/5653310\", \"parent_html\": \"https://zenodo.org/records/5653310\", \"parent_doi\": \"https://doi.org/10.5281/zenodo.5653310\", \"parent_doi_html\": \"https://zenodo.org/doi/10.5281/zenodo.5653310\", \"self_iiif_manifest\": \"https://zenodo.org/api/iiif/record:10053903/manifest\", \"self_iiif_sequence\": \"https://zenodo.org/api/iiif/record:10053903/sequence/default\", \"files\": \"https://zenodo.org/api/records/10053903/files\", \"media_files\": \"https://zenodo.org/api/records/10053903/media-files\", \"archive\": \"https://zenodo.org/api/records/10053903/files-archive\", \"archive_media\": \"https://zenodo.org/api/records/10053903/media-files-archive\", \"latest\": \"https://zenodo.org/api/records/10053903/versions/latest\", \"latest_html\": \"https://zenodo.org/records/10053903/latest\", \"versions\": \"https://zenodo.org/api/records/10053903/versions\", \"draft\": \"https://zenodo.org/api/records/10053903/draft\", \"access_links\": \"https://zenodo.org/api/records/10053903/access/links\", \"access_grants\": \"https://zenodo.org/api/records/10053903/access/grants\", \"access_users\": \"https://zenodo.org/api/records/10053903/access/users\", \"access_request\": \"https://zenodo.org/api/records/10053903/access/request\", \"access\": \"https://zenodo.org/api/records/10053903/access\", \"communities\": \"https://zenodo.org/api/records/10053903/communities\", \"communities-suggestions\": \"https://zenodo.org/api/records/10053903/communities-suggestions\", \"requests\": \"https://zenodo.org/api/records/10053903/requests\"}, \"updated\": \"2025-05-15T09:13:47.716113+00:00\", \"recid\": \"10053903\", \"revision\": 16, \"files\": [{\"id\": \"5ffb4935-0a78-4fd5-af2a-f0f5113ca7c0\", \"key\": \"datapackage.json\", \"size\": 42936, \"checksum\": \"md5:5ea86b7a222ac843e3833a2e50a2056b\", \"links\": {\"self\": \"https://zenodo.org/api/records/10053903/files/datapackage.json/content\"}}, {\"id\": \"f09bbed1-50e3-46b3-a1d9-10e8adeec337\", \"key\": \"O_ASSEN-acceleration-2018.csv.gz\", \"size\": 4067780, \"checksum\": \"md5:40b771d9bd53ee5cfae0266209364d9c\", \"links\": {\"self\": \"https://zenodo.org/api/records/10053903/files/O_ASSEN-acceleration-2018.csv.gz/content\"}}, {\"id\": \"73a7a2a5-cdfc-48a7-be65-641144ef9c38\", \"key\": \"O_ASSEN-acceleration-2019.csv.gz\", \"size\": 3696729, \"checksum\": \"md5:9d2cbbaece00ef8cf8b4fff87cf07079\", \"links\": {\"self\": \"https://zenodo.org/api/records/10053903/files/O_ASSEN-acceleration-2019.csv.gz/content\"}}, {\"id\": \"93e83695-a619-4f09-aa93-edb2fadcd586\", \"key\": \"O_ASSEN-gps-2018.csv.gz\", \"size\": 751448, \"checksum\": \"md5:f5e041ea66ebe8708a46fe57f383f549\", \"links\": {\"self\": \"https://zenodo.org/api/records/10053903/files/O_ASSEN-gps-2018.csv.gz/content\"}}, {\"id\": \"e64508be-1860-415f-bbbd-fc4353f33278\", \"key\": \"O_ASSEN-gps-2019.csv.gz\", \"size\": 263439, \"checksum\": \"md5:cec4550a769d4f053e92565b44d0409c\", \"links\": {\"self\": \"https://zenodo.org/api/records/10053903/files/O_ASSEN-gps-2019.csv.gz/content\"}}, {\"id\": \"05581f20-937c-4180-824c-c3068b641d34\", \"key\": \"O_ASSEN-reference-data.csv\", \"size\": 2082, \"checksum\": \"md5:5a5da155f404d5a3b4e50c350ad089d1\", \"links\": {\"self\": \"https://zenodo.org/api/records/10053903/files/O_ASSEN-reference-data.csv/content\"}}], \"swh\": {}, \"owners\": [{\"id\": \"6828\"}], \"status\": \"published\", \"stats\": {\"downloads\": 54743, \"unique_downloads\": 19482, \"views\": 1351, \"unique_views\": 1248, \"version_downloads\": 44749, \"version_unique_downloads\": 17507, \"version_unique_views\": 392, \"version_views\": 430}, \"state\": \"done\", \"submitted\": true}" + }, + "cookies": [ + { + "httpOnly": true, + "name": "5569e5a730cade8ff2b54f1e815f3670", + "path": "/", + "sameSite": "None", + "secure": true, + "value": "fe8892fcd3384dc73cd2eb4e9b901329" + } + ], + "headers": [ + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "access-control-expose-headers", + "value": "Content-Type, ETag, Link, X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Reset" + }, + { + "name": "cache-control", + "value": "private" + }, + { + "name": "content-encoding", + "value": "gzip" + }, + { + "name": "content-security-policy", + "value": "default-src 'self' fonts.googleapis.com *.gstatic.com data: 'unsafe-inline' 'unsafe-eval' blob: zenodo-broker.web.cern.ch zenodo-broker-qa.web.cern.ch maxcdn.bootstrapcdn.com cdnjs.cloudflare.com ajax.googleapis.com webanalytics.web.cern.ch" + }, + { + "name": "content-type", + "value": "application/json" + }, + { + "name": "date", + "value": "Tue, 27 May 2025 07:50:50 GMT" + }, + { + "name": "etag", + "value": "W/\"16\"" + }, + { + "name": "link", + "value": " ; rel=\"linkset\" ; type=\"application/linkset+json\"" + }, + { + "name": "permissions-policy", + "value": "interest-cohort=()" + }, + { + "name": "referrer-policy", + "value": "strict-origin-when-cross-origin" + }, + { + "name": "retry-after", + "value": "60" + }, + { + "name": "server", + "value": "nginx" + }, + { + "name": "set-cookie", + "value": "5569e5a730cade8ff2b54f1e815f3670=fe8892fcd3384dc73cd2eb4e9b901329; path=/; HttpOnly; Secure; SameSite=None" + }, + { + "name": "strict-transport-security", + "value": "max-age=31556926; includeSubDomains, max-age=15768000" + }, + { + "name": "transfer-encoding", + "value": "chunked" + }, + { + "name": "vary", + "value": "Accept-Encoding" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "x-frame-options", + "value": "sameorigin" + }, + { + "name": "x-ratelimit-limit", + "value": "133" + }, + { + "name": "x-ratelimit-remaining", + "value": "132" + }, + { + "name": "x-ratelimit-reset", + "value": "1748332311" + }, + { + "name": "x-request-id", + "value": "8c371249a0b01a15e4baa71551bdeed2" + }, + { + "name": "x-xss-protection", + "value": "1; mode=block" + } + ], + "headersSize": 1250, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-05-27T07:50:48.946Z", + "time": 848, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 848 + } + }, + { + "_id": "e41c8f62376401a90fe9fd1f22af5e8b", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [], + "headersSize": 77, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://zenodo.org/records/10053903/files/datapackage.json" + }, + "response": { + "bodySize": 42936, + "content": { + "mimeType": "text/plain; charset=utf-8, text/plain; charset=utf-8", + "size": 42936, + "text": "{\n \"id\": \"https://doi.org/10.5281/zenodo.10053903\",\n \"profile\": \"tabular-data-package\",\n \"resources\": [\n {\n \"name\": \"reference-data\",\n \"path\": \"O_ASSEN-reference-data.csv\",\n \"profile\": \"tabular-data-resource\",\n \"format\": \"csv\",\n \"mediatype\": \"text/csv\",\n \"encoding\": \"UTF-8\",\n \"schema\": {\n \"fields\": [\n {\n \"name\": \"tag-id\",\n \"title\": \"tag ID\",\n \"description\": \"A unique identifier for the tag, provided by the data owner. If the data owner does not provide a tag ID, an internal Movebank tag identifier may sometimes be shown. Example: '2342'; Units: none; Entity described: tag\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000181/2/\"\n },\n {\n \"name\": \"animal-id\",\n \"title\": \"animal ID\",\n \"description\": \"An individual identifier for the animal, provided by the data owner. Values are unique within the study. If the data owner does not provide an Animal ID, an internal Movebank animal identifier is sometimes shown. Example: 'TUSC_CV5'; Units: none; Entity described: individual\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000016/3/\"\n },\n {\n \"name\": \"animal-taxon\",\n \"title\": \"animal taxon\",\n \"description\": \"The scientific name of the taxon on which the tag was deployed, as defined by the Integrated Taxonomic Information System www.itis.gov. If the species name can not be provided, this should be the lowest level taxonomic rank that can be determined and that is used in the ITIS taxonomy. Additional information can be provided using the term 'taxon detail'. The values 'test' and 'calibration' identify events relevant to animal tracking studies that should not be associated with a taxon. Format: controlled list; Entity described: individual\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000024/4/\"\n },\n {\n \"name\": \"deploy-on-date\",\n \"title\": \"deploy on timestamp\",\n \"description\": \"The timestamp when the tag deployment started. Data records recorded before this day and time are not associated with the animal related to the deployment. Values are typically defined by the data owner, and in some cases are created automatically during data import. Example: '2008-08-30 18:00:00.000'; Format: yyyy-MM-dd HH:mm:ss.SSS; Units: UTC or GPS time; Entity described: deployment\",\n \"type\": \"datetime\",\n \"format\": \"%Y-%m-%d %H:%M:%S.%f\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000081/3/\"\n },\n {\n \"name\": \"deploy-off-date\",\n \"title\": \"deploy off timestamp\",\n \"description\": \"The timestamp when the tag deployment ended. Data records recorded after this day and time are not associated with the animal related to the deployment. Values are typically defined by the data owner, and in some cases are created automatically during data import. Further information can be provided in 'deployment end type' and 'deployment end comments'. Example: '2009-10-01 12:00:00.000'; Format: yyyy-MM-dd HH:mm:ss.SSS; Units: UTC or GPS time; Entity described: deployment\",\n \"type\": \"datetime\",\n \"format\": \"%Y-%m-%d %H:%M:%S.%f\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000077/4/\"\n },\n {\n \"name\": \"alt-project-id\",\n \"title\": \"alt project ID\",\n \"description\": \"A name or unique identifier for a project associated with the deployment, for example a monitoring program or another data platform. Best practice is to include the name of the related database or organization followed by the project identifier. Example: 'MOTUS145'; Units: none; Entity described: deployment\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000244/2/\"\n },\n {\n \"name\": \"animal-comments\",\n \"title\": \"animal comments\",\n \"description\": \"Additional information about the animal. Example: 'first to fledge from nest'; Units: none; Entity described: individual\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000012/3/\"\n },\n {\n \"name\": \"animal-life-stage\",\n \"title\": \"animal life stage\",\n \"description\": \"The age class or life stage of the animal at the beginning of the deployment. Can be years or months of age or terms such as 'adult', 'subadult' and 'juvenile'. Best practice is to define units in the values if needed (e.g. '2 years'). Example: 'juvenile, adult'; Units: none; Entity described: deployment\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000018/3/\"\n },\n {\n \"name\": \"animal-mass\",\n \"title\": \"animal mass\",\n \"description\": \"The mass of the animal, typically at the beginning of the deployment. Example: '500'; Units: grams; Entity described: deployment\",\n \"type\": \"number\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000019/2/\"\n },\n {\n \"name\": \"animal-nickname\",\n \"title\": \"animal nickname\",\n \"description\": \"An alternate identifier for the animal. Used as the display name for animals shown in the Animal Tracker App. Example: 'Ali'; Units: none; Entity described: individual\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000020/2/\"\n },\n {\n \"name\": \"animal-ring-id\",\n \"title\": \"animal ring ID\",\n \"description\": \"A number or color scheme for a band or ring attached to the animal. Color bands and other markings can be stored in 'animal marker ID'. Example: '26225'; Units: none; Entity described: individual\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000022/3/\"\n },\n {\n \"name\": \"animal-sex\",\n \"title\": \"animal sex\",\n \"description\": \"The sex of the animal. Allowed values are m = male; f = female; u = unknown. Format: controlled list; Entity described: individual\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000023/3/\"\n },\n {\n \"name\": \"attachment-type\",\n \"title\": \"attachment type\",\n \"description\": \"The way a tag is attached to an animal. Details can be provided in 'attachment comments'. Values are chosen from a controlled list: backpack-harness = The tag is attached to the animal using a backpack-style harness; collar = The tag is attached by a collar around the animal's neck; ear-tag = The tag is attached to the animal's ear; fin mount = The tag is attached to the animal's fin; glue = The tag is attached to the animal using glue; harness = The tag is attached to the animal using a harness; implant = The tag is placed under the skin of the animal; leg-band = The tag is attached as a leg band or ring; leg-loop-harness = The tag is attached to the animal using a leg-loop-style harness; none = No tag was attached, e.g., for observations using natural markings; other = The tag is attached using another method; subcutaneous-anchor = The tag is attached using one or more anchors attached underneath the animal's skin; suction-cup = The tag is attached using one or more suction cups; sutures = The tag is attached by one or more sutures; tape = The tag is attached to the animal using tape. Format: controlled list; Entity described: deployment\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000052/5/\"\n },\n {\n \"name\": \"deploy-on-latitude\",\n \"title\": \"deploy on latitude\",\n \"description\": \"The geographic latitude of the location where the animal was released. Intended primarily for cases in which the animal release location has higher accuracy than that derived from sensor data. Example: '27.3516'; Units: decimal degrees, WGS84 reference system; Entity described: deployment\",\n \"type\": \"number\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000078/3/\"\n },\n {\n \"name\": \"deploy-on-longitude\",\n \"title\": \"deploy on longitude\",\n \"description\": \"The geographic longitude of the location where the animal was released. Intended primarily for cases in which the animal release location has higher accuracy than that derived from sensor data. Example: '-97.3321'; Units: decimal degrees, WGS84 reference system; Entity described: deployment\",\n \"type\": \"number\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000079/3/\"\n },\n {\n \"name\": \"deploy-on-measurements\",\n \"title\": \"deploy on measurements\",\n \"description\": \"A list of additional measurements taken during capture of the animal at the start of the deployment. Recommended best practice is to define units and use a key:value encoding schema for a data interchange format such as JSON. Example: \\\"{tarsusLengthInMillimeters:17.3, wingChordInMillimeters:125}\\\"; Units: not defined; Entity described: deployment\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000356/2/\"\n },\n {\n \"name\": \"deployment-comments\",\n \"title\": \"deployment comments\",\n \"description\": \"Additional information about the tag deployment that is not described by other reference data terms. Example: 'body length 154 cm; condition good'; Units: none; Entity described: deployment\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000082/2/\"\n },\n {\n \"name\": \"deployment-end-type\",\n \"title\": \"deployment end type\",\n \"description\": \"A categorical classification describing the end of the tag deployment on the animal. Best practice is to clarify how the 'deploy-off timestamp', if present, was chosen. Values are chosen from a controlled list: analysis-end = the end time represents the end of the period of interest; captured = The tag remained on the animal but the animal was captured or confined; dead = The deployment ended with the death of the animal that was carrying the tag; dead/fall-off = The tag stopped moving, and it is not possible to determine whether it is due to death of the animal or unscheduled tag detachment; equipment-failure = The tag stopped working; fall-off = The attachment of the tag to the animal failed, and it fell of accidentally; other = other; released = The tag remained on the animal but the animal was released from captivity or confinement; removal = The tag was purposefully removed from the animal; scheduled-detachment = The tag was programmed to detach from the animal; transmission-end = The tag stopped transmitting usable data; unknown = The cause of the end of data availability or transmission is unknown. Format: controlled list; Entity described: deployment\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000084/5/\"\n },\n {\n \"name\": \"deployment-id\",\n \"title\": \"deployment ID\",\n \"description\": \"A unique identifier for the deployment of a tag on animal, provided by the data owner. If the data owner does not provide a Deployment ID, an internal Movebank deployment identifier may sometimes be shown. Example: 'Jane_42818'; Units: none; Entity described: deployment\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000085/3/\"\n },\n {\n \"name\": \"location-accuracy-comments\",\n \"title\": \"location accuracy comments\",\n \"description\": \"Comments about the location accuracy. This can further describe values provided in 'location error text', 'location error numerical', 'vertical error numerical', 'lat lower', 'lat upper', 'long lower' and/or 'long upper'. The percentile uncertainty can be provided using 'location error percentile'. Example: '1 standard deviation errors, assuming normal distribution, provided by the GPS unit'; Units: none; Entity described: deployment\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000141/3/\"\n },\n {\n \"name\": \"manipulation-type\",\n \"title\": \"manipulation type\",\n \"description\": \"The way in which the animal was manipulated during the deployment. Additional information can be provided using 'manipulation comments'. Changes in manipulation status during deployment can be identified using 'manipulation status'. Values are chosen from a controlled list: confined = The animal's movement was restricted to within a defined area; domesticated = The animal is domesticated, for example, is a house pet or part of a managed herd; manipulated-other = The animal was manipulated in some other way, such as a physiological manipulation; none = The animal received no treatment other than tag attachment and related measurements and sampling (if applicable); reintroduction = The animal has been reintroduced as part of wildlife conservation or management efforts; relocated = The animal was released from a site other than the one at which it was captured. Format: controlled list; Entity described: deployment\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000155/6/\"\n },\n {\n \"name\": \"study-site\",\n \"title\": \"study site\",\n \"description\": \"A location such as the deployment site, study site, or colony name. Example: 'Pickerel Island North'; Units: none; Entity described: deployment\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000175/3/\"\n },\n {\n \"name\": \"tag-firmware\",\n \"title\": \"tag firmware\",\n \"description\": \"The tag firmware and version used during the deployment. If needed, identify the relevant sensors on the tag. Units: none; Entity described: deployment\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000380/1/\"\n },\n {\n \"name\": \"tag-manufacturer-name\",\n \"title\": \"tag manufacturer name\",\n \"description\": \"The company or person that produced the tag. Example: 'Holohil'; Units: none; Entity described: tag\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000183/3/\"\n },\n {\n \"name\": \"tag-mass\",\n \"title\": \"tag mass\",\n \"description\": \"The mass of the tag. Can be used with 'tag mass total' to define the mass of the tag separately from that of the tag with additional hardware. Example: '24'; Units: grams; Entity described: tag\",\n \"type\": \"number\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000184/4/\"\n },\n {\n \"name\": \"tag-readout-method\",\n \"title\": \"tag readout method\",\n \"description\": \"The way the data are received from the tag. Values are chosen from a controlled list: ISS = Data are transferred via the International Space Station; LPWAN = Data are transferred through a low-power wide-area network, such as LoRa or Sigfox; multiple = Data are acquired using multiple methods; none = Data are obtained without use of an animal-borne tag, such as by observing a unique marking; other-wireless = Data are transferred via another form of wireless data transfer, such as a VHF transmitter/receiver; phone-network = Data are transferred via a phone network, such as GSM or AMPS; satellite = Data are transferred via satellite; tag-retrieval = The tag must be physically retrieved in order to obtain the data; telemetry-network = Data are obtained through a radio or acoustic telemetry network; Wi-Fi/Bluetooth = Data are transferred via a local Wi-Fi or Bluetooth system. Format: controlled list; Entity described: deployment\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000188/4/\"\n },\n {\n \"name\": \"tag-serial-no\",\n \"title\": \"tag serial no\",\n \"description\": \"The serial number of the tag. Example: 'MN93-33243'; Units: none; Entity described: tag\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000189/3/\"\n }\n ],\n \"primaryKey\": [\"animal-id\", \"tag-id\"]\n }\n },\n {\n \"name\": \"gps\",\n \"path\": [\"O_ASSEN-gps-2018.csv.gz\", \"O_ASSEN-gps-2019.csv.gz\"],\n \"profile\": \"tabular-data-resource\",\n \"format\": \"csv\",\n \"mediatype\": \"text/csv\",\n \"encoding\": \"UTF-8\",\n \"schema\": {\n \"fields\": [\n {\n \"name\": \"event-id\",\n \"title\": \"event ID\",\n \"description\": \"An identifier for the set of values associated with each event, i.e. sensor measurement. A unique event ID is assigned to every time-location or other time-measurement record in Movebank. If multiple measurements are included within a single row of a data file, they will share an event ID. If users import the same sensor measurement to Movebank multiple times, a separate event ID will be assigned to each. Example: '14328243575'; Units: none; Entity described: event\",\n \"type\": \"integer\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000103/3/\"\n },\n {\n \"name\": \"visible\",\n \"title\": \"visible\",\n \"description\": \"Determines whether an event is visible on the Movebank map. Allowed values are TRUE or FALSE. Values are calculated automatically, with TRUE indicating the event has not been flagged as an outlier by 'algorithm marked outlier', 'import marked outlier' or 'manually marked outlier', or that the user has overridden the results of these outlier attributes using 'manually marked valid' = TRUE. Units: none; Entity described: event\",\n \"type\": \"boolean\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000209/3/\"\n },\n {\n \"name\": \"timestamp\",\n \"title\": \"timestamp\",\n \"description\": \"The date and time corresponding to a sensor measurement or an estimate derived from sensor measurements. Example: '2008-08-14 18:31:00.000'; Format: yyyy-MM-dd HH:mm:ss.SSS; Units: UTC or GPS time; Entity described: event\",\n \"type\": \"datetime\",\n \"format\": \"%Y-%m-%d %H:%M:%S.%f\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000200/2/\"\n },\n {\n \"name\": \"location-long\",\n \"title\": \"location long\",\n \"description\": \"The geographic longitude of the location as estimated by the sensor. Positive values are east of the Greenwich Meridian, negative values are west of it. Example: '-121.1761111'; Units: decimal degrees, WGS84 reference system; Entity described: event\",\n \"type\": \"number\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000146/2/\"\n },\n {\n \"name\": \"location-lat\",\n \"title\": \"location lat\",\n \"description\": \"The geographic latitude of the location as estimated by the sensor. Example: '-41.0982423'; Units: decimal degrees, WGS84 reference system; Entity described: event\",\n \"type\": \"number\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000145/4/\"\n },\n {\n \"name\": \"bar:barometric-pressure\",\n \"title\": \"barometric pressure\",\n \"description\": \"The barometric air or water pressure. Example: '32536.0'; Units: mbar (hPa); Entity described: event\",\n \"type\": \"number\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000055/3/\"\n },\n {\n \"name\": \"external-temperature\",\n \"title\": \"external temperature\",\n \"description\": \"The temperature measured by the tag (different from ambient temperature or internal body temperature of the animal). Example: '32.1'; Units: degrees Celsius; Entity described: event\",\n \"type\": \"number\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000104/2/\"\n },\n {\n \"name\": \"gps:dop\",\n \"title\": \"GPS DOP\",\n \"description\": \"Dilution of precision provided by the GPS. Example: '1.8'; Units: unitless; Entity described: event\",\n \"type\": \"number\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000115/2/\"\n },\n {\n \"name\": \"gps:satellite-count\",\n \"title\": \"GPS satellite count\",\n \"description\": \"The number of GPS satellites used to estimate the location. Example: '8'; Units: count; Entity described: event\",\n \"type\": \"integer\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000120/3/\"\n },\n {\n \"name\": \"gps-time-to-fix\",\n \"title\": \"GPS time to fix\",\n \"description\": \"The time required to obtain the GPS location fix. Example: '36'; Units: seconds; Entity described: event\",\n \"type\": \"number\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000121/3/\"\n },\n {\n \"name\": \"ground-speed\",\n \"title\": \"ground speed\",\n \"description\": \"The estimated ground speed provided by the sensor or calculated between consecutive locations. Example: '7.22'; Units: m/s; Entity described: event\",\n \"type\": \"number\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000124/2/\"\n },\n {\n \"name\": \"heading\",\n \"title\": \"heading\",\n \"description\": \"The direction in which the tag is moving, in decimal degrees clockwise from north, as provided by the sensor or calculated between consecutive locations. Values range from 0-360: 0 = north, 90 = east, 180 = south, 270 = west. Example: '315.88'; Units: degrees clockwise from north; Entity described: event\",\n \"type\": \"number\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000129/2/\"\n },\n {\n \"name\": \"height-above-msl\",\n \"title\": \"height above mean sea level\",\n \"description\": \"The estimated height of the tag above mean sea level, typically estimated by the tag. If altitudes are calculated as height above an ellipsoid, use 'height above ellipsoid'. Example: '34'; Units: meters; Entity described: event\",\n \"type\": \"number\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000131/3/\"\n },\n {\n \"name\": \"import-marked-outlier\",\n \"title\": \"import marked outlier\",\n \"description\": \"Identifies events as outliers. Outliers have the value TRUE. Typically used to import a record of outliers that were identified by the data provider or owner with automated methods outside of Movebank. Information about how outliers were defined can be provided in 'outlier comments'. Units: none; Entity described: event\",\n \"type\": \"boolean\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000133/3/\"\n },\n {\n \"name\": \"location-error-numerical\",\n \"title\": \"location error numerical\",\n \"description\": \"An estimate of the horizontal error of the location including only numbers. (If the error estimates include non-numerical characters such as '>' use 'location error text'.) These values can be described using 'location error percentile' and 'location accuracy comments'. Example: '50'; Units: meters; Entity described: event\",\n \"type\": \"number\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000142/3/\"\n },\n {\n \"name\": \"manually-marked-outlier\",\n \"title\": \"manually marked outlier\",\n \"description\": \"Identifies events flagged manually as outliers, typically using the Event Editor in Movebank, and may also include outliers identified using other methods. Outliers have the value TRUE. Information about how outliers were defined can be provided in 'outlier comments'. Units: none; Entity described: event\",\n \"type\": \"boolean\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000156/3/\"\n },\n {\n \"name\": \"vertical-error-numerical\",\n \"title\": \"vertical error numerical\",\n \"description\": \"An estimate of the vertical error of the location. These values can be described using 'location error percentile' and 'location accuracy comments'. Example: '12'; Units: meters; Entity described: event\",\n \"type\": \"number\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000208/3/\"\n },\n {\n \"name\": \"sensor-type\",\n \"title\": \"sensor type\",\n \"description\": \"The type of sensor with which data were collected. All sensors are associated with a tag id, and tags can contain multiple sensor types. Each event record in Movebank is assigned one sensor type. If values from multiple sensors are reported in a single event, the primary sensor is used. Values are chosen from a controlled list: acceleration = The sensor collects acceleration data; accessory-measurements = The sensor collects accessory measurements, such as battery voltage; acoustic-telemetry = The sensor transmits an acoustic signal that is detected by receivers to determine location; argos-doppler-shift = The sensor location is estimated by Argos using Doppler shift; barometer = The sensor records air or water pressure; bird-ring = The animal is identified by a band or ring that has a unique identifier; gps = The sensor uses GPS to determine location; gyroscope = The sensor records angular velocity; heart-rate = The sensor records or is used to calculate heart rate; magnetometer = The sensor records the magnetic field; natural-mark = The animal is identified by a unique natural marking; orientation = Quaternion components describing the orientation of the tag are derived from accelerometer and gyroscope measurements; proximity = The sensor identifies proximity to other tags; radio-transmitter = The sensor transmits a radio signal that is detected by receivers to determine location; sigfox-geolocation = The sensor location is determined by Sigfox using the received signal strength indicator; solar-geolocator = The sensor collects light levels, which are used to determine position (for processed locations); solar-geolocator-raw = The sensor collects light levels, which are used to determine position (for raw light-level measurements); solar-geolocator-twilight = The sensor collects light levels, which are used to determine position (for twilights calculated from light-level measurements). Format: controlled list; Entity described: event\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000170/6/\"\n },\n {\n \"name\": \"individual-taxon-canonical-name\",\n \"title\": \"individual taxon canonical name\",\n \"description\": \"This attribute has been merged with 'animal taxon'. The scientific name of the species on which the tag was deployed, as defined by the Integrated Taxonomic Information System (ITIS, www.itis.gov). If the species name can not be provided, this should be the lowest level taxonomic rank that can be determined and that is used in the ITIS taxonomy. Additional information can be provided using the term 'taxon detail'. Format: controlled list; Entity described: individual\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000135/5/\"\n },\n {\n \"name\": \"tag-local-identifier\",\n \"title\": \"tag local identifier\",\n \"description\": \"This attribute has been merged with 'tag ID'. An identifier for the tag, provided by the data owner. Values are unique within the study. If the data owner does not provide a tag ID, an internal Movebank tag identifier may sometimes be shown. Example: '2342'; Units: none; Entity described: tag\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000182/5/\"\n },\n {\n \"name\": \"individual-local-identifier\",\n \"title\": \"individual local identifier\",\n \"description\": \"This attribute has been merged with 'animal ID'. An individual identifier for the animal, provided by the data owner. Values are unique within the study. If the data owner does not provide an Animal ID, an internal Movebank animal identifier is sometimes shown. Example: '91876A, Gary'; Units: none; Entity described: individual\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000134/4/\"\n },\n {\n \"name\": \"study-name\",\n \"title\": \"study name\",\n \"description\": \"The name of the study in Movebank. Example: 'Coyotes, Kays and Bogan, Albany NY'; Units: none; Entity described: study\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000173/3/\"\n }\n ],\n \"primaryKey\": \"event-id\",\n \"foreignKeys\": [\n {\n \"fields\": [\"individual-local-identifier\", \"tag-local-identifier\"],\n \"reference\": {\n \"resource\": \"reference-data\",\n \"fields\": [\"animal-id\", \"tag-id\"]\n }\n }\n ]\n }\n },\n {\n \"name\": \"acceleration\",\n \"path\": [\"O_ASSEN-acceleration-2018.csv.gz\", \"O_ASSEN-acceleration-2019.csv.gz\"],\n \"profile\": \"tabular-data-resource\",\n \"format\": \"csv\",\n \"mediatype\": \"text/csv\",\n \"encoding\": \"UTF-8\",\n \"schema\": {\n \"fields\": [\n {\n \"name\": \"event-id\",\n \"title\": \"event ID\",\n \"description\": \"An identifier for the set of values associated with each event, i.e. sensor measurement. A unique event ID is assigned to every time-location or other time-measurement record in Movebank. If multiple measurements are included within a single row of a data file, they will share an event ID. If users import the same sensor measurement to Movebank multiple times, a separate event ID will be assigned to each. Example: '14328243575'; Units: none; Entity described: event\",\n \"type\": \"integer\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000103/3/\"\n },\n {\n \"name\": \"visible\",\n \"title\": \"visible\",\n \"description\": \"Determines whether an event is visible on the Movebank map. Allowed values are TRUE or FALSE. Values are calculated automatically, with TRUE indicating the event has not been flagged as an outlier by 'algorithm marked outlier', 'import marked outlier' or 'manually marked outlier', or that the user has overridden the results of these outlier attributes using 'manually marked valid' = TRUE. Units: none; Entity described: event\",\n \"type\": \"boolean\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000209/3/\"\n },\n {\n \"name\": \"timestamp\",\n \"title\": \"timestamp\",\n \"description\": \"The date and time corresponding to a sensor measurement or an estimate derived from sensor measurements. Example: '2008-08-14 18:31:00.000'; Format: yyyy-MM-dd HH:mm:ss.SSS; Units: UTC or GPS time; Entity described: event\",\n \"type\": \"datetime\",\n \"format\": \"%Y-%m-%d %H:%M:%S.%f\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000200/2/\"\n },\n {\n \"name\": \"acceleration-raw-x\",\n \"title\": \"acceleration raw x\",\n \"description\": \"Raw acceleration values provided by the tag for the X axis. Range and units may vary by provider, tag, and orientation of the sensor on the animal. Example: '0.556641'; Units: not defined; Entity described: event\",\n \"type\": \"number\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000002/2/\"\n },\n {\n \"name\": \"acceleration-raw-y\",\n \"title\": \"acceleration raw y\",\n \"description\": \"Raw acceleration values provided by the tag for the Y axis. Range and units may vary by provider, tag, and orientation of the sensor on the animal. Example: '0.09375'; Units: not defined; Entity described: event\",\n \"type\": \"number\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000003/2/\"\n },\n {\n \"name\": \"acceleration-raw-z\",\n \"title\": \"acceleration raw z\",\n \"description\": \"Raw acceleration values provided by the tag for the Z axis. Range and units may vary by provider, tag, and orientation of the sensor on the animal. Example: '-0.84375'; Units: not defined; Entity described: event\",\n \"type\": \"number\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000004/2/\"\n },\n {\n \"name\": \"start-timestamp\",\n \"title\": \"start timestamp\",\n \"description\": \"The date and time when the sampling interval or burst began. Example: '2011-01-03 13:45:00.000'; Format: yyyy-MM-dd HH:mm:ss.SSS; Units: UTC or GPS time; Entity described: event\",\n \"type\": \"datetime\",\n \"format\": \"%Y-%m-%d %H:%M:%S.%f\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000171/2/\"\n },\n {\n \"name\": \"tilt-x\",\n \"title\": \"tilt x\",\n \"description\": \"Tilt provided by the accelerometer for the X axis. Example: '0'; Units: g forces (1 g = 9.8 m s^-2); Entity described: event\",\n \"type\": \"number\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000197/2/\"\n },\n {\n \"name\": \"tilt-y\",\n \"title\": \"tilt y\",\n \"description\": \"Tilt provided by the accelerometer for the Y axis. Example: '0'; Units: g forces (1 g = 9.8 m s^-2); Entity described: event\",\n \"type\": \"number\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000198/2/\"\n },\n {\n \"name\": \"tilt-z\",\n \"title\": \"tilt z\",\n \"description\": \"Tilt provided by the accelerometer for the Z axis. Example: '1'; Units: g forces (1 g = 9.8 m s^-2); Entity described: event\",\n \"type\": \"number\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000199/2/\"\n },\n {\n \"name\": \"sensor-type\",\n \"title\": \"sensor type\",\n \"description\": \"The type of sensor with which data were collected. All sensors are associated with a tag id, and tags can contain multiple sensor types. Each event record in Movebank is assigned one sensor type. If values from multiple sensors are reported in a single event, the primary sensor is used. Values are chosen from a controlled list: acceleration = The sensor collects acceleration data; accessory-measurements = The sensor collects accessory measurements, such as battery voltage; acoustic-telemetry = The sensor transmits an acoustic signal that is detected by receivers to determine location; argos-doppler-shift = The sensor location is estimated by Argos using Doppler shift; barometer = The sensor records air or water pressure; bird-ring = The animal is identified by a band or ring that has a unique identifier; gps = The sensor uses GPS to determine location; gyroscope = The sensor records angular velocity; heart-rate = The sensor records or is used to calculate heart rate; magnetometer = The sensor records the magnetic field; natural-mark = The animal is identified by a unique natural marking; orientation = Quaternion components describing the orientation of the tag are derived from accelerometer and gyroscope measurements; proximity = The sensor identifies proximity to other tags; radio-transmitter = The sensor transmits a radio signal that is detected by receivers to determine location; sigfox-geolocation = The sensor location is determined by Sigfox using the received signal strength indicator; solar-geolocator = The sensor collects light levels, which are used to determine position (for processed locations); solar-geolocator-raw = The sensor collects light levels, which are used to determine position (for raw light-level measurements); solar-geolocator-twilight = The sensor collects light levels, which are used to determine position (for twilights calculated from light-level measurements). Format: controlled list; Entity described: event\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000170/6/\"\n },\n {\n \"name\": \"individual-taxon-canonical-name\",\n \"title\": \"individual taxon canonical name\",\n \"description\": \"This attribute has been merged with 'animal taxon'. The scientific name of the species on which the tag was deployed, as defined by the Integrated Taxonomic Information System (ITIS, www.itis.gov). If the species name can not be provided, this should be the lowest level taxonomic rank that can be determined and that is used in the ITIS taxonomy. Additional information can be provided using the term 'taxon detail'. Format: controlled list; Entity described: individual\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000135/5/\"\n },\n {\n \"name\": \"tag-local-identifier\",\n \"title\": \"tag local identifier\",\n \"description\": \"This attribute has been merged with 'tag ID'. An identifier for the tag, provided by the data owner. Values are unique within the study. If the data owner does not provide a tag ID, an internal Movebank tag identifier may sometimes be shown. Example: '2342'; Units: none; Entity described: tag\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000182/5/\"\n },\n {\n \"name\": \"individual-local-identifier\",\n \"title\": \"individual local identifier\",\n \"description\": \"This attribute has been merged with 'animal ID'. An individual identifier for the animal, provided by the data owner. Values are unique within the study. If the data owner does not provide an Animal ID, an internal Movebank animal identifier is sometimes shown. Example: '91876A, Gary'; Units: none; Entity described: individual\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000134/4/\"\n },\n {\n \"name\": \"study-name\",\n \"title\": \"study name\",\n \"description\": \"The name of the study in Movebank. Example: 'Coyotes, Kays and Bogan, Albany NY'; Units: none; Entity described: study\",\n \"type\": \"string\",\n \"format\": \"default\",\n \"skos:exactMatch\": \"http://vocab.nerc.ac.uk/collection/MVB/current/MVB000173/3/\"\n }\n ],\n \"primaryKey\": \"event-id\",\n \"foreignKeys\": [\n {\n \"fields\": [\"individual-local-identifier\", \"tag-local-identifier\"],\n \"reference\": {\n \"resource\": \"reference-data\",\n \"fields\": [\"animal-id\", \"tag-id\"]\n }\n }\n ]\n }\n }\n ]\n}\n" + }, + "cookies": [ + { + "httpOnly": true, + "name": "5569e5a730cade8ff2b54f1e815f3670", + "path": "/", + "sameSite": "None", + "secure": true, + "value": "0acb936b975d164e9af26483a08df23e" + } + ], + "headers": [ + { + "name": "accept-ranges", + "value": "bytes" + }, + { + "name": "content-disposition", + "value": "inline" + }, + { + "name": "content-encoding", + "value": "gzip" + }, + { + "name": "content-security-policy", + "value": "default-src 'self' fonts.googleapis.com *.gstatic.com data: 'unsafe-inline' 'unsafe-eval' blob: zenodo-broker.web.cern.ch zenodo-broker-qa.web.cern.ch maxcdn.bootstrapcdn.com cdnjs.cloudflare.com ajax.googleapis.com webanalytics.web.cern.ch" + }, + { + "name": "content-type", + "value": "text/plain; charset=utf-8, text/plain; charset=utf-8" + }, + { + "name": "date", + "value": "Tue, 27 May 2025 09:55:24 GMT" + }, + { + "name": "oc-checksum", + "value": "MD5:5ea86b7a222ac843e3833a2e50a2056b" + }, + { + "name": "referrer-policy", + "value": "strict-origin-when-cross-origin" + }, + { + "name": "retry-after", + "value": "60" + }, + { + "name": "server", + "value": "nginx" + }, + { + "name": "set-cookie", + "value": "5569e5a730cade8ff2b54f1e815f3670=0acb936b975d164e9af26483a08df23e; path=/; HttpOnly; Secure; SameSite=None" + }, + { + "name": "strict-transport-security", + "value": "max-age=31556926; includeSubDomains" + }, + { + "name": "transfer-encoding", + "value": "chunked" + }, + { + "name": "vary", + "value": "Accept-Encoding" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "x-download-options", + "value": "noopen" + }, + { + "name": "x-frame-options", + "value": "sameorigin" + }, + { + "name": "x-permitted-cross-domain-policies", + "value": "none" + }, + { + "name": "x-ratelimit-limit", + "value": "133" + }, + { + "name": "x-ratelimit-remaining", + "value": "132" + }, + { + "name": "x-ratelimit-reset", + "value": "1748339785" + }, + { + "name": "x-xss-protection", + "value": "1; mode=block" + } + ], + "headersSize": 1064, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-05-27T09:55:23.201Z", + "time": 1212, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 1212 + } + } + ], + "pages": [], + "version": "1.2" + } +} diff --git a/zenodo/package/load.spec.ts b/zenodo/package/load.spec.ts index 2a4d2146..1f41d674 100644 --- a/zenodo/package/load.spec.ts +++ b/zenodo/package/load.spec.ts @@ -7,7 +7,15 @@ describe("loadPackageFromZenodo", () => { it("should load a package", async () => { const datapackage = await loadPackageFromZenodo({ - datasetUrl: "https://sandbox.zenodo.org/records/260528", + datasetUrl: "https://zenodo.org/records/15525711", + }) + + expect(datapackage).toMatchSnapshot() + }) + + it("shoule merge datapackage.json if present", async () => { + const datapackage = await loadPackageFromZenodo({ + datasetUrl: "https://zenodo.org/records/10053903", }) expect(datapackage).toMatchSnapshot() diff --git a/zenodo/package/load.ts b/zenodo/package/load.ts index 08cb10f5..a05eb59a 100644 --- a/zenodo/package/load.ts +++ b/zenodo/package/load.ts @@ -1,3 +1,4 @@ +import { mergePackages } from "@dpkit/core" import { makeZenodoApiRequest } from "../general/index.js" import type { ZenodoPackage } from "./Package.js" import { normalizeZenodoPackage } from "./process/normalize.js" @@ -19,13 +20,23 @@ export async function loadPackageFromZenodo(props: { throw new Error(`Failed to extract record ID from URL: ${datasetUrl}`) } - const zenodoPackage = (await makeZenodoApiRequest({ + const zenodoPackage = await makeZenodoApiRequest({ endpoint: `/records/${recordId}`, apiKey, sandbox, - })) as ZenodoPackage + }) + + const systemPackage = normalizeZenodoPackage({ zenodoPackage }) + const userPackagePath = systemPackage.resources + .filter(resource => resource["zenodo:key"] === "datapackage.json") + .map(resource => resource["zenodo:url"]) + .at(0) + + const datapackage = await mergePackages({ systemPackage, userPackagePath }) + datapackage.resources = datapackage.resources.map(resource => { + return { ...resource, "zenodo:key": undefined, "zenodo:url": undefined } + }) - const datapackage = normalizeZenodoPackage({ zenodoPackage }) return datapackage } diff --git a/zenodo/plugin.ts b/zenodo/plugin.ts new file mode 100644 index 00000000..afec0898 --- /dev/null +++ b/zenodo/plugin.ts @@ -0,0 +1,20 @@ +import type { Descriptor, Plugin } from "@dpkit/core" +import { isRemotePath } from "@dpkit/core" +import { loadPackageFromZenodo } from "./package/load.js" + +export class ZenodoPlugin implements Plugin { + async loadPackage(props: { source: string; options?: Descriptor }) { + const isRemote = isRemotePath({ path: props.source }) + if (!isRemote) return undefined + + const isZenodo = new URL(props.source).hostname.endsWith("zenodo.org") + if (!isZenodo) return undefined + + const cleanup = async () => {} + const datapackage = await loadPackageFromZenodo({ + datasetUrl: props.source, + }) + + return { datapackage, cleanup } + } +} diff --git a/zenodo/resource/process/normalize.ts b/zenodo/resource/process/normalize.ts index f18ed4ac..670e0c78 100644 --- a/zenodo/resource/process/normalize.ts +++ b/zenodo/resource/process/normalize.ts @@ -1,4 +1,3 @@ -import type { Resource } from "@dpkit/core" import { getFormat, getName } from "@dpkit/core" import type { ZenodoResource } from "../Resource.js" @@ -11,16 +10,21 @@ export function normalizeZenodoResource(props: { zenodoResource: ZenodoResource }) { const { zenodoResource } = props + const path = normalizeZenodoPath({ link: zenodoResource.links.self }) - const resource: Resource = { + const resource = { + path, name: getName({ filename: zenodoResource.key }) ?? zenodoResource.id, format: getFormat({ filename: zenodoResource.key }), - path: zenodoResource.links.self - .replace("/api/", "/") - .replace(/\/content$/, ""), bytes: zenodoResource.size, hash: zenodoResource.checksum, + "zenodo:key": zenodoResource.key, + "zenodo:url": path, } return resource } + +function normalizeZenodoPath(props: { link: string }) { + return props.link.replace("/api/", "/").replace(/\/content$/, "") +} diff --git a/zenodo/typedoc.json b/zenodo/typedoc.json new file mode 100644 index 00000000..fc33b815 --- /dev/null +++ b/zenodo/typedoc.json @@ -0,0 +1,5 @@ +{ + "entryPoints": ["index.ts"], + "projectDocuments": ["CHANGELOG.md", "OVERVIEW.md"], + "skipErrorChecking": true +} diff --git a/zip/OVERVIEW.md b/zip/OVERVIEW.md new file mode 100644 index 00000000..a704ff69 --- /dev/null +++ b/zip/OVERVIEW.md @@ -0,0 +1,2 @@ +# @dpkit/zip + diff --git a/zip/README.md b/zip/README.md index 5c7d66a0..c5425c56 100644 --- a/zip/README.md +++ b/zip/README.md @@ -1,3 +1,3 @@ # @dpkit/zip -Data Package implementation in TypeScript. Data Package is a standard consisting of a set of simple yet extensible specifications to describe datasets, data files and tabular data. It is a data definition language (DDL) and data API that facilitates findability, accessibility, interoperability, and reusability (FAIR) of data. For more information, please visit the [documentation portal](https://dpkit.datist.io). +dpkit is a fast TypeScript data management framework based on the Data Package standard. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/zip/package.json b/zip/package.json index 07fe4fa8..98c4ea75 100644 --- a/zip/package.json +++ b/zip/package.json @@ -6,7 +6,7 @@ "license": "MIT", "author": "Evgeny Karev", "repository": "https://github.com/datisthq/dpkit", - "description": "Data Package implementation in TypeScript.", + "description": "Fast TypeScript data management framework based on the Data Package standard", "keywords": [ "data", "package", @@ -16,15 +16,15 @@ "fair", "zip" ], + "scripts": { + "build": "tsc --build" + }, "dependencies": { "@dpkit/core": "workspace:*", "@dpkit/file": "workspace:*", "yauzl-promise": "^4.0.0", "yazl": "^3.3.1" }, - "scripts": { - "build": "tsc --build" - }, "devDependencies": { "@types/yauzl-promise": "^4.0.1", "@types/yazl": "^2.4.6" diff --git a/zip/plugin.ts b/zip/plugin.ts index 903c4252..1db5cabf 100644 --- a/zip/plugin.ts +++ b/zip/plugin.ts @@ -24,7 +24,7 @@ export class ZipPlugin implements Plugin { await savePackageToZip({ datapackage: props.datapackage, archivePath: props.target, - withRemote: props.options?.withRemote, + withRemote: !!props.options?.withRemote, }) return { path: undefined } diff --git a/zip/typedoc.json b/zip/typedoc.json new file mode 100644 index 00000000..fc33b815 --- /dev/null +++ b/zip/typedoc.json @@ -0,0 +1,5 @@ +{ + "entryPoints": ["index.ts"], + "projectDocuments": ["CHANGELOG.md", "OVERVIEW.md"], + "skipErrorChecking": true +}