diff --git a/browser/components/Report/Error/Cell.tsx b/browser/components/Report/Error/Cell.tsx index 7e9b440d..dd99239a 100644 --- a/browser/components/Report/Error/Cell.tsx +++ b/browser/components/Report/Error/Cell.tsx @@ -215,6 +215,7 @@ export function CellMaxLengthError(props: { {error.rowNumber} {" "} + {t("is more than")} {error.maxLength} {" "} diff --git a/browser/components/Report/Error/Data.tsx b/browser/components/Report/Error/Data.tsx new file mode 100644 index 00000000..e728f4fd --- /dev/null +++ b/browser/components/Report/Error/Data.tsx @@ -0,0 +1,9 @@ +import type * as errorTypes from "@dpkit/lib" +import { Text } from "@mantine/core" +import { useTranslation } from "react-i18next" + +export function DataError(props: { error: errorTypes.DataError }) { + const { t } = useTranslation() + + return {t(props.error.message as any)} +} diff --git a/browser/components/Report/Error/Error.tsx b/browser/components/Report/Error/Error.tsx index 49d85b74..de7acc12 100644 --- a/browser/components/Report/Error/Error.tsx +++ b/browser/components/Report/Error/Error.tsx @@ -13,21 +13,29 @@ import { CellTypeError, CellUniqueError, } from "./Cell.tsx" +import { DataError } from "./Data.tsx" import { FieldNameError, FieldTypeError } from "./Field.tsx" import { FieldsExtraError, FieldsMissingError } from "./Fields.tsx" import { BytesError, EncodingError, HashError } from "./File.tsx" +import { ForeignKeyError } from "./ForeignKey.tsx" import { MetadataError } from "./Metadata.tsx" import { RowUniqueError } from "./Row.tsx" export function Error(props: { // TODO: should be lib.Error - error: errorTypes.MetadataError | errorTypes.FileError | errorTypes.TableError + error: + | errorTypes.MetadataError + | errorTypes.DataError + | errorTypes.FileError + | errorTypes.TableError }) { const { error } = props switch (error.type) { case "metadata": return + case "data": + return case "file/bytes": return case "file/hash": @@ -68,5 +76,9 @@ export function Error(props: { return case "cell/jsonSchema": return + case "foreignKey": + return + default: + return null } } diff --git a/browser/components/Report/Error/ForeignKey.tsx b/browser/components/Report/Error/ForeignKey.tsx new file mode 100644 index 00000000..bdd2bf8f --- /dev/null +++ b/browser/components/Report/Error/ForeignKey.tsx @@ -0,0 +1,20 @@ +import type * as errorTypes from "@dpkit/lib" +import { Code, Text } from "@mantine/core" +import { useTranslation } from "react-i18next" + +// TODO: improve error message +export function ForeignKeyError(props: { error: errorTypes.ForeignKeyError }) { + const { t } = useTranslation() + return ( + + {t("Foreign key violation in field(s)")}{" "} + + {props.error.foreignKey.fields.join(", ")} + + {": "} + + {props.error.cells.join(", ")} + + + ) +} diff --git a/browser/components/Report/Report.tsx b/browser/components/Report/Report.tsx index 5dfed360..23e7431f 100644 --- a/browser/components/Report/Report.tsx +++ b/browser/components/Report/Report.tsx @@ -1,4 +1,9 @@ -import type { FileError, MetadataError, TableError } from "@dpkit/lib" +import type { + DataError, + FileError, + MetadataError, + TableError, +} from "@dpkit/lib" import { Card, Divider, ScrollArea, Stack, Tabs } from "@mantine/core" import { groupBy } from "es-toolkit" import { useState } from "react" @@ -7,7 +12,7 @@ import { objectKeys } from "ts-extras" import { Error } from "./Error/Error.tsx" export function Report(props: { - errors?: (MetadataError | FileError | TableError)[] + errors?: (MetadataError | DataError | FileError | TableError)[] }) { const { t } = useTranslation() const { errors } = props diff --git a/browser/locales/de.json b/browser/locales/de.json index 329d656a..22c98b19 100644 --- a/browser/locales/de.json +++ b/browser/locales/de.json @@ -70,6 +70,7 @@ "at": "bei", "The cell values of the fields": "Die Zellwerte der Felder", "are not unique": "sind nicht eindeutig", + "Foreign key violation in field(s)": "Fremdschlüsselverletzung in Feld(ern)", "Under Construction": "In Bearbeitung", "This tool is currently under construction and not yet available": "Dieses Werkzeug befindet sich derzeit in der Entwicklung und ist noch nicht verfügbar", "Brought to you by": "Präsentiert von", diff --git a/browser/locales/en.json b/browser/locales/en.json index bc65af93..e98c9181 100644 --- a/browser/locales/en.json +++ b/browser/locales/en.json @@ -70,6 +70,7 @@ "at": "at", "The cell values of the fields": "The cell values of the fields", "are not unique": "are not unique", + "Foreign key violation in field(s)": "Foreign key violation in field(s)", "Under Construction": "Under Construction", "This tool is currently under construction and not yet available": "This tool is currently under construction and not yet available", "Brought to you by": "Brought to you by", diff --git a/browser/locales/es.json b/browser/locales/es.json index 8ab2f48d..dba2b9b2 100644 --- a/browser/locales/es.json +++ b/browser/locales/es.json @@ -70,6 +70,7 @@ "at": "en", "The cell values of the fields": "Los valores de celda de los campos", "are not unique": "no son únicos", + "Foreign key violation in field(s)": "Violación de clave foránea en campo(s)", "Under Construction": "En construcción", "This tool is currently under construction and not yet available": "Esta herramienta está actualmente en construcción y aún no está disponible", "Brought to you by": "Presentado por", diff --git a/browser/locales/fr.json b/browser/locales/fr.json index 4fdb5125..6f394401 100644 --- a/browser/locales/fr.json +++ b/browser/locales/fr.json @@ -70,6 +70,7 @@ "at": "à", "The cell values of the fields": "Les valeurs de cellule des champs", "are not unique": "ne sont pas uniques", + "Foreign key violation in field(s)": "Violation de clé étrangère dans le(s) champ(s)", "Under Construction": "En construction", "This tool is currently under construction and not yet available": "Cet outil est actuellement en construction et n'est pas encore disponible", "Brought to you by": "Proposé par", diff --git a/browser/locales/it.json b/browser/locales/it.json index 38fdd254..39ba5265 100644 --- a/browser/locales/it.json +++ b/browser/locales/it.json @@ -70,6 +70,7 @@ "at": "a", "The cell values of the fields": "I valori delle celle dei campi", "are not unique": "non sono univoci", + "Foreign key violation in field(s)": "Violazione di chiave esterna nel/i campo/i", "Under Construction": "In costruzione", "This tool is currently under construction and not yet available": "Questo strumento è attualmente in costruzione e non ancora disponibile", "Brought to you by": "Offerto da", diff --git a/browser/locales/pt.json b/browser/locales/pt.json index fbafa1c8..70cc5de6 100644 --- a/browser/locales/pt.json +++ b/browser/locales/pt.json @@ -70,6 +70,7 @@ "at": "em", "The cell values of the fields": "Os valores das células dos campos", "are not unique": "não são únicos", + "Foreign key violation in field(s)": "Violação de chave estrangeira no(s) campo(s)", "Under Construction": "Em construção", "This tool is currently under construction and not yet available": "Esta ferramenta está atualmente em construção e ainda não está disponível", "Brought to you by": "Apresentado por", diff --git a/browser/locales/ru.json b/browser/locales/ru.json index 12e47c9e..1322f1df 100644 --- a/browser/locales/ru.json +++ b/browser/locales/ru.json @@ -70,6 +70,7 @@ "at": "в", "The cell values of the fields": "Значения ячеек полей", "are not unique": "не уникальны", + "Foreign key violation in field(s)": "Нарушение внешнего ключа в поле(ях)", "Under Construction": "В разработке", "This tool is currently under construction and not yet available": "Этот инструмент в настоящее время находится в разработке и пока недоступен", "Brought to you by": "Представлено", diff --git a/browser/locales/uk.json b/browser/locales/uk.json index 805c0c70..5761d0ab 100644 --- a/browser/locales/uk.json +++ b/browser/locales/uk.json @@ -70,6 +70,7 @@ "at": "в", "The cell values of the fields": "Значення комірок полів", "are not unique": "не унікальні", + "Foreign key violation in field(s)": "Порушення зовнішнього ключа в полі(ях)", "Under Construction": "В розробці", "This tool is currently under construction and not yet available": "Цей інструмент наразі перебуває в розробці і ще недоступний", "Brought to you by": "Представлено", diff --git a/cli/commands/dialect/validate.tsx b/cli/commands/dialect/validate.tsx index 19f63932..59bdb329 100644 --- a/cli/commands/dialect/validate.tsx +++ b/cli/commands/dialect/validate.tsx @@ -39,7 +39,7 @@ export const validateDialectCommand = new Command("validate") ? await session.task("Loading dialect", resolveDialect(resource.dialect)) : undefined - const { descriptor } = path + const descriptor = path ? await session.task("Loading descriptor", loadDescriptor(path)) : dialect ? { descriptor: dialect } diff --git a/cli/commands/resource/validate.spec.ts b/cli/commands/resource/validate.spec.ts index 4dbc78da..c52e3780 100644 --- a/cli/commands/resource/validate.spec.ts +++ b/cli/commands/resource/validate.spec.ts @@ -104,11 +104,11 @@ describe("resource validate", () => { expect(result.errors.length).toBeGreaterThan(0) }) - it("should validate resource with schema", async () => { + it.skip("should validate resource with schema", async () => { const csvPath = await writeTempFile("id,name\n1,alice\n2,bob") const resourceContent = JSON.stringify({ name: "test-resource", - path: basename(csvPath), + path: `${basename(csvPath)}.csv`, schema: { fields: [ { name: "id", type: "integer" }, diff --git a/cli/commands/schema/convert.tsx b/cli/commands/schema/convert.tsx index 892bf178..6405ac20 100644 --- a/cli/commands/schema/convert.tsx +++ b/cli/commands/schema/convert.tsx @@ -71,8 +71,7 @@ export const convertSchemaCommand = new Command("convert") converter = convertSchemaFromJsonSchema } - const loaded = await session.task("Loading schema", loadDescriptor(path)) - const source = loaded.descriptor + const source = await session.task("Loading schema", loadDescriptor(path)) const target = await session.task("Converting schema", converter(source)) if (!options.toPath) { diff --git a/cli/commands/schema/validate.tsx b/cli/commands/schema/validate.tsx index 2ea00a21..0e97e623 100644 --- a/cli/commands/schema/validate.tsx +++ b/cli/commands/schema/validate.tsx @@ -39,7 +39,7 @@ export const validateSchemaCommand = new Command("validate") ? await session.task("Loading schema", resolveSchema(resource.schema)) : undefined - const { descriptor } = path + const descriptor = path ? await session.task("Loading descriptor", loadDescriptor(path)) : schema ? { descriptor: schema } diff --git a/cli/components/ErrorGrid.tsx b/cli/components/ErrorGrid.tsx index c57bf19e..c4dc40b0 100644 --- a/cli/components/ErrorGrid.tsx +++ b/cli/components/ErrorGrid.tsx @@ -1,10 +1,15 @@ -import type { FileError, MetadataError, TableError } from "@dpkit/lib" +import type { + DataError, + FileError, + MetadataError, + TableError, +} from "@dpkit/lib" import { DataFrame } from "nodejs-polars" import React from "react" import { TableGrid } from "./TableGrid.tsx" export function ErrorGrid(props: { - errors: (TableError | MetadataError | FileError)[] + errors: (TableError | DataError | MetadataError | FileError)[] quit?: boolean }) { // TODO: Property process errors diff --git a/cli/helpers/error.ts b/cli/helpers/error.ts index 25999188..36b9b687 100644 --- a/cli/helpers/error.ts +++ b/cli/helpers/error.ts @@ -1,10 +1,17 @@ -import type { FileError, MetadataError, TableError } from "@dpkit/lib" +import type { + DataError, + FileError, + MetadataError, + TableError, +} from "@dpkit/lib" import { countBy } from "es-toolkit" import type { Session } from "./session.ts" export async function selectErrorResource( session: Session, - errors: ((TableError | MetadataError | FileError) & { resource: string })[], + errors: ((TableError | MetadataError | DataError | FileError) & { + resource: string + })[], ) { const groups = countBy(errors, error => error.resource) @@ -25,7 +32,7 @@ export async function selectErrorResource( export async function selectErrorType( session: Session, - errors: (TableError | MetadataError | FileError)[], + errors: (TableError | MetadataError | DataError | FileError)[], ) { const groups = countBy(errors, error => error.type) diff --git a/core/general/descriptor/Descriptor.ts b/core/descriptor/Descriptor.ts similarity index 100% rename from core/general/descriptor/Descriptor.ts rename to core/descriptor/Descriptor.ts diff --git a/core/general/descriptor/fixtures/schema.json b/core/descriptor/fixtures/schema.json similarity index 100% rename from core/general/descriptor/fixtures/schema.json rename to core/descriptor/fixtures/schema.json diff --git a/core/descriptor/index.ts b/core/descriptor/index.ts new file mode 100644 index 00000000..5a5049c0 --- /dev/null +++ b/core/descriptor/index.ts @@ -0,0 +1,6 @@ +export { parseDescriptor } from "./process/parse.ts" +export { stringifyDescriptor } from "./process/stringify.ts" +export { loadDescriptor } from "./load.ts" +export { saveDescriptor } from "./save.ts" +export type { Descriptor } from "./Descriptor.ts" +export { isDescriptor } from "./Descriptor.ts" diff --git a/core/general/descriptor/load.spec.ts b/core/descriptor/load.spec.ts similarity index 85% rename from core/general/descriptor/load.spec.ts rename to core/descriptor/load.spec.ts index b6e428f1..ce7ad7e2 100644 --- a/core/general/descriptor/load.spec.ts +++ b/core/descriptor/load.spec.ts @@ -29,11 +29,8 @@ describe("loadDescriptor", () => { }) it("loads a local descriptor from a file path", async () => { - const { basepath, descriptor } = await loadDescriptor( - getFixturePath("schema.json"), - ) + const descriptor = await loadDescriptor(getFixturePath("schema.json")) - expect(basepath).toEqual(getFixturePath()) expect(descriptor).toEqual(expectedDescriptor) }) @@ -44,9 +41,8 @@ describe("loadDescriptor", () => { json: () => Promise.resolve(expectedDescriptor), }) - const { basepath, descriptor } = await loadDescriptor(testUrl) + const descriptor = await loadDescriptor(testUrl) - expect(basepath).toEqual("https://example.com") expect(descriptor).toEqual(expectedDescriptor) expect(fetch).toHaveBeenCalledWith(testUrl) }) diff --git a/core/general/descriptor/load.ts b/core/descriptor/load.ts similarity index 66% rename from core/general/descriptor/load.ts rename to core/descriptor/load.ts index b9771743..4b9e8978 100644 --- a/core/general/descriptor/load.ts +++ b/core/descriptor/load.ts @@ -1,6 +1,5 @@ -import { node } from "../node.ts" -import { getProtocol } from "../path.ts" -import { getBasepath, isRemotePath } from "../path.ts" +import { node } from "../node/index.ts" +import { getProtocol, isRemotePath } from "../path/index.ts" import { parseDescriptor } from "./process/parse.ts" /** @@ -19,36 +18,32 @@ export async function loadDescriptor( throw new Error("Cannot load descriptor for security reasons") } - return isRemote - ? await loadRemoteDescriptor(path) - : await loadLocalDescriptor(path) + return !isRemote + ? await loadLocalDescriptor(path) + : await loadRemoteDescriptor(path) } -const ALLOWED_REMOTE_PROTOCOLS = ["http", "https", "ftp", "ftps"] +async function loadLocalDescriptor(path: string) { + if (!node) { + throw new Error("File system is not supported in this environment") + } + + const text = await node.fs.readFile(path, "utf-8") + const descriptor = parseDescriptor(text) + + return descriptor +} async function loadRemoteDescriptor(path: string) { const url = new URL(path) const protocol = getProtocol(path) - if (!ALLOWED_REMOTE_PROTOCOLS.includes(protocol)) { + if (!["http", "https", "ftp", "ftps"].includes(protocol)) { throw new Error(`Unsupported remote protocol: ${protocol}`) } const response = await fetch(url.toString()) const descriptor = (await response.json()) as Record - const basepath = getBasepath(response.url ?? path) // supports redirects - - return { descriptor, basepath } -} - -async function loadLocalDescriptor(path: string) { - if (!node) { - throw new Error("File system is not supported in this environment") - } - - const text = await node.fs.readFile(path, "utf-8") - const descriptor = parseDescriptor(text) - const basepath = getBasepath(path) - return { descriptor, basepath } + return descriptor } diff --git a/core/general/descriptor/process/parse.ts b/core/descriptor/process/parse.ts similarity index 100% rename from core/general/descriptor/process/parse.ts rename to core/descriptor/process/parse.ts diff --git a/core/general/descriptor/process/stringify.ts b/core/descriptor/process/stringify.ts similarity index 100% rename from core/general/descriptor/process/stringify.ts rename to core/descriptor/process/stringify.ts diff --git a/core/general/descriptor/save.spec.ts b/core/descriptor/save.spec.ts similarity index 100% rename from core/general/descriptor/save.spec.ts rename to core/descriptor/save.spec.ts diff --git a/core/general/descriptor/save.ts b/core/descriptor/save.ts similarity index 94% rename from core/general/descriptor/save.ts rename to core/descriptor/save.ts index a3bd13ff..d5733e06 100644 --- a/core/general/descriptor/save.ts +++ b/core/descriptor/save.ts @@ -1,4 +1,4 @@ -import { node } from "../node.ts" +import { node } from "../node/index.ts" import type { Descriptor } from "./Descriptor.ts" import { stringifyDescriptor } from "./process/stringify.ts" diff --git a/core/dialect/Dialect.ts b/core/dialect/Dialect.ts index c2c87ebf..94b713d3 100644 --- a/core/dialect/Dialect.ts +++ b/core/dialect/Dialect.ts @@ -1,4 +1,4 @@ -import type { Metadata } from "../general/index.ts" +import type { Metadata } from "../metadata/index.ts" /** * Descriptor that describes the structure of tabular data, such as delimiters, diff --git a/core/dialect/assert.ts b/core/dialect/assert.ts index 32b82733..471131f5 100644 --- a/core/dialect/assert.ts +++ b/core/dialect/assert.ts @@ -1,5 +1,5 @@ +import type { Descriptor } from "../descriptor/index.ts" import { AssertionError } from "../error/index.ts" -import type { Descriptor } from "../general/index.ts" import type { Dialect } from "./Dialect.ts" import { validateDialect } from "./validate.ts" diff --git a/core/dialect/convert/fromDescriptor.ts b/core/dialect/convert/fromDescriptor.ts index 5896ad2b..39c1bc15 100644 --- a/core/dialect/convert/fromDescriptor.ts +++ b/core/dialect/convert/fromDescriptor.ts @@ -1,4 +1,4 @@ -import type { Descriptor } from "../../general/index.ts" +import type { Descriptor } from "../../descriptor/index.ts" export function convertDialectFromDescriptor(descriptor: Descriptor) { descriptor = globalThis.structuredClone(descriptor) diff --git a/core/dialect/convert/toDescriptor.ts b/core/dialect/convert/toDescriptor.ts index 859066cb..2387e0e0 100644 --- a/core/dialect/convert/toDescriptor.ts +++ b/core/dialect/convert/toDescriptor.ts @@ -1,4 +1,4 @@ -import type { Descriptor } from "../../general/index.ts" +import type { Descriptor } from "../../descriptor/index.ts" import type { Dialect } from "../Dialect.ts" export function convertDialectToDescriptor(dialect: Dialect) { diff --git a/core/dialect/load.ts b/core/dialect/load.ts index 6beba344..9de14a4a 100644 --- a/core/dialect/load.ts +++ b/core/dialect/load.ts @@ -1,4 +1,4 @@ -import { loadDescriptor } from "../general/index.ts" +import { loadDescriptor } from "../descriptor/index.ts" import { assertDialect } from "./assert.ts" /** @@ -6,7 +6,7 @@ import { assertDialect } from "./assert.ts" * Ensures the descriptor is valid against its profile */ export async function loadDialect(path: string) { - const { descriptor } = await loadDescriptor(path) + const descriptor = await loadDescriptor(path) const dialect = await assertDialect(descriptor) return dialect } diff --git a/core/dialect/save.ts b/core/dialect/save.ts index 01481819..8dbb01b4 100644 --- a/core/dialect/save.ts +++ b/core/dialect/save.ts @@ -1,4 +1,4 @@ -import { saveDescriptor } from "../general/index.ts" +import { saveDescriptor } from "../descriptor/index.ts" import type { Dialect } from "./Dialect.ts" import { convertDialectToDescriptor } from "./convert/toDescriptor.ts" diff --git a/core/dialect/validate.ts b/core/dialect/validate.ts index a59cfe4c..01bbf8d9 100644 --- a/core/dialect/validate.ts +++ b/core/dialect/validate.ts @@ -1,5 +1,5 @@ -import { type Descriptor, validateDescriptor } from "../general/index.ts" -import { loadProfile } from "../general/index.ts" +import type { Descriptor } from "../descriptor/index.ts" +import { validateDescriptor } from "../profile/index.ts" import type { Dialect } from "./Dialect.ts" import { convertDialectFromDescriptor } from "./convert/fromDescriptor.ts" @@ -11,12 +11,11 @@ const DEFAULT_PROFILE = "https://datapackage.org/profiles/1.0/tabledialect.json" export async function validateDialect(source: Descriptor | Dialect) { const descriptor = source as Descriptor - const $schema = + const profile = typeof descriptor.$schema === "string" ? descriptor.$schema : DEFAULT_PROFILE - const profile = await loadProfile($schema) const { valid, errors } = await validateDescriptor(descriptor, { profile }) let dialect: Dialect | undefined = undefined diff --git a/core/error/Data.ts b/core/error/Data.ts new file mode 100644 index 00000000..d5dbbdba --- /dev/null +++ b/core/error/Data.ts @@ -0,0 +1,6 @@ +import type { BaseError } from "./Base.ts" + +export interface DataError extends BaseError { + type: "data" + message: string +} diff --git a/core/error/index.ts b/core/error/index.ts index 414f1fee..cf171be0 100644 --- a/core/error/index.ts +++ b/core/error/index.ts @@ -1,3 +1,4 @@ export { AssertionError } from "./Assertion.ts" export type { BaseError } from "./Base.ts" +export type { DataError } from "./Data.ts" export type { MetadataError } from "./Metadata.ts" diff --git a/core/field/convert/fromDescriptor.ts b/core/field/convert/fromDescriptor.ts index b094d8b5..ceae4a6e 100644 --- a/core/field/convert/fromDescriptor.ts +++ b/core/field/convert/fromDescriptor.ts @@ -1,4 +1,4 @@ -import type { Descriptor } from "../../general/index.ts" +import type { Descriptor } from "../../descriptor/index.ts" export function convertFieldFromDescriptor(descriptor: Descriptor) { descriptor = globalThis.structuredClone(descriptor) diff --git a/core/field/convert/toDescriptor.ts b/core/field/convert/toDescriptor.ts index 52737938..4f225cdb 100644 --- a/core/field/convert/toDescriptor.ts +++ b/core/field/convert/toDescriptor.ts @@ -1,4 +1,4 @@ -import type { Descriptor } from "../../general/index.ts" +import type { Descriptor } from "../../descriptor/index.ts" import type { Field } from "../Field.ts" export function convertFieldToDescriptor(field: Field) { diff --git a/core/field/types/Base.ts b/core/field/types/Base.ts index 2e77c9f8..ba32a92d 100644 --- a/core/field/types/Base.ts +++ b/core/field/types/Base.ts @@ -1,4 +1,4 @@ -import type { Metadata } from "../../general/index.ts" +import type { Metadata } from "../../metadata/index.ts" /** * Base field properties common to all field types diff --git a/core/general/index.ts b/core/general/index.ts deleted file mode 100644 index d3d66287..00000000 --- a/core/general/index.ts +++ /dev/null @@ -1,20 +0,0 @@ -// TODO: split this general module into more focused descriptor/metadata/etc -export { parseDescriptor } from "./descriptor/process/parse.ts" -export { stringifyDescriptor } from "./descriptor/process/stringify.ts" -export { loadDescriptor } from "./descriptor/load.ts" -export { saveDescriptor } from "./descriptor/save.ts" -export { validateDescriptor } from "./descriptor/validate.ts" -export type { Descriptor } from "./descriptor/Descriptor.ts" -export type { Metadata } from "./metadata/Metadata.ts" -export { isDescriptor } from "./descriptor/Descriptor.ts" -export { loadProfile } from "./profile/load.ts" -export { - getName, - getFormat, - isRemotePath, - getBasepath, - normalizePath, - denormalizePath, - getFilename, - getProtocol, -} from "./path.ts" diff --git a/core/general/path.spec.ts b/core/general/path.spec.ts deleted file mode 100644 index 7ac14e75..00000000 --- a/core/general/path.spec.ts +++ /dev/null @@ -1,283 +0,0 @@ -import { join, relative } from "node:path" -import { describe, expect, it } from "vitest" -import { - denormalizePath, - getBasepath, - getFilename, - isRemotePath, - normalizePath, -} from "./path.ts" - -describe("isRemotePath", () => { - it.each([ - { - description: "http URL", - path: "http://example.com/path/to/file.txt", - isRemote: true, - }, - { - description: "https URL", - path: "https://example.com/path/to/file.txt", - isRemote: true, - }, - { - description: "ftp URL", - path: "ftp://example.com/path/to/file.txt", - isRemote: true, - }, - { - description: "file URL", - path: "file:///path/to/file.txt", - isRemote: false, - }, - { - description: "absolute path", - path: "/path/to/file.txt", - isRemote: false, - }, - { - description: "relative path", - path: "path/to/file.txt", - isRemote: false, - }, - { - description: "current directory path", - path: "./file.txt", - isRemote: false, - }, - { - description: "parent directory path", - path: "../file.txt", - isRemote: false, - }, - { - description: "empty string", - path: "", - isRemote: false, - }, - { - // new URL considers this to be a valid URL - description: "protocol without slashes", - path: "http:example.com", - isRemote: true, - }, - ])("$description", ({ path, isRemote }) => { - expect(isRemotePath(path)).toBe(isRemote) - }) -}) - -describe("getFilename", () => { - it.each([ - { - description: "simple filename", - path: "file.txt", - filename: "file.txt", - }, - { - description: "directory path with filename", - path: "some/path/to/file.txt", - filename: "file.txt", - }, - { - description: "remote HTTP URL", - path: "http://example.com/path/to/file.txt", - filename: "file.txt", - }, - { - description: "remote HTTPS URL", - path: "https://example.com/path/to/file.txt", - filename: "file.txt", - }, - { - description: "URL with query parameters", - path: "https://example.com/path/to/file.txt?query=param", - filename: "file.txt", - }, - { - description: "URL with hash", - path: "https://example.com/path/to/file.txt#section", - filename: "file.txt", - }, - { - description: "URL with query and hash", - path: "https://example.com/path/to/file.txt?query=param#section", - filename: "file.txt", - }, - { - description: "URL with no filename", - path: "https://example.com/path/", - filename: undefined, - }, - { - description: "local path with no filename", - path: "some/path/", - filename: undefined, - }, - ])("$description", ({ path, filename }) => { - expect(getFilename(path)).toEqual(filename) - }) -}) - -describe("getBasepath", () => { - it.each([ - { - description: "http URL with file", - path: "http://example.com/path/to/file.txt", - basepath: "http://example.com/path/to", - }, - { - description: "https URL with file", - path: "https://example.com/path/to/file.txt", - basepath: "https://example.com/path/to", - }, - { - description: "URL with query parameters", - path: "https://example.com/path/to/file.txt?query=param", - basepath: "https://example.com/path/to", - }, - { - description: "URL with hash", - path: "https://example.com/path/to/file.txt#section", - basepath: "https://example.com/path/to", - }, - { - description: "URL with no file", - path: "https://example.com/path/to/", - basepath: "https://example.com/path/to", - }, - { - description: "URL with only domain", - path: "https://example.com", - basepath: "https://example.com", - }, - { - description: "local file path", - path: "some/path/to/file.txt", - basepath: join("some", "path", "to"), - }, - { - description: "local path with no file", - path: "some/path/to/", - basepath: join("some", "path"), - }, - { - description: "root level file", - path: "file.txt", - basepath: "", - }, - ])("$description", ({ path, basepath }) => { - expect(getBasepath(path)).toEqual(basepath) - }) -}) - -describe("normalizePath", () => { - it.each([ - { - description: "local path without basepath", - path: "path/to/file.txt", - basepath: undefined, - normalizedPath: join("path", "to", "file.txt"), - }, - { - description: "local path with local basepath", - path: "file.txt", - basepath: "path/to", - normalizedPath: join("path", "to", "file.txt"), - }, - { - description: "remote path", - path: "http://example.com/path/to/file.txt", - basepath: undefined, - normalizedPath: "http://example.com/path/to/file.txt", - }, - { - description: "remote path with query string", - path: "http://example.com/path/to/file.txt?query=param", - basepath: undefined, - normalizedPath: "http://example.com/path/to/file.txt?query=param", - }, - { - description: "local path with remote basepath", - path: "path/to/file.txt", - basepath: "http://example.com", - normalizedPath: "http://example.com/path/to/file.txt", - }, - { - description: "local path with absolute basepath", - path: "file.txt", - basepath: "/absolute/path", - normalizedPath: relative(process.cwd(), "/absolute/path/file.txt"), - }, - { - description: "path with empty basepath", - path: "path/to/file.txt", - basepath: "", - normalizedPath: join("path", "to", "file.txt"), - }, - ])("$description", ({ path, basepath, normalizedPath }) => { - expect(normalizePath(path, { basepath })).toEqual(normalizedPath) - }) - - it.each([ - { - description: "absolute path", - path: "/absolute/path/to/file.txt", - basepath: undefined, - }, - { - description: "local traversed path", - path: "../file.txt", - basepath: "/folder", - }, - { - description: "remote traversed path", - path: "../file.txt", - basepath: "http://example.com/data", - }, - ])("$description -- throw", ({ path, basepath }) => { - expect(() => normalizePath(path, { basepath })).toThrow() - }) -}) - -describe("denormalizePath", () => { - it.each([ - { - description: "remote URL without basepath", - path: "http://example.com/path/to/file.txt", - basepath: undefined, - denormalizedPath: "http://example.com/path/to/file.txt", - }, - { - description: "remote URL with basepath", - path: "http://example.com/path/to/file.txt", - basepath: "data", - denormalizedPath: "http://example.com/path/to/file.txt", - }, - { - description: "local file in subfolder", - path: "/tmp/data/file.csv", - basepath: "/tmp", - denormalizedPath: "data/file.csv", - }, - { - description: "local file in direct child folder", - path: "/tmp/file.csv", - basepath: "/tmp", - denormalizedPath: "file.csv", - }, - { - description: "local file with deeply nested basepath", - path: "/tmp/data/nested/deep/file.csv", - basepath: "/tmp/data/nested", - denormalizedPath: "deep/file.csv", - }, - { - description: "local file with multi-level basepath", - path: "/home/user/projects/data/file.csv", - basepath: "/home/user/projects", - denormalizedPath: "data/file.csv", - }, - ])("$description", ({ path, basepath, denormalizedPath }) => { - expect(denormalizePath(path, { basepath })).toEqual(denormalizedPath) - }) -}) diff --git a/core/general/profile/Profile.ts b/core/general/profile/Profile.ts deleted file mode 100644 index 5ec52525..00000000 --- a/core/general/profile/Profile.ts +++ /dev/null @@ -1,3 +0,0 @@ -import type { Descriptor } from "../descriptor/Descriptor.ts" - -export type Profile = Descriptor diff --git a/core/general/profile/load.ts b/core/general/profile/load.ts deleted file mode 100644 index 0da7c4d9..00000000 --- a/core/general/profile/load.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { loadDescriptor } from "../descriptor/load.ts" -import { cache } from "./cache.ts" -import type { ProfileType } from "./registry.ts" -import { validateProfile } from "./validate.ts" - -// TODO: Narrow return type to valid JSON Schema - -export async function loadProfile( - path: string, - options?: { type?: ProfileType }, -) { - let profile = cache.get(path) - - if (!profile) { - const descriptor = await loadDescriptor(path, { onlyRemote: true }) - const result = await validateProfile({ - descriptor, - path, - type: options?.type, - }) - - if (!result.profile) { - throw new Error(`Profile at path ${path} is invalid`) - } - - profile = result.profile - cache.set(path, profile) - } - - return profile -} diff --git a/core/general/profile/validate.spec.ts b/core/general/profile/validate.spec.ts deleted file mode 100644 index aa768def..00000000 --- a/core/general/profile/validate.spec.ts +++ /dev/null @@ -1,213 +0,0 @@ -import { beforeAll, describe, expect, it, vi } from "vitest" -import { ajv } from "./ajv.ts" -import { validateProfile } from "./validate.ts" - -describe("validateProfile", () => { - beforeAll(() => { - vi.spyOn(ajv, "validateSchema") - }) - - it("returns valid result for valid descriptor without options", async () => { - const descriptor = { - name: "test", - } - - const result = await validateProfile(descriptor) - - expect(result.valid).toBe(true) - expect(result.errors).toEqual([]) - expect(result.profile).toEqual(descriptor) - }) - - it("returns validation errors for invalid schema", async () => { - const descriptor = { - name: "test", - } - - vi.mocked(ajv.validateSchema).mockImplementationOnce(async () => { - ajv.errors = [ - { - keyword: "type", - instancePath: "/name", - schemaPath: "#/properties/name/type", - params: {}, - message: "must be string", - }, - ] - }) - - const result = await validateProfile(descriptor) - - expect(result.valid).toBe(false) - expect(result.errors.length).toBe(1) - expect(result.errors[0]?.message).toBe("must be string") - expect(result.profile).toBeUndefined() - }) - - it("uses keyword when error message is not available", async () => { - const descriptor = { - name: "test", - } - - vi.mocked(ajv.validateSchema).mockImplementationOnce(async () => { - ajv.errors = [ - { - keyword: "required", - instancePath: "", - schemaPath: "#/required", - params: {}, - message: undefined, - }, - ] - }) - - const result = await validateProfile(descriptor) - - expect(result.valid).toBe(false) - expect(result.errors.length).toBe(1) - expect(result.errors[0]?.message).toBe("required") - expect(result.profile).toBeUndefined() - }) - - it("returns valid result for official profile path", async () => { - const descriptor = { - name: "test", - } - - const result = await validateProfile(descriptor, { - path: "https://datapackage.org/profiles/1.0/datapackage.json", - type: "package", - }) - - expect(result.valid).toBe(true) - expect(result.errors).toEqual([]) - expect(result.profile).toEqual(descriptor) - }) - - it("returns valid result when path matches alternate official profile", async () => { - const descriptor = { - name: "test", - } - - const result = await validateProfile(descriptor, { - path: "https://specs.frictionlessdata.io/schemas/data-package.json", - type: "package", - }) - - expect(result.valid).toBe(true) - expect(result.errors).toEqual([]) - expect(result.profile).toEqual(descriptor) - }) - - it("returns error for profile type mismatch", async () => { - const descriptor = { - name: "test", - } - - const result = await validateProfile(descriptor, { - path: "custom-profile.json", - type: "package", - }) - - expect(result.valid).toBe(false) - expect(result.errors.length).toBe(1) - expect(result.errors[0]?.message).toBe( - "Profile at custom-profile.json is not a valid package profile", - ) - expect(result.profile).toBeUndefined() - }) - - it("returns valid result when only path is provided", async () => { - const descriptor = { - name: "test", - } - - const result = await validateProfile(descriptor, { - path: "custom-profile.json", - }) - - expect(result.valid).toBe(true) - expect(result.errors).toEqual([]) - expect(result.profile).toEqual(descriptor) - }) - - it("returns valid result when only type is provided", async () => { - const descriptor = { - name: "test", - } - - const result = await validateProfile(descriptor, { - type: "package", - }) - - expect(result.valid).toBe(true) - expect(result.errors).toEqual([]) - expect(result.profile).toEqual(descriptor) - }) - - it("returns multiple errors when both schema and type validation fail", async () => { - const descriptor = { - name: "test", - } - - vi.mocked(ajv.validateSchema).mockImplementationOnce(async () => { - ajv.errors = [ - { - keyword: "type", - instancePath: "/name", - schemaPath: "#/properties/name/type", - params: {}, - message: "must be string", - }, - ] - }) - - const result = await validateProfile(descriptor, { - path: "custom-profile.json", - type: "package", - }) - - expect(result.valid).toBe(false) - expect(result.errors.length).toBe(2) - expect(result.errors[0]?.message).toBe("must be string") - expect(result.errors[1]?.message).toBe( - "Profile at custom-profile.json is not a valid package profile", - ) - expect(result.profile).toBeUndefined() - }) - - it("validates different profile types correctly", async () => { - const descriptorDialect = { - delimiter: ",", - } - - const resultDialect = await validateProfile(descriptorDialect, { - path: "https://datapackage.org/profiles/1.0/tabledialect.json", - type: "dialect", - }) - - expect(resultDialect.valid).toBe(true) - - const descriptorResource = { - name: "test-resource", - } - - const resultResource = await validateProfile(descriptorResource, { - path: "https://datapackage.org/profiles/1.0/dataresource.json", - type: "resource", - }) - - expect(resultResource.valid).toBe(true) - - const descriptorSchema = { - fields: [], - } - - const resultSchema = await validateProfile(descriptorSchema, { - path: "https://datapackage.org/profiles/1.0/tableschema.json", - type: "schema", - }) - - expect(resultSchema.valid).toBe(true) - }) -}) diff --git a/core/index.ts b/core/index.ts index 807f6b4d..c5c07974 100644 --- a/core/index.ts +++ b/core/index.ts @@ -1,4 +1,8 @@ -export * from "./general/index.ts" +export * from "./descriptor/index.ts" +export * from "./metadata/index.ts" +export * from "./path/index.ts" +export * from "./profile/index.ts" +export * from "./node/index.ts" export * from "./error/index.ts" export * from "./dialect/index.ts" export * from "./field/index.ts" diff --git a/core/general/metadata/Metadata.ts b/core/metadata/Metadata.ts similarity index 100% rename from core/general/metadata/Metadata.ts rename to core/metadata/Metadata.ts diff --git a/core/metadata/index.ts b/core/metadata/index.ts new file mode 100644 index 00000000..6e26ee58 --- /dev/null +++ b/core/metadata/index.ts @@ -0,0 +1 @@ +export type { Metadata } from "./Metadata.ts" diff --git a/core/node/index.ts b/core/node/index.ts new file mode 100644 index 00000000..2b0b5dd5 --- /dev/null +++ b/core/node/index.ts @@ -0,0 +1 @@ +export { node } from "./node.ts" diff --git a/core/node/load.spec.ts b/core/node/load.spec.ts new file mode 100644 index 00000000..159da363 --- /dev/null +++ b/core/node/load.spec.ts @@ -0,0 +1,24 @@ +import { describe, expect, it } from "vitest" +import { loadNodeApis } from "./load.ts" + +describe("loadNodeApis", () => { + it("should return node APIs when running in Node.js environment", async () => { + const result = await loadNodeApis() + + expect(result).toBeDefined() + expect(result?.fs).toBeDefined() + expect(result?.path).toBeDefined() + }) + + it("should have fs.readFile function", async () => { + const result = await loadNodeApis() + + expect(typeof result?.fs.readFile).toBe("function") + }) + + it("should have path.join function", async () => { + const result = await loadNodeApis() + + expect(typeof result?.path.join).toBe("function") + }) +}) diff --git a/core/general/node.ts b/core/node/load.ts similarity index 83% rename from core/general/node.ts rename to core/node/load.ts index 5c7576b1..c628b02b 100644 --- a/core/general/node.ts +++ b/core/node/load.ts @@ -1,5 +1,3 @@ -export const node = await loadNodeApis() - export async function loadNodeApis() { if (globalThis.process) { const fs = await import("node:fs/promises") diff --git a/core/node/node.ts b/core/node/node.ts new file mode 100644 index 00000000..7f4fc626 --- /dev/null +++ b/core/node/node.ts @@ -0,0 +1,3 @@ +import { loadNodeApis } from "./load.ts" + +export const node = await loadNodeApis() diff --git a/core/package/Package.ts b/core/package/Package.ts index 992db912..1f4e44c4 100644 --- a/core/package/Package.ts +++ b/core/package/Package.ts @@ -1,4 +1,4 @@ -import type { Metadata } from "../general/index.ts" +import type { Metadata } from "../metadata/index.ts" import type { License, Resource, Source } from "../resource/index.ts" import type { Contributor } from "./Contributor.ts" diff --git a/core/package/assert.ts b/core/package/assert.ts index d8eb1595..b4980d35 100644 --- a/core/package/assert.ts +++ b/core/package/assert.ts @@ -1,5 +1,5 @@ +import type { Descriptor } from "../descriptor/index.ts" import { AssertionError } from "../error/index.ts" -import type { Descriptor } from "../general/index.ts" import type { Package } from "./Package.ts" import { validatePackageMetadata } from "./validate.ts" diff --git a/core/package/convert/fromDescriptor.ts b/core/package/convert/fromDescriptor.ts index f4143f30..1e948d76 100644 --- a/core/package/convert/fromDescriptor.ts +++ b/core/package/convert/fromDescriptor.ts @@ -1,4 +1,5 @@ -import type { Descriptor } from "../../general/index.ts" +import type { Descriptor } from "../../descriptor/index.ts" +import { isRemotePath } from "../../path/index.ts" import { convertResourceFromDescriptor } from "../../resource/index.ts" export function convertPackageFromDescriptor( @@ -17,7 +18,12 @@ export function convertPackageFromDescriptor( } function convertProfile(descriptor: Descriptor) { - descriptor.$schema = descriptor.$schema ?? descriptor.profile + const remoteProfile = + typeof descriptor.profile === "string" && isRemotePath(descriptor.profile) + ? descriptor.profile + : undefined + + descriptor.$schema = descriptor.$schema ?? remoteProfile } function convertResources( diff --git a/core/package/convert/toDescriptor.ts b/core/package/convert/toDescriptor.ts index 665b9251..efb57d0e 100644 --- a/core/package/convert/toDescriptor.ts +++ b/core/package/convert/toDescriptor.ts @@ -1,4 +1,4 @@ -import type { Descriptor } from "../../general/index.ts" +import type { Descriptor } from "../../descriptor/index.ts" import { convertResourceToDescriptor } from "../../resource/index.ts" import type { Package } from "../Package.ts" diff --git a/core/package/index.ts b/core/package/index.ts index 0d6e37bc..c5596f4d 100644 --- a/core/package/index.ts +++ b/core/package/index.ts @@ -7,6 +7,3 @@ export { convertPackageFromDescriptor } from "./convert/fromDescriptor.ts" export { convertPackageToDescriptor } from "./convert/toDescriptor.ts" export type { Contributor } from "./Contributor.ts" export { mergePackages } from "./merge.ts" - -// TODO: Remove in v2 -export { validatePackageMetadata as validatePackageDescriptor } from "./validate.ts" diff --git a/core/package/load.ts b/core/package/load.ts index 0cdf374f..acf82f01 100644 --- a/core/package/load.ts +++ b/core/package/load.ts @@ -1,4 +1,5 @@ -import { loadDescriptor } from "../general/index.ts" +import { loadDescriptor } from "../descriptor/index.ts" +import { resolveBasepath } from "../path/index.ts" import { assertPackage } from "./assert.ts" /** @@ -6,7 +7,8 @@ import { assertPackage } from "./assert.ts" * Ensures the descriptor is valid against its profile */ export async function loadPackageDescriptor(path: string) { - const { basepath, descriptor } = await loadDescriptor(path) + const basepath = await resolveBasepath(path) + const descriptor = await loadDescriptor(path) const dataPackage = await assertPackage(descriptor, { basepath }) return dataPackage } diff --git a/core/package/merge.ts b/core/package/merge.ts index a8765208..88d841d4 100644 --- a/core/package/merge.ts +++ b/core/package/merge.ts @@ -1,6 +1,8 @@ import type { Package } from "./Package.ts" import { loadPackageDescriptor } from "./load.ts" +// TODO: Move to @dpkit/dataset? + /** * Merges a system data package into a user data package if provided */ diff --git a/core/package/save.ts b/core/package/save.ts index 3441c5c0..831167ab 100644 --- a/core/package/save.ts +++ b/core/package/save.ts @@ -1,4 +1,5 @@ -import { getBasepath, saveDescriptor } from "../general/index.ts" +import { saveDescriptor } from "../descriptor/index.ts" +import { getBasepath } from "../path/index.ts" import type { Package } from "./Package.ts" import { convertPackageToDescriptor } from "./convert/toDescriptor.ts" diff --git a/core/package/validate.spec.ts b/core/package/validate.spec.ts index 4f54e79e..a4233ea3 100644 --- a/core/package/validate.spec.ts +++ b/core/package/validate.spec.ts @@ -1,6 +1,6 @@ import { useRecording } from "@dpkit/test" import { describe, expect, it } from "vitest" -import { loadDescriptor } from "../general/index.ts" +import { loadDescriptor } from "../descriptor/index.ts" import { validatePackageMetadata } from "./validate.ts" useRecording() @@ -42,7 +42,7 @@ describe("validatePackageMetadata", () => { }) it("should validate camtrap dp (#144)", async () => { - const { descriptor } = await loadDescriptor( + const descriptor = await loadDescriptor( "https://raw.githubusercontent.com/tdwg/camtrap-dp/refs/tags/1.0.2/example/datapackage.json", ) diff --git a/core/package/validate.ts b/core/package/validate.ts index ef36e28f..d58ecf63 100644 --- a/core/package/validate.ts +++ b/core/package/validate.ts @@ -1,5 +1,5 @@ -import { type Descriptor, validateDescriptor } from "../general/index.ts" -import { loadProfile } from "../general/index.ts" +import type { Descriptor } from "../descriptor/index.ts" +import { validateDescriptor } from "../profile/index.ts" import type { Package } from "./Package.ts" import { convertPackageFromDescriptor } from "./convert/fromDescriptor.ts" @@ -16,12 +16,11 @@ export async function validatePackageMetadata( ) { const descriptor = source as Descriptor - const $schema = + const profile = typeof descriptor.$schema === "string" ? descriptor.$schema : DEFAULT_PROFILE - const profile = await loadProfile($schema) const { valid, errors } = await validateDescriptor(descriptor, { profile }) let dataPackage: Package | undefined = undefined diff --git a/core/path/basepath.spec.ts b/core/path/basepath.spec.ts new file mode 100644 index 00000000..2ebfb21f --- /dev/null +++ b/core/path/basepath.spec.ts @@ -0,0 +1,55 @@ +import { join } from "node:path" +import { describe, expect, it } from "vitest" +import { getBasepath } from "./basepath.ts" + +describe("getBasepath", () => { + it.each([ + { + description: "http URL with file", + path: "http://example.com/path/to/file.txt", + basepath: "http://example.com/path/to", + }, + { + description: "https URL with file", + path: "https://example.com/path/to/file.txt", + basepath: "https://example.com/path/to", + }, + { + description: "URL with query parameters", + path: "https://example.com/path/to/file.txt?query=param", + basepath: "https://example.com/path/to", + }, + { + description: "URL with hash", + path: "https://example.com/path/to/file.txt#section", + basepath: "https://example.com/path/to", + }, + { + description: "URL with no file", + path: "https://example.com/path/to/", + basepath: "https://example.com/path/to", + }, + { + description: "URL with only domain", + path: "https://example.com", + basepath: "https://example.com", + }, + { + description: "local file path", + path: "some/path/to/file.txt", + basepath: join("some", "path", "to"), + }, + { + description: "local path with no file", + path: "some/path/to/", + basepath: join("some", "path"), + }, + { + description: "root level file", + path: "file.txt", + basepath: "", + }, + ])("$description", ({ path, basepath }) => { + expect(getBasepath(path)).toEqual(basepath) + }) +}) diff --git a/core/path/basepath.ts b/core/path/basepath.ts new file mode 100644 index 00000000..6cfc93ae --- /dev/null +++ b/core/path/basepath.ts @@ -0,0 +1,31 @@ +import { node } from "../node/index.ts" +import { isRemotePath } from "./path.ts" + +export async function resolveBasepath(path: string) { + const isRemote = isRemotePath(path) + + // Resolves redirects + if (isRemote) { + const url = new URL(path) + const response = await fetch(url.toString(), { method: "HEAD" }) + path = response.url + } + + return getBasepath(path) +} + +export function getBasepath(path: string) { + const isRemote = isRemotePath(path) + + if (isRemote) { + const normalizedPath = new URL(path).toString() + return normalizedPath.split("/").slice(0, -1).join("/") + } + + if (!node) { + throw new Error("File system is not supported in this environment") + } + + const resolvedPath = node.path.resolve(path) + return node.path.relative(process.cwd(), node.path.parse(resolvedPath).dir) +} diff --git a/core/path/denormalize.spec.ts b/core/path/denormalize.spec.ts new file mode 100644 index 00000000..254df38a --- /dev/null +++ b/core/path/denormalize.spec.ts @@ -0,0 +1,45 @@ +import { describe, expect, it } from "vitest" +import { denormalizePath } from "./denormalize.ts" + +describe("denormalizePath", () => { + it.each([ + { + description: "remote URL without basepath", + path: "http://example.com/path/to/file.txt", + basepath: undefined, + denormalizedPath: "http://example.com/path/to/file.txt", + }, + { + description: "remote URL with basepath", + path: "http://example.com/path/to/file.txt", + basepath: "data", + denormalizedPath: "http://example.com/path/to/file.txt", + }, + { + description: "local file in subfolder", + path: "/tmp/data/file.csv", + basepath: "/tmp", + denormalizedPath: "data/file.csv", + }, + { + description: "local file in direct child folder", + path: "/tmp/file.csv", + basepath: "/tmp", + denormalizedPath: "file.csv", + }, + { + description: "local file with deeply nested basepath", + path: "/tmp/data/nested/deep/file.csv", + basepath: "/tmp/data/nested", + denormalizedPath: "deep/file.csv", + }, + { + description: "local file with multi-level basepath", + path: "/home/user/projects/data/file.csv", + basepath: "/home/user/projects", + denormalizedPath: "data/file.csv", + }, + ])("$description", ({ path, basepath, denormalizedPath }) => { + expect(denormalizePath(path, { basepath })).toEqual(denormalizedPath) + }) +}) diff --git a/core/path/denormalize.ts b/core/path/denormalize.ts new file mode 100644 index 00000000..06f29a52 --- /dev/null +++ b/core/path/denormalize.ts @@ -0,0 +1,37 @@ +import { node } from "../node/index.ts" +import { isRemotePath } from "./path.ts" + +export function denormalizePath(path: string, options: { basepath?: string }) { + const isPathRemote = isRemotePath(path) + const isBasepathRemote = isRemotePath(options.basepath ?? "") + + if (isPathRemote) { + return new URL(path).toString() + } + + if (isBasepathRemote) { + const basepath = new URL(options.basepath ?? "").toString() + + if (!path.startsWith(basepath)) { + throw new Error(`Path ${path} is not a subpath of ${options.basepath}`) + } + + const relative = path.replace(`${basepath}/`, "") + return relative + } + + if (!node) { + throw new Error("File system is not supported in this environment") + } + + const normalizedPath = node.path.resolve(path) + const normalizedBasepath = node.path.resolve(options.basepath ?? "") + + if (!normalizedPath.startsWith(normalizedBasepath)) { + throw new Error(`Path ${path} is not a subpath of ${options.basepath}`) + } + + // The Data Package standard requires "/" as the path separator + const relative = node.path.relative(normalizedBasepath, normalizedPath) + return relative.split(node.path.sep).join("/") +} diff --git a/core/path/index.ts b/core/path/index.ts new file mode 100644 index 00000000..095db746 --- /dev/null +++ b/core/path/index.ts @@ -0,0 +1,5 @@ +export { getName, getFormat, getFilename, getProtocol } from "./path.ts" +export { getBasepath, resolveBasepath } from "./basepath.ts" +export { denormalizePath } from "./denormalize.ts" +export { normalizePath } from "./normalize.ts" +export { isRemotePath } from "./path.ts" diff --git a/core/path/normalize.spec.ts b/core/path/normalize.spec.ts new file mode 100644 index 00000000..5db2e08f --- /dev/null +++ b/core/path/normalize.spec.ts @@ -0,0 +1,72 @@ +import { join, relative } from "node:path" +import { describe, expect, it } from "vitest" +import { normalizePath } from "./normalize.ts" + +describe("normalizePath", () => { + it.each([ + { + description: "local path without basepath", + path: "path/to/file.txt", + basepath: undefined, + normalizedPath: join("path", "to", "file.txt"), + }, + { + description: "local path with local basepath", + path: "file.txt", + basepath: "path/to", + normalizedPath: join("path", "to", "file.txt"), + }, + { + description: "remote path", + path: "http://example.com/path/to/file.txt", + basepath: undefined, + normalizedPath: "http://example.com/path/to/file.txt", + }, + { + description: "remote path with query string", + path: "http://example.com/path/to/file.txt?query=param", + basepath: undefined, + normalizedPath: "http://example.com/path/to/file.txt?query=param", + }, + { + description: "local path with remote basepath", + path: "path/to/file.txt", + basepath: "http://example.com", + normalizedPath: "http://example.com/path/to/file.txt", + }, + { + description: "local path with absolute basepath", + path: "file.txt", + basepath: "/absolute/path", + normalizedPath: relative(process.cwd(), "/absolute/path/file.txt"), + }, + { + description: "path with empty basepath", + path: "path/to/file.txt", + basepath: "", + normalizedPath: join("path", "to", "file.txt"), + }, + ])("$description", ({ path, basepath, normalizedPath }) => { + expect(normalizePath(path, { basepath })).toEqual(normalizedPath) + }) + + it.each([ + { + description: "absolute path", + path: "/absolute/path/to/file.txt", + basepath: undefined, + }, + { + description: "local traversed path", + path: "../file.txt", + basepath: "/folder", + }, + { + description: "remote traversed path", + path: "../file.txt", + basepath: "http://example.com/data", + }, + ])("$description -- throw", ({ path, basepath }) => { + expect(() => normalizePath(path, { basepath })).toThrow() + }) +}) diff --git a/core/general/path.ts b/core/path/normalize.ts similarity index 55% rename from core/general/path.ts rename to core/path/normalize.ts index d8c2d97a..dd91dbfb 100644 --- a/core/general/path.ts +++ b/core/path/normalize.ts @@ -1,77 +1,5 @@ -import slugify from "@sindresorhus/slugify" -import { node } from "./node.ts" - -export function isRemotePath(path: string) { - const protocol = getProtocol(path) - return protocol !== "file" -} - -export function getName(filename?: string) { - if (!filename) { - return undefined - } - - const name = filename.split(".")[0] - if (!name) { - return undefined - } - - return slugify(name) -} - -export function getProtocol(path: string) { - try { - const url = new URL(path) - const protocol = url.protocol.replace(":", "") - - // Handle Windows drive letters - if (protocol.length < 2) { - return "file" - } - - return protocol - } catch { - return "file" - } -} - -export function getFormat(filename?: string) { - return filename?.split(".").slice(-1)[0]?.toLowerCase() -} - -export function getFilename(path: string) { - const isRemote = isRemotePath(path) - - if (isRemote) { - const pathname = new URL(path).pathname - const filename = pathname.split("/").slice(-1)[0] - return filename?.includes(".") ? filename : undefined - } - - if (!node) { - throw new Error("File system is not supported in this environment") - } - - const resolvedPath = node.path.resolve(path) - const filename = node.path.parse(resolvedPath).base - return filename?.includes(".") ? filename : undefined -} - -export function getBasepath(path: string) { - const isRemote = isRemotePath(path) - - if (isRemote) { - const normalizedPath = new URL(path).toString() - return normalizedPath.split("/").slice(0, -1).join("/") - } - - if (!node) { - throw new Error("File system is not supported in this environment") - } - - const resolvedPath = node.path.resolve(path) - return node.path.relative(process.cwd(), node.path.parse(resolvedPath).dir) -} +import { node } from "../node/index.ts" +import { isRemotePath } from "./path.ts" export function normalizePath(path: string, options: { basepath?: string }) { const isPathRemote = isRemotePath(path) diff --git a/core/path/path.spec.ts b/core/path/path.spec.ts new file mode 100644 index 00000000..d3f550e4 --- /dev/null +++ b/core/path/path.spec.ts @@ -0,0 +1,112 @@ +import { describe, expect, it } from "vitest" +import { getFilename, isRemotePath } from "./path.ts" + +describe("isRemotePath", () => { + it.each([ + { + description: "http URL", + path: "http://example.com/path/to/file.txt", + isRemote: true, + }, + { + description: "https URL", + path: "https://example.com/path/to/file.txt", + isRemote: true, + }, + { + description: "ftp URL", + path: "ftp://example.com/path/to/file.txt", + isRemote: true, + }, + { + description: "file URL", + path: "file:///path/to/file.txt", + isRemote: false, + }, + { + description: "absolute path", + path: "/path/to/file.txt", + isRemote: false, + }, + { + description: "relative path", + path: "path/to/file.txt", + isRemote: false, + }, + { + description: "current directory path", + path: "./file.txt", + isRemote: false, + }, + { + description: "parent directory path", + path: "../file.txt", + isRemote: false, + }, + { + description: "empty string", + path: "", + isRemote: false, + }, + { + // new URL considers this to be a valid URL + description: "protocol without slashes", + path: "http:example.com", + isRemote: true, + }, + ])("$description", ({ path, isRemote }) => { + expect(isRemotePath(path)).toBe(isRemote) + }) +}) + +describe("getFilename", () => { + it.each([ + { + description: "simple filename", + path: "file.txt", + filename: "file.txt", + }, + { + description: "directory path with filename", + path: "some/path/to/file.txt", + filename: "file.txt", + }, + { + description: "remote HTTP URL", + path: "http://example.com/path/to/file.txt", + filename: "file.txt", + }, + { + description: "remote HTTPS URL", + path: "https://example.com/path/to/file.txt", + filename: "file.txt", + }, + { + description: "URL with query parameters", + path: "https://example.com/path/to/file.txt?query=param", + filename: "file.txt", + }, + { + description: "URL with hash", + path: "https://example.com/path/to/file.txt#section", + filename: "file.txt", + }, + { + description: "URL with query and hash", + path: "https://example.com/path/to/file.txt?query=param#section", + filename: "file.txt", + }, + { + description: "URL with no filename", + path: "https://example.com/path/", + filename: undefined, + }, + { + description: "local path with no filename", + path: "some/path/", + filename: undefined, + }, + ])("$description", ({ path, filename }) => { + expect(getFilename(path)).toEqual(filename) + }) +}) diff --git a/core/path/path.ts b/core/path/path.ts new file mode 100644 index 00000000..79732851 --- /dev/null +++ b/core/path/path.ts @@ -0,0 +1,58 @@ +import slugify from "@sindresorhus/slugify" +import { node } from "../node/index.ts" + +export function isRemotePath(path: string) { + const protocol = getProtocol(path) + return protocol !== "file" +} + +export function getName(filename?: string) { + if (!filename) { + return undefined + } + + const name = filename.split(".")[0] + if (!name) { + return undefined + } + + return slugify(name) +} + +export function getProtocol(path: string) { + try { + const url = new URL(path) + const protocol = url.protocol.replace(":", "") + + // Handle Windows drive letters + if (protocol.length < 2) { + return "file" + } + + return protocol + } catch { + return "file" + } +} + +export function getFormat(filename?: string) { + return filename?.split(".").slice(-1)[0]?.toLowerCase() +} + +export function getFilename(path: string) { + const isRemote = isRemotePath(path) + + if (isRemote) { + const pathname = new URL(path).pathname + const filename = pathname.split("/").slice(-1)[0] + return filename?.includes(".") ? filename : undefined + } + + if (!node) { + throw new Error("File system is not supported in this environment") + } + + const resolvedPath = node.path.resolve(path) + const filename = node.path.parse(resolvedPath).base + return filename?.includes(".") ? filename : undefined +} diff --git a/core/plugin.ts b/core/plugin.ts index e805322c..22f91c64 100644 --- a/core/plugin.ts +++ b/core/plugin.ts @@ -6,8 +6,10 @@ export type SavePackageOptions = { } export interface Plugin { + // TODO: move to @dpkit/dataset? loadPackage?(source: string): Promise + // TODO: move to @dpkit/dataset? savePackage?( dataPackage: Package, options: SavePackageOptions, diff --git a/core/profile/Profile.ts b/core/profile/Profile.ts new file mode 100644 index 00000000..2da02e2b --- /dev/null +++ b/core/profile/Profile.ts @@ -0,0 +1,11 @@ +import type { Descriptor } from "../descriptor/index.ts" + +// TODO: Narrow to valid JSON Schema +export type Profile = Descriptor +export type ProfileType = "dialect" | "package" | "resource" | "schema" +export type ProfileRegistry = { + type: ProfileType + path: string + version: string + profile: Profile +}[] diff --git a/core/general/profile/ajv.ts b/core/profile/ajv.ts similarity index 100% rename from core/general/profile/ajv.ts rename to core/profile/ajv.ts diff --git a/core/profile/assert.spec.ts b/core/profile/assert.spec.ts new file mode 100644 index 00000000..038ad5e3 --- /dev/null +++ b/core/profile/assert.spec.ts @@ -0,0 +1,186 @@ +import { beforeAll, describe, expect, it, vi } from "vitest" +import { ajv } from "./ajv.ts" +import { assertProfile } from "./assert.ts" + +describe("assertProfile", () => { + beforeAll(() => { + vi.spyOn(ajv, "validateSchema") + }) + + it("returns profile for valid descriptor without options", async () => { + const descriptor = { + name: "test", + } + + const profile = await assertProfile(descriptor) + + expect(profile).toEqual(descriptor) + }) + + it("throws error for invalid schema", async () => { + const descriptor = { + name: "test", + } + + vi.mocked(ajv.validateSchema).mockImplementationOnce(async () => { + ajv.errors = [ + { + keyword: "type", + instancePath: "/name", + schemaPath: "#/properties/name/type", + params: {}, + message: "must be string", + }, + ] + }) + + await expect(assertProfile(descriptor)).rejects.toThrow( + "Profile at path undefined is invalid", + ) + }) + + it("throws error when error message is not available", async () => { + const descriptor = { + name: "test", + } + + vi.mocked(ajv.validateSchema).mockImplementationOnce(async () => { + ajv.errors = [ + { + keyword: "required", + instancePath: "", + schemaPath: "#/required", + params: {}, + message: undefined, + }, + ] + }) + + await expect(assertProfile(descriptor)).rejects.toThrow( + "Profile at path undefined is invalid", + ) + }) + + it("returns profile for official profile path", async () => { + const descriptor = { + name: "test", + } + + const profile = await assertProfile(descriptor, { + path: "https://datapackage.org/profiles/1.0/datapackage.json", + type: "package", + }) + + expect(profile).toEqual(descriptor) + }) + + it("returns profile when path matches alternate official profile", async () => { + const descriptor = { + name: "test", + } + + const profile = await assertProfile(descriptor, { + path: "https://specs.frictionlessdata.io/schemas/data-package.json", + type: "package", + }) + + expect(profile).toEqual(descriptor) + }) + + it("throws error for profile type mismatch", async () => { + const descriptor = { + name: "test", + } + + await expect( + assertProfile(descriptor, { + path: "custom-profile.json", + type: "package", + }), + ).rejects.toThrow("Profile at path custom-profile.json is invalid") + }) + + it("returns profile when only path is provided", async () => { + const descriptor = { + name: "test", + } + + const profile = await assertProfile(descriptor, { + path: "custom-profile.json", + }) + + expect(profile).toEqual(descriptor) + }) + + it("returns profile when only type is provided", async () => { + const descriptor = { + name: "test", + } + + const profile = await assertProfile(descriptor, { + type: "package", + }) + + expect(profile).toEqual(descriptor) + }) + + it("throws error when both schema and type validation fail", async () => { + const descriptor = { + name: "test", + } + + vi.mocked(ajv.validateSchema).mockImplementationOnce(async () => { + ajv.errors = [ + { + keyword: "type", + instancePath: "/name", + schemaPath: "#/properties/name/type", + params: {}, + message: "must be string", + }, + ] + }) + + await expect( + assertProfile(descriptor, { + path: "custom-profile.json", + type: "package", + }), + ).rejects.toThrow("Profile at path custom-profile.json is invalid") + }) + + it("validates different profile types correctly", async () => { + const descriptorDialect = { + delimiter: ",", + } + + const profileDialect = await assertProfile(descriptorDialect, { + path: "https://datapackage.org/profiles/1.0/tabledialect.json", + type: "dialect", + }) + + expect(profileDialect).toEqual(descriptorDialect) + + const descriptorResource = { + name: "test-resource", + } + + const profileResource = await assertProfile(descriptorResource, { + path: "https://datapackage.org/profiles/1.0/dataresource.json", + type: "resource", + }) + + expect(profileResource).toEqual(descriptorResource) + + const descriptorSchema = { + fields: [], + } + + const profileSchema = await assertProfile(descriptorSchema, { + path: "https://datapackage.org/profiles/1.0/tableschema.json", + type: "schema", + }) + + expect(profileSchema).toEqual(descriptorSchema) + }) +}) diff --git a/core/general/profile/validate.ts b/core/profile/assert.ts similarity index 80% rename from core/general/profile/validate.ts rename to core/profile/assert.ts index 193211f3..0ca8e7b1 100644 --- a/core/general/profile/validate.ts +++ b/core/profile/assert.ts @@ -1,10 +1,10 @@ -import type { Descriptor } from "../descriptor/Descriptor.ts" +import type { Descriptor } from "../descriptor/index.ts" import type { Profile } from "./Profile.ts" +import type { ProfileType } from "./Profile.ts" import { ajv } from "./ajv.ts" -import type { ProfileType } from "./registry.ts" import { profileRegistry } from "./registry.ts" -export async function validateProfile( +export async function assertProfile( descriptor: Descriptor, options?: { path?: string @@ -24,11 +24,12 @@ export async function validateProfile( }) } - return { - errors, - valid: !errors.length, - profile: !errors.length ? (descriptor as Profile) : undefined, + // TODO: Improve consolidated error message + if (errors.length) { + throw new Error(`Profile at path ${options?.path} is invalid`) } + + return descriptor as Profile } function checkProfileType( diff --git a/core/general/profile/cache.ts b/core/profile/cache.ts similarity index 56% rename from core/general/profile/cache.ts rename to core/profile/cache.ts index 6044a49b..a5b0d1af 100644 --- a/core/general/profile/cache.ts +++ b/core/profile/cache.ts @@ -1,8 +1,8 @@ import QuickLRU from "quick-lru" -import type { Descriptor } from "../descriptor/Descriptor.ts" +import type { Profile } from "./Profile.ts" import { profileRegistry } from "./registry.ts" -export const cache = new QuickLRU({ maxSize: 100 }) +export const cache = new QuickLRU({ maxSize: 100 }) for (const { path, profile } of Object.values(profileRegistry)) { cache.set(path, profile) } diff --git a/core/profile/index.ts b/core/profile/index.ts new file mode 100644 index 00000000..9184eb8a --- /dev/null +++ b/core/profile/index.ts @@ -0,0 +1 @@ +export { validateDescriptor } from "./validate.ts" diff --git a/core/profile/load.ts b/core/profile/load.ts new file mode 100644 index 00000000..0f79d9d8 --- /dev/null +++ b/core/profile/load.ts @@ -0,0 +1,19 @@ +import { loadDescriptor } from "../descriptor/index.ts" +import { cache } from "./cache.ts" +import type { ProfileType } from "./Profile.ts" +import { assertProfile } from "./assert.ts" + +export async function loadProfile( + path: string, + options?: { type?: ProfileType }, +) { + let profile = cache.get(path) + + if (!profile) { + const descriptor = await loadDescriptor(path, { onlyRemote: true }) + profile = await assertProfile(descriptor, { path, type: options?.type }) + cache.set(path, profile) + } + + return profile +} diff --git a/core/general/profile/registry.ts b/core/profile/registry.ts similarity index 95% rename from core/general/profile/registry.ts rename to core/profile/registry.ts index 521df8c1..36d74293 100644 --- a/core/general/profile/registry.ts +++ b/core/profile/registry.ts @@ -1,3 +1,4 @@ +import type { ProfileRegistry } from "./Profile.ts" import dialect_1_0 from "./registry/dialect-1.0.json" with { type: "json" } import dialect_2_0 from "./registry/dialect-2.0.json" with { type: "json" } import package_1_0 from "./registry/package-1.0.json" with { type: "json" } @@ -7,9 +8,7 @@ import resource_2_0 from "./registry/resource-2.0.json" with { type: "json" } import schema_1_0 from "./registry/schema-1.0.json" with { type: "json" } import schema_2_0 from "./registry/schema-2.0.json" with { type: "json" } -export type ProfileType = (typeof profileRegistry)[number]["type"] - -export const profileRegistry = [ +export const profileRegistry: ProfileRegistry = [ { type: "dialect", path: "https://specs.frictionlessdata.io/schemas/csv-dialect.json", diff --git a/core/general/profile/registry/dialect-1.0.json b/core/profile/registry/dialect-1.0.json similarity index 100% rename from core/general/profile/registry/dialect-1.0.json rename to core/profile/registry/dialect-1.0.json diff --git a/core/general/profile/registry/dialect-2.0.json b/core/profile/registry/dialect-2.0.json similarity index 100% rename from core/general/profile/registry/dialect-2.0.json rename to core/profile/registry/dialect-2.0.json diff --git a/core/general/profile/registry/package-1.0.json b/core/profile/registry/package-1.0.json similarity index 100% rename from core/general/profile/registry/package-1.0.json rename to core/profile/registry/package-1.0.json diff --git a/core/general/profile/registry/package-2.0.json b/core/profile/registry/package-2.0.json similarity index 100% rename from core/general/profile/registry/package-2.0.json rename to core/profile/registry/package-2.0.json diff --git a/core/general/profile/registry/resource-1.0.json b/core/profile/registry/resource-1.0.json similarity index 100% rename from core/general/profile/registry/resource-1.0.json rename to core/profile/registry/resource-1.0.json diff --git a/core/general/profile/registry/resource-2.0.json b/core/profile/registry/resource-2.0.json similarity index 100% rename from core/general/profile/registry/resource-2.0.json rename to core/profile/registry/resource-2.0.json diff --git a/core/general/profile/registry/schema-1.0.json b/core/profile/registry/schema-1.0.json similarity index 100% rename from core/general/profile/registry/schema-1.0.json rename to core/profile/registry/schema-1.0.json diff --git a/core/general/profile/registry/schema-2.0.json b/core/profile/registry/schema-2.0.json similarity index 100% rename from core/general/profile/registry/schema-2.0.json rename to core/profile/registry/schema-2.0.json diff --git a/core/general/profile/scripts/prune.ts b/core/profile/scripts/prune.ts similarity index 100% rename from core/general/profile/scripts/prune.ts rename to core/profile/scripts/prune.ts diff --git a/core/general/descriptor/validate.spec.ts b/core/profile/validate.spec.ts similarity index 100% rename from core/general/descriptor/validate.spec.ts rename to core/profile/validate.spec.ts diff --git a/core/general/descriptor/validate.ts b/core/profile/validate.ts similarity index 55% rename from core/general/descriptor/validate.ts rename to core/profile/validate.ts index 92c575c4..8badeb2a 100644 --- a/core/general/descriptor/validate.ts +++ b/core/profile/validate.ts @@ -1,6 +1,8 @@ -import type { MetadataError } from "../../error/index.ts" +import type { Descriptor } from "../descriptor/index.ts" +import type { MetadataError } from "../error/index.ts" import { ajv } from "../profile/ajv.ts" -import type { Descriptor } from "./Descriptor.ts" +import type { Profile } from "./Profile.ts" +import { loadProfile } from "./load.ts" /** * Validate a descriptor (JSON Object) against a JSON Schema @@ -10,10 +12,15 @@ import type { Descriptor } from "./Descriptor.ts" export async function validateDescriptor( descriptor: Descriptor, options: { - profile: Descriptor + profile: Profile | string }, ) { - const validate = await ajv.compileAsync(options.profile) + const profile = + typeof options.profile === "string" + ? await loadProfile(options.profile) + : options.profile + + const validate = await ajv.compileAsync(profile) const valid = validate(descriptor) const errors: MetadataError[] = validate.errors diff --git a/core/resource/Resource.ts b/core/resource/Resource.ts index aa4078e1..1df4a803 100644 --- a/core/resource/Resource.ts +++ b/core/resource/Resource.ts @@ -1,5 +1,5 @@ import type { Dialect } from "../dialect/Dialect.ts" -import type { Metadata } from "../general/index.ts" +import type { Metadata } from "../metadata/index.ts" import type { Schema } from "../schema/Schema.ts" import type { License } from "./License.ts" import type { Source } from "./Source.ts" diff --git a/core/resource/assert.ts b/core/resource/assert.ts index a05c0ae1..50a3ebb5 100644 --- a/core/resource/assert.ts +++ b/core/resource/assert.ts @@ -1,5 +1,5 @@ +import type { Descriptor } from "../descriptor/index.ts" import { AssertionError } from "../error/index.ts" -import type { Descriptor } from "../general/index.ts" import type { Resource } from "./Resource.ts" import { validateResourceMetadata } from "./validate.ts" diff --git a/core/resource/convert/fromDescriptor.ts b/core/resource/convert/fromDescriptor.ts index 3e51b99a..cd98dbce 100644 --- a/core/resource/convert/fromDescriptor.ts +++ b/core/resource/convert/fromDescriptor.ts @@ -1,7 +1,7 @@ +import { isDescriptor } from "../../descriptor/index.ts" +import type { Descriptor } from "../../descriptor/index.ts" import { convertDialectFromDescriptor } from "../../dialect/index.ts" -import { isDescriptor, normalizePath } from "../../general/index.ts" -import type { Descriptor } from "../../general/index.ts" -import { isRemotePath } from "../../general/index.ts" +import { isRemotePath, normalizePath } from "../../path/index.ts" import { convertSchemaFromDescriptor } from "../../schema/index.ts" export function convertResourceFromDescriptor( diff --git a/core/resource/convert/toDescriptor.ts b/core/resource/convert/toDescriptor.ts index b51f4a8a..109250ae 100644 --- a/core/resource/convert/toDescriptor.ts +++ b/core/resource/convert/toDescriptor.ts @@ -1,6 +1,6 @@ +import type { Descriptor } from "../../descriptor/index.ts" import { convertDialectToDescriptor } from "../../dialect/index.ts" -import { denormalizePath } from "../../general/index.ts" -import type { Descriptor } from "../../general/index.ts" +import { denormalizePath } from "../../path/index.ts" import { convertSchemaToDescriptor } from "../../schema/index.ts" import type { Resource } from "../Resource.ts" diff --git a/core/resource/helpers.ts b/core/resource/helpers.ts index 5ca5987c..9e1dbc89 100644 --- a/core/resource/helpers.ts +++ b/core/resource/helpers.ts @@ -1,4 +1,4 @@ -import { isRemotePath } from "../general/index.ts" +import { isRemotePath } from "../path/index.ts" import type { Resource } from "./Resource.ts" export function isRemoteResource(resource: Resource) { diff --git a/core/resource/infer.ts b/core/resource/infer.ts index 02e06ff6..a025b64f 100644 --- a/core/resource/infer.ts +++ b/core/resource/infer.ts @@ -1,9 +1,4 @@ -import { - getFilename, - getFormat, - getName, - getProtocol, -} from "../general/index.ts" +import { getFilename, getFormat, getName, getProtocol } from "../path/index.ts" import type { Resource } from "./Resource.ts" export function inferResourceName(resource: Partial) { diff --git a/core/resource/load.ts b/core/resource/load.ts index 1d82e88b..ea135e5b 100644 --- a/core/resource/load.ts +++ b/core/resource/load.ts @@ -1,4 +1,5 @@ -import { loadDescriptor } from "../general/index.ts" +import { loadDescriptor } from "../descriptor/index.ts" +import { resolveBasepath } from "../path/index.ts" import { assertResource } from "./assert.ts" /** @@ -6,7 +7,8 @@ import { assertResource } from "./assert.ts" * Ensures the descriptor is valid against its profile */ export async function loadResourceDescriptor(path: string) { - const { descriptor, basepath } = await loadDescriptor(path) + const basepath = await resolveBasepath(path) + const descriptor = await loadDescriptor(path) const resource = await assertResource(descriptor, { basepath }) return resource } diff --git a/core/resource/save.ts b/core/resource/save.ts index 82e4e72b..0099770f 100644 --- a/core/resource/save.ts +++ b/core/resource/save.ts @@ -1,4 +1,5 @@ -import { getBasepath, saveDescriptor } from "../general/index.ts" +import { saveDescriptor } from "../descriptor/index.ts" +import { getBasepath } from "../path/index.ts" import type { Resource } from "./Resource.ts" import { convertResourceToDescriptor } from "./convert/toDescriptor.ts" diff --git a/core/resource/validate.ts b/core/resource/validate.ts index 7ef1d3eb..28bab261 100644 --- a/core/resource/validate.ts +++ b/core/resource/validate.ts @@ -1,7 +1,7 @@ +import type { Descriptor } from "../descriptor/index.ts" import { loadDialect } from "../dialect/index.ts" import { AssertionError } from "../error/index.ts" -import { type Descriptor, validateDescriptor } from "../general/index.ts" -import { loadProfile } from "../general/index.ts" +import { validateDescriptor } from "../profile/index.ts" import { loadSchema } from "../schema/index.ts" import type { Resource } from "./Resource.ts" import { convertResourceFromDescriptor } from "./convert/fromDescriptor.ts" @@ -19,12 +19,11 @@ export async function validateResourceMetadata( ) { const descriptor = source as Descriptor - const $schema = + const profile = typeof descriptor.$schema === "string" ? descriptor.$schema : DEFAULT_PROFILE - const profile = await loadProfile($schema) let { valid, errors } = await validateDescriptor(descriptor, { profile }) let resource: Resource | undefined = undefined diff --git a/core/schema/Schema.ts b/core/schema/Schema.ts index f7bce413..81825c74 100644 --- a/core/schema/Schema.ts +++ b/core/schema/Schema.ts @@ -1,5 +1,5 @@ import type { Field } from "../field/index.ts" -import type { Metadata } from "../general/index.ts" +import type { Metadata } from "../metadata/index.ts" import type { ForeignKey } from "./ForeignKey.ts" /** diff --git a/core/schema/assert.ts b/core/schema/assert.ts index c2b62372..da7ded6b 100644 --- a/core/schema/assert.ts +++ b/core/schema/assert.ts @@ -1,5 +1,5 @@ +import type { Descriptor } from "../descriptor/index.ts" import { AssertionError } from "../error/index.ts" -import type { Descriptor } from "../general/index.ts" import type { Schema } from "./Schema.ts" import { validateSchema } from "./validate.ts" diff --git a/core/schema/convert/fromDescriptor.ts b/core/schema/convert/fromDescriptor.ts index 960592c0..79bc2305 100644 --- a/core/schema/convert/fromDescriptor.ts +++ b/core/schema/convert/fromDescriptor.ts @@ -1,6 +1,6 @@ import invariant from "tiny-invariant" +import type { Descriptor } from "../../descriptor/index.ts" import { convertFieldFromDescriptor } from "../../field/index.ts" -import type { Descriptor } from "../../general/index.ts" export function convertSchemaFromDescriptor(descriptor: Descriptor) { descriptor = globalThis.structuredClone(descriptor) diff --git a/core/schema/convert/toDescriptor.ts b/core/schema/convert/toDescriptor.ts index 3c57424e..0babc5b0 100644 --- a/core/schema/convert/toDescriptor.ts +++ b/core/schema/convert/toDescriptor.ts @@ -1,4 +1,4 @@ -import type { Descriptor } from "../../general/index.ts" +import type { Descriptor } from "../../descriptor/index.ts" import type { Schema } from "../Schema.ts" export function convertSchemaToDescriptor(schema: Schema) { diff --git a/core/schema/load.ts b/core/schema/load.ts index 00bd37b0..04f84d42 100644 --- a/core/schema/load.ts +++ b/core/schema/load.ts @@ -1,4 +1,4 @@ -import { loadDescriptor } from "../general/index.ts" +import { loadDescriptor } from "../descriptor/index.ts" import { assertSchema } from "./assert.ts" /** @@ -6,7 +6,7 @@ import { assertSchema } from "./assert.ts" * Ensures the descriptor is valid against its profile */ export async function loadSchema(path: string) { - const { descriptor } = await loadDescriptor(path) + const descriptor = await loadDescriptor(path) const schema = await assertSchema(descriptor) return schema } diff --git a/core/schema/save.ts b/core/schema/save.ts index 9a1b235b..6c50399a 100644 --- a/core/schema/save.ts +++ b/core/schema/save.ts @@ -1,4 +1,4 @@ -import { saveDescriptor } from "../general/index.ts" +import { saveDescriptor } from "../descriptor/index.ts" import type { Schema } from "./Schema.ts" import { convertSchemaToDescriptor } from "./convert/toDescriptor.ts" diff --git a/core/schema/validate.ts b/core/schema/validate.ts index bc6dbed3..6d470da7 100644 --- a/core/schema/validate.ts +++ b/core/schema/validate.ts @@ -1,5 +1,5 @@ -import { type Descriptor, validateDescriptor } from "../general/index.ts" -import { loadProfile } from "../general/index.ts" +import type { Descriptor } from "../descriptor/index.ts" +import { validateDescriptor } from "../profile/index.ts" import type { Schema } from "./Schema.ts" import { convertSchemaFromDescriptor } from "./convert/fromDescriptor.ts" @@ -11,12 +11,11 @@ const DEFAULT_PROFILE = "https://datapackage.org/profiles/1.0/tableschema.json" export async function validateSchema(source: Descriptor | Schema) { const descriptor = source as Descriptor - const $schema = + const profile = typeof descriptor.$schema === "string" ? descriptor.$schema : DEFAULT_PROFILE - const profile = await loadProfile($schema) const { valid, errors } = await validateDescriptor(descriptor, { profile }) let schema: Schema | undefined = undefined diff --git a/file/file/validate.ts b/file/file/validate.ts index ea0e5cbb..3535a467 100644 --- a/file/file/validate.ts +++ b/file/file/validate.ts @@ -10,40 +10,47 @@ export async function validateFile( const localPaths = await prefetchFiles(path) if (options?.bytes) { - const bytes = await inferFileBytes(localPaths) - if (bytes !== options.bytes) { + const bytes = options.bytes + const actualBytes = await inferFileBytes(localPaths) + + if (bytes !== actualBytes) { errors.push({ type: "file/bytes", - bytes: options.bytes, - actualBytes: bytes, + bytes, + actualBytes, }) } } if (options?.hash) { - const [_hashValue, hashType = "md5"] = options.hash.split(":").toReversed() - const hash = await inferFileHash(localPaths, { + const [hashValue, hashType = "md5"] = options.hash.split(":").toReversed() + + const hash = `${hashType}:${hashValue}` + const actualHash = await inferFileHash(localPaths, { hashType: hashType as any, }) - if (hash !== options.hash) { + if (hash !== actualHash) { errors.push({ type: "file/hash", - hash: options.hash, - actualHash: hash, + hash, + actualHash, }) } } if (options?.encoding) { - const encoding = await inferFileEncoding(localPaths) + const encoding = options.encoding + const actualEncoding = await inferFileEncoding(localPaths) - if (encoding && encoding !== options.encoding) { - errors.push({ - type: "file/encoding", - encoding: options.encoding, - actualEncoding: encoding, - }) + if (actualEncoding) { + if (encoding !== actualEncoding) { + errors.push({ + type: "file/encoding", + encoding, + actualEncoding, + }) + } } } diff --git a/github/package/fixtures/generated/loadPackageFromGithub-should-merge-datapackage-json-if-present_618226504/recording.har b/github/package/fixtures/generated/loadPackageFromGithub-should-merge-datapackage-json-if-present_618226504/recording.har index f89c040d..e1ba3813 100644 --- a/github/package/fixtures/generated/loadPackageFromGithub-should-merge-datapackage-json-if-present_618226504/recording.har +++ b/github/package/fixtures/generated/loadPackageFromGithub-should-merge-datapackage-json-if-present_618226504/recording.har @@ -445,6 +445,147 @@ "ssl": -1, "wait": 346 } + }, + { + "_id": "2fc9ef7063eaa13e9b46bdfb36e801d0", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [], + "headersSize": 108, + "httpVersion": "HTTP/1.1", + "method": "HEAD", + "queryString": [], + "url": "https://raw.githubusercontent.com/roll/currency-codes/refs/heads/master/datapackage.json" + }, + "response": { + "bodySize": 679, + "content": { + "mimeType": "text/plain; charset=utf-8", + "size": 679 + }, + "cookies": [], + "headers": [ + { + "name": "accept-ranges", + "value": "bytes" + }, + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "cache-control", + "value": "max-age=300" + }, + { + "name": "connection", + "value": "close" + }, + { + "name": "content-encoding", + "value": "gzip" + }, + { + "name": "content-length", + "value": "679" + }, + { + "name": "content-security-policy", + "value": "default-src 'none'; style-src 'unsafe-inline'; sandbox" + }, + { + "name": "content-type", + "value": "text/plain; charset=utf-8" + }, + { + "name": "cross-origin-resource-policy", + "value": "cross-origin" + }, + { + "name": "date", + "value": "Fri, 31 Oct 2025 07:31:18 GMT" + }, + { + "name": "etag", + "value": "W/\"18535666ca34d8d5f92e12776d3673e8e31c05c2be5c50a439cee1f3213ab9cb\"" + }, + { + "name": "expires", + "value": "Fri, 31 Oct 2025 07:36:18 GMT" + }, + { + "name": "source-age", + "value": "0" + }, + { + "name": "strict-transport-security", + "value": "max-age=31536000" + }, + { + "name": "vary", + "value": "Authorization,Accept-Encoding" + }, + { + "name": "via", + "value": "1.1 varnish" + }, + { + "name": "x-cache", + "value": "MISS" + }, + { + "name": "x-cache-hits", + "value": "0" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "x-fastly-request-id", + "value": "e044ba96c2be9646f2b98f8df059609ba43674fd" + }, + { + "name": "x-frame-options", + "value": "deny" + }, + { + "name": "x-github-request-id", + "value": "44CE:1FAB18:CCC9A7:DF847E:690465C5" + }, + { + "name": "x-served-by", + "value": "cache-lis1490043-LIS" + }, + { + "name": "x-timer", + "value": "S1761895878.877871,VS0,VE185" + }, + { + "name": "x-xss-protection", + "value": "1; mode=block" + } + ], + "headersSize": 897, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-10-31T07:31:17.615Z", + "time": 581, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 581 + } } ], "pages": [], diff --git a/lib/package/validate.ts b/lib/package/validate.ts index cf8f2109..8ed390d3 100644 --- a/lib/package/validate.ts +++ b/lib/package/validate.ts @@ -1,7 +1,10 @@ import type { Descriptor, Package } from "@dpkit/core" -import { loadDescriptor, validatePackageDescriptor } from "@dpkit/core" +import { loadDescriptor, validatePackageMetadata } from "@dpkit/core" +import { resolveBasepath } from "@dpkit/core" +import { validatePackageForeignKeys } from "@dpkit/table" import { dpkit } from "../plugin.ts" import { validateResourceData } from "../resource/index.ts" +import { loadTable } from "../table/index.ts" export async function validatePackage( source: string | Descriptor | Partial, @@ -22,25 +25,33 @@ export async function validatePackage( } if (!descriptor) { - const result = await loadDescriptor(source) - descriptor = result.descriptor - basepath = result.basepath + basepath = await resolveBasepath(source) + descriptor = await loadDescriptor(source) } } - const { valid, errors, dataPackage } = await validatePackageDescriptor( - descriptor, - { basepath }, - ) + const metadataReport = await validatePackageMetadata(descriptor, { + basepath, + }) - if (!dataPackage) { + if (!metadataReport.dataPackage) { return { - valid, - errors: errors.map(error => ({ ...error, resource: undefined })), + valid: metadataReport.valid, + errors: metadataReport.errors.map(error => ({ + ...error, + resource: undefined, + })), } } - return await validatePackageData(dataPackage) + const dataReport = await validatePackageData(metadataReport.dataPackage) + const fkReport = await validatePackageForeignKeys( + metadataReport.dataPackage, + { loadTable }, + ) + + const errors = [...dataReport.errors, ...fkReport.errors] + return { valid: errors.length === 0, errors } } export async function validatePackageData(dataPackage: Package) { diff --git a/lib/resource/validate.spec.ts b/lib/resource/validate.spec.ts index 9d397ec3..9cff0490 100644 --- a/lib/resource/validate.spec.ts +++ b/lib/resource/validate.spec.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest" -import { validateResource } from "./validate.ts" +import { validateResource, validateResourceData } from "./validate.ts" describe("validateResource", () => { it("should catch validation errors for invalid tabular data", async () => { @@ -74,4 +74,27 @@ describe("validateResource", () => { expect(report.valid).toBe(false) expect(report.errors.length).toEqual(3) }) + + it("should catch missing table error when schema is defined but table cannot be loaded", async () => { + const resource = { + name: "test", + path: "https://example.com/table.bad", + schema: { + fields: [ + { name: "id", type: "integer" as const }, + { name: "name", type: "string" as const }, + ], + }, + } + + const report = await validateResource(resource) + + expect(report.valid).toBe(false) + expect(report.errors).toEqual([ + { + type: "data", + message: "missing table", + }, + ]) + }) }) diff --git a/lib/resource/validate.ts b/lib/resource/validate.ts index 725863e7..c46631bc 100644 --- a/lib/resource/validate.ts +++ b/lib/resource/validate.ts @@ -1,6 +1,7 @@ -import type { Descriptor, Resource } from "@dpkit/core" +import type { DataError, Descriptor, Resource } from "@dpkit/core" import { resolveSchema } from "@dpkit/core" import { loadDescriptor, validateResourceMetadata } from "@dpkit/core" +import { resolveBasepath } from "@dpkit/core" import { validateFile } from "@dpkit/file" import { validateTable } from "@dpkit/table" import type { InferSchemaOptions } from "@dpkit/table" @@ -15,9 +16,8 @@ export async function validateResource( let basepath = options?.basepath if (typeof descriptor === "string") { - const result = await loadDescriptor(descriptor) - descriptor = result.descriptor - basepath = result.basepath + basepath = await resolveBasepath(descriptor) + descriptor = await loadDescriptor(descriptor) } const { valid, errors, resource } = await validateResourceMetadata( @@ -36,6 +36,9 @@ export async function validateResourceData( resource: Partial, options?: InferSchemaOptions, ) { + const errors: DataError[] = [] + + // TODO: validateFileFromResource? const fileReport = await validateFile(resource.path, { bytes: resource.bytes, hash: resource.hash, @@ -46,6 +49,7 @@ export async function validateResourceData( return fileReport } + // TODO: validateTableFromResource? const table = await loadTable(resource, { denormalized: true }) if (table) { let schema = await resolveSchema(resource.schema) @@ -57,5 +61,14 @@ export async function validateResourceData( } } - return { valid: true, errors: [] } + // TODO: Add document validation here + + if (!table && resource.schema) { + errors.push({ + type: "data", + message: "missing table", + }) + } + + return { valid: errors.length === 0, errors } } diff --git a/table/error/ForeignKey.ts b/table/error/ForeignKey.ts new file mode 100644 index 00000000..a7142810 --- /dev/null +++ b/table/error/ForeignKey.ts @@ -0,0 +1,8 @@ +import type { ForeignKey } from "@dpkit/core" +import type { BaseTableError } from "./Base.ts" + +export interface ForeignKeyError extends BaseTableError { + type: "foreignKey" + foreignKey: ForeignKey + cells: string[] +} diff --git a/table/error/Row.ts b/table/error/Row.ts index aa9833ec..c29b1ffe 100644 --- a/table/error/Row.ts +++ b/table/error/Row.ts @@ -9,4 +9,5 @@ export interface BaseRowError extends BaseTableError { export interface RowUniqueError extends BaseRowError { type: "row/unique" fieldNames: string[] + // TODO: add cells? } diff --git a/table/error/Table.ts b/table/error/Table.ts index b9be6b7f..e5cbe328 100644 --- a/table/error/Table.ts +++ b/table/error/Table.ts @@ -1,6 +1,12 @@ import type { CellError } from "./Cell.ts" import type { FieldError } from "./Field.ts" import type { FieldsError } from "./Fields.ts" +import type { ForeignKeyError } from "./ForeignKey.ts" import type { RowError } from "./Row.ts" -export type TableError = FieldsError | FieldError | RowError | CellError +export type TableError = + | FieldsError + | FieldError + | RowError + | CellError + | ForeignKeyError diff --git a/table/error/index.ts b/table/error/index.ts index 1bfa9f82..e2c5759d 100644 --- a/table/error/index.ts +++ b/table/error/index.ts @@ -2,4 +2,5 @@ export type * from "./Table.ts" export type * from "./Cell.ts" export type * from "./Field.ts" export type * from "./Fields.ts" +export type * from "./ForeignKey.ts" export type * from "./Row.ts" diff --git a/table/index.ts b/table/index.ts index 206f8f09..a7b99757 100644 --- a/table/index.ts +++ b/table/index.ts @@ -2,6 +2,7 @@ export * from "./data/index.ts" export * from "./dialect/index.ts" export * from "./error/index.ts" export * from "./field/index.ts" +export * from "./package/index.ts" export * from "./schema/index.ts" export * from "./table/index.ts" export * from "./plugin.ts" diff --git a/table/package/index.ts b/table/package/index.ts new file mode 100644 index 00000000..eb1b4c45 --- /dev/null +++ b/table/package/index.ts @@ -0,0 +1 @@ +export { validatePackageForeignKeys } from "./validate.ts" diff --git a/table/package/validate.spec.ts b/table/package/validate.spec.ts new file mode 100644 index 00000000..c73a0b76 --- /dev/null +++ b/table/package/validate.spec.ts @@ -0,0 +1,691 @@ +import type { Package, Resource } from "@dpkit/core" +import { DataFrame } from "nodejs-polars" +import { describe, expect, it } from "vitest" +import type { Table } from "../table/Table.ts" +import { validatePackageForeignKeys } from "./validate.ts" + +describe("validatePackageForeignKeys", () => { + it("should validate package with valid foreign keys", async () => { + const dataPackage: Package = { + name: "test-package", + resources: [ + { + name: "users", + type: "table" as const, + schema: { + fields: [ + { name: "id", type: "integer" as const }, + { name: "name", type: "string" as const }, + ], + }, + }, + { + name: "posts", + type: "table" as const, + schema: { + fields: [ + { name: "id", type: "integer" as const }, + { name: "user_id", type: "integer" as const }, + { name: "title", type: "string" as const }, + ], + foreignKeys: [ + { + fields: ["user_id"], + reference: { + resource: "users", + fields: ["id"], + }, + }, + ], + }, + }, + ], + } + + const tables: Record = { + users: DataFrame({ + id: [1, 2, 3], + name: ["Alice", "Bob", "Charlie"], + }).lazy(), + posts: DataFrame({ + id: [1, 2, 3], + user_id: [1, 2, 3], + title: ["Post 1", "Post 2", "Post 3"], + }).lazy(), + } + + const loadTable = async (resource: Resource): Promise => { + return tables[resource.name] + } + + const result = await validatePackageForeignKeys(dataPackage, { loadTable }) + + expect(result.valid).toBe(true) + expect(result.errors).toEqual([]) + }) + + it("should detect foreign key violations", async () => { + const dataPackage: Package = { + name: "test-package", + resources: [ + { + name: "users", + type: "table" as const, + schema: { + fields: [ + { name: "id", type: "integer" as const }, + { name: "name", type: "string" as const }, + ], + }, + }, + { + name: "posts", + type: "table" as const, + schema: { + fields: [ + { name: "id", type: "integer" as const }, + { name: "user_id", type: "integer" as const }, + { name: "title", type: "string" as const }, + ], + foreignKeys: [ + { + fields: ["user_id"], + reference: { + resource: "users", + fields: ["id"], + }, + }, + ], + }, + }, + ], + } + + const tables: Record = { + users: DataFrame({ + id: [1, 2], + name: ["Alice", "Bob"], + }).lazy(), + posts: DataFrame({ + id: [1, 2, 3], + user_id: [1, 2, 999], + title: ["Post 1", "Post 2", "Post 3"], + }).lazy(), + } + + const loadTable = async (resource: Resource): Promise
=> { + return tables[resource.name] + } + + const result = await validatePackageForeignKeys(dataPackage, { loadTable }) + + expect(result.valid).toBe(false) + expect(result.errors).toEqual([ + { + type: "foreignKey", + foreignKey: { + fields: ["user_id"], + reference: { + resource: "users", + fields: ["id"], + }, + }, + cells: ["999"], + }, + ]) + }) + + it("should handle self-referencing foreign keys", async () => { + const dataPackage: Package = { + name: "test-package", + resources: [ + { + name: "categories", + type: "table" as const, + schema: { + fields: [ + { name: "id", type: "integer" as const }, + { name: "parent_id", type: "integer" as const }, + { name: "name", type: "string" as const }, + ], + foreignKeys: [ + { + fields: ["parent_id"], + reference: { + fields: ["id"], + }, + }, + ], + }, + }, + ], + } + + const tables: Record = { + categories: DataFrame({ + id: [1, 2, 3], + parent_id: [1, 1, 2], + name: ["Root", "Child 1", "Child 2"], + }).lazy(), + } + + const loadTable = async (resource: Resource): Promise
=> { + return tables[resource.name] + } + + const result = await validatePackageForeignKeys(dataPackage, { loadTable }) + + expect(result.valid).toBe(true) + expect(result.errors).toEqual([]) + }) + + it("should detect violations in self-referencing foreign keys", async () => { + const dataPackage: Package = { + name: "test-package", + resources: [ + { + name: "categories", + type: "table" as const, + schema: { + fields: [ + { name: "id", type: "integer" as const }, + { name: "parent_id", type: "integer" as const }, + { name: "name", type: "string" as const }, + ], + foreignKeys: [ + { + fields: ["parent_id"], + reference: { + fields: ["id"], + }, + }, + ], + }, + }, + ], + } + + const tables: Record = { + categories: DataFrame({ + id: [1, 2, 3], + parent_id: [1, 1, 999], + name: ["Root", "Child 1", "Child 2"], + }).lazy(), + } + + const loadTable = async (resource: Resource): Promise
=> { + return tables[resource.name] + } + + const result = await validatePackageForeignKeys(dataPackage, { loadTable }) + + expect(result.valid).toBe(false) + expect(result.errors).toEqual([ + { + type: "foreignKey", + foreignKey: { + fields: ["parent_id"], + reference: { + fields: ["id"], + }, + }, + cells: ["999"], + }, + ]) + }) + + it("should respect maxErrors limit", async () => { + const dataPackage: Package = { + name: "test-package", + resources: [ + { + name: "users", + type: "table" as const, + schema: { + fields: [{ name: "id", type: "integer" as const }], + }, + }, + { + name: "posts", + type: "table" as const, + schema: { + fields: [ + { name: "id", type: "integer" as const }, + { name: "user_id", type: "integer" as const }, + ], + foreignKeys: [ + { + fields: ["user_id"], + reference: { + resource: "users", + fields: ["id"], + }, + }, + ], + }, + }, + ], + } + + const tables: Record = { + users: DataFrame({ + id: [1], + }).lazy(), + posts: DataFrame({ + id: [1, 2, 3, 4, 5], + user_id: [999, 998, 997, 996, 995], + }).lazy(), + } + + const loadTable = async (resource: Resource): Promise
=> { + return tables[resource.name] + } + + const result = await validatePackageForeignKeys(dataPackage, { + loadTable, + maxErrors: 3, + }) + + expect(result.valid).toBe(false) + expect(result.errors).toEqual([ + { + type: "foreignKey", + foreignKey: { + fields: ["user_id"], + reference: { + resource: "users", + fields: ["id"], + }, + }, + cells: ["999"], + }, + { + type: "foreignKey", + foreignKey: { + fields: ["user_id"], + reference: { + resource: "users", + fields: ["id"], + }, + }, + cells: ["998"], + }, + { + type: "foreignKey", + foreignKey: { + fields: ["user_id"], + reference: { + resource: "users", + fields: ["id"], + }, + }, + cells: ["997"], + }, + ]) + }) + + it("should handle multiple foreign keys in a resource", async () => { + const dataPackage: Package = { + name: "test-package", + resources: [ + { + name: "users", + type: "table" as const, + schema: { + fields: [{ name: "id", type: "integer" as const }], + }, + }, + { + name: "categories", + type: "table" as const, + schema: { + fields: [{ name: "id", type: "integer" as const }], + }, + }, + { + name: "posts", + type: "table" as const, + schema: { + fields: [ + { name: "id", type: "integer" as const }, + { name: "user_id", type: "integer" as const }, + { name: "category_id", type: "integer" as const }, + ], + foreignKeys: [ + { + fields: ["user_id"], + reference: { + resource: "users", + fields: ["id"], + }, + }, + { + fields: ["category_id"], + reference: { + resource: "categories", + fields: ["id"], + }, + }, + ], + }, + }, + ], + } + + const tables: Record = { + users: DataFrame({ + id: [1, 2], + }).lazy(), + categories: DataFrame({ + id: [10, 20], + }).lazy(), + posts: DataFrame({ + id: [1, 2], + user_id: [1, 2], + category_id: [10, 20], + }).lazy(), + } + + const loadTable = async (resource: Resource): Promise
=> { + return tables[resource.name] + } + + const result = await validatePackageForeignKeys(dataPackage, { loadTable }) + + expect(result.valid).toBe(true) + expect(result.errors).toEqual([]) + }) + + it("should detect violations in multiple foreign keys", async () => { + const dataPackage: Package = { + name: "test-package", + resources: [ + { + name: "users", + type: "table" as const, + schema: { + fields: [{ name: "id", type: "integer" as const }], + }, + }, + { + name: "categories", + type: "table" as const, + schema: { + fields: [{ name: "id", type: "integer" as const }], + }, + }, + { + name: "posts", + type: "table" as const, + schema: { + fields: [ + { name: "id", type: "integer" as const }, + { name: "user_id", type: "integer" as const }, + { name: "category_id", type: "integer" as const }, + ], + foreignKeys: [ + { + fields: ["user_id"], + reference: { + resource: "users", + fields: ["id"], + }, + }, + { + fields: ["category_id"], + reference: { + resource: "categories", + fields: ["id"], + }, + }, + ], + }, + }, + ], + } + + const tables: Record = { + users: DataFrame({ + id: [1], + }).lazy(), + categories: DataFrame({ + id: [10], + }).lazy(), + posts: DataFrame({ + id: [1, 2], + user_id: [999, 1], + category_id: [10, 888], + }).lazy(), + } + + const loadTable = async (resource: Resource): Promise
=> { + return tables[resource.name] + } + + const result = await validatePackageForeignKeys(dataPackage, { loadTable }) + + expect(result.valid).toBe(false) + expect(result.errors).toEqual([ + { + type: "foreignKey", + foreignKey: { + fields: ["user_id"], + reference: { + resource: "users", + fields: ["id"], + }, + }, + cells: ["999"], + }, + { + type: "foreignKey", + foreignKey: { + fields: ["category_id"], + reference: { + resource: "categories", + fields: ["id"], + }, + }, + cells: ["888"], + }, + ]) + }) + + it("should skip resources without schema", async () => { + const dataPackage: Package = { + name: "test-package", + resources: [ + { + name: "no-schema", + type: "table" as const, + }, + { + name: "users", + type: "table" as const, + schema: { + fields: [{ name: "id", type: "integer" as const }], + }, + }, + ], + } + + const tables: Record = { + users: DataFrame({ + id: [1], + }).lazy(), + } + + const loadTable = async (resource: Resource): Promise
=> { + return tables[resource.name] + } + + const result = await validatePackageForeignKeys(dataPackage, { loadTable }) + + expect(result.valid).toBe(true) + expect(result.errors).toEqual([]) + }) + + it("should skip resources without foreign keys", async () => { + const dataPackage: Package = { + name: "test-package", + resources: [ + { + name: "users", + type: "table" as const, + schema: { + fields: [ + { name: "id", type: "integer" as const }, + { name: "name", type: "string" as const }, + ], + }, + }, + ], + } + + const tables: Record = { + users: DataFrame({ + id: [1, 2], + name: ["Alice", "Bob"], + }).lazy(), + } + + const loadTable = async (resource: Resource): Promise
=> { + return tables[resource.name] + } + + const result = await validatePackageForeignKeys(dataPackage, { loadTable }) + + expect(result.valid).toBe(true) + expect(result.errors).toEqual([]) + }) + + it("should handle composite foreign keys", async () => { + const dataPackage: Package = { + name: "test-package", + resources: [ + { + name: "users", + type: "table" as const, + schema: { + fields: [ + { name: "first_name", type: "string" as const }, + { name: "last_name", type: "string" as const }, + ], + }, + }, + { + name: "posts", + type: "table" as const, + schema: { + fields: [ + { name: "id", type: "integer" as const }, + { name: "author_first", type: "string" as const }, + { name: "author_last", type: "string" as const }, + ], + foreignKeys: [ + { + fields: ["author_first", "author_last"], + reference: { + resource: "users", + fields: ["first_name", "last_name"], + }, + }, + ], + }, + }, + ], + } + + const tables: Record = { + users: DataFrame({ + first_name: ["Alice", "Bob"], + last_name: ["Smith", "Jones"], + }).lazy(), + posts: DataFrame({ + id: [1, 2], + author_first: ["Alice", "Bob"], + author_last: ["Smith", "Jones"], + }).lazy(), + } + + const loadTable = async (resource: Resource): Promise
=> { + return tables[resource.name] + } + + const result = await validatePackageForeignKeys(dataPackage, { loadTable }) + + expect(result.valid).toBe(true) + expect(result.errors).toEqual([]) + }) + + it("should detect violations in composite foreign keys", async () => { + const dataPackage: Package = { + name: "test-package", + resources: [ + { + name: "users", + type: "table" as const, + schema: { + fields: [ + { name: "first_name", type: "string" as const }, + { name: "last_name", type: "string" as const }, + ], + }, + }, + { + name: "posts", + type: "table" as const, + schema: { + fields: [ + { name: "id", type: "integer" as const }, + { name: "author_first", type: "string" as const }, + { name: "author_last", type: "string" as const }, + ], + foreignKeys: [ + { + fields: ["author_first", "author_last"], + reference: { + resource: "users", + fields: ["first_name", "last_name"], + }, + }, + ], + }, + }, + ], + } + + const tables: Record = { + users: DataFrame({ + first_name: ["Alice", "Bob"], + last_name: ["Smith", "Jones"], + }).lazy(), + posts: DataFrame({ + id: [1, 2], + author_first: ["Alice", "Charlie"], + author_last: ["Smith", "Brown"], + }).lazy(), + } + + const loadTable = async (resource: Resource): Promise
=> { + return tables[resource.name] + } + + const result = await validatePackageForeignKeys(dataPackage, { loadTable }) + + expect(result.valid).toBe(false) + expect(result.errors).toEqual([ + { + type: "foreignKey", + foreignKey: { + fields: ["author_first", "author_last"], + reference: { + resource: "users", + fields: ["first_name", "last_name"], + }, + }, + cells: ["Charlie", "Brown"], + }, + ]) + }) +}) diff --git a/table/package/validate.ts b/table/package/validate.ts new file mode 100644 index 00000000..cec50a7f --- /dev/null +++ b/table/package/validate.ts @@ -0,0 +1,94 @@ +import type { Package, Resource } from "@dpkit/core" +import { resolveSchema } from "@dpkit/core" +import type { DataError } from "@dpkit/core" +import type { ForeignKeyError } from "../error/index.ts" +import type { Table } from "../table/Table.ts" + +// TODO: foreign key fields definition should be validated as well (metadata/here?) +// TODO: review temporary files creation from validatePackage call + +export async function validatePackageForeignKeys( + dataPackage: Package, + options: { + maxErrors?: number + loadTable: (resource: Resource) => Promise
+ }, +) { + const { loadTable, maxErrors = 1000 } = options + + const errors: (DataError | ForeignKeyError)[] = [] + const tables: Record = {} + + for (const resource of dataPackage.resources) { + const schema = await resolveSchema(resource.schema) + if (!schema) continue + + const foreignKeys = schema.foreignKeys + if (!foreignKeys) continue + + const names = [ + resource.name, + ...foreignKeys.map(it => it.reference.resource), + ].filter(Boolean) as string[] + + for (const name of names) { + const resource = dataPackage.resources.find(r => r.name === name) + + if (!resource) { + errors.push({ + type: "data", + message: `missing ${name} resource`, + }) + + continue + } + + if (!tables[name]) { + const table = await loadTable(resource) + + if (!table) { + errors.push({ + type: "data", + message: `missing ${resource.name} table`, + }) + + continue + } + + tables[name] = table + } + } + + for (const foreignKey of foreignKeys) { + const left = tables[resource.name] as Table + const right = tables[ + foreignKey.reference.resource ?? resource.name + ] as Table + + const foreignKeyCheckTable = left + .select(...foreignKey.fields) + .join(right, { + how: "anti", + leftOn: foreignKey.fields, + rightOn: foreignKey.reference.fields, + }) + + const foreignKeyCheckFrame = await foreignKeyCheckTable + .head(maxErrors) + .collect() + + for (const row of foreignKeyCheckFrame.toRecords() as any[]) { + errors.push({ + type: "foreignKey", + foreignKey, + cells: Object.values(row).map(String), + }) + } + } + } + + return { + errors: errors.slice(0, maxErrors), + valid: errors.length === 0, + } +} diff --git a/table/profiles/geojson.json b/table/profiles/geojson.json index 5e9039c5..4351e4b2 100644 --- a/table/profiles/geojson.json +++ b/table/profiles/geojson.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", + "$schema": "http://json-schema.org/draft-07/schema", "title": "Geo JSON object", "description": "Schema for a Geo JSON object", "type": "object", diff --git a/table/profiles/topojson.json b/table/profiles/topojson.json index eea22837..a4e5b09c 100644 --- a/table/profiles/topojson.json +++ b/table/profiles/topojson.json @@ -1,5 +1,5 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", + "$schema": "http://json-schema.org/draft-07/schema", "title": "TopoJSON object", "description": "Schema for a TopoJSON object", "type": "object", diff --git a/zenodo/package/fixtures/generated/load.spec.ts.snap b/zenodo/package/fixtures/generated/load.spec.ts.snap index f6185b80..6ca24442 100644 --- a/zenodo/package/fixtures/generated/load.spec.ts.snap +++ b/zenodo/package/fixtures/generated/load.spec.ts.snap @@ -41,7 +41,7 @@ exports[`loadPackageFromZenodo > should load a package 1`] = ` exports[`loadPackageFromZenodo > shoule merge datapackage.json if present 1`] = ` { - "$schema": "tabular-data-package", + "$schema": undefined, "contributors": [ { "path": "Vogelwerkgroep Assen", diff --git a/zenodo/package/fixtures/generated/loadPackageFromZenodo-shoule-merge-datapackage-json-if-present_2160001855/recording.har b/zenodo/package/fixtures/generated/loadPackageFromZenodo-shoule-merge-datapackage-json-if-present_2160001855/recording.har index a2386e7d..2dd8b10d 100644 --- a/zenodo/package/fixtures/generated/loadPackageFromZenodo-shoule-merge-datapackage-json-if-present_2160001855/recording.har +++ b/zenodo/package/fixtures/generated/loadPackageFromZenodo-shoule-merge-datapackage-json-if-present_2160001855/recording.har @@ -292,6 +292,156 @@ "ssl": -1, "wait": 273 } + }, + { + "_id": "bd54488f667a8e2a2e0c71f5914bee02", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [], + "headersSize": 78, + "httpVersion": "HTTP/1.1", + "method": "HEAD", + "queryString": [], + "url": "https://zenodo.org/records/10053903/files/datapackage.json" + }, + "response": { + "bodySize": 0, + "content": { + "mimeType": "text/plain; charset=utf-8", + "size": 0 + }, + "cookies": [ + { + "httpOnly": true, + "name": "5569e5a730cade8ff2b54f1e815f3670", + "path": "/", + "sameSite": "None", + "secure": true, + "value": "3c2fa475c5dd7f217e458bf5ddeda192" + } + ], + "headers": [ + { + "name": "cache-control", + "value": "private" + }, + { + "name": "connection", + "value": "close" + }, + { + "name": "content-disposition", + "value": "inline" + }, + { + "name": "content-encoding", + "value": "gzip" + }, + { + "name": "content-security-policy", + "value": "default-src 'self' fonts.googleapis.com *.gstatic.com data: 'unsafe-inline' 'unsafe-eval' blob: zenodo-broker.web.cern.ch zenodo-broker-qa.web.cern.ch maxcdn.bootstrapcdn.com cdnjs.cloudflare.com ajax.googleapis.com webanalytics.web.cern.ch" + }, + { + "name": "content-type", + "value": "text/plain; charset=utf-8" + }, + { + "name": "date", + "value": "Fri, 31 Oct 2025 07:31:16 GMT" + }, + { + "name": "last-modified", + "value": "Wed, 23 Apr 2025 15:34:34 GMT" + }, + { + "name": "link", + "value": " ; rel=\"collection\" ; type=\"text/html\" , ; rel=\"linkset\" ; type=\"application/linkset+json\"" + }, + { + "name": "permissions-policy", + "value": "interest-cohort=()" + }, + { + "name": "referrer-policy", + "value": "strict-origin-when-cross-origin" + }, + { + "name": "retry-after", + "value": "60" + }, + { + "name": "server", + "value": "nginx" + }, + { + "name": "set-cookie", + "value": "5569e5a730cade8ff2b54f1e815f3670=3c2fa475c5dd7f217e458bf5ddeda192; path=/; HttpOnly; Secure; SameSite=None" + }, + { + "name": "strict-transport-security", + "value": "max-age=31556926; includeSubDomains, max-age=15768000" + }, + { + "name": "vary", + "value": "Accept-Encoding" + }, + { + "name": "x-content-type-options", + "value": "nosniff" + }, + { + "name": "x-download-options", + "value": "noopen" + }, + { + "name": "x-frame-options", + "value": "sameorigin" + }, + { + "name": "x-permitted-cross-domain-policies", + "value": "none" + }, + { + "name": "x-ratelimit-limit", + "value": "133" + }, + { + "name": "x-ratelimit-remaining", + "value": "132" + }, + { + "name": "x-ratelimit-reset", + "value": "1761895937" + }, + { + "name": "x-request-id", + "value": "dcb3f4cab21b0dbb46c56b858ff2a8b4" + }, + { + "name": "x-xss-protection", + "value": "1; mode=block" + } + ], + "headersSize": 1308, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2025-10-31T07:31:16.093Z", + "time": 777, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 777 + } } ], "pages": [],