From 712bddd44e50a15fedbee9f4a49b610dc681de56 Mon Sep 17 00:00:00 2001 From: roll Date: Wed, 1 Oct 2025 10:21:17 +0100 Subject: [PATCH 01/10] Bootstrapped jsonschema package --- jsonschema/README.md | 3 +++ jsonschema/index.ts | 0 jsonschema/package.json | 31 +++++++++++++++++++++++++++++++ jsonschema/schema/denormalize.ts | 0 jsonschema/schema/index.ts | 0 jsonschema/schema/normalize.ts | 0 jsonschema/tsconfig.json | 3 +++ jsonschema/typedoc.json | 4 ++++ pnpm-workspace.yaml | 1 + 9 files changed, 42 insertions(+) create mode 100644 jsonschema/README.md create mode 100644 jsonschema/index.ts create mode 100644 jsonschema/package.json create mode 100644 jsonschema/schema/denormalize.ts create mode 100644 jsonschema/schema/index.ts create mode 100644 jsonschema/schema/normalize.ts create mode 100644 jsonschema/tsconfig.json create mode 100644 jsonschema/typedoc.json diff --git a/jsonschema/README.md b/jsonschema/README.md new file mode 100644 index 00000000..f812b1f3 --- /dev/null +++ b/jsonschema/README.md @@ -0,0 +1,3 @@ +# @dpkit/jsonschema + +dpkit is a fast data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/jsonschema/index.ts b/jsonschema/index.ts new file mode 100644 index 00000000..e69de29b diff --git a/jsonschema/package.json b/jsonschema/package.json new file mode 100644 index 00000000..db4e050b --- /dev/null +++ b/jsonschema/package.json @@ -0,0 +1,31 @@ +{ + "name": "@dpkit/jsonschema", + "type": "module", + "version": "0.0.0-development", + "exports": "./build/index.js", + "license": "MIT", + "author": "Evgeny Karev", + "repository": "https://github.com/datisthq/dpkit", + "description": "Fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames", + "keywords": [ + "data", + "polars", + "dataframe", + "datapackage", + "tableschema", + "typescript", + "validation", + "quality", + "fair", + "jsonschema" + ], + "scripts": { + "build": "tsc" + }, + "dependencies": { + "@dpkit/core": "workspace:*" + }, + "devDependencies": { + "@dpkit/test": "workspace:*" + } +} diff --git a/jsonschema/schema/denormalize.ts b/jsonschema/schema/denormalize.ts new file mode 100644 index 00000000..e69de29b diff --git a/jsonschema/schema/index.ts b/jsonschema/schema/index.ts new file mode 100644 index 00000000..e69de29b diff --git a/jsonschema/schema/normalize.ts b/jsonschema/schema/normalize.ts new file mode 100644 index 00000000..e69de29b diff --git a/jsonschema/tsconfig.json b/jsonschema/tsconfig.json new file mode 100644 index 00000000..3c43903c --- /dev/null +++ b/jsonschema/tsconfig.json @@ -0,0 +1,3 @@ +{ + "extends": "../tsconfig.json" +} diff --git a/jsonschema/typedoc.json b/jsonschema/typedoc.json new file mode 100644 index 00000000..f8e49f3a --- /dev/null +++ b/jsonschema/typedoc.json @@ -0,0 +1,4 @@ +{ + "entryPoints": ["index.ts"], + "skipErrorChecking": true +} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 5a522aff..3d9e4d8d 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -15,6 +15,7 @@ packages: - github - inline - json + - jsonschema - ods - parquet - site From e60a4d61a1e72b6cea5d63794a3c1fd395f8f797 Mon Sep 17 00:00:00 2001 From: roll Date: Wed, 1 Oct 2025 10:42:48 +0100 Subject: [PATCH 02/10] Bootstraped functions --- jsonschema/index.ts | 1 + jsonschema/package.json | 3 ++- jsonschema/schema/denormalize.ts | 6 ++++++ jsonschema/schema/index.ts | 2 ++ jsonschema/schema/normalize.ts | 5 +++++ pnpm-lock.yaml | 18 ++++++++++++++++++ 6 files changed, 34 insertions(+), 1 deletion(-) diff --git a/jsonschema/index.ts b/jsonschema/index.ts index e69de29b..352ef322 100644 --- a/jsonschema/index.ts +++ b/jsonschema/index.ts @@ -0,0 +1 @@ +export * from "./schema/index.ts" diff --git a/jsonschema/package.json b/jsonschema/package.json index db4e050b..c89d6ebd 100644 --- a/jsonschema/package.json +++ b/jsonschema/package.json @@ -23,7 +23,8 @@ "build": "tsc" }, "dependencies": { - "@dpkit/core": "workspace:*" + "@dpkit/core": "workspace:*", + "@types/json-schema": "^7.0.15" }, "devDependencies": { "@dpkit/test": "workspace:*" diff --git a/jsonschema/schema/denormalize.ts b/jsonschema/schema/denormalize.ts index e69de29b..21a5d9f1 100644 --- a/jsonschema/schema/denormalize.ts +++ b/jsonschema/schema/denormalize.ts @@ -0,0 +1,6 @@ +import type { Schema } from "@dpkit/core" +import type { JSONSchema7 } from "json-schema" + +export function denormalizeJsonSchema(schema: Schema): JSONSchema7 { + return {} +} diff --git a/jsonschema/schema/index.ts b/jsonschema/schema/index.ts index e69de29b..dc8f9904 100644 --- a/jsonschema/schema/index.ts +++ b/jsonschema/schema/index.ts @@ -0,0 +1,2 @@ +export { denormalizeJsonSchema } from "./denormalize.ts" +export { normalizeJsonSchema } from "./normalize.ts" diff --git a/jsonschema/schema/normalize.ts b/jsonschema/schema/normalize.ts index e69de29b..ab6732bd 100644 --- a/jsonschema/schema/normalize.ts +++ b/jsonschema/schema/normalize.ts @@ -0,0 +1,5 @@ +import type { JSONSchema7 } from "json-schema" + +export function normalizeJsonSchema(jsonSchema: JSONSchema7) { + jsonSchema.allOf +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a949310b..33c58959 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -405,6 +405,19 @@ importers: specifier: workspace:* version: link:../test + jsonschema: + dependencies: + '@dpkit/core': + specifier: workspace:* + version: link:../core + '@types/json-schema': + specifier: ^7.0.15 + version: 7.0.15 + devDependencies: + '@dpkit/test': + specifier: workspace:* + version: link:../test + ods: dependencies: '@dpkit/core': @@ -1576,6 +1589,9 @@ packages: '@types/js-yaml@4.0.9': resolution: {integrity: sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg==} + '@types/json-schema@7.0.15': + resolution: {integrity: sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==} + '@types/mdast@4.0.4': resolution: {integrity: sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==} @@ -5832,6 +5848,8 @@ snapshots: '@types/js-yaml@4.0.9': {} + '@types/json-schema@7.0.15': {} + '@types/mdast@4.0.4': dependencies: '@types/unist': 3.0.3 From 8445350b4b429852be71869732bc64b7f1b95366 Mon Sep 17 00:00:00 2001 From: roll Date: Wed, 1 Oct 2025 10:50:25 +0100 Subject: [PATCH 03/10] Implemented normalize --- jsonschema/package.json | 3 +- jsonschema/schema/denormalize.ts | 2 +- jsonschema/schema/normalize.spec.ts | 202 ++++++++++++++++++++++++++++ jsonschema/schema/normalize.ts | 134 +++++++++++++++++- pnpm-lock.yaml | 8 ++ 5 files changed, 345 insertions(+), 4 deletions(-) create mode 100644 jsonschema/schema/normalize.spec.ts diff --git a/jsonschema/package.json b/jsonschema/package.json index c89d6ebd..194fdaa2 100644 --- a/jsonschema/package.json +++ b/jsonschema/package.json @@ -24,7 +24,8 @@ }, "dependencies": { "@dpkit/core": "workspace:*", - "@types/json-schema": "^7.0.15" + "@types/json-schema": "^7.0.15", + "json-schema": "^0.4.0" }, "devDependencies": { "@dpkit/test": "workspace:*" diff --git a/jsonschema/schema/denormalize.ts b/jsonschema/schema/denormalize.ts index 21a5d9f1..db1887ab 100644 --- a/jsonschema/schema/denormalize.ts +++ b/jsonschema/schema/denormalize.ts @@ -1,6 +1,6 @@ import type { Schema } from "@dpkit/core" import type { JSONSchema7 } from "json-schema" -export function denormalizeJsonSchema(schema: Schema): JSONSchema7 { +export function denormalizeJsonSchema(_schema: Schema): JSONSchema7 { return {} } diff --git a/jsonschema/schema/normalize.spec.ts b/jsonschema/schema/normalize.spec.ts new file mode 100644 index 00000000..e098c2a8 --- /dev/null +++ b/jsonschema/schema/normalize.spec.ts @@ -0,0 +1,202 @@ +import type { JSONSchema7 } from "json-schema" +import { describe, expect, it } from "vitest" +import { normalizeJsonSchema } from "./normalize.ts" + +describe("normalizeJsonSchema", () => { + it("converts JSONSchema object to Table Schema", () => { + const jsonSchema: JSONSchema7 = { + type: "object", + title: "User Schema", + description: "Schema for user data", + required: ["id", "name"], + properties: { + id: { + type: "integer", + title: "User ID", + description: "Unique identifier", + minimum: 1, + }, + name: { + type: "string", + title: "Full Name", + maxLength: 100, + }, + email: { + type: "string", + pattern: "^[^@]+@[^@]+\\.[^@]+$", + }, + age: { + type: "integer", + minimum: 0, + maximum: 150, + }, + isActive: { + type: "boolean", + }, + tags: { + type: "array", + minItems: 0, + maxItems: 10, + }, + metadata: { + type: "object", + }, + }, + } + + const tableSchema = normalizeJsonSchema(jsonSchema) + + expect((tableSchema as any).title).toBe("User Schema") + expect((tableSchema as any).description).toBe("Schema for user data") + expect(tableSchema.fields).toHaveLength(7) + + const idField = tableSchema.fields.find(f => f.name === "id") + expect(idField).toEqual({ + name: "id", + type: "integer", + title: "User ID", + description: "Unique identifier", + constraints: { + required: true, + minimum: 1, + }, + }) + + const nameField = tableSchema.fields.find(f => f.name === "name") + expect(nameField).toEqual({ + name: "name", + type: "string", + title: "Full Name", + constraints: { + required: true, + maxLength: 100, + }, + }) + + const emailField = tableSchema.fields.find(f => f.name === "email") + expect(emailField).toEqual({ + name: "email", + type: "string", + constraints: { + pattern: "^[^@]+@[^@]+\\.[^@]+$", + }, + }) + + const ageField = tableSchema.fields.find(f => f.name === "age") + expect(ageField).toEqual({ + name: "age", + type: "integer", + constraints: { + minimum: 0, + maximum: 150, + }, + }) + + const isActiveField = tableSchema.fields.find(f => f.name === "isActive") + expect(isActiveField).toEqual({ + name: "isActive", + type: "boolean", + }) + + const tagsField = tableSchema.fields.find(f => f.name === "tags") + expect(tagsField).toEqual({ + name: "tags", + type: "array", + constraints: { + minLength: 0, + maxLength: 10, + }, + }) + + const metadataField = tableSchema.fields.find(f => f.name === "metadata") + expect(metadataField).toEqual({ + name: "metadata", + type: "object", + constraints: { + jsonSchema: { + type: "object", + }, + }, + }) + }) + + it("handles union types by picking first non-null type", () => { + const jsonSchema: JSONSchema7 = { + type: "object", + properties: { + nullableString: { + type: ["string", "null"], + }, + multiType: { + type: ["number", "string"], + }, + }, + } + + const tableSchema = normalizeJsonSchema(jsonSchema) + + const nullableField = tableSchema.fields.find( + f => f.name === "nullableString", + ) + expect(nullableField?.type).toBe("string") + + const multiField = tableSchema.fields.find(f => f.name === "multiType") + expect(multiField?.type).toBe("number") + }) + + it("defaults to string type for unknown types", () => { + const jsonSchema: JSONSchema7 = { + type: "object", + properties: { + unknownField: { + // No type specified + }, + }, + } + + const tableSchema = normalizeJsonSchema(jsonSchema) + + const unknownField = tableSchema.fields.find(f => f.name === "unknownField") + expect(unknownField?.type).toBe("string") + }) + + it("throws error for non-object schemas", () => { + const jsonSchema: JSONSchema7 = { + type: "string", + } + + expect(() => normalizeJsonSchema(jsonSchema)).toThrow( + "JSONSchema must be an object type with properties", + ) + }) + + it("throws error for objects without properties", () => { + const jsonSchema: JSONSchema7 = { + type: "object", + // No properties + } + + expect(() => normalizeJsonSchema(jsonSchema)).toThrow( + "JSONSchema must be an object type with properties", + ) + }) + + it("skips boolean schema properties", () => { + const jsonSchema: JSONSchema7 = { + type: "object", + properties: { + validField: { + type: "string", + }, + booleanSchema: true, // This should be skipped + falseBooleanSchema: false, // This should also be skipped + }, + } + + const tableSchema = normalizeJsonSchema(jsonSchema) + + expect(tableSchema.fields).toHaveLength(1) + expect(tableSchema.fields[0]?.name).toBe("validField") + }) +}) + diff --git a/jsonschema/schema/normalize.ts b/jsonschema/schema/normalize.ts index ab6732bd..8da83feb 100644 --- a/jsonschema/schema/normalize.ts +++ b/jsonschema/schema/normalize.ts @@ -1,5 +1,135 @@ import type { JSONSchema7 } from "json-schema" +import type { Schema, Field } from "@dpkit/core" -export function normalizeJsonSchema(jsonSchema: JSONSchema7) { - jsonSchema.allOf +export function normalizeJsonSchema(jsonSchema: JSONSchema7): Schema { + if (jsonSchema.type !== "object" || !jsonSchema.properties) { + throw new Error("JSONSchema must be an object type with properties") + } + + const fields: Field[] = [] + const requiredFields = new Set(jsonSchema.required || []) + + for (const [name, property] of Object.entries(jsonSchema.properties)) { + if (typeof property === "boolean") { + continue // Skip boolean schemas + } + + const field = convertJsonSchemaPropertyToField(name, property, requiredFields.has(name)) + if (field) { + fields.push(field) + } + } + + const schema: Schema & { title?: string; description?: string } = { + fields, + } + + if (jsonSchema.title) { + schema.title = jsonSchema.title + } + + if (jsonSchema.description) { + schema.description = jsonSchema.description + } + + return schema +} + +function convertJsonSchemaPropertyToField( + name: string, + property: JSONSchema7, + isRequired: boolean +): Field | null { + if (typeof property === "boolean") { + return null + } + + const baseField = { + name, + title: property.title, + description: property.description, + constraints: isRequired ? { required: true } : undefined, + } + + // Handle different JSONSchema types + switch (property.type) { + case "string": + return { + ...baseField, + type: "string", + constraints: { + ...baseField.constraints, + minLength: property.minLength, + maxLength: property.maxLength, + pattern: property.pattern, + enum: property.enum as string[] | undefined, + }, + } as Field + + case "number": + return { + ...baseField, + type: "number", + constraints: { + ...baseField.constraints, + minimum: property.minimum, + maximum: property.maximum, + }, + } as Field + + case "integer": + return { + ...baseField, + type: "integer", + constraints: { + ...baseField.constraints, + minimum: property.minimum, + maximum: property.maximum, + }, + } as Field + + case "boolean": + return { + ...baseField, + type: "boolean", + } as Field + + case "array": + return { + ...baseField, + type: "array", + constraints: { + ...baseField.constraints, + minLength: property.minItems, + maxLength: property.maxItems, + jsonSchema: property.items as Record, + }, + } as Field + + case "object": + return { + ...baseField, + type: "object", + constraints: { + ...baseField.constraints, + jsonSchema: property as Record, + }, + } as Field + + default: + // Handle union types or unspecified types + if (Array.isArray(property.type)) { + // For union types, pick the first non-null type or default to string + const nonNullType = property.type.find(t => t !== "null") + if (nonNullType) { + return convertJsonSchemaPropertyToField(name, { ...property, type: nonNullType }, isRequired) + } + } + + // Default to string type for unknown/unspecified types + return { + ...baseField, + type: "string", + } as Field + } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 33c58959..f675aa3a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -413,6 +413,9 @@ importers: '@types/json-schema': specifier: ^7.0.15 version: 7.0.15 + json-schema: + specifier: ^0.4.0 + version: 0.4.0 devDependencies: '@dpkit/test': specifier: workspace:* @@ -2898,6 +2901,9 @@ packages: json-schema-traverse@1.0.0: resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==} + json-schema@0.4.0: + resolution: {integrity: sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==} + jsonfile@6.2.0: resolution: {integrity: sha512-FGuPw30AdOIUTRMC2OMRtQV+jkVj2cfPqSeWXv1NEAJ1qZ5zb1X6z1mFhbfOB/iy3ssJCD+3KuZ8r8C3uVFlAg==} @@ -7426,6 +7432,8 @@ snapshots: json-schema-traverse@1.0.0: {} + json-schema@0.4.0: {} + jsonfile@6.2.0: dependencies: universalify: 2.0.1 From a61b1b0a31e958c22062e9b72176b62e48075fdc Mon Sep 17 00:00:00 2001 From: roll Date: Wed, 1 Oct 2025 11:02:21 +0100 Subject: [PATCH 04/10] Added name/title/description to Schema --- core/dialect/Dialect.ts | 8 +++---- core/package/Package.ts | 10 ++++---- core/resource/Resource.ts | 5 ++++ core/schema/Schema.ts | 23 ++++++++++++++---- jsonschema/schema/denormalize.ts | 4 ++-- jsonschema/schema/normalize.spec.ts | 36 ++++++----------------------- jsonschema/schema/normalize.ts | 18 +++++++-------- 7 files changed, 50 insertions(+), 54 deletions(-) diff --git a/core/dialect/Dialect.ts b/core/dialect/Dialect.ts index 73249daf..c2c87ebf 100644 --- a/core/dialect/Dialect.ts +++ b/core/dialect/Dialect.ts @@ -7,14 +7,14 @@ import type { Metadata } from "../general/index.ts" */ export interface Dialect extends Metadata { /** - * The name of this dialect + * JSON schema profile URL for validation */ - name?: string + $schema?: string /** - * JSON schema profile URL for validation + * The name of this dialect */ - $schema?: string + name?: string /** * Whether the file includes a header row with field names diff --git a/core/package/Package.ts b/core/package/Package.ts index a4ff6852..992db912 100644 --- a/core/package/Package.ts +++ b/core/package/Package.ts @@ -7,6 +7,11 @@ import type { Contributor } from "./Contributor.ts" * @see https://datapackage.org/standard/data-package/ */ export interface Package extends Metadata { + /** + * URL of profile (optional) + */ + $schema?: string + /** * Data resources in this package (required) */ @@ -18,11 +23,6 @@ export interface Package extends Metadata { */ name?: string - /** - * Package schema URL for validation - */ - $schema?: string - /** * Human-readable title */ diff --git a/core/resource/Resource.ts b/core/resource/Resource.ts index 00792e03..aa4078e1 100644 --- a/core/resource/Resource.ts +++ b/core/resource/Resource.ts @@ -9,6 +9,11 @@ import type { Source } from "./Source.ts" * @see https://datapackage.org/standard/data-resource/ */ export interface Resource extends Metadata { + /** + * JSON schema profile URL for validation + */ + $schema?: string + /** * Unique resource identifier * Should use lowercase alphanumeric characters, periods, hyphens, and underscores diff --git a/core/schema/Schema.ts b/core/schema/Schema.ts index 508edcc7..f7bce413 100644 --- a/core/schema/Schema.ts +++ b/core/schema/Schema.ts @@ -8,14 +8,29 @@ import type { ForeignKey } from "./ForeignKey.ts" */ export interface Schema extends Metadata { /** - * Fields in this schema (required) + * URL of profile (optional) */ - fields: Field[] + $schema?: string /** - * URL of schema (optional) + * Name of schema (optional) */ - $schema?: string + name?: string + + /** + * Title of schema (optional) + */ + title?: string + + /** + * Description of schema (optional) + */ + description?: string + + /** + * Fields in this schema (required) + */ + fields: Field[] /** * Field matching rule (optional) diff --git a/jsonschema/schema/denormalize.ts b/jsonschema/schema/denormalize.ts index db1887ab..116180bd 100644 --- a/jsonschema/schema/denormalize.ts +++ b/jsonschema/schema/denormalize.ts @@ -1,6 +1,6 @@ import type { Schema } from "@dpkit/core" -import type { JSONSchema7 } from "json-schema" +import type { JSONSchema7Object } from "json-schema" -export function denormalizeJsonSchema(_schema: Schema): JSONSchema7 { +export function denormalizeJsonSchema(_schema: Schema): JSONSchema7Object { return {} } diff --git a/jsonschema/schema/normalize.spec.ts b/jsonschema/schema/normalize.spec.ts index e098c2a8..f8717778 100644 --- a/jsonschema/schema/normalize.spec.ts +++ b/jsonschema/schema/normalize.spec.ts @@ -1,10 +1,10 @@ -import type { JSONSchema7 } from "json-schema" +import type { JSONSchema7Object } from "json-schema" import { describe, expect, it } from "vitest" import { normalizeJsonSchema } from "./normalize.ts" describe("normalizeJsonSchema", () => { it("converts JSONSchema object to Table Schema", () => { - const jsonSchema: JSONSchema7 = { + const jsonSchema: JSONSchema7Object = { type: "object", title: "User Schema", description: "Schema for user data", @@ -46,8 +46,8 @@ describe("normalizeJsonSchema", () => { const tableSchema = normalizeJsonSchema(jsonSchema) - expect((tableSchema as any).title).toBe("User Schema") - expect((tableSchema as any).description).toBe("Schema for user data") + expect(tableSchema.title).toBe("User Schema") + expect(tableSchema.description).toBe("Schema for user data") expect(tableSchema.fields).toHaveLength(7) const idField = tableSchema.fields.find(f => f.name === "id") @@ -121,7 +121,7 @@ describe("normalizeJsonSchema", () => { }) it("handles union types by picking first non-null type", () => { - const jsonSchema: JSONSchema7 = { + const jsonSchema: JSONSchema7Object = { type: "object", properties: { nullableString: { @@ -145,7 +145,7 @@ describe("normalizeJsonSchema", () => { }) it("defaults to string type for unknown types", () => { - const jsonSchema: JSONSchema7 = { + const jsonSchema: JSONSchema7Object = { type: "object", properties: { unknownField: { @@ -160,29 +160,8 @@ describe("normalizeJsonSchema", () => { expect(unknownField?.type).toBe("string") }) - it("throws error for non-object schemas", () => { - const jsonSchema: JSONSchema7 = { - type: "string", - } - - expect(() => normalizeJsonSchema(jsonSchema)).toThrow( - "JSONSchema must be an object type with properties", - ) - }) - - it("throws error for objects without properties", () => { - const jsonSchema: JSONSchema7 = { - type: "object", - // No properties - } - - expect(() => normalizeJsonSchema(jsonSchema)).toThrow( - "JSONSchema must be an object type with properties", - ) - }) - it("skips boolean schema properties", () => { - const jsonSchema: JSONSchema7 = { + const jsonSchema: JSONSchema7Object = { type: "object", properties: { validField: { @@ -199,4 +178,3 @@ describe("normalizeJsonSchema", () => { expect(tableSchema.fields[0]?.name).toBe("validField") }) }) - diff --git a/jsonschema/schema/normalize.ts b/jsonschema/schema/normalize.ts index 8da83feb..dd5965b9 100644 --- a/jsonschema/schema/normalize.ts +++ b/jsonschema/schema/normalize.ts @@ -1,15 +1,13 @@ -import type { JSONSchema7 } from "json-schema" +import type { JSONSchema7, JSONSchema7Object } from "json-schema" import type { Schema, Field } from "@dpkit/core" -export function normalizeJsonSchema(jsonSchema: JSONSchema7): Schema { - if (jsonSchema.type !== "object" || !jsonSchema.properties) { - throw new Error("JSONSchema must be an object type with properties") - } - +export function normalizeJsonSchema(jsonSchema: JSONSchema7Object): Schema { const fields: Field[] = [] - const requiredFields = new Set(jsonSchema.required || []) + const requiredFields = new Set( + Array.isArray(jsonSchema.required) ? jsonSchema.required : [] + ) - for (const [name, property] of Object.entries(jsonSchema.properties)) { + for (const [name, property] of Object.entries(jsonSchema.properties || {})) { if (typeof property === "boolean") { continue // Skip boolean schemas } @@ -24,11 +22,11 @@ export function normalizeJsonSchema(jsonSchema: JSONSchema7): Schema { fields, } - if (jsonSchema.title) { + if (typeof jsonSchema.title === "string") { schema.title = jsonSchema.title } - if (jsonSchema.description) { + if (typeof jsonSchema.description === "string") { schema.description = jsonSchema.description } From 5d7b04248f527cb8c0a4dcb472c99c4595f8d47a Mon Sep 17 00:00:00 2001 From: roll Date: Wed, 1 Oct 2025 11:10:19 +0100 Subject: [PATCH 05/10] Implemented denormalize --- jsonschema/schema/denormalize.spec.ts | 454 ++++++++++++++++++++++++++ jsonschema/schema/denormalize.ts | 216 +++++++++++- jsonschema/schema/normalize.spec.ts | 10 +- jsonschema/schema/normalize.ts | 22 +- 4 files changed, 686 insertions(+), 16 deletions(-) create mode 100644 jsonschema/schema/denormalize.spec.ts diff --git a/jsonschema/schema/denormalize.spec.ts b/jsonschema/schema/denormalize.spec.ts new file mode 100644 index 00000000..fa4baf47 --- /dev/null +++ b/jsonschema/schema/denormalize.spec.ts @@ -0,0 +1,454 @@ +import { describe, expect, it } from "vitest" +import type { Schema } from "@dpkit/core" +import { denormalizeJsonSchema } from "./denormalize.ts" + +describe("denormalizeJsonSchema", () => { + it("converts Table Schema to JSONSchema object", () => { + const tableSchema: Schema & { title?: string; description?: string } = { + title: "User Schema", + description: "Schema for user data", + fields: [ + { + name: "id", + type: "integer", + title: "User ID", + description: "Unique identifier", + constraints: { + required: true, + minimum: 1, + }, + }, + { + name: "name", + type: "string", + title: "Full Name", + constraints: { + required: true, + maxLength: 100, + }, + }, + { + name: "email", + type: "string", + constraints: { + pattern: "^[^@]+@[^@]+\\.[^@]+$", + }, + }, + { + name: "age", + type: "integer", + constraints: { + minimum: 0, + maximum: 150, + }, + }, + { + name: "isActive", + type: "boolean", + }, + { + name: "tags", + type: "array", + constraints: { + minLength: 0, + maxLength: 10, + }, + }, + { + name: "metadata", + type: "object", + constraints: { + jsonSchema: { + type: "object", + properties: { + category: { type: "string" }, + }, + }, + }, + }, + ], + } + + const jsonSchema = denormalizeJsonSchema(tableSchema) + + expect(jsonSchema.title).toBe("User Schema") + expect(jsonSchema.description).toBe("Schema for user data") + expect(jsonSchema.type).toBe("object") + expect(jsonSchema.required).toEqual(["id", "name"]) + expect(Object.keys(jsonSchema.properties || {})).toHaveLength(7) + + expect((jsonSchema.properties as any)?.id).toEqual({ + type: "integer", + title: "User ID", + description: "Unique identifier", + minimum: 1, + }) + + expect((jsonSchema.properties as any)?.name).toEqual({ + type: "string", + title: "Full Name", + maxLength: 100, + }) + + expect((jsonSchema.properties as any)?.email).toEqual({ + type: "string", + pattern: "^[^@]+@[^@]+\\.[^@]+$", + }) + + expect((jsonSchema.properties as any)?.age).toEqual({ + type: "integer", + minimum: 0, + maximum: 150, + }) + + expect((jsonSchema.properties as any)?.isActive).toEqual({ + type: "boolean", + }) + + expect((jsonSchema.properties as any)?.tags).toEqual({ + type: "array", + minItems: 0, + maxItems: 10, + }) + + expect((jsonSchema.properties as any)?.metadata).toEqual({ + type: "object", + properties: { + category: { type: "string" }, + }, + }) + }) + + it("handles string field types with various constraints", () => { + const tableSchema: Schema = { + fields: [ + { + name: "basicString", + type: "string", + }, + { + name: "constrainedString", + type: "string", + constraints: { + minLength: 5, + maxLength: 50, + pattern: "^[A-Z]", + enum: ["Option1", "Option2", "Option3"], + }, + }, + ], + } + + const jsonSchema = denormalizeJsonSchema(tableSchema) + + expect((jsonSchema.properties as any)?.basicString).toEqual({ + type: "string", + }) + + expect((jsonSchema.properties as any)?.constrainedString).toEqual({ + type: "string", + minLength: 5, + maxLength: 50, + pattern: "^[A-Z]", + enum: ["Option1", "Option2", "Option3"], + }) + }) + + it("handles numeric field types", () => { + const tableSchema: Schema = { + fields: [ + { + name: "simpleNumber", + type: "number", + }, + { + name: "constrainedNumber", + type: "number", + constraints: { + minimum: 0.1, + maximum: 99.9, + }, + }, + { + name: "simpleInteger", + type: "integer", + }, + { + name: "constrainedInteger", + type: "integer", + constraints: { + minimum: 1, + maximum: 100, + }, + }, + ], + } + + const jsonSchema = denormalizeJsonSchema(tableSchema) + + expect((jsonSchema.properties as any)?.simpleNumber).toEqual({ + type: "number", + }) + + expect((jsonSchema.properties as any)?.constrainedNumber).toEqual({ + type: "number", + minimum: 0.1, + maximum: 99.9, + }) + + expect((jsonSchema.properties as any)?.simpleInteger).toEqual({ + type: "integer", + }) + + expect((jsonSchema.properties as any)?.constrainedInteger).toEqual({ + type: "integer", + minimum: 1, + maximum: 100, + }) + }) + + it("handles array and object field types", () => { + const tableSchema: Schema = { + fields: [ + { + name: "simpleArray", + type: "array", + }, + { + name: "constrainedArray", + type: "array", + constraints: { + minLength: 1, + maxLength: 5, + jsonSchema: { + type: "string", + }, + }, + }, + { + name: "simpleObject", + type: "object", + }, + { + name: "constrainedObject", + type: "object", + constraints: { + jsonSchema: { + type: "object", + properties: { + nested: { type: "string" }, + }, + required: ["nested"], + }, + }, + }, + ], + } + + const jsonSchema = denormalizeJsonSchema(tableSchema) + + expect((jsonSchema.properties as any)?.simpleArray).toEqual({ + type: "array", + }) + + expect((jsonSchema.properties as any)?.constrainedArray).toEqual({ + type: "array", + minItems: 1, + maxItems: 5, + items: { + type: "string", + }, + }) + + expect((jsonSchema.properties as any)?.simpleObject).toEqual({ + type: "object", + }) + + expect((jsonSchema.properties as any)?.constrainedObject).toEqual({ + type: "object", + properties: { + nested: { type: "string" }, + }, + required: ["nested"], + }) + }) + + it("handles date and time field types", () => { + const tableSchema: Schema = { + fields: [ + { + name: "dateField", + type: "date", + title: "Date Field", + }, + { + name: "datetimeField", + type: "datetime", + description: "DateTime Field", + }, + { + name: "timeField", + type: "time", + }, + { + name: "yearField", + type: "year", + }, + { + name: "yearmonthField", + type: "yearmonth", + }, + { + name: "durationField", + type: "duration", + }, + ], + } + + const jsonSchema = denormalizeJsonSchema(tableSchema) + + expect((jsonSchema.properties as any)?.dateField).toEqual({ + type: "string", + format: "date", + title: "Date Field", + }) + + expect((jsonSchema.properties as any)?.datetimeField).toEqual({ + type: "string", + format: "date-time", + description: "DateTime Field", + }) + + expect((jsonSchema.properties as any)?.timeField).toEqual({ + type: "string", + format: "time", + }) + + expect((jsonSchema.properties as any)?.yearField).toEqual({ + type: "string", + format: "year", + }) + + expect((jsonSchema.properties as any)?.yearmonthField).toEqual({ + type: "string", + format: "yearmonth", + }) + + expect((jsonSchema.properties as any)?.durationField).toEqual({ + type: "string", + format: "duration", + }) + }) + + it("handles geospatial field types", () => { + const tableSchema: Schema = { + fields: [ + { + name: "geopointField", + type: "geopoint", + }, + { + name: "geojsonField", + type: "geojson", + }, + ], + } + + const jsonSchema = denormalizeJsonSchema(tableSchema) + + expect((jsonSchema.properties as any)?.geopointField).toEqual({ + type: "string", + format: "geopoint", + }) + + expect((jsonSchema.properties as any)?.geojsonField).toEqual({ + type: "object", + format: "geojson", + }) + }) + + it("handles special field types", () => { + const tableSchema: Schema = { + fields: [ + { + name: "listField", + type: "list", + }, + { + name: "anyField", + type: "any", + }, + ], + } + + const jsonSchema = denormalizeJsonSchema(tableSchema) + + expect((jsonSchema.properties as any)?.listField).toEqual({ + type: "array", + format: "list", + }) + + expect((jsonSchema.properties as any)?.anyField).toEqual({ + type: "string", + }) + }) + + it("handles schema without title and description", () => { + const tableSchema: Schema = { + fields: [ + { + name: "field1", + type: "string", + }, + ], + } + + const jsonSchema = denormalizeJsonSchema(tableSchema) + + expect(jsonSchema.title).toBeUndefined() + expect(jsonSchema.description).toBeUndefined() + expect(jsonSchema.type).toBe("object") + expect((jsonSchema.properties as any)?.field1).toEqual({ + type: "string", + }) + }) + + it("handles schema with no required fields", () => { + const tableSchema: Schema = { + fields: [ + { + name: "optionalField1", + type: "string", + }, + { + name: "optionalField2", + type: "integer", + }, + ], + } + + const jsonSchema = denormalizeJsonSchema(tableSchema) + + expect(jsonSchema.required).toBeUndefined() + expect((jsonSchema.properties as any)?.optionalField1).toEqual({ + type: "string", + }) + expect((jsonSchema.properties as any)?.optionalField2).toEqual({ + type: "integer", + }) + }) + + it("handles empty schema", () => { + const tableSchema: Schema = { + fields: [], + } + + const jsonSchema = denormalizeJsonSchema(tableSchema) + + expect(jsonSchema.type).toBe("object") + expect(jsonSchema.properties).toEqual({}) + expect(jsonSchema.required).toBeUndefined() + expect(jsonSchema.title).toBeUndefined() + expect(jsonSchema.description).toBeUndefined() + }) +}) \ No newline at end of file diff --git a/jsonschema/schema/denormalize.ts b/jsonschema/schema/denormalize.ts index 116180bd..f23dacd3 100644 --- a/jsonschema/schema/denormalize.ts +++ b/jsonschema/schema/denormalize.ts @@ -1,6 +1,214 @@ -import type { Schema } from "@dpkit/core" -import type { JSONSchema7Object } from "json-schema" +import type { Field, Schema } from "@dpkit/core" +import type { JSONSchema7 } from "json-schema" -export function denormalizeJsonSchema(_schema: Schema): JSONSchema7Object { - return {} +export function denormalizeJsonSchema(schema: Schema): JSONSchema7 { + const properties: Record = {} + const required: string[] = [] + + for (const field of schema.fields) { + const property = convertFieldToJsonSchemaProperty(field) + if (property) { + properties[field.name] = property + + // Check if field is required + if (field.constraints?.required) { + required.push(field.name) + } + } + } + + const jsonSchema: JSONSchema7 = { + type: "object", + properties, + } + + if (required.length > 0) { + jsonSchema.required = required + } + + if (schema.title) { + jsonSchema.title = schema.title + } + + if (schema.description) { + jsonSchema.description = schema.description + } + + return jsonSchema +} + +function convertFieldToJsonSchemaProperty(field: Field): JSONSchema7 | null { + const baseProperty: Partial = {} + + // Handle different Table Schema field types + switch (field.type) { + case "string": { + const property: JSONSchema7 = { + ...baseProperty, + type: "string", + } + + if (field.constraints?.minLength !== undefined) { + property.minLength = field.constraints.minLength + } + if (field.constraints?.maxLength !== undefined) { + property.maxLength = field.constraints.maxLength + } + if (field.constraints?.pattern) { + property.pattern = field.constraints.pattern + } + if (field.constraints?.enum) { + property.enum = field.constraints.enum + } + + return property + } + + case "number": { + const property: JSONSchema7 = { + ...baseProperty, + type: "number", + } + + if ( + field.constraints?.minimum !== undefined && + typeof field.constraints.minimum === "number" + ) { + property.minimum = field.constraints.minimum + } + if ( + field.constraints?.maximum !== undefined && + typeof field.constraints.maximum === "number" + ) { + property.maximum = field.constraints.maximum + } + + return property + } + + case "integer": { + const property: JSONSchema7 = { + ...baseProperty, + type: "integer", + } + + if ( + field.constraints?.minimum !== undefined && + typeof field.constraints.minimum === "number" + ) { + property.minimum = field.constraints.minimum + } + if ( + field.constraints?.maximum !== undefined && + typeof field.constraints.maximum === "number" + ) { + property.maximum = field.constraints.maximum + } + + return property + } + + case "boolean": { + return { + ...baseProperty, + type: "boolean", + } + } + + case "array": { + const property: JSONSchema7 = { + ...baseProperty, + type: "array", + } + + if (field.constraints?.minLength !== undefined) { + property.minItems = field.constraints.minLength + } + if (field.constraints?.maxLength !== undefined) { + property.maxItems = field.constraints.maxLength + } + if (field.constraints?.jsonSchema) { + property.items = field.constraints.jsonSchema + } + + return property + } + + case "object": { + const property: JSONSchema7 = { + ...baseProperty, + type: "object", + } + + // If there's a jsonSchema constraint, merge it + if (field.constraints?.jsonSchema) { + return { + ...property, + ...field.constraints.jsonSchema, + } + } + + return property + } + + // For other field types that don't have direct JSONSchema equivalents, + // convert them to string type with format information + case "date": + return { + ...baseProperty, + type: "string", + format: "date", + } + + case "datetime": + return { + ...baseProperty, + type: "string", + format: "date-time", + } + + case "time": + return { + ...baseProperty, + type: "string", + format: "time", + } + + case "year": + case "yearmonth": + case "duration": + return { + ...baseProperty, + type: "string", + format: field.type, + } + + case "geopoint": + return { + ...baseProperty, + type: "string", + format: "geopoint", + } + + case "geojson": + return { + ...baseProperty, + type: "object", + format: "geojson", + } + + case "list": + return { + ...baseProperty, + type: "array", + format: "list", + } + + default: + // Default to string type for unknown field types + return { + ...baseProperty, + type: "string", + } + } } diff --git a/jsonschema/schema/normalize.spec.ts b/jsonschema/schema/normalize.spec.ts index f8717778..89cdf787 100644 --- a/jsonschema/schema/normalize.spec.ts +++ b/jsonschema/schema/normalize.spec.ts @@ -1,10 +1,10 @@ -import type { JSONSchema7Object } from "json-schema" +import type { JSONSchema7 } from "json-schema" import { describe, expect, it } from "vitest" import { normalizeJsonSchema } from "./normalize.ts" describe("normalizeJsonSchema", () => { it("converts JSONSchema object to Table Schema", () => { - const jsonSchema: JSONSchema7Object = { + const jsonSchema: JSONSchema7 = { type: "object", title: "User Schema", description: "Schema for user data", @@ -121,7 +121,7 @@ describe("normalizeJsonSchema", () => { }) it("handles union types by picking first non-null type", () => { - const jsonSchema: JSONSchema7Object = { + const jsonSchema: JSONSchema7 = { type: "object", properties: { nullableString: { @@ -145,7 +145,7 @@ describe("normalizeJsonSchema", () => { }) it("defaults to string type for unknown types", () => { - const jsonSchema: JSONSchema7Object = { + const jsonSchema: JSONSchema7 = { type: "object", properties: { unknownField: { @@ -161,7 +161,7 @@ describe("normalizeJsonSchema", () => { }) it("skips boolean schema properties", () => { - const jsonSchema: JSONSchema7Object = { + const jsonSchema: JSONSchema7 = { type: "object", properties: { validField: { diff --git a/jsonschema/schema/normalize.ts b/jsonschema/schema/normalize.ts index dd5965b9..a21540d4 100644 --- a/jsonschema/schema/normalize.ts +++ b/jsonschema/schema/normalize.ts @@ -1,10 +1,10 @@ -import type { JSONSchema7, JSONSchema7Object } from "json-schema" -import type { Schema, Field } from "@dpkit/core" +import type { Field, Schema } from "@dpkit/core" +import type { JSONSchema7 } from "json-schema" -export function normalizeJsonSchema(jsonSchema: JSONSchema7Object): Schema { +export function normalizeJsonSchema(jsonSchema: JSONSchema7): Schema { const fields: Field[] = [] const requiredFields = new Set( - Array.isArray(jsonSchema.required) ? jsonSchema.required : [] + Array.isArray(jsonSchema.required) ? jsonSchema.required : [], ) for (const [name, property] of Object.entries(jsonSchema.properties || {})) { @@ -12,7 +12,11 @@ export function normalizeJsonSchema(jsonSchema: JSONSchema7Object): Schema { continue // Skip boolean schemas } - const field = convertJsonSchemaPropertyToField(name, property, requiredFields.has(name)) + const field = convertJsonSchemaPropertyToField( + name, + property, + requiredFields.has(name), + ) if (field) { fields.push(field) } @@ -36,7 +40,7 @@ export function normalizeJsonSchema(jsonSchema: JSONSchema7Object): Schema { function convertJsonSchemaPropertyToField( name: string, property: JSONSchema7, - isRequired: boolean + isRequired: boolean, ): Field | null { if (typeof property === "boolean") { return null @@ -120,7 +124,11 @@ function convertJsonSchemaPropertyToField( // For union types, pick the first non-null type or default to string const nonNullType = property.type.find(t => t !== "null") if (nonNullType) { - return convertJsonSchemaPropertyToField(name, { ...property, type: nonNullType }, isRequired) + return convertJsonSchemaPropertyToField( + name, + { ...property, type: nonNullType }, + isRequired, + ) } } From 76277e708a33567a31fc2c3909e9a01168281315 Mon Sep 17 00:00:00 2001 From: roll Date: Wed, 1 Oct 2025 11:34:56 +0100 Subject: [PATCH 06/10] Added jsonschema to all --- all/index.ts | 10 ++++++++-- all/package.json | 1 + pnpm-lock.yaml | 3 +++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/all/index.ts b/all/index.ts index 965d9d51..ff159001 100644 --- a/all/index.ts +++ b/all/index.ts @@ -1,15 +1,21 @@ +export * from "@dpkit/arrow" export * from "@dpkit/camtrap" -export * from "@dpkit/csv" export * from "@dpkit/ckan" export * from "@dpkit/core" +export * from "@dpkit/csv" +export * from "@dpkit/database" export * from "@dpkit/datahub" export * from "@dpkit/file" export * from "@dpkit/folder" export * from "@dpkit/github" export * from "@dpkit/inline" +export * from "@dpkit/json" +export * from "@dpkit/jsonschema" +export * from "@dpkit/ods" +export * from "@dpkit/parquet" export * from "@dpkit/table" +export * from "@dpkit/xlsx" export * from "@dpkit/zenodo" - // TODO: Enable after migrated away from yauzl-promise (uses native crc32) //export * from "@dpkit/zip" diff --git a/all/package.json b/all/package.json index 2c35c396..4781def5 100644 --- a/all/package.json +++ b/all/package.json @@ -32,6 +32,7 @@ "@dpkit/github": "workspace:*", "@dpkit/inline": "workspace:*", "@dpkit/json": "workspace:*", + "@dpkit/jsonschema": "workspace:*", "@dpkit/ods": "workspace:*", "@dpkit/parquet": "workspace:*", "@dpkit/table": "workspace:*", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f675aa3a..1013f231 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -86,6 +86,9 @@ importers: '@dpkit/json': specifier: workspace:* version: link:../json + '@dpkit/jsonschema': + specifier: workspace:* + version: link:../jsonschema '@dpkit/ods': specifier: workspace:* version: link:../ods From 171cd5cc179122894a007539ab42f46c8098d189 Mon Sep 17 00:00:00 2001 From: roll Date: Wed, 1 Oct 2025 11:56:40 +0100 Subject: [PATCH 07/10] Added covnertSchema command --- dpkit/commands/schema/convert.tsx | 71 +++++++++++++++++++++++++++ dpkit/commands/schema/index.ts | 2 + jsonschema/schema/denormalize.spec.ts | 4 +- 3 files changed, 75 insertions(+), 2 deletions(-) create mode 100644 dpkit/commands/schema/convert.tsx diff --git a/dpkit/commands/schema/convert.tsx b/dpkit/commands/schema/convert.tsx new file mode 100644 index 00000000..840eceeb --- /dev/null +++ b/dpkit/commands/schema/convert.tsx @@ -0,0 +1,71 @@ +import { denormalizeJsonSchema, normalizeJsonSchema } from "@dpkit/all" +import { loadDescriptor, saveDescriptor } from "@dpkit/all" +import { Command, Option } from "commander" +import { helpConfiguration } from "../../helpers/help.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +const format = new Option("--format ", "source schema format").choices([ + "schema", + "jsonschema", +]) + +const toFormat = new Option( + "--to-format ", + "target schema format", +).choices(["schema", "jsonschema"]) + +export const convertSchemaCommand = new Command("convert") + .configureHelp(helpConfiguration) + .description("Convert schema between different formats") + + .addArgument(params.positionalDescriptorPath) + .addOption(format) + .addOption(toFormat) + .addOption(params.toPath) + .addOption(params.json) + .addOption(params.debug) + + .action(async (path, options) => { + const session = Session.create({ + title: "Convert schema", + json: options.json || !options.toPath, + debug: options.debug, + }) + + if (!options.format && !options.toFormat) { + session.terminate("Either --format or --to-format must be specified") + process.exit(1) + } + + if (options.format === options.toFormat) { + session.terminate("Source and target formats must be different") + process.exit(1) + } + + const converter = + options.format === "schema" || options.toFormat === "jsonschema" + ? denormalizeJsonSchema + : normalizeJsonSchema + + const source = await session.task("Loading schema", loadDescriptor(path)) + const target = await session.task( + "Converting schema", + // @ts-ignore + converter(source.descriptor), + ) + + if (!options.toPath) { + session.render(target) + return + } + + await session.task( + "Saving schema", + saveDescriptor(target as any, { + path: options.toPath, + }), + ) + + session.success(`Converted schema from ${path} to ${options.toPath}`) + }) diff --git a/dpkit/commands/schema/index.ts b/dpkit/commands/schema/index.ts index cb081347..aa289456 100644 --- a/dpkit/commands/schema/index.ts +++ b/dpkit/commands/schema/index.ts @@ -1,5 +1,6 @@ import { Command } from "commander" import { helpConfiguration } from "../../helpers/help.ts" +import { convertSchemaCommand } from "./convert.tsx" import { exploreSchemaCommand } from "./explore.tsx" import { inferSchemaCommand } from "./infer.tsx" import { scriptSchemaCommand } from "./script.tsx" @@ -10,6 +11,7 @@ export const schemaCommand = new Command("schema") .description("Table Schema related commands") .addCommand(inferSchemaCommand) + .addCommand(convertSchemaCommand) .addCommand(exploreSchemaCommand) .addCommand(scriptSchemaCommand) .addCommand(validateSchemaCommand) diff --git a/jsonschema/schema/denormalize.spec.ts b/jsonschema/schema/denormalize.spec.ts index fa4baf47..9a9d810a 100644 --- a/jsonschema/schema/denormalize.spec.ts +++ b/jsonschema/schema/denormalize.spec.ts @@ -1,5 +1,5 @@ -import { describe, expect, it } from "vitest" import type { Schema } from "@dpkit/core" +import { describe, expect, it } from "vitest" import { denormalizeJsonSchema } from "./denormalize.ts" describe("denormalizeJsonSchema", () => { @@ -451,4 +451,4 @@ describe("denormalizeJsonSchema", () => { expect(jsonSchema.title).toBeUndefined() expect(jsonSchema.description).toBeUndefined() }) -}) \ No newline at end of file +}) From ad5739e6fd6305437c4d41c084c5771eda23ebd0 Mon Sep 17 00:00:00 2001 From: roll Date: Wed, 1 Oct 2025 12:16:30 +0100 Subject: [PATCH 08/10] Fixed tests --- jsonschema/schema/denormalize.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/jsonschema/schema/denormalize.ts b/jsonschema/schema/denormalize.ts index f23dacd3..dad9c04b 100644 --- a/jsonschema/schema/denormalize.ts +++ b/jsonschema/schema/denormalize.ts @@ -40,6 +40,14 @@ export function denormalizeJsonSchema(schema: Schema): JSONSchema7 { function convertFieldToJsonSchemaProperty(field: Field): JSONSchema7 | null { const baseProperty: Partial = {} + if (field.title) { + baseProperty.title = field.title + } + + if (field.description) { + baseProperty.description = field.description + } + // Handle different Table Schema field types switch (field.type) { case "string": { From 8546f165603fa9be6601c11b25501dbc0fe43c03 Mon Sep 17 00:00:00 2001 From: roll Date: Wed, 1 Oct 2025 12:23:17 +0100 Subject: [PATCH 09/10] Added jsonschema to api reference --- docs/astro.config.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/astro.config.ts b/docs/astro.config.ts index 973b9cca..cc87c489 100644 --- a/docs/astro.config.ts +++ b/docs/astro.config.ts @@ -19,6 +19,7 @@ const PACKAGES = { "@dpkit/github": "../github", "@dpkit/inline": "../inline", "@dpkit/json": "../json", + "@dpkit/jsonschema": "../jsonschema", "@dpkit/ods": "../ods", "@dpkit/parquet": "../parquet", "@dpkit/table": "../table", From a7ae98386782c40c27648a957849f6a6d2832561 Mon Sep 17 00:00:00 2001 From: roll Date: Wed, 1 Oct 2025 12:25:12 +0100 Subject: [PATCH 10/10] Added schema convert docs --- site/content/docs/guides/schema.md | 65 ++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/site/content/docs/guides/schema.md b/site/content/docs/guides/schema.md index e7c43e04..907f07ef 100644 --- a/site/content/docs/guides/schema.md +++ b/site/content/docs/guides/schema.md @@ -84,6 +84,40 @@ dp schema infer --from-package datapackage.json --from-resource "users" dp schema infer data.csv --json > schema.json ``` +### `dp schema convert` + +Convert table schemas between different formats, supporting bidirectional conversion between Table Schema and JSONSchema formats. + +```bash +dp schema convert +``` + +**Options:** +- `--format `: Source schema format (`schema`, `jsonschema`) +- `--to-format `: Target schema format (`schema`, `jsonschema`) +- `--to-path `: Output path for converted schema +- `-j, --json`: Output as JSON (automatically enabled when no --to-path) +- `-d, --debug`: Enable debug mode + +**Supported Formats:** +- **`schema`**: Data Package Table Schema format +- **`jsonschema`**: JSON Schema format + +**Examples:** +```bash +# Convert Table Schema to JSONSchema +dp schema convert schema.json --to-format jsonschema + +# Convert JSONSchema to Table Schema +dp schema convert schema.jsonschema.json --format jsonschema + +# Save converted schema to file +dp schema convert schema.json --to-format jsonschema --to-path converted.jsonschema.json + +# Convert from JSONSchema and save as Table Schema +dp schema convert input.jsonschema.json --format jsonschema --to-path output.schema.json +``` + ### `dp schema explore` Explore a table schema from a local or remote path to view its field definitions and constraints in an interactive format. @@ -199,6 +233,20 @@ dp> schema.primaryKey dp schema explore schema.json ``` +### Schema Format Conversion + +```bash +# Convert Table Schema to JSONSchema for JSON Schema validation tools +dp schema infer data.csv --json > table.schema.json +dp schema convert table.schema.json --to-format jsonschema --to-path api.jsonschema.json + +# Convert JSONSchema back to Table Schema for dpkit tools +dp schema convert api.jsonschema.json --format jsonschema --to-path converted.schema.json + +# Validate the round-trip conversion +dp schema validate converted.schema.json +``` + ### Schema Analysis and Refinement ```bash @@ -293,6 +341,23 @@ All schema commands support multiple output formats: - **JSON**: Use `--json` flag for machine-readable output - **Debug Mode**: Use `--debug` for detailed operation logs +## Schema Format Interoperability + +The `convert` command enables seamless integration with other schema ecosystems: + +```bash +# Use with JSON Schema validation libraries +dp schema infer data.csv --json > table.schema.json +dp schema convert table.schema.json --to-format jsonschema --to-path validation.jsonschema.json + +# Import existing JSONSchema into dpkit workflow +dp schema convert external.jsonschema.json --format jsonschema --to-path dpkit.schema.json +dp table validate data.csv --schema dpkit.schema.json + +# Cross-platform schema sharing +dp schema convert schema.json --to-format jsonschema --to-path api-spec.jsonschema.json +``` + ## Integration with Other Commands Schema commands work seamlessly with other dpkit commands: