diff --git a/all/index.ts b/all/index.ts index 965d9d51..ff159001 100644 --- a/all/index.ts +++ b/all/index.ts @@ -1,15 +1,21 @@ +export * from "@dpkit/arrow" export * from "@dpkit/camtrap" -export * from "@dpkit/csv" export * from "@dpkit/ckan" export * from "@dpkit/core" +export * from "@dpkit/csv" +export * from "@dpkit/database" export * from "@dpkit/datahub" export * from "@dpkit/file" export * from "@dpkit/folder" export * from "@dpkit/github" export * from "@dpkit/inline" +export * from "@dpkit/json" +export * from "@dpkit/jsonschema" +export * from "@dpkit/ods" +export * from "@dpkit/parquet" export * from "@dpkit/table" +export * from "@dpkit/xlsx" export * from "@dpkit/zenodo" - // TODO: Enable after migrated away from yauzl-promise (uses native crc32) //export * from "@dpkit/zip" diff --git a/all/package.json b/all/package.json index 2c35c396..4781def5 100644 --- a/all/package.json +++ b/all/package.json @@ -32,6 +32,7 @@ "@dpkit/github": "workspace:*", "@dpkit/inline": "workspace:*", "@dpkit/json": "workspace:*", + "@dpkit/jsonschema": "workspace:*", "@dpkit/ods": "workspace:*", "@dpkit/parquet": "workspace:*", "@dpkit/table": "workspace:*", diff --git a/core/dialect/Dialect.ts b/core/dialect/Dialect.ts index 73249daf..c2c87ebf 100644 --- a/core/dialect/Dialect.ts +++ b/core/dialect/Dialect.ts @@ -7,14 +7,14 @@ import type { Metadata } from "../general/index.ts" */ export interface Dialect extends Metadata { /** - * The name of this dialect + * JSON schema profile URL for validation */ - name?: string + $schema?: string /** - * JSON schema profile URL for validation + * The name of this dialect */ - $schema?: string + name?: string /** * Whether the file includes a header row with field names diff --git a/core/package/Package.ts b/core/package/Package.ts index a4ff6852..992db912 100644 --- a/core/package/Package.ts +++ b/core/package/Package.ts @@ -7,6 +7,11 @@ import type { Contributor } from "./Contributor.ts" * @see https://datapackage.org/standard/data-package/ */ export interface Package extends Metadata { + /** + * URL of profile (optional) + */ + $schema?: string + /** * Data resources in this package (required) */ @@ -18,11 +23,6 @@ export interface Package extends Metadata { */ name?: string - /** - * Package schema URL for validation - */ - $schema?: string - /** * Human-readable title */ diff --git a/core/resource/Resource.ts b/core/resource/Resource.ts index 00792e03..aa4078e1 100644 --- a/core/resource/Resource.ts +++ b/core/resource/Resource.ts @@ -9,6 +9,11 @@ import type { Source } from "./Source.ts" * @see https://datapackage.org/standard/data-resource/ */ export interface Resource extends Metadata { + /** + * JSON schema profile URL for validation + */ + $schema?: string + /** * Unique resource identifier * Should use lowercase alphanumeric characters, periods, hyphens, and underscores diff --git a/core/schema/Schema.ts b/core/schema/Schema.ts index 508edcc7..f7bce413 100644 --- a/core/schema/Schema.ts +++ b/core/schema/Schema.ts @@ -8,14 +8,29 @@ import type { ForeignKey } from "./ForeignKey.ts" */ export interface Schema extends Metadata { /** - * Fields in this schema (required) + * URL of profile (optional) */ - fields: Field[] + $schema?: string /** - * URL of schema (optional) + * Name of schema (optional) */ - $schema?: string + name?: string + + /** + * Title of schema (optional) + */ + title?: string + + /** + * Description of schema (optional) + */ + description?: string + + /** + * Fields in this schema (required) + */ + fields: Field[] /** * Field matching rule (optional) diff --git a/docs/astro.config.ts b/docs/astro.config.ts index 973b9cca..cc87c489 100644 --- a/docs/astro.config.ts +++ b/docs/astro.config.ts @@ -19,6 +19,7 @@ const PACKAGES = { "@dpkit/github": "../github", "@dpkit/inline": "../inline", "@dpkit/json": "../json", + "@dpkit/jsonschema": "../jsonschema", "@dpkit/ods": "../ods", "@dpkit/parquet": "../parquet", "@dpkit/table": "../table", diff --git a/dpkit/commands/schema/convert.tsx b/dpkit/commands/schema/convert.tsx new file mode 100644 index 00000000..840eceeb --- /dev/null +++ b/dpkit/commands/schema/convert.tsx @@ -0,0 +1,71 @@ +import { denormalizeJsonSchema, normalizeJsonSchema } from "@dpkit/all" +import { loadDescriptor, saveDescriptor } from "@dpkit/all" +import { Command, Option } from "commander" +import { helpConfiguration } from "../../helpers/help.ts" +import { Session } from "../../helpers/session.ts" +import * as params from "../../params/index.ts" + +const format = new Option("--format ", "source schema format").choices([ + "schema", + "jsonschema", +]) + +const toFormat = new Option( + "--to-format ", + "target schema format", +).choices(["schema", "jsonschema"]) + +export const convertSchemaCommand = new Command("convert") + .configureHelp(helpConfiguration) + .description("Convert schema between different formats") + + .addArgument(params.positionalDescriptorPath) + .addOption(format) + .addOption(toFormat) + .addOption(params.toPath) + .addOption(params.json) + .addOption(params.debug) + + .action(async (path, options) => { + const session = Session.create({ + title: "Convert schema", + json: options.json || !options.toPath, + debug: options.debug, + }) + + if (!options.format && !options.toFormat) { + session.terminate("Either --format or --to-format must be specified") + process.exit(1) + } + + if (options.format === options.toFormat) { + session.terminate("Source and target formats must be different") + process.exit(1) + } + + const converter = + options.format === "schema" || options.toFormat === "jsonschema" + ? denormalizeJsonSchema + : normalizeJsonSchema + + const source = await session.task("Loading schema", loadDescriptor(path)) + const target = await session.task( + "Converting schema", + // @ts-ignore + converter(source.descriptor), + ) + + if (!options.toPath) { + session.render(target) + return + } + + await session.task( + "Saving schema", + saveDescriptor(target as any, { + path: options.toPath, + }), + ) + + session.success(`Converted schema from ${path} to ${options.toPath}`) + }) diff --git a/dpkit/commands/schema/index.ts b/dpkit/commands/schema/index.ts index cb081347..aa289456 100644 --- a/dpkit/commands/schema/index.ts +++ b/dpkit/commands/schema/index.ts @@ -1,5 +1,6 @@ import { Command } from "commander" import { helpConfiguration } from "../../helpers/help.ts" +import { convertSchemaCommand } from "./convert.tsx" import { exploreSchemaCommand } from "./explore.tsx" import { inferSchemaCommand } from "./infer.tsx" import { scriptSchemaCommand } from "./script.tsx" @@ -10,6 +11,7 @@ export const schemaCommand = new Command("schema") .description("Table Schema related commands") .addCommand(inferSchemaCommand) + .addCommand(convertSchemaCommand) .addCommand(exploreSchemaCommand) .addCommand(scriptSchemaCommand) .addCommand(validateSchemaCommand) diff --git a/jsonschema/README.md b/jsonschema/README.md new file mode 100644 index 00000000..f812b1f3 --- /dev/null +++ b/jsonschema/README.md @@ -0,0 +1,3 @@ +# @dpkit/jsonschema + +dpkit is a fast data management framework built on top of the Data Package standard and Polars DataFrames. It supports various formats like CSV, JSON, and Parquet and integrates with data platforms such as CKAN, Zenodo, and GitHub. For more information, please visit the [documentation portal](https://dpkit.datist.io). diff --git a/jsonschema/index.ts b/jsonschema/index.ts new file mode 100644 index 00000000..352ef322 --- /dev/null +++ b/jsonschema/index.ts @@ -0,0 +1 @@ +export * from "./schema/index.ts" diff --git a/jsonschema/package.json b/jsonschema/package.json new file mode 100644 index 00000000..194fdaa2 --- /dev/null +++ b/jsonschema/package.json @@ -0,0 +1,33 @@ +{ + "name": "@dpkit/jsonschema", + "type": "module", + "version": "0.0.0-development", + "exports": "./build/index.js", + "license": "MIT", + "author": "Evgeny Karev", + "repository": "https://github.com/datisthq/dpkit", + "description": "Fast TypeScript data management framework built on top of the Data Package standard and Polars DataFrames", + "keywords": [ + "data", + "polars", + "dataframe", + "datapackage", + "tableschema", + "typescript", + "validation", + "quality", + "fair", + "jsonschema" + ], + "scripts": { + "build": "tsc" + }, + "dependencies": { + "@dpkit/core": "workspace:*", + "@types/json-schema": "^7.0.15", + "json-schema": "^0.4.0" + }, + "devDependencies": { + "@dpkit/test": "workspace:*" + } +} diff --git a/jsonschema/schema/denormalize.spec.ts b/jsonschema/schema/denormalize.spec.ts new file mode 100644 index 00000000..9a9d810a --- /dev/null +++ b/jsonschema/schema/denormalize.spec.ts @@ -0,0 +1,454 @@ +import type { Schema } from "@dpkit/core" +import { describe, expect, it } from "vitest" +import { denormalizeJsonSchema } from "./denormalize.ts" + +describe("denormalizeJsonSchema", () => { + it("converts Table Schema to JSONSchema object", () => { + const tableSchema: Schema & { title?: string; description?: string } = { + title: "User Schema", + description: "Schema for user data", + fields: [ + { + name: "id", + type: "integer", + title: "User ID", + description: "Unique identifier", + constraints: { + required: true, + minimum: 1, + }, + }, + { + name: "name", + type: "string", + title: "Full Name", + constraints: { + required: true, + maxLength: 100, + }, + }, + { + name: "email", + type: "string", + constraints: { + pattern: "^[^@]+@[^@]+\\.[^@]+$", + }, + }, + { + name: "age", + type: "integer", + constraints: { + minimum: 0, + maximum: 150, + }, + }, + { + name: "isActive", + type: "boolean", + }, + { + name: "tags", + type: "array", + constraints: { + minLength: 0, + maxLength: 10, + }, + }, + { + name: "metadata", + type: "object", + constraints: { + jsonSchema: { + type: "object", + properties: { + category: { type: "string" }, + }, + }, + }, + }, + ], + } + + const jsonSchema = denormalizeJsonSchema(tableSchema) + + expect(jsonSchema.title).toBe("User Schema") + expect(jsonSchema.description).toBe("Schema for user data") + expect(jsonSchema.type).toBe("object") + expect(jsonSchema.required).toEqual(["id", "name"]) + expect(Object.keys(jsonSchema.properties || {})).toHaveLength(7) + + expect((jsonSchema.properties as any)?.id).toEqual({ + type: "integer", + title: "User ID", + description: "Unique identifier", + minimum: 1, + }) + + expect((jsonSchema.properties as any)?.name).toEqual({ + type: "string", + title: "Full Name", + maxLength: 100, + }) + + expect((jsonSchema.properties as any)?.email).toEqual({ + type: "string", + pattern: "^[^@]+@[^@]+\\.[^@]+$", + }) + + expect((jsonSchema.properties as any)?.age).toEqual({ + type: "integer", + minimum: 0, + maximum: 150, + }) + + expect((jsonSchema.properties as any)?.isActive).toEqual({ + type: "boolean", + }) + + expect((jsonSchema.properties as any)?.tags).toEqual({ + type: "array", + minItems: 0, + maxItems: 10, + }) + + expect((jsonSchema.properties as any)?.metadata).toEqual({ + type: "object", + properties: { + category: { type: "string" }, + }, + }) + }) + + it("handles string field types with various constraints", () => { + const tableSchema: Schema = { + fields: [ + { + name: "basicString", + type: "string", + }, + { + name: "constrainedString", + type: "string", + constraints: { + minLength: 5, + maxLength: 50, + pattern: "^[A-Z]", + enum: ["Option1", "Option2", "Option3"], + }, + }, + ], + } + + const jsonSchema = denormalizeJsonSchema(tableSchema) + + expect((jsonSchema.properties as any)?.basicString).toEqual({ + type: "string", + }) + + expect((jsonSchema.properties as any)?.constrainedString).toEqual({ + type: "string", + minLength: 5, + maxLength: 50, + pattern: "^[A-Z]", + enum: ["Option1", "Option2", "Option3"], + }) + }) + + it("handles numeric field types", () => { + const tableSchema: Schema = { + fields: [ + { + name: "simpleNumber", + type: "number", + }, + { + name: "constrainedNumber", + type: "number", + constraints: { + minimum: 0.1, + maximum: 99.9, + }, + }, + { + name: "simpleInteger", + type: "integer", + }, + { + name: "constrainedInteger", + type: "integer", + constraints: { + minimum: 1, + maximum: 100, + }, + }, + ], + } + + const jsonSchema = denormalizeJsonSchema(tableSchema) + + expect((jsonSchema.properties as any)?.simpleNumber).toEqual({ + type: "number", + }) + + expect((jsonSchema.properties as any)?.constrainedNumber).toEqual({ + type: "number", + minimum: 0.1, + maximum: 99.9, + }) + + expect((jsonSchema.properties as any)?.simpleInteger).toEqual({ + type: "integer", + }) + + expect((jsonSchema.properties as any)?.constrainedInteger).toEqual({ + type: "integer", + minimum: 1, + maximum: 100, + }) + }) + + it("handles array and object field types", () => { + const tableSchema: Schema = { + fields: [ + { + name: "simpleArray", + type: "array", + }, + { + name: "constrainedArray", + type: "array", + constraints: { + minLength: 1, + maxLength: 5, + jsonSchema: { + type: "string", + }, + }, + }, + { + name: "simpleObject", + type: "object", + }, + { + name: "constrainedObject", + type: "object", + constraints: { + jsonSchema: { + type: "object", + properties: { + nested: { type: "string" }, + }, + required: ["nested"], + }, + }, + }, + ], + } + + const jsonSchema = denormalizeJsonSchema(tableSchema) + + expect((jsonSchema.properties as any)?.simpleArray).toEqual({ + type: "array", + }) + + expect((jsonSchema.properties as any)?.constrainedArray).toEqual({ + type: "array", + minItems: 1, + maxItems: 5, + items: { + type: "string", + }, + }) + + expect((jsonSchema.properties as any)?.simpleObject).toEqual({ + type: "object", + }) + + expect((jsonSchema.properties as any)?.constrainedObject).toEqual({ + type: "object", + properties: { + nested: { type: "string" }, + }, + required: ["nested"], + }) + }) + + it("handles date and time field types", () => { + const tableSchema: Schema = { + fields: [ + { + name: "dateField", + type: "date", + title: "Date Field", + }, + { + name: "datetimeField", + type: "datetime", + description: "DateTime Field", + }, + { + name: "timeField", + type: "time", + }, + { + name: "yearField", + type: "year", + }, + { + name: "yearmonthField", + type: "yearmonth", + }, + { + name: "durationField", + type: "duration", + }, + ], + } + + const jsonSchema = denormalizeJsonSchema(tableSchema) + + expect((jsonSchema.properties as any)?.dateField).toEqual({ + type: "string", + format: "date", + title: "Date Field", + }) + + expect((jsonSchema.properties as any)?.datetimeField).toEqual({ + type: "string", + format: "date-time", + description: "DateTime Field", + }) + + expect((jsonSchema.properties as any)?.timeField).toEqual({ + type: "string", + format: "time", + }) + + expect((jsonSchema.properties as any)?.yearField).toEqual({ + type: "string", + format: "year", + }) + + expect((jsonSchema.properties as any)?.yearmonthField).toEqual({ + type: "string", + format: "yearmonth", + }) + + expect((jsonSchema.properties as any)?.durationField).toEqual({ + type: "string", + format: "duration", + }) + }) + + it("handles geospatial field types", () => { + const tableSchema: Schema = { + fields: [ + { + name: "geopointField", + type: "geopoint", + }, + { + name: "geojsonField", + type: "geojson", + }, + ], + } + + const jsonSchema = denormalizeJsonSchema(tableSchema) + + expect((jsonSchema.properties as any)?.geopointField).toEqual({ + type: "string", + format: "geopoint", + }) + + expect((jsonSchema.properties as any)?.geojsonField).toEqual({ + type: "object", + format: "geojson", + }) + }) + + it("handles special field types", () => { + const tableSchema: Schema = { + fields: [ + { + name: "listField", + type: "list", + }, + { + name: "anyField", + type: "any", + }, + ], + } + + const jsonSchema = denormalizeJsonSchema(tableSchema) + + expect((jsonSchema.properties as any)?.listField).toEqual({ + type: "array", + format: "list", + }) + + expect((jsonSchema.properties as any)?.anyField).toEqual({ + type: "string", + }) + }) + + it("handles schema without title and description", () => { + const tableSchema: Schema = { + fields: [ + { + name: "field1", + type: "string", + }, + ], + } + + const jsonSchema = denormalizeJsonSchema(tableSchema) + + expect(jsonSchema.title).toBeUndefined() + expect(jsonSchema.description).toBeUndefined() + expect(jsonSchema.type).toBe("object") + expect((jsonSchema.properties as any)?.field1).toEqual({ + type: "string", + }) + }) + + it("handles schema with no required fields", () => { + const tableSchema: Schema = { + fields: [ + { + name: "optionalField1", + type: "string", + }, + { + name: "optionalField2", + type: "integer", + }, + ], + } + + const jsonSchema = denormalizeJsonSchema(tableSchema) + + expect(jsonSchema.required).toBeUndefined() + expect((jsonSchema.properties as any)?.optionalField1).toEqual({ + type: "string", + }) + expect((jsonSchema.properties as any)?.optionalField2).toEqual({ + type: "integer", + }) + }) + + it("handles empty schema", () => { + const tableSchema: Schema = { + fields: [], + } + + const jsonSchema = denormalizeJsonSchema(tableSchema) + + expect(jsonSchema.type).toBe("object") + expect(jsonSchema.properties).toEqual({}) + expect(jsonSchema.required).toBeUndefined() + expect(jsonSchema.title).toBeUndefined() + expect(jsonSchema.description).toBeUndefined() + }) +}) diff --git a/jsonschema/schema/denormalize.ts b/jsonschema/schema/denormalize.ts new file mode 100644 index 00000000..dad9c04b --- /dev/null +++ b/jsonschema/schema/denormalize.ts @@ -0,0 +1,222 @@ +import type { Field, Schema } from "@dpkit/core" +import type { JSONSchema7 } from "json-schema" + +export function denormalizeJsonSchema(schema: Schema): JSONSchema7 { + const properties: Record = {} + const required: string[] = [] + + for (const field of schema.fields) { + const property = convertFieldToJsonSchemaProperty(field) + if (property) { + properties[field.name] = property + + // Check if field is required + if (field.constraints?.required) { + required.push(field.name) + } + } + } + + const jsonSchema: JSONSchema7 = { + type: "object", + properties, + } + + if (required.length > 0) { + jsonSchema.required = required + } + + if (schema.title) { + jsonSchema.title = schema.title + } + + if (schema.description) { + jsonSchema.description = schema.description + } + + return jsonSchema +} + +function convertFieldToJsonSchemaProperty(field: Field): JSONSchema7 | null { + const baseProperty: Partial = {} + + if (field.title) { + baseProperty.title = field.title + } + + if (field.description) { + baseProperty.description = field.description + } + + // Handle different Table Schema field types + switch (field.type) { + case "string": { + const property: JSONSchema7 = { + ...baseProperty, + type: "string", + } + + if (field.constraints?.minLength !== undefined) { + property.minLength = field.constraints.minLength + } + if (field.constraints?.maxLength !== undefined) { + property.maxLength = field.constraints.maxLength + } + if (field.constraints?.pattern) { + property.pattern = field.constraints.pattern + } + if (field.constraints?.enum) { + property.enum = field.constraints.enum + } + + return property + } + + case "number": { + const property: JSONSchema7 = { + ...baseProperty, + type: "number", + } + + if ( + field.constraints?.minimum !== undefined && + typeof field.constraints.minimum === "number" + ) { + property.minimum = field.constraints.minimum + } + if ( + field.constraints?.maximum !== undefined && + typeof field.constraints.maximum === "number" + ) { + property.maximum = field.constraints.maximum + } + + return property + } + + case "integer": { + const property: JSONSchema7 = { + ...baseProperty, + type: "integer", + } + + if ( + field.constraints?.minimum !== undefined && + typeof field.constraints.minimum === "number" + ) { + property.minimum = field.constraints.minimum + } + if ( + field.constraints?.maximum !== undefined && + typeof field.constraints.maximum === "number" + ) { + property.maximum = field.constraints.maximum + } + + return property + } + + case "boolean": { + return { + ...baseProperty, + type: "boolean", + } + } + + case "array": { + const property: JSONSchema7 = { + ...baseProperty, + type: "array", + } + + if (field.constraints?.minLength !== undefined) { + property.minItems = field.constraints.minLength + } + if (field.constraints?.maxLength !== undefined) { + property.maxItems = field.constraints.maxLength + } + if (field.constraints?.jsonSchema) { + property.items = field.constraints.jsonSchema + } + + return property + } + + case "object": { + const property: JSONSchema7 = { + ...baseProperty, + type: "object", + } + + // If there's a jsonSchema constraint, merge it + if (field.constraints?.jsonSchema) { + return { + ...property, + ...field.constraints.jsonSchema, + } + } + + return property + } + + // For other field types that don't have direct JSONSchema equivalents, + // convert them to string type with format information + case "date": + return { + ...baseProperty, + type: "string", + format: "date", + } + + case "datetime": + return { + ...baseProperty, + type: "string", + format: "date-time", + } + + case "time": + return { + ...baseProperty, + type: "string", + format: "time", + } + + case "year": + case "yearmonth": + case "duration": + return { + ...baseProperty, + type: "string", + format: field.type, + } + + case "geopoint": + return { + ...baseProperty, + type: "string", + format: "geopoint", + } + + case "geojson": + return { + ...baseProperty, + type: "object", + format: "geojson", + } + + case "list": + return { + ...baseProperty, + type: "array", + format: "list", + } + + default: + // Default to string type for unknown field types + return { + ...baseProperty, + type: "string", + } + } +} diff --git a/jsonschema/schema/index.ts b/jsonschema/schema/index.ts new file mode 100644 index 00000000..dc8f9904 --- /dev/null +++ b/jsonschema/schema/index.ts @@ -0,0 +1,2 @@ +export { denormalizeJsonSchema } from "./denormalize.ts" +export { normalizeJsonSchema } from "./normalize.ts" diff --git a/jsonschema/schema/normalize.spec.ts b/jsonschema/schema/normalize.spec.ts new file mode 100644 index 00000000..89cdf787 --- /dev/null +++ b/jsonschema/schema/normalize.spec.ts @@ -0,0 +1,180 @@ +import type { JSONSchema7 } from "json-schema" +import { describe, expect, it } from "vitest" +import { normalizeJsonSchema } from "./normalize.ts" + +describe("normalizeJsonSchema", () => { + it("converts JSONSchema object to Table Schema", () => { + const jsonSchema: JSONSchema7 = { + type: "object", + title: "User Schema", + description: "Schema for user data", + required: ["id", "name"], + properties: { + id: { + type: "integer", + title: "User ID", + description: "Unique identifier", + minimum: 1, + }, + name: { + type: "string", + title: "Full Name", + maxLength: 100, + }, + email: { + type: "string", + pattern: "^[^@]+@[^@]+\\.[^@]+$", + }, + age: { + type: "integer", + minimum: 0, + maximum: 150, + }, + isActive: { + type: "boolean", + }, + tags: { + type: "array", + minItems: 0, + maxItems: 10, + }, + metadata: { + type: "object", + }, + }, + } + + const tableSchema = normalizeJsonSchema(jsonSchema) + + expect(tableSchema.title).toBe("User Schema") + expect(tableSchema.description).toBe("Schema for user data") + expect(tableSchema.fields).toHaveLength(7) + + const idField = tableSchema.fields.find(f => f.name === "id") + expect(idField).toEqual({ + name: "id", + type: "integer", + title: "User ID", + description: "Unique identifier", + constraints: { + required: true, + minimum: 1, + }, + }) + + const nameField = tableSchema.fields.find(f => f.name === "name") + expect(nameField).toEqual({ + name: "name", + type: "string", + title: "Full Name", + constraints: { + required: true, + maxLength: 100, + }, + }) + + const emailField = tableSchema.fields.find(f => f.name === "email") + expect(emailField).toEqual({ + name: "email", + type: "string", + constraints: { + pattern: "^[^@]+@[^@]+\\.[^@]+$", + }, + }) + + const ageField = tableSchema.fields.find(f => f.name === "age") + expect(ageField).toEqual({ + name: "age", + type: "integer", + constraints: { + minimum: 0, + maximum: 150, + }, + }) + + const isActiveField = tableSchema.fields.find(f => f.name === "isActive") + expect(isActiveField).toEqual({ + name: "isActive", + type: "boolean", + }) + + const tagsField = tableSchema.fields.find(f => f.name === "tags") + expect(tagsField).toEqual({ + name: "tags", + type: "array", + constraints: { + minLength: 0, + maxLength: 10, + }, + }) + + const metadataField = tableSchema.fields.find(f => f.name === "metadata") + expect(metadataField).toEqual({ + name: "metadata", + type: "object", + constraints: { + jsonSchema: { + type: "object", + }, + }, + }) + }) + + it("handles union types by picking first non-null type", () => { + const jsonSchema: JSONSchema7 = { + type: "object", + properties: { + nullableString: { + type: ["string", "null"], + }, + multiType: { + type: ["number", "string"], + }, + }, + } + + const tableSchema = normalizeJsonSchema(jsonSchema) + + const nullableField = tableSchema.fields.find( + f => f.name === "nullableString", + ) + expect(nullableField?.type).toBe("string") + + const multiField = tableSchema.fields.find(f => f.name === "multiType") + expect(multiField?.type).toBe("number") + }) + + it("defaults to string type for unknown types", () => { + const jsonSchema: JSONSchema7 = { + type: "object", + properties: { + unknownField: { + // No type specified + }, + }, + } + + const tableSchema = normalizeJsonSchema(jsonSchema) + + const unknownField = tableSchema.fields.find(f => f.name === "unknownField") + expect(unknownField?.type).toBe("string") + }) + + it("skips boolean schema properties", () => { + const jsonSchema: JSONSchema7 = { + type: "object", + properties: { + validField: { + type: "string", + }, + booleanSchema: true, // This should be skipped + falseBooleanSchema: false, // This should also be skipped + }, + } + + const tableSchema = normalizeJsonSchema(jsonSchema) + + expect(tableSchema.fields).toHaveLength(1) + expect(tableSchema.fields[0]?.name).toBe("validField") + }) +}) diff --git a/jsonschema/schema/normalize.ts b/jsonschema/schema/normalize.ts new file mode 100644 index 00000000..a21540d4 --- /dev/null +++ b/jsonschema/schema/normalize.ts @@ -0,0 +1,141 @@ +import type { Field, Schema } from "@dpkit/core" +import type { JSONSchema7 } from "json-schema" + +export function normalizeJsonSchema(jsonSchema: JSONSchema7): Schema { + const fields: Field[] = [] + const requiredFields = new Set( + Array.isArray(jsonSchema.required) ? jsonSchema.required : [], + ) + + for (const [name, property] of Object.entries(jsonSchema.properties || {})) { + if (typeof property === "boolean") { + continue // Skip boolean schemas + } + + const field = convertJsonSchemaPropertyToField( + name, + property, + requiredFields.has(name), + ) + if (field) { + fields.push(field) + } + } + + const schema: Schema & { title?: string; description?: string } = { + fields, + } + + if (typeof jsonSchema.title === "string") { + schema.title = jsonSchema.title + } + + if (typeof jsonSchema.description === "string") { + schema.description = jsonSchema.description + } + + return schema +} + +function convertJsonSchemaPropertyToField( + name: string, + property: JSONSchema7, + isRequired: boolean, +): Field | null { + if (typeof property === "boolean") { + return null + } + + const baseField = { + name, + title: property.title, + description: property.description, + constraints: isRequired ? { required: true } : undefined, + } + + // Handle different JSONSchema types + switch (property.type) { + case "string": + return { + ...baseField, + type: "string", + constraints: { + ...baseField.constraints, + minLength: property.minLength, + maxLength: property.maxLength, + pattern: property.pattern, + enum: property.enum as string[] | undefined, + }, + } as Field + + case "number": + return { + ...baseField, + type: "number", + constraints: { + ...baseField.constraints, + minimum: property.minimum, + maximum: property.maximum, + }, + } as Field + + case "integer": + return { + ...baseField, + type: "integer", + constraints: { + ...baseField.constraints, + minimum: property.minimum, + maximum: property.maximum, + }, + } as Field + + case "boolean": + return { + ...baseField, + type: "boolean", + } as Field + + case "array": + return { + ...baseField, + type: "array", + constraints: { + ...baseField.constraints, + minLength: property.minItems, + maxLength: property.maxItems, + jsonSchema: property.items as Record, + }, + } as Field + + case "object": + return { + ...baseField, + type: "object", + constraints: { + ...baseField.constraints, + jsonSchema: property as Record, + }, + } as Field + + default: + // Handle union types or unspecified types + if (Array.isArray(property.type)) { + // For union types, pick the first non-null type or default to string + const nonNullType = property.type.find(t => t !== "null") + if (nonNullType) { + return convertJsonSchemaPropertyToField( + name, + { ...property, type: nonNullType }, + isRequired, + ) + } + } + + // Default to string type for unknown/unspecified types + return { + ...baseField, + type: "string", + } as Field + } +} diff --git a/jsonschema/tsconfig.json b/jsonschema/tsconfig.json new file mode 100644 index 00000000..3c43903c --- /dev/null +++ b/jsonschema/tsconfig.json @@ -0,0 +1,3 @@ +{ + "extends": "../tsconfig.json" +} diff --git a/jsonschema/typedoc.json b/jsonschema/typedoc.json new file mode 100644 index 00000000..f8e49f3a --- /dev/null +++ b/jsonschema/typedoc.json @@ -0,0 +1,4 @@ +{ + "entryPoints": ["index.ts"], + "skipErrorChecking": true +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a949310b..1013f231 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -86,6 +86,9 @@ importers: '@dpkit/json': specifier: workspace:* version: link:../json + '@dpkit/jsonschema': + specifier: workspace:* + version: link:../jsonschema '@dpkit/ods': specifier: workspace:* version: link:../ods @@ -405,6 +408,22 @@ importers: specifier: workspace:* version: link:../test + jsonschema: + dependencies: + '@dpkit/core': + specifier: workspace:* + version: link:../core + '@types/json-schema': + specifier: ^7.0.15 + version: 7.0.15 + json-schema: + specifier: ^0.4.0 + version: 0.4.0 + devDependencies: + '@dpkit/test': + specifier: workspace:* + version: link:../test + ods: dependencies: '@dpkit/core': @@ -1576,6 +1595,9 @@ packages: '@types/js-yaml@4.0.9': resolution: {integrity: sha512-k4MGaQl5TGo/iipqb2UDG2UwjXziSWkh0uysQelTlJpX1qGlpUZYm8PnO4DxG1qBomtJUdYJ6qR6xdIah10JLg==} + '@types/json-schema@7.0.15': + resolution: {integrity: sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==} + '@types/mdast@4.0.4': resolution: {integrity: sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==} @@ -2882,6 +2904,9 @@ packages: json-schema-traverse@1.0.0: resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==} + json-schema@0.4.0: + resolution: {integrity: sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==} + jsonfile@6.2.0: resolution: {integrity: sha512-FGuPw30AdOIUTRMC2OMRtQV+jkVj2cfPqSeWXv1NEAJ1qZ5zb1X6z1mFhbfOB/iy3ssJCD+3KuZ8r8C3uVFlAg==} @@ -5832,6 +5857,8 @@ snapshots: '@types/js-yaml@4.0.9': {} + '@types/json-schema@7.0.15': {} + '@types/mdast@4.0.4': dependencies: '@types/unist': 3.0.3 @@ -7408,6 +7435,8 @@ snapshots: json-schema-traverse@1.0.0: {} + json-schema@0.4.0: {} + jsonfile@6.2.0: dependencies: universalify: 2.0.1 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 5a522aff..3d9e4d8d 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -15,6 +15,7 @@ packages: - github - inline - json + - jsonschema - ods - parquet - site diff --git a/site/content/docs/guides/schema.md b/site/content/docs/guides/schema.md index e7c43e04..907f07ef 100644 --- a/site/content/docs/guides/schema.md +++ b/site/content/docs/guides/schema.md @@ -84,6 +84,40 @@ dp schema infer --from-package datapackage.json --from-resource "users" dp schema infer data.csv --json > schema.json ``` +### `dp schema convert` + +Convert table schemas between different formats, supporting bidirectional conversion between Table Schema and JSONSchema formats. + +```bash +dp schema convert +``` + +**Options:** +- `--format `: Source schema format (`schema`, `jsonschema`) +- `--to-format `: Target schema format (`schema`, `jsonschema`) +- `--to-path `: Output path for converted schema +- `-j, --json`: Output as JSON (automatically enabled when no --to-path) +- `-d, --debug`: Enable debug mode + +**Supported Formats:** +- **`schema`**: Data Package Table Schema format +- **`jsonschema`**: JSON Schema format + +**Examples:** +```bash +# Convert Table Schema to JSONSchema +dp schema convert schema.json --to-format jsonschema + +# Convert JSONSchema to Table Schema +dp schema convert schema.jsonschema.json --format jsonschema + +# Save converted schema to file +dp schema convert schema.json --to-format jsonschema --to-path converted.jsonschema.json + +# Convert from JSONSchema and save as Table Schema +dp schema convert input.jsonschema.json --format jsonschema --to-path output.schema.json +``` + ### `dp schema explore` Explore a table schema from a local or remote path to view its field definitions and constraints in an interactive format. @@ -199,6 +233,20 @@ dp> schema.primaryKey dp schema explore schema.json ``` +### Schema Format Conversion + +```bash +# Convert Table Schema to JSONSchema for JSON Schema validation tools +dp schema infer data.csv --json > table.schema.json +dp schema convert table.schema.json --to-format jsonschema --to-path api.jsonschema.json + +# Convert JSONSchema back to Table Schema for dpkit tools +dp schema convert api.jsonschema.json --format jsonschema --to-path converted.schema.json + +# Validate the round-trip conversion +dp schema validate converted.schema.json +``` + ### Schema Analysis and Refinement ```bash @@ -293,6 +341,23 @@ All schema commands support multiple output formats: - **JSON**: Use `--json` flag for machine-readable output - **Debug Mode**: Use `--debug` for detailed operation logs +## Schema Format Interoperability + +The `convert` command enables seamless integration with other schema ecosystems: + +```bash +# Use with JSON Schema validation libraries +dp schema infer data.csv --json > table.schema.json +dp schema convert table.schema.json --to-format jsonschema --to-path validation.jsonschema.json + +# Import existing JSONSchema into dpkit workflow +dp schema convert external.jsonschema.json --format jsonschema --to-path dpkit.schema.json +dp table validate data.csv --schema dpkit.schema.json + +# Cross-platform schema sharing +dp schema convert schema.json --to-format jsonschema --to-path api-spec.jsonschema.json +``` + ## Integration with Other Commands Schema commands work seamlessly with other dpkit commands: