From 13ddf7689e67880fc40f12fd1ebfc03bad6e8258 Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 30 Oct 2025 10:03:17 +0000 Subject: [PATCH 01/11] Fixed validate type --- table/field/validate.ts | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/table/field/validate.ts b/table/field/validate.ts index 963c02b1..144f9bd7 100644 --- a/table/field/validate.ts +++ b/table/field/validate.ts @@ -54,9 +54,13 @@ function validateName(mapping: FieldMapping) { function validateType(mapping: FieldMapping) { const errors: FieldError[] = [] + const variant = mapping.source.type.variant + // TODO: Rebase on proper polars type definition when available + // https://github.com/pola-rs/nodejs-polars/issues/372 const compatMapping: Record = { Bool: ["boolean"], + Categorical: ["string"], Date: ["date"], Datetime: ["datetime"], Float32: ["number", "integer"], @@ -66,18 +70,21 @@ function validateType(mapping: FieldMapping) { Int64: ["integer"], Int8: ["integer"], List: ["list"], + String: ["any"], Time: ["time"], UInt16: ["integer"], UInt32: ["integer"], UInt64: ["integer"], UInt8: ["integer"], - Utf8: ["string"], + Utf8: ["any"], } - const compatTypes = compatMapping[mapping.source.type.variant] - if (!compatTypes) return errors + const compatTypes = compatMapping[variant] ?? [] + const isCompatible = !!new Set(compatTypes).intersection( + new Set([mapping.target.type, "any"]), + ).size - if (!compatTypes.includes(mapping.target.type)) { + if (!isCompatible) { errors.push({ type: "field/type", fieldName: mapping.target.name, From 25939a1f220e45f4c0e73731867973d5bf91d9ec Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 30 Oct 2025 10:29:31 +0000 Subject: [PATCH 02/11] Removed array/geojson/object parse/stringify --- table/field/parse.ts | 9 --- table/field/stringify.ts | 9 --- table/field/types/array.spec.ts | 71 ------------------------ table/field/types/array.ts | 20 ++----- table/field/types/geojson.spec.ts | 91 ------------------------------- table/field/types/geojson.ts | 19 ++----- table/field/types/object.spec.ts | 74 ------------------------- table/field/types/object.ts | 17 +----- 8 files changed, 11 insertions(+), 299 deletions(-) diff --git a/table/field/parse.ts b/table/field/parse.ts index 896417fa..c59042b6 100644 --- a/table/field/parse.ts +++ b/table/field/parse.ts @@ -1,17 +1,14 @@ import type { Expr } from "nodejs-polars" import { DataType } from "nodejs-polars" import type { FieldMapping } from "./Mapping.ts" -import { parseArrayField } from "./types/array.ts" import { parseBooleanField } from "./types/boolean.ts" import { parseDateField } from "./types/date.ts" import { parseDatetimeField } from "./types/datetime.ts" import { parseDurationField } from "./types/duration.ts" -import { parseGeojsonField } from "./types/geojson.ts" import { parseGeopointField } from "./types/geopoint.ts" import { parseIntegerField } from "./types/integer.ts" import { parseListField } from "./types/list.ts" import { parseNumberField } from "./types/number.ts" -import { parseObjectField } from "./types/object.ts" import { parseStringField } from "./types/string.ts" import { parseTimeField } from "./types/time.ts" import { parseYearField } from "./types/year.ts" @@ -22,8 +19,6 @@ export function parseField(mapping: FieldMapping, fieldExpr: Expr) { const field = mapping.target switch (field.type) { - case "array": - return parseArrayField(field, fieldExpr) case "boolean": return parseBooleanField(field, fieldExpr) case "date": @@ -32,8 +27,6 @@ export function parseField(mapping: FieldMapping, fieldExpr: Expr) { return parseDatetimeField(field, fieldExpr) case "duration": return parseDurationField(field, fieldExpr) - case "geojson": - return parseGeojsonField(field, fieldExpr) case "geopoint": return parseGeopointField(field, fieldExpr) case "integer": @@ -42,8 +35,6 @@ export function parseField(mapping: FieldMapping, fieldExpr: Expr) { return parseListField(field, fieldExpr) case "number": return parseNumberField(field, fieldExpr) - case "object": - return parseObjectField(field, fieldExpr) case "string": return parseStringField(field, fieldExpr) case "time": diff --git a/table/field/stringify.ts b/table/field/stringify.ts index f03b4dce..185410a3 100644 --- a/table/field/stringify.ts +++ b/table/field/stringify.ts @@ -1,16 +1,13 @@ import type { Field } from "@dpkit/core" import type { Expr } from "nodejs-polars" -import { stringifyArrayField } from "./types/array.ts" import { stringifyBooleanField } from "./types/boolean.ts" import { stringifyDateField } from "./types/date.ts" import { stringifyDatetimeField } from "./types/datetime.ts" import { stringifyDurationField } from "./types/duration.ts" -import { stringifyGeojsonField } from "./types/geojson.ts" import { stringifyGeopointField } from "./types/geopoint.ts" import { stringifyIntegerField } from "./types/integer.ts" import { stringifyListField } from "./types/list.ts" import { stringifyNumberField } from "./types/number.ts" -import { stringifyObjectField } from "./types/object.ts" import { stringifyStringField } from "./types/string.ts" import { stringifyTimeField } from "./types/time.ts" import { stringifyYearField } from "./types/year.ts" @@ -18,8 +15,6 @@ import { stringifyYearmonthField } from "./types/yearmonth.ts" export function stringifyField(field: Field, fieldExpr: Expr) { switch (field.type) { - case "array": - return stringifyArrayField(field, fieldExpr) case "boolean": return stringifyBooleanField(field, fieldExpr) case "date": @@ -28,8 +23,6 @@ export function stringifyField(field: Field, fieldExpr: Expr) { return stringifyDatetimeField(field, fieldExpr) case "duration": return stringifyDurationField(field, fieldExpr) - case "geojson": - return stringifyGeojsonField(field, fieldExpr) case "geopoint": return stringifyGeopointField(field, fieldExpr) case "integer": @@ -38,8 +31,6 @@ export function stringifyField(field: Field, fieldExpr: Expr) { return stringifyListField(field, fieldExpr) case "number": return stringifyNumberField(field, fieldExpr) - case "object": - return stringifyObjectField(field, fieldExpr) case "string": return stringifyStringField(field, fieldExpr) case "time": diff --git a/table/field/types/array.spec.ts b/table/field/types/array.spec.ts index 01d2ec1f..e69de29b 100644 --- a/table/field/types/array.spec.ts +++ b/table/field/types/array.spec.ts @@ -1,71 +0,0 @@ -import { DataFrame, DataType, Series } from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { denormalizeTable, normalizeTable } from "../../table/index.ts" - -describe("parseArrayField", () => { - it.each([ - // Valid JSON arrays - ["[1,2,3]", [1, 2, 3]], - ['["a","b","c"]', ["a", "b", "c"]], - ['[{"name":"John"},{"name":"Jane"}]', [{ name: "John" }, { name: "Jane" }]], - ["[]", []], - - // JSON but not an array - ['{"name":"John"}', null], - ["{}", null], - - // Trimming whitespace - //[" [1,2,3] ", [1, 2, 3]], - //['\t["a","b","c"]\n', ["a", "b", "c"]], - - // Invalid JSON - skip test that's causing issues - // ["[invalid]", null], - ["not json", null], - ["", null], - ["null", null], - ["undefined", null], - ])("%s -> %s", async (cell, value) => { - const table = DataFrame([Series("name", [cell], DataType.String)]).lazy() - - const schema = { - fields: [{ name: "name", type: "array" as const }], - } - - const ldf = await normalizeTable(table, schema) - const df = await ldf.collect() - const res = df.getColumn("name").get(0) - expect(res ? JSON.parse(res) : res).toEqual(value) - }) -}) - -describe("stringifyArrayField", () => { - it.each([ - // JSON array strings should be returned as-is - ["[1,2,3]", "[1,2,3]"], - ['["a","b","c"]', '["a","b","c"]'], - ['[{"name":"John"},{"name":"Jane"}]', '[{"name":"John"},{"name":"Jane"}]'], - ["[]", "[]"], - - // Complex nested arrays - [ - '[{"users":[{"id":1,"name":"Alice"}],"count":2}]', - '[{"users":[{"id":1,"name":"Alice"}],"count":2}]', - ], - ["[[1,2],[3,4]]", "[[1,2],[3,4]]"], - ['[true,false,null,"text",42,3.14]', '[true,false,null,"text",42,3.14]'], - - // Null handling - [null, ""], - ])("%s -> %s", async (value, expected) => { - const table = DataFrame([Series("name", [value], DataType.String)]).lazy() - - const schema = { - fields: [{ name: "name", type: "array" as const }], - } - - const ldf = await denormalizeTable(table, schema) - const df = await ldf.collect() - - expect(df.toRecords()[0]?.name).toEqual(expected) - }) -}) diff --git a/table/field/types/array.ts b/table/field/types/array.ts index f3c7ddb3..fcefba7a 100644 --- a/table/field/types/array.ts +++ b/table/field/types/array.ts @@ -1,17 +1,5 @@ -import type { ArrayField } from "@dpkit/core" -import { lit, when } from "nodejs-polars" -import type { Expr } from "nodejs-polars" +import type { ObjectField } from "@dpkit/core" +import type { Table } from "../../table/index.ts" -// TODO: Is there a better way to do this? -// Polars does not support really support free-form JSON -// So we just make a basic check and return as it is -export function parseArrayField(_field: ArrayField, fieldExpr: Expr) { - return when(fieldExpr.str.contains("^\\[")) - .then(fieldExpr) - .otherwise(lit(null)) -} - -export function stringifyArrayField(_field: ArrayField, fieldExpr: Expr) { - // TODO: implement - return fieldExpr -} +// @ts-ignore +export async function validateArrayField(field: ObjectField, table: Table) {} diff --git a/table/field/types/geojson.spec.ts b/table/field/types/geojson.spec.ts index 93da3adb..e69de29b 100644 --- a/table/field/types/geojson.spec.ts +++ b/table/field/types/geojson.spec.ts @@ -1,91 +0,0 @@ -import { DataFrame, DataType, Series } from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { denormalizeTable, normalizeTable } from "../../table/index.ts" - -describe("parseGeojsonField", () => { - it.each([ - // Valid JSON objects - ['{"name":"John","age":30}', { name: "John", age: 30 }], - ['{"numbers":[1,2,3]}', { numbers: [1, 2, 3] }], - ['{"nested":{"prop":"value"}}', { nested: { prop: "value" } }], - ["{}", {}], - - // JSON but not an object - ["[1,2,3]", null], - ['["a","b","c"]', null], - - // Trimming whitespace - //[' {"name":"John"} ', { name: "John" }], - //['\t{"name":"John"}\n', { name: "John" }], - - // Invalid JSON - //["{invalid}", null], - ["not json", null], - ["", null], - ["null", null], - ["undefined", null], - ])("%s -> %s", async (cell, value) => { - const table = DataFrame([Series("name", [cell], DataType.String)]).lazy() - - const schema = { - fields: [{ name: "name", type: "geojson" as const }], - } - - const ldf = await normalizeTable(table, schema) - const df = await ldf.collect() - - const res = df.getColumn("name").get(0) - expect(res ? JSON.parse(res) : res).toEqual(value) - }) -}) - -describe("stringifyGeojsonField", () => { - it.each([ - // GeoJSON Point - [ - '{"type":"Point","coordinates":[125.6,10.1]}', - '{"type":"Point","coordinates":[125.6,10.1]}', - ], - - // GeoJSON LineString - [ - '{"type":"LineString","coordinates":[[125.6,10.1],[125.7,10.2]]}', - '{"type":"LineString","coordinates":[[125.6,10.1],[125.7,10.2]]}', - ], - - // GeoJSON Polygon - [ - '{"type":"Polygon","coordinates":[[[125.6,10.1],[125.7,10.1],[125.7,10.2],[125.6,10.2],[125.6,10.1]]]}', - '{"type":"Polygon","coordinates":[[[125.6,10.1],[125.7,10.1],[125.7,10.2],[125.6,10.2],[125.6,10.1]]]}', - ], - - // GeoJSON Feature - [ - '{"type":"Feature","geometry":{"type":"Point","coordinates":[125.6,10.1]},"properties":{"name":"Sample Point"}}', - '{"type":"Feature","geometry":{"type":"Point","coordinates":[125.6,10.1]},"properties":{"name":"Sample Point"}}', - ], - - // GeoJSON FeatureCollection - [ - '{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[125.6,10.1]},"properties":{"name":"Point 1"}}]}', - '{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[125.6,10.1]},"properties":{"name":"Point 1"}}]}', - ], - - // Empty GeoJSON object - ["{}", "{}"], - - // Null handling - [null, ""], - ])("%s -> %s", async (value, expected) => { - const table = DataFrame([Series("name", [value], DataType.String)]).lazy() - - const schema = { - fields: [{ name: "name", type: "geojson" as const }], - } - - const ldf = await denormalizeTable(table, schema) - const df = await ldf.collect() - - expect(df.toRecords()[0]?.name).toEqual(expected) - }) -}) diff --git a/table/field/types/geojson.ts b/table/field/types/geojson.ts index da7a404f..9991129b 100644 --- a/table/field/types/geojson.ts +++ b/table/field/types/geojson.ts @@ -1,16 +1,5 @@ -import type { GeojsonField } from "@dpkit/core" -import { lit, when } from "nodejs-polars" -import type { Expr } from "nodejs-polars" +import type { ObjectField } from "@dpkit/core" +import type { Table } from "../../table/index.ts" -// TODO: Is there a better way to do this? -// Polars does not support really support free-form JSON -// So we just make a basic check and return as it is -export function parseGeojsonField(_field: GeojsonField, fieldExpr: Expr) { - return when(fieldExpr.str.contains("^\\{")) - .then(fieldExpr) - .otherwise(lit(null)) -} - -export function stringifyGeojsonField(_field: GeojsonField, fieldExpr: Expr) { - return fieldExpr -} +// @ts-ignore +export async function validateGeojsonField(field: ObjectField, table: Table) {} diff --git a/table/field/types/object.spec.ts b/table/field/types/object.spec.ts index 36cdbf30..e69de29b 100644 --- a/table/field/types/object.spec.ts +++ b/table/field/types/object.spec.ts @@ -1,74 +0,0 @@ -import { DataFrame, DataType, Series } from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { denormalizeTable, normalizeTable } from "../../table/index.ts" - -describe("parseObjectField", () => { - it.each([ - // Valid JSON objects - ['{"name":"John","age":30}', { name: "John", age: 30 }], - ['{"numbers":[1,2,3]}', { numbers: [1, 2, 3] }], - ['{"nested":{"prop":"value"}}', { nested: { prop: "value" } }], - ["{}", {}], - - // JSON but not an object - ["[1,2,3]", null], - ['["a","b","c"]', null], - - // Trimming whitespace - //[' {"name":"John"} ', { name: "John" }], - //['\t{"name":"John"}\n', { name: "John" }], - - // Invalid JSON - //["{invalid}", null], - ["not json", null], - ["", null], - ["null", null], - ["undefined", null], - ])("%s -> %s", async (cell, value) => { - const table = DataFrame([Series("name", [cell], DataType.String)]).lazy() - - const schema = { - fields: [{ name: "name", type: "object" as const }], - } - - const ldf = await normalizeTable(table, schema) - const df = await ldf.collect() - - const res = df.getColumn("name").get(0) - expect(res ? JSON.parse(res) : res).toEqual(value) - }) -}) - -describe("stringifyObjectField", () => { - it.each([ - // JSON strings should be returned as-is - ['{"name":"John","age":30}', '{"name":"John","age":30}'], - ['{"numbers":[1,2,3]}', '{"numbers":[1,2,3]}'], - ['{"nested":{"prop":"value"}}', '{"nested":{"prop":"value"}}'], - ["{}", "{}"], - - // Complex nested objects as JSON strings - [ - '{"users":[{"id":1,"name":"Alice"},{"id":2,"name":"Bob"}]}', - '{"users":[{"id":1,"name":"Alice"},{"id":2,"name":"Bob"}]}', - ], - [ - '{"config":{"debug":true,"timeout":5000}}', - '{"config":{"debug":true,"timeout":5000}}', - ], - - // Null handling - [null, ""], - ])("%s -> %s", async (value, expected) => { - const table = DataFrame([Series("name", [value], DataType.String)]).lazy() - - const schema = { - fields: [{ name: "name", type: "object" as const }], - } - - const ldf = await denormalizeTable(table, schema) - const df = await ldf.collect() - - expect(df.toRecords()[0]?.name).toEqual(expected) - }) -}) diff --git a/table/field/types/object.ts b/table/field/types/object.ts index 02e0c8c5..7801275c 100644 --- a/table/field/types/object.ts +++ b/table/field/types/object.ts @@ -1,16 +1,5 @@ import type { ObjectField } from "@dpkit/core" -import { lit, when } from "nodejs-polars" -import type { Expr } from "nodejs-polars" +import type { Table } from "../../table/index.ts" -// TODO: Is there a better way to do this? -// Polars does not support really support free-form JSON -// So we just make a basic check and return as it is -export function parseObjectField(_field: ObjectField, fieldExpr: Expr) { - return when(fieldExpr.str.contains("^\\{")) - .then(fieldExpr) - .otherwise(lit(null)) -} - -export function stringifyObjectField(_field: ObjectField, fieldExpr: Expr) { - return fieldExpr -} +// @ts-ignore +export async function validateObjectField(field: ObjectField, table: Table) {} From 9aefddb0ab2e6d9e6ba374136c86c36701993af3 Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 30 Oct 2025 11:18:34 +0000 Subject: [PATCH 03/11] Implemented validateObjectField --- table/field/types/object.spec.ts | 209 +++++++++++++++++++++++++++++++ table/field/types/object.ts | 40 +++++- table/field/validate.ts | 7 ++ table/helpers.ts | 4 + 4 files changed, 258 insertions(+), 2 deletions(-) diff --git a/table/field/types/object.spec.ts b/table/field/types/object.spec.ts index e69de29b..558752b5 100644 --- a/table/field/types/object.spec.ts +++ b/table/field/types/object.spec.ts @@ -0,0 +1,209 @@ +import type { Schema } from "@dpkit/core" +import { DataFrame } from "nodejs-polars" +import { describe, expect, it } from "vitest" +import { validateTable } from "../../table/index.ts" + +describe("validateTable (object field)", () => { + it("should not report errors for valid JSON objects", async () => { + const table = DataFrame({ + metadata: ['{"key":"value"}', '{"num":123}', '{"arr":[1,2,3]}'], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "metadata", + type: "object", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(0) + }) + + it("should report errors for JSON arrays", async () => { + const table = DataFrame({ + data: ['[1,2,3]', '{"key":"value"}', '["a","b","c"]'], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "data", + type: "object", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toEqual([ + { + type: "cell/type", + fieldName: "data", + fieldType: "object", + rowNumber: 1, + cell: "[1,2,3]", + }, + { + type: "cell/type", + fieldName: "data", + fieldType: "object", + rowNumber: 3, + cell: '["a","b","c"]', + }, + ]) + }) + + it("should not report errors for null values", async () => { + const table = DataFrame({ + config: ['{"key":"value"}', null, '{"num":123}'], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "config", + type: "object", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(0) + }) + + it("should report errors for invalid JSON", async () => { + const table = DataFrame({ + data: ['{"valid":true}', 'invalid json', '{"key":"value"}', '{broken}'], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "data", + type: "object", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors.filter(e => e.type === "cell/type")).toHaveLength(2) + expect(errors).toContainEqual({ + type: "cell/type", + fieldName: "data", + fieldType: "object", + rowNumber: 2, + cell: "invalid json", + }) + expect(errors).toContainEqual({ + type: "cell/type", + fieldName: "data", + fieldType: "object", + rowNumber: 4, + cell: "{broken}", + }) + }) + + it("should handle complex nested JSON structures", async () => { + const table = DataFrame({ + complex: [ + '{"user":{"name":"John","age":30,"tags":["admin","user"]}}', + '{"nested":{"deep":{"value":true}}}', + '{"array":[{"id":1},{"id":2}]}', + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "complex", + type: "object", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(0) + }) + + it("should report errors for empty strings", async () => { + const table = DataFrame({ + data: ['{"valid":true}', "", '{"key":"value"}'], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "data", + type: "object", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toEqual([ + { + type: "cell/type", + fieldName: "data", + fieldType: "object", + rowNumber: 2, + cell: "", + }, + ]) + }) + + it("should report errors for JSON primitives", async () => { + const table = DataFrame({ + data: ['"string"', "123", "true", "false", "null"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "data", + type: "object", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toEqual([ + { + type: "cell/type", + fieldName: "data", + fieldType: "object", + rowNumber: 1, + cell: '"string"', + }, + { + type: "cell/type", + fieldName: "data", + fieldType: "object", + rowNumber: 2, + cell: "123", + }, + { + type: "cell/type", + fieldName: "data", + fieldType: "object", + rowNumber: 3, + cell: "true", + }, + { + type: "cell/type", + fieldName: "data", + fieldType: "object", + rowNumber: 4, + cell: "false", + }, + { + type: "cell/type", + fieldName: "data", + fieldType: "object", + rowNumber: 5, + cell: "null", + }, + ]) + }) +}) diff --git a/table/field/types/object.ts b/table/field/types/object.ts index 7801275c..2cfff6ad 100644 --- a/table/field/types/object.ts +++ b/table/field/types/object.ts @@ -1,5 +1,41 @@ import type { ObjectField } from "@dpkit/core" +import * as pl from "nodejs-polars" +import type { CellTypeError } from "../../error/index.ts" +import { isObject } from "../../helpers.ts" import type { Table } from "../../table/index.ts" -// @ts-ignore -export async function validateObjectField(field: ObjectField, table: Table) {} +// TODO: Improve the implementation +// Make unblocking / handle large data / process in parallel / move processing to Rust? + +export async function validateObjectField(field: ObjectField, table: Table) { + const errors: CellTypeError[] = [] + + const frame = await table + .withRowCount() + .select( + pl.col("row_nr").add(1).alias("number"), + pl.col(field.name).alias("source"), + ) + .collect() + + for (const row of frame.toRecords() as any[]) { + if (row.source === null) continue + let target: Record | undefined + + try { + target = JSON.parse(row.source) + } catch (error) {} + + if (!target || !isObject(target)) { + errors.push({ + type: "cell/type", + cell: String(row.source), + fieldName: field.name, + fieldType: "object", + rowNumber: row.number, + }) + } + } + + return errors +} diff --git a/table/field/validate.ts b/table/field/validate.ts index 144f9bd7..27894a20 100644 --- a/table/field/validate.ts +++ b/table/field/validate.ts @@ -13,6 +13,7 @@ import { checkCellRequired } from "./checks/required.ts" import { checkCellType } from "./checks/type.ts" import { checkCellUnique } from "./checks/unique.ts" import { normalizeField } from "./normalize.ts" +import { validateObjectField } from "./types/object.ts" export async function validateField( mapping: FieldMapping, @@ -106,6 +107,12 @@ async function validateCells( const { maxErrors } = options const errors: CellError[] = [] + // Types that require non-polars validation + switch (mapping.target.type) { + case "object": + return await validateObjectField(mapping.target, table) + } + let fieldCheckTable = table .withRowCount() .select( diff --git a/table/helpers.ts b/table/helpers.ts index 032b745c..f5c85729 100644 --- a/table/helpers.ts +++ b/table/helpers.ts @@ -1,6 +1,10 @@ import type { Expr } from "nodejs-polars" import * as pl from "nodejs-polars" +export function isObject(value: any): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value) +} + export function arrayDiff(a: string[], b: string[]) { return a.filter(x => !b.includes(x)) } From df962cbee8f606039502f1b1621055b56987a485 Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 30 Oct 2025 11:37:03 +0000 Subject: [PATCH 04/11] Implemented jsonSchema constraint --- table/error/Cell.ts | 6 + table/field/types/object.spec.ts | 182 +++++++++++++++++++++++++++++++ table/field/types/object.ts | 22 +++- 3 files changed, 208 insertions(+), 2 deletions(-) diff --git a/table/error/Cell.ts b/table/error/Cell.ts index 09bc7412..ae179dde 100644 --- a/table/error/Cell.ts +++ b/table/error/Cell.ts @@ -13,6 +13,7 @@ export type CellError = | CellPatternError | CellUniqueError | CellEnumError + | CellJsonSchemaError export interface BaseCellError extends BaseTableError { fieldName: string @@ -73,3 +74,8 @@ export interface CellEnumError extends BaseCellError { type: "cell/enum" enum: string[] } + +export interface CellJsonSchemaError extends BaseCellError { + type: "cell/jsonSchema" + jsonSchema: Record +} diff --git a/table/field/types/object.spec.ts b/table/field/types/object.spec.ts index 558752b5..ff5bd203 100644 --- a/table/field/types/object.spec.ts +++ b/table/field/types/object.spec.ts @@ -206,4 +206,186 @@ describe("validateTable (object field)", () => { }, ]) }) + + it("should not report errors for objects matching jsonSchema", async () => { + const table = DataFrame({ + user: [ + '{"name":"John","age":30}', + '{"name":"Jane","age":25}', + '{"name":"Bob","age":35}', + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "user", + type: "object", + constraints: { + jsonSchema: { + type: "object", + properties: { + name: { type: "string" }, + age: { type: "number" }, + }, + required: ["name", "age"], + }, + }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(0) + }) + + it("should report errors for objects not matching jsonSchema", async () => { + const jsonSchema = { + type: "object", + properties: { + name: { type: "string" }, + age: { type: "number" }, + }, + required: ["name", "age"], + } + + const table = DataFrame({ + user: [ + '{"name":"John","age":30}', + '{"name":"Jane"}', + '{"age":25}', + '{"name":"Bob","age":"invalid"}', + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "user", + type: "object", + constraints: { + jsonSchema, + }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors.filter(e => e.type === "cell/jsonSchema")).toHaveLength(3) + expect(errors).toContainEqual({ + type: "cell/jsonSchema", + fieldName: "user", + jsonSchema, + rowNumber: 2, + cell: '{"name":"Jane"}', + }) + expect(errors).toContainEqual({ + type: "cell/jsonSchema", + fieldName: "user", + jsonSchema, + rowNumber: 3, + cell: '{"age":25}', + }) + expect(errors).toContainEqual({ + type: "cell/jsonSchema", + fieldName: "user", + jsonSchema, + rowNumber: 4, + cell: '{"name":"Bob","age":"invalid"}', + }) + }) + + it("should validate complex jsonSchema with nested objects", async () => { + const table = DataFrame({ + config: [ + '{"database":{"host":"localhost","port":5432},"cache":{"enabled":true}}', + '{"database":{"host":"localhost","port":"invalid"},"cache":{"enabled":true}}', + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "config", + type: "object", + constraints: { + jsonSchema: { + type: "object", + properties: { + database: { + type: "object", + properties: { + host: { type: "string" }, + port: { type: "number" }, + }, + required: ["host", "port"], + }, + cache: { + type: "object", + properties: { + enabled: { type: "boolean" }, + }, + required: ["enabled"], + }, + }, + required: ["database", "cache"], + }, + }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toEqual([ + { + type: "cell/jsonSchema", + fieldName: "config", + jsonSchema: schema.fields[0].constraints?.jsonSchema, + rowNumber: 2, + cell: '{"database":{"host":"localhost","port":"invalid"},"cache":{"enabled":true}}', + }, + ]) + }) + + it("should validate jsonSchema with array properties", async () => { + const table = DataFrame({ + data: [ + '{"items":[1,2,3],"name":"test"}', + '{"items":["not","numbers"],"name":"test"}', + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "data", + type: "object", + constraints: { + jsonSchema: { + type: "object", + properties: { + items: { + type: "array", + items: { type: "number" }, + }, + name: { type: "string" }, + }, + required: ["items", "name"], + }, + }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toEqual([ + { + type: "cell/jsonSchema", + fieldName: "data", + jsonSchema: schema.fields[0].constraints?.jsonSchema, + rowNumber: 2, + cell: '{"items":["not","numbers"],"name":"test"}', + }, + ]) + }) }) diff --git a/table/field/types/object.ts b/table/field/types/object.ts index 2cfff6ad..49b93131 100644 --- a/table/field/types/object.ts +++ b/table/field/types/object.ts @@ -1,6 +1,7 @@ import type { ObjectField } from "@dpkit/core" +import { validateDescriptor } from "@dpkit/core" import * as pl from "nodejs-polars" -import type { CellTypeError } from "../../error/index.ts" +import type { CellError } from "../../error/index.ts" import { isObject } from "../../helpers.ts" import type { Table } from "../../table/index.ts" @@ -8,7 +9,8 @@ import type { Table } from "../../table/index.ts" // Make unblocking / handle large data / process in parallel / move processing to Rust? export async function validateObjectField(field: ObjectField, table: Table) { - const errors: CellTypeError[] = [] + const errors: CellError[] = [] + const jsonSchema = field.constraints?.jsonSchema const frame = await table .withRowCount() @@ -34,6 +36,22 @@ export async function validateObjectField(field: ObjectField, table: Table) { fieldType: "object", rowNumber: row.number, }) + + continue + } + + if (jsonSchema) { + const report = await validateDescriptor(target, { profile: jsonSchema }) + + if (!report.valid) { + errors.push({ + type: "cell/jsonSchema", + cell: String(row.source), + fieldName: field.name, + rowNumber: row.number, + jsonSchema, + }) + } } } From ecdfc425abcdab51889f1101286cbcbc4e7005ee Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 30 Oct 2025 11:46:54 +0000 Subject: [PATCH 05/11] Extracted validateJsonField --- core/field/types/Geojson.ts | 5 +++ table/field/types/json.ts | 66 +++++++++++++++++++++++++++++++++++++ table/field/types/object.ts | 56 ++----------------------------- table/field/validate.ts | 4 +-- 4 files changed, 75 insertions(+), 56 deletions(-) create mode 100644 table/field/types/json.ts diff --git a/core/field/types/Geojson.ts b/core/field/types/Geojson.ts index 1151dea0..2da68193 100644 --- a/core/field/types/Geojson.ts +++ b/core/field/types/Geojson.ts @@ -21,6 +21,11 @@ export interface GeojsonField extends BaseField { * GeoJSON-specific constraints */ export interface GeojsonConstraints extends BaseConstraints { + /** + * JSON Schema object for validating the object structure and properties + */ + jsonSchema?: Record + /** * Restrict values to a specified set of GeoJSON objects * Serialized as strings or GeoJSON object literals diff --git a/table/field/types/json.ts b/table/field/types/json.ts new file mode 100644 index 00000000..edc6f34a --- /dev/null +++ b/table/field/types/json.ts @@ -0,0 +1,66 @@ +import type { ArrayField, GeojsonField, ObjectField } from "@dpkit/core" +import { validateDescriptor } from "@dpkit/core" +import * as pl from "nodejs-polars" +import type { CellError } from "../../error/index.ts" +import { isObject } from "../../helpers.ts" +import type { Table } from "../../table/index.ts" + +// TODO: Improve the implementation +// Make unblocking / handle large data / process in parallel / move processing to Rust? + +export async function validateJsonField( + field: ArrayField | GeojsonField | ObjectField, + table: Table, +) { + const errors: CellError[] = [] + const jsonSchema = field.constraints?.jsonSchema + + const frame = await table + .withRowCount() + .select( + pl.col("row_nr").add(1).alias("number"), + pl.col(field.name).alias("source"), + ) + .collect() + + for (const row of frame.toRecords() as any[]) { + if (row.source === null) continue + + let target: Record | undefined + const checkCompat = field.type === "array" ? Array.isArray : isObject + + try { + target = JSON.parse(row.source) + } catch (error) {} + + if (!target || !checkCompat(target)) { + errors.push({ + type: "cell/type", + cell: String(row.source), + fieldName: field.name, + fieldType: "object", + rowNumber: row.number, + }) + + continue + } + + if (jsonSchema) { + // TODO: Extract more generic function validateJson? + // @ts-ignore + const report = await validateDescriptor(target, { profile: jsonSchema }) + + if (!report.valid) { + errors.push({ + type: "cell/jsonSchema", + cell: String(row.source), + fieldName: field.name, + rowNumber: row.number, + jsonSchema, + }) + } + } + } + + return errors +} diff --git a/table/field/types/object.ts b/table/field/types/object.ts index 49b93131..312f8240 100644 --- a/table/field/types/object.ts +++ b/table/field/types/object.ts @@ -1,59 +1,7 @@ import type { ObjectField } from "@dpkit/core" -import { validateDescriptor } from "@dpkit/core" -import * as pl from "nodejs-polars" -import type { CellError } from "../../error/index.ts" -import { isObject } from "../../helpers.ts" import type { Table } from "../../table/index.ts" - -// TODO: Improve the implementation -// Make unblocking / handle large data / process in parallel / move processing to Rust? +import { validateJsonField } from "./json.ts" export async function validateObjectField(field: ObjectField, table: Table) { - const errors: CellError[] = [] - const jsonSchema = field.constraints?.jsonSchema - - const frame = await table - .withRowCount() - .select( - pl.col("row_nr").add(1).alias("number"), - pl.col(field.name).alias("source"), - ) - .collect() - - for (const row of frame.toRecords() as any[]) { - if (row.source === null) continue - let target: Record | undefined - - try { - target = JSON.parse(row.source) - } catch (error) {} - - if (!target || !isObject(target)) { - errors.push({ - type: "cell/type", - cell: String(row.source), - fieldName: field.name, - fieldType: "object", - rowNumber: row.number, - }) - - continue - } - - if (jsonSchema) { - const report = await validateDescriptor(target, { profile: jsonSchema }) - - if (!report.valid) { - errors.push({ - type: "cell/jsonSchema", - cell: String(row.source), - fieldName: field.name, - rowNumber: row.number, - jsonSchema, - }) - } - } - } - - return errors + return validateJsonField(field, table) } diff --git a/table/field/validate.ts b/table/field/validate.ts index 27894a20..db4ca9c4 100644 --- a/table/field/validate.ts +++ b/table/field/validate.ts @@ -81,11 +81,11 @@ function validateType(mapping: FieldMapping) { } const compatTypes = compatMapping[variant] ?? [] - const isCompatible = !!new Set(compatTypes).intersection( + const isCompat = !!new Set(compatTypes).intersection( new Set([mapping.target.type, "any"]), ).size - if (!isCompatible) { + if (!isCompat) { errors.push({ type: "field/type", fieldName: mapping.target.name, From a5fe754e6e1697dcf27b6428915a92c34d279b0b Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 30 Oct 2025 11:48:09 +0000 Subject: [PATCH 06/11] Implemented array validation --- table/field/types/array.spec.ts | 360 +++++++++++++++++++++++++++++++ table/field/types/array.ts | 8 +- table/field/types/object.spec.ts | 6 +- table/field/validate.ts | 3 + 4 files changed, 371 insertions(+), 6 deletions(-) diff --git a/table/field/types/array.spec.ts b/table/field/types/array.spec.ts index e69de29b..6b7bd07d 100644 --- a/table/field/types/array.spec.ts +++ b/table/field/types/array.spec.ts @@ -0,0 +1,360 @@ +import type { Schema } from "@dpkit/core" +import { DataFrame } from "nodejs-polars" +import { describe, expect, it } from "vitest" +import { validateTable } from "../../table/index.ts" + +describe("validateArrayField", () => { + it("should not report errors for valid JSON arrays", async () => { + const table = DataFrame({ + tags: ['["tag1","tag2"]', "[1,2,3]", '["a","b","c"]'], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "tags", + type: "array", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(0) + }) + + it("should not report errors for empty arrays", async () => { + const table = DataFrame({ + items: ["[]", "[]", "[]"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "items", + type: "array", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(0) + }) + + it("should not report errors for null values", async () => { + const table = DataFrame({ + data: ['["value"]', null, "[1,2,3]"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "data", + type: "array", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(0) + }) + + it("should report errors for JSON objects", async () => { + const table = DataFrame({ + data: ["[1,2,3]", '{"key":"value"}', '["a","b"]'], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "data", + type: "array", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toEqual([ + { + type: "cell/type", + fieldName: "data", + fieldType: "object", + rowNumber: 2, + cell: '{"key":"value"}', + }, + ]) + }) + + it("should report errors for invalid JSON", async () => { + const table = DataFrame({ + data: ['["valid"]', "invalid json", "[1,2,3]", "[broken"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "data", + type: "array", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors.filter(e => e.type === "cell/type")).toHaveLength(2) + expect(errors).toContainEqual({ + type: "cell/type", + fieldName: "data", + fieldType: "object", + rowNumber: 2, + cell: "invalid json", + }) + expect(errors).toContainEqual({ + type: "cell/type", + fieldName: "data", + fieldType: "object", + rowNumber: 4, + cell: "[broken", + }) + }) + + it("should handle nested arrays", async () => { + const table = DataFrame({ + matrix: ["[[1,2],[3,4]]", "[[5,6],[7,8]]", '[["a","b"],["c","d"]]'], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "matrix", + type: "array", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(0) + }) + + it("should report errors for empty strings", async () => { + const table = DataFrame({ + data: ['["valid"]', "", "[1,2,3]"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "data", + type: "array", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toEqual([ + { + type: "cell/type", + fieldName: "data", + fieldType: "object", + rowNumber: 2, + cell: "", + }, + ]) + }) + + it("should report errors for JSON primitives", async () => { + const table = DataFrame({ + data: ['"string"', "123", "true", "false", "null"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "data", + type: "array", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toEqual([ + { + type: "cell/type", + fieldName: "data", + fieldType: "object", + rowNumber: 1, + cell: '"string"', + }, + { + type: "cell/type", + fieldName: "data", + fieldType: "object", + rowNumber: 2, + cell: "123", + }, + { + type: "cell/type", + fieldName: "data", + fieldType: "object", + rowNumber: 3, + cell: "true", + }, + { + type: "cell/type", + fieldName: "data", + fieldType: "object", + rowNumber: 4, + cell: "false", + }, + { + type: "cell/type", + fieldName: "data", + fieldType: "object", + rowNumber: 5, + cell: "null", + }, + ]) + }) + + it("should not report errors for arrays matching jsonSchema", async () => { + const table = DataFrame({ + scores: ["[80,90,100]", "[75,85,95]", "[90,95,100]"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "scores", + type: "array", + constraints: { + jsonSchema: { + type: "array", + items: { type: "number" }, + minItems: 3, + maxItems: 3, + }, + }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(0) + }) + + it("should report errors for arrays not matching jsonSchema", async () => { + const jsonSchema = { + type: "array", + items: { type: "number" }, + minItems: 2, + } + + const table = DataFrame({ + numbers: ["[1,2,3]", '["not","numbers"]', "[1]", "[4,5,6]"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "numbers", + type: "array", + constraints: { + jsonSchema, + }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors.filter(e => e.type === "cell/jsonSchema")).toHaveLength(2) + expect(errors).toContainEqual({ + type: "cell/jsonSchema", + fieldName: "numbers", + jsonSchema, + rowNumber: 2, + cell: '["not","numbers"]', + }) + expect(errors).toContainEqual({ + type: "cell/jsonSchema", + fieldName: "numbers", + jsonSchema, + rowNumber: 3, + cell: "[1]", + }) + }) + + it("should validate complex jsonSchema with array of objects", async () => { + const table = DataFrame({ + users: [ + '[{"name":"John","age":30},{"name":"Jane","age":25}]', + '[{"name":"Bob","age":"invalid"}]', + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "users", + type: "array", + constraints: { + jsonSchema: { + type: "array", + items: { + type: "object", + properties: { + name: { type: "string" }, + age: { type: "number" }, + }, + required: ["name", "age"], + }, + }, + }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toEqual([ + { + type: "cell/jsonSchema", + fieldName: "users", + // @ts-ignore + jsonSchema: schema.fields[0].constraints?.jsonSchema, + rowNumber: 2, + cell: '[{"name":"Bob","age":"invalid"}]', + }, + ]) + }) + + it("should validate jsonSchema with unique items constraint", async () => { + const table = DataFrame({ + tags: ['["unique","values"]', '["duplicate","duplicate"]'], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "tags", + type: "array", + constraints: { + jsonSchema: { + type: "array", + items: { type: "string" }, + uniqueItems: true, + }, + }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toEqual([ + { + type: "cell/jsonSchema", + fieldName: "tags", + // @ts-ignore + jsonSchema: schema.fields[0].constraints?.jsonSchema, + rowNumber: 2, + cell: '["duplicate","duplicate"]', + }, + ]) + }) +}) diff --git a/table/field/types/array.ts b/table/field/types/array.ts index fcefba7a..76d672f0 100644 --- a/table/field/types/array.ts +++ b/table/field/types/array.ts @@ -1,5 +1,7 @@ -import type { ObjectField } from "@dpkit/core" +import type { ArrayField } from "@dpkit/core" import type { Table } from "../../table/index.ts" +import { validateJsonField } from "./json.ts" -// @ts-ignore -export async function validateArrayField(field: ObjectField, table: Table) {} +export async function validateArrayField(field: ArrayField, table: Table) { + return validateJsonField(field, table) +} diff --git a/table/field/types/object.spec.ts b/table/field/types/object.spec.ts index ff5bd203..aa302b1c 100644 --- a/table/field/types/object.spec.ts +++ b/table/field/types/object.spec.ts @@ -3,7 +3,7 @@ import { DataFrame } from "nodejs-polars" import { describe, expect, it } from "vitest" import { validateTable } from "../../table/index.ts" -describe("validateTable (object field)", () => { +describe("validateObjectField", () => { it("should not report errors for valid JSON objects", async () => { const table = DataFrame({ metadata: ['{"key":"value"}', '{"num":123}', '{"arr":[1,2,3]}'], @@ -24,7 +24,7 @@ describe("validateTable (object field)", () => { it("should report errors for JSON arrays", async () => { const table = DataFrame({ - data: ['[1,2,3]', '{"key":"value"}', '["a","b","c"]'], + data: ["[1,2,3]", '{"key":"value"}', '["a","b","c"]'], }).lazy() const schema: Schema = { @@ -75,7 +75,7 @@ describe("validateTable (object field)", () => { it("should report errors for invalid JSON", async () => { const table = DataFrame({ - data: ['{"valid":true}', 'invalid json', '{"key":"value"}', '{broken}'], + data: ['{"valid":true}', "invalid json", '{"key":"value"}', "{broken}"], }).lazy() const schema: Schema = { diff --git a/table/field/validate.ts b/table/field/validate.ts index db4ca9c4..dee798dd 100644 --- a/table/field/validate.ts +++ b/table/field/validate.ts @@ -13,6 +13,7 @@ import { checkCellRequired } from "./checks/required.ts" import { checkCellType } from "./checks/type.ts" import { checkCellUnique } from "./checks/unique.ts" import { normalizeField } from "./normalize.ts" +import { validateArrayField } from "./types/array.ts" import { validateObjectField } from "./types/object.ts" export async function validateField( @@ -109,6 +110,8 @@ async function validateCells( // Types that require non-polars validation switch (mapping.target.type) { + case "array": + return await validateArrayField(mapping.target, table) case "object": return await validateObjectField(mapping.target, table) } From 5f79f67ddd7e64de8608862ad9e298755e0a37b4 Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 30 Oct 2025 11:58:57 +0000 Subject: [PATCH 07/11] Implemented geojson validation --- table/field/types/geojson.spec.ts | 463 ++++++++++++++++++++++++++++++ table/field/types/geojson.ts | 8 +- table/field/validate.ts | 3 + 3 files changed, 471 insertions(+), 3 deletions(-) diff --git a/table/field/types/geojson.spec.ts b/table/field/types/geojson.spec.ts index e69de29b..04bcd4a0 100644 --- a/table/field/types/geojson.spec.ts +++ b/table/field/types/geojson.spec.ts @@ -0,0 +1,463 @@ +import type { Schema } from "@dpkit/core" +import { DataFrame } from "nodejs-polars" +import { describe, expect, it } from "vitest" +import { validateTable } from "../../table/index.ts" + +describe("validateGeojsonField", () => { + it("should not report errors for valid GeoJSON Point", async () => { + const table = DataFrame({ + location: [ + '{"type":"Point","coordinates":[0,0]}', + '{"type":"Point","coordinates":[12.5,41.9]}', + '{"type":"Point","coordinates":[-73.9,40.7]}', + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "location", + type: "geojson", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(0) + }) + + it("should not report errors for valid GeoJSON geometries", async () => { + const table = DataFrame({ + geometry: [ + '{"type":"LineString","coordinates":[[0,0],[1,1]]}', + '{"type":"Polygon","coordinates":[[[0,0],[1,0],[1,1],[0,1],[0,0]]]}', + '{"type":"MultiPoint","coordinates":[[0,0],[1,1]]}', + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "geometry", + type: "geojson", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(0) + }) + + it("should not report errors for valid GeoJSON Feature", async () => { + const table = DataFrame({ + feature: [ + '{"type":"Feature","geometry":{"type":"Point","coordinates":[0,0]},"properties":{"name":"Test"}}', + '{"type":"Feature","geometry":{"type":"LineString","coordinates":[[0,0],[1,1]]},"properties":{"id":1}}', + '{"type":"Feature","geometry":null,"properties":{}}', + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "feature", + type: "geojson", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(0) + }) + + it("should not report errors for valid GeoJSON FeatureCollection", async () => { + const table = DataFrame({ + collection: [ + '{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[0,0]},"properties":{}}]}', + '{"type":"FeatureCollection","features":[]}', + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "collection", + type: "geojson", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(0) + }) + + it("should not report errors for null values", async () => { + const table = DataFrame({ + location: [ + '{"type":"Point","coordinates":[0,0]}', + null, + '{"type":"Feature","geometry":null,"properties":{}}', + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "location", + type: "geojson", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(0) + }) + + it("should report errors for JSON arrays", async () => { + const table = DataFrame({ + data: [ + '{"type":"Point","coordinates":[0,0]}', + "[[0,0],[1,1]]", + '{"type":"Feature","geometry":null,"properties":{}}', + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "data", + type: "geojson", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toEqual([ + { + type: "cell/type", + fieldName: "data", + fieldType: "object", + rowNumber: 2, + cell: "[[0,0],[1,1]]", + }, + ]) + }) + + it("should report errors for invalid JSON", async () => { + const table = DataFrame({ + data: [ + '{"type":"Point","coordinates":[0,0]}', + "invalid json", + '{"type":"Feature"}', + "{broken}", + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "data", + type: "geojson", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors.filter(e => e.type === "cell/type")).toHaveLength(2) + expect(errors).toContainEqual({ + type: "cell/type", + fieldName: "data", + fieldType: "object", + rowNumber: 2, + cell: "invalid json", + }) + expect(errors).toContainEqual({ + type: "cell/type", + fieldName: "data", + fieldType: "object", + rowNumber: 4, + cell: "{broken}", + }) + }) + + it("should report errors for empty strings", async () => { + const table = DataFrame({ + data: [ + '{"type":"Point","coordinates":[0,0]}', + "", + '{"type":"Feature","geometry":null,"properties":{}}', + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "data", + type: "geojson", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toEqual([ + { + type: "cell/type", + fieldName: "data", + fieldType: "object", + rowNumber: 2, + cell: "", + }, + ]) + }) + + it("should report errors for JSON primitives", async () => { + const table = DataFrame({ + data: ['"string"', "123", "true", "false", "null"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "data", + type: "geojson", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toEqual([ + { + type: "cell/type", + fieldName: "data", + fieldType: "object", + rowNumber: 1, + cell: '"string"', + }, + { + type: "cell/type", + fieldName: "data", + fieldType: "object", + rowNumber: 2, + cell: "123", + }, + { + type: "cell/type", + fieldName: "data", + fieldType: "object", + rowNumber: 3, + cell: "true", + }, + { + type: "cell/type", + fieldName: "data", + fieldType: "object", + rowNumber: 4, + cell: "false", + }, + { + type: "cell/type", + fieldName: "data", + fieldType: "object", + rowNumber: 5, + cell: "null", + }, + ]) + }) + + it("should not report errors for GeoJSON matching jsonSchema", async () => { + const table = DataFrame({ + location: [ + '{"type":"Point","coordinates":[0,0]}', + '{"type":"Point","coordinates":[12.5,41.9]}', + '{"type":"Point","coordinates":[-73.9,40.7]}', + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "location", + type: "geojson", + constraints: { + jsonSchema: { + type: "object", + properties: { + type: { type: "string", const: "Point" }, + coordinates: { + type: "array", + items: { type: "number" }, + minItems: 2, + maxItems: 2, + }, + }, + required: ["type", "coordinates"], + }, + }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(0) + }) + + it("should report errors for GeoJSON not matching jsonSchema", async () => { + const jsonSchema = { + type: "object", + properties: { + type: { type: "string", const: "Point" }, + coordinates: { + type: "array", + items: { type: "number" }, + minItems: 2, + maxItems: 2, + }, + }, + required: ["type", "coordinates"], + } + + const table = DataFrame({ + location: [ + '{"type":"Point","coordinates":[0,0]}', + '{"type":"LineString","coordinates":[[0,0],[1,1]]}', + '{"type":"Point"}', + '{"type":"Point","coordinates":[0,0,0]}', + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "location", + type: "geojson", + constraints: { + jsonSchema, + }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors.filter(e => e.type === "cell/jsonSchema")).toHaveLength(3) + expect(errors).toContainEqual({ + type: "cell/jsonSchema", + fieldName: "location", + jsonSchema, + rowNumber: 2, + cell: '{"type":"LineString","coordinates":[[0,0],[1,1]]}', + }) + expect(errors).toContainEqual({ + type: "cell/jsonSchema", + fieldName: "location", + jsonSchema, + rowNumber: 3, + cell: '{"type":"Point"}', + }) + expect(errors).toContainEqual({ + type: "cell/jsonSchema", + fieldName: "location", + jsonSchema, + rowNumber: 4, + cell: '{"type":"Point","coordinates":[0,0,0]}', + }) + }) + + it("should validate complex GeoJSON Feature with jsonSchema", async () => { + const table = DataFrame({ + feature: [ + '{"type":"Feature","geometry":{"type":"Point","coordinates":[0,0]},"properties":{"name":"Valid","category":"test"}}', + '{"type":"Feature","geometry":{"type":"Point","coordinates":[0,0]},"properties":{"name":"Missing category"}}', + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "feature", + type: "geojson", + constraints: { + jsonSchema: { + type: "object", + properties: { + type: { type: "string", const: "Feature" }, + geometry: { type: "object" }, + properties: { + type: "object", + properties: { + name: { type: "string" }, + category: { type: "string" }, + }, + required: ["name", "category"], + }, + }, + required: ["type", "geometry", "properties"], + }, + }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toEqual([ + { + type: "cell/jsonSchema", + fieldName: "feature", + // @ts-ignore + jsonSchema: schema.fields[0].constraints?.jsonSchema, + rowNumber: 2, + cell: '{"type":"Feature","geometry":{"type":"Point","coordinates":[0,0]},"properties":{"name":"Missing category"}}', + }, + ]) + }) + + it("should validate GeoJSON FeatureCollection with jsonSchema", async () => { + const table = DataFrame({ + collection: [ + '{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[0,0]},"properties":{}}]}', + '{"type":"FeatureCollection","features":"invalid"}', + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "collection", + type: "geojson", + constraints: { + jsonSchema: { + type: "object", + properties: { + type: { type: "string", const: "FeatureCollection" }, + features: { + type: "array", + items: { + type: "object", + properties: { + type: { type: "string", const: "Feature" }, + geometry: { type: "object" }, + properties: { type: "object" }, + }, + required: ["type", "geometry", "properties"], + }, + }, + }, + required: ["type", "features"], + }, + }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toEqual([ + { + type: "cell/jsonSchema", + fieldName: "collection", + // @ts-ignore + jsonSchema: schema.fields[0].constraints?.jsonSchema, + rowNumber: 2, + cell: '{"type":"FeatureCollection","features":"invalid"}', + }, + ]) + }) +}) diff --git a/table/field/types/geojson.ts b/table/field/types/geojson.ts index 9991129b..909a7438 100644 --- a/table/field/types/geojson.ts +++ b/table/field/types/geojson.ts @@ -1,5 +1,7 @@ -import type { ObjectField } from "@dpkit/core" +import type { GeojsonField } from "@dpkit/core" import type { Table } from "../../table/index.ts" +import { validateJsonField } from "./json.ts" -// @ts-ignore -export async function validateGeojsonField(field: ObjectField, table: Table) {} +export async function validateGeojsonField(field: GeojsonField, table: Table) { + return validateJsonField(field, table) +} diff --git a/table/field/validate.ts b/table/field/validate.ts index dee798dd..b544d227 100644 --- a/table/field/validate.ts +++ b/table/field/validate.ts @@ -14,6 +14,7 @@ import { checkCellType } from "./checks/type.ts" import { checkCellUnique } from "./checks/unique.ts" import { normalizeField } from "./normalize.ts" import { validateArrayField } from "./types/array.ts" +import { validateGeojsonField } from "./types/geojson.ts" import { validateObjectField } from "./types/object.ts" export async function validateField( @@ -112,6 +113,8 @@ async function validateCells( switch (mapping.target.type) { case "array": return await validateArrayField(mapping.target, table) + case "geojson": + return await validateGeojsonField(mapping.target, table) case "object": return await validateObjectField(mapping.target, table) } From 0c357c58b3f6f069a055c5fcb4e17b4901248a5f Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 30 Oct 2025 12:10:40 +0000 Subject: [PATCH 08/11] Updated browser --- browser/components/Report/Error/Cell.tsx | 25 +++++++++++++++++++++++ browser/components/Report/Error/Error.tsx | 3 +++ package.json | 2 +- 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/browser/components/Report/Error/Cell.tsx b/browser/components/Report/Error/Cell.tsx index 1a900d8f..7e9b440d 100644 --- a/browser/components/Report/Error/Cell.tsx +++ b/browser/components/Report/Error/Cell.tsx @@ -301,3 +301,28 @@ export function CellEnumError(props: { error: errorTypes.CellEnumError }) { ) } + +export function CellJsonSchemaError(props: { + error: errorTypes.CellJsonSchemaError +}) { + const { t } = useTranslation() + const { error } = props + + return ( + + {t("Value of the cell")}{" "} + + {error.cell} + {" "} + {t("in field")}{" "} + + {error.fieldName} + {" "} + {t("of row")}{" "} + + {error.rowNumber} + {" "} + {t("does not match the")} JSON schema + + ) +} diff --git a/browser/components/Report/Error/Error.tsx b/browser/components/Report/Error/Error.tsx index 3c2a8471..49d85b74 100644 --- a/browser/components/Report/Error/Error.tsx +++ b/browser/components/Report/Error/Error.tsx @@ -3,6 +3,7 @@ import { CellEnumError, CellExclusiveMaximumError, CellExclusiveMinimumError, + CellJsonSchemaError, CellMaxLengthError, CellMaximumError, CellMinLengthError, @@ -65,5 +66,7 @@ export function Error(props: { return case "cell/enum": return + case "cell/jsonSchema": + return } } diff --git a/package.json b/package.json index af581bfc..36e92c12 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "spec": "vitest run", "test": "pnpm lint && pnpm type && pnpm spec", "test:win": "pnpm spec", - "type": "tsc --noEmit && pnpm -F browser type && pnpm -F cloud type", + "type": "tsc --noEmit && pnpm -F browser type", "version:ci": "pnpm --include-workspace-root -r exec pnpm version --no-git-tag-version" }, "devDependencies": { From 9e3642483687089110738bad724ac473bb0cde6a Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 30 Oct 2025 12:12:55 +0000 Subject: [PATCH 09/11] Added geojson profiles --- table/profiles/geojson.json | 217 +++++++++++++++++++++++++++++ table/profiles/topojson.json | 259 +++++++++++++++++++++++++++++++++++ 2 files changed, 476 insertions(+) create mode 100644 table/profiles/geojson.json create mode 100644 table/profiles/topojson.json diff --git a/table/profiles/geojson.json b/table/profiles/geojson.json new file mode 100644 index 00000000..4e4ae100 --- /dev/null +++ b/table/profiles/geojson.json @@ -0,0 +1,217 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "id": "https://raw.githubusercontent.com/fge/sample-json-schemas/master/geojson/geojson.json#", + "title": "Geo JSON object", + "description": "Schema for a Geo JSON object", + "type": "object", + "required": ["type"], + "properties": { + "crs": { "$ref": "#/definitions/crs" }, + "bbox": { "$ref": "#/definitions/bbox" } + }, + "oneOf": [ + { "$ref": "#/definitions/geometry" }, + { "$ref": "#/definitions/geometryCollection" }, + { "$ref": "#/definitions/feature" }, + { "$ref": "#/definitions/featureCollection" } + ], + "definitions": { + "geometryCollection": { + "title": "GeometryCollection", + "description": "A collection of geometry objects", + "required": ["geometries"], + "properties": { + "type": { "enum": ["GeometryCollection"] }, + "geometries": { + "type": "array", + "items": { "$ref": "#/definitions/geometry" } + } + } + }, + "feature": { + "title": "Feature", + "description": "A Geo JSON feature object", + "required": ["geometry", "properties"], + "properties": { + "type": { "enum": ["Feature"] }, + "geometry": { + "oneOf": [{ "type": "null" }, { "$ref": "#/definitions/geometry" }] + }, + "properties": { "type": ["object", "null"] }, + "id": { "FIXME": "may be there, type not known (string? number?)" } + } + }, + "featureCollection": { + "title": "FeatureCollection", + "description": "A Geo JSON feature collection", + "required": ["features"], + "properties": { + "type": { "enum": ["FeatureCollection"] }, + "features": { + "type": "array", + "items": { "$ref": "#/definitions/feature" } + } + } + }, + "geometry": { + "title": "geometry", + "description": "One geometry as defined by GeoJSON", + "type": "object", + "required": ["type", "coordinates"], + "oneOf": [ + { + "title": "Point", + "properties": { + "type": { "enum": ["Point"] }, + "coordinates": { + "$ref": "#/definitions/geometry/definitions/position" + } + } + }, + { + "title": "MultiPoint", + "properties": { + "type": { "enum": ["MultiPoint"] }, + "coordinates": { + "$ref": "#/definitions/geometry/definitions/positionArray" + } + } + }, + { + "title": "LineString", + "properties": { + "type": { "enum": ["LineString"] }, + "coordinates": { + "$ref": "#/definitions/geometry/definitions/lineString" + } + } + }, + { + "title": "MultiLineString", + "properties": { + "type": { "enum": ["MultiLineString"] }, + "coordinates": { + "type": "array", + "items": { + "$ref": "#/definitions/geometry/definitions/lineString" + } + } + } + }, + { + "title": "Polygon", + "properties": { + "type": { "enum": ["Polygon"] }, + "coordinates": { + "$ref": "#/definitions/geometry/definitions/polygon" + } + } + }, + { + "title": "MultiPolygon", + "properties": { + "type": { "enum": ["MultiPolygon"] }, + "coordinates": { + "type": "array", + "items": { "$ref": "#/definitions/geometry/definitions/polygon" } + } + } + } + ], + "definitions": { + "position": { + "description": "A single position", + "type": "array", + "minItems": 2, + "items": [{ "type": "number" }, { "type": "number" }], + "additionalItems": false + }, + "positionArray": { + "description": "An array of positions", + "type": "array", + "items": { "$ref": "#/definitions/geometry/definitions/position" } + }, + "lineString": { + "description": "An array of two or more positions", + "allOf": [ + { "$ref": "#/definitions/geometry/definitions/positionArray" }, + { "minItems": 2 } + ] + }, + "linearRing": { + "description": "An array of four positions where the first equals the last", + "allOf": [ + { "$ref": "#/definitions/geometry/definitions/positionArray" }, + { "minItems": 4 } + ] + }, + "polygon": { + "description": "An array of linear rings", + "type": "array", + "items": { "$ref": "#/definitions/geometry/definitions/linearRing" } + } + } + }, + "crs": { + "title": "crs", + "description": "a Coordinate Reference System object", + "type": ["object", "null"], + "required": ["type", "properties"], + "properties": { + "type": { "type": "string" }, + "properties": { "type": "object" } + }, + "additionalProperties": false, + "oneOf": [ + { "$ref": "#/definitions/crs/definitions/namedCrs" }, + { "$ref": "#/definitions/crs/definitions/linkedCrs" } + ], + "definitions": { + "namedCrs": { + "properties": { + "type": { "enum": ["name"] }, + "properties": { + "required": ["name"], + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "FIXME": "semantic validation necessary" + } + } + } + } + }, + "linkedObject": { + "type": "object", + "required": ["href"], + "properties": { + "href": { + "type": "string", + "format": "uri", + "FIXME": "spec says \"dereferenceable\", cannot enforce that" + }, + "type": { + "type": "string", + "description": "Suggested values: proj4, ogjwkt, esriwkt" + } + } + }, + "linkedCrs": { + "properties": { + "type": { "enum": ["link"] }, + "properties": { + "$ref": "#/definitions/crs/definitions/linkedObject" + } + } + } + } + }, + "bbox": { + "description": "A bounding box as defined by GeoJSON", + "FIXME": "unenforceable constraint: even number of elements in array", + "type": "array", + "items": { "type": "number" } + } + } +} diff --git a/table/profiles/topojson.json b/table/profiles/topojson.json new file mode 100644 index 00000000..eea22837 --- /dev/null +++ b/table/profiles/topojson.json @@ -0,0 +1,259 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "TopoJSON object", + "description": "Schema for a TopoJSON object", + "type": "object", + "required": ["type"], + "properties": { + "bbox": { + "$ref": "#/definitions/bbox" + } + }, + "oneOf": [ + { + "$ref": "#/definitions/topology" + }, + { + "$ref": "#/definitions/geometry" + } + ], + "definitions": { + "bbox": { + "title": "TopoJSON bounding box", + "description": "A bounding box as defined by TopoJSON", + "type": "array", + "items": { + "$ref": "#/definitions/bbox/definitions/dimension" + }, + "minItems": 2, + "maxItems": 2, + "definitions": { + "dimension": { + "type": "array", + "description": "This array should have an entry per dimension in the geometries", + "items": { + "type": "number" + } + } + } + }, + "geometry": { + "title": "Geometry objects", + "description": "A Geometry object as defined by TopoJSON", + "type": "object", + "required": ["type"], + "properties": { + "id": { + "type": ["string", "integer"] + }, + "properties": { + "type": "object" + } + }, + "oneOf": [ + { + "title": "Point", + "description": "A Point Geometry object as defined by TopoJSON", + "required": ["type", "coordinates"], + "properties": { + "type": { + "enum": ["Point"] + }, + "coordinates": { + "$ref": "#/definitions/geometry/definitions/position" + } + } + }, + { + "title": "MultiPoint", + "description": "A MultiPoint Geometry object as defined by TopoJSON", + "required": ["type", "coordinates"], + "properties": { + "type": { + "enum": ["MultiPoint"] + }, + "coordinates": { + "type": "array", + "items": { + "$ref": "#/definitions/geometry/definitions/position" + } + } + } + }, + { + "title": "LineString", + "description": "A LineString Geometry object as defined by TopoJSON", + "required": ["type", "arcs"], + "properties": { + "type": { + "enum": ["LineString"] + }, + "arcs": { + "type": "array", + "items": { + "type": "integer" + } + } + } + }, + { + "title": "MultiLineString", + "description": "A MultiLineString Geometry object as defined by TopoJSON", + "required": ["type", "arcs"], + "properties": { + "type": { + "enum": ["MultiLineString"] + }, + "arcs": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "integer" + } + } + } + } + }, + { + "title": "Polygon", + "description": "A Polygon Geometry object as defined by TopoJSON", + "required": ["type", "arcs"], + "properties": { + "type": { + "enum": ["Polygon"] + }, + "arcs": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "integer" + } + } + } + } + }, + { + "title": "MultiPolygon", + "description": "A MultiPolygon Geometry object as defined by TopoJSON", + "required": ["type", "arcs"], + "properties": { + "type": { + "enum": ["MultiPolygon"] + }, + "arcs": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "integer" + } + } + } + } + } + }, + { + "title": "GeometryCollection", + "description": "A MultiPolygon Geometry object as defined by TopoJSON", + "required": ["type", "geometries"], + "properties": { + "type": { + "enum": ["GeometryCollection"] + }, + "geometries": { + "type": "array", + "items": { + "$ref": "#/definitions/geometry" + } + } + } + } + ], + "definitions": { + "position": { + "type": "array", + "items": { + "type": "number" + }, + "minItems": 2 + } + } + }, + "topology": { + "title": "Topology", + "description": "A Topology object as defined by TopoJSON", + "type": "object", + "required": ["objects", "arcs"], + "properties": { + "type": { + "enum": ["Topology"] + }, + "objects": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/geometry" + } + }, + "arcs": { + "$ref": "#/definitions/topology/definitions/arcs" + }, + "transform": { + "$ref": "#/definitions/topology/definitions/transform" + }, + "bbox": { + "$ref": "#/definitions/bbox" + } + }, + "definitions": { + "transform": { + "type": "object", + "required": ["scale", "translate"], + "properties": { + "scale": { + "type": "array", + "items": { + "type": "number" + }, + "minItems": 2 + }, + "translate": { + "type": "array", + "items": { + "type": "number" + }, + "minItems": 2 + } + } + }, + "arcs": { + "type": "array", + "items": { + "type": "array", + "items": { + "oneOf": [ + { + "$ref": "#/definitions/topology/definitions/position" + }, + { + "type": "null" + } + ] + }, + "minItems": 2 + } + }, + "position": { + "type": "array", + "items": { + "type": "number" + }, + "minItems": 2 + } + } + } + } +} From 019d425bab5640a39a61d14d9460591010395ad9 Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 30 Oct 2025 12:20:30 +0000 Subject: [PATCH 10/11] Added suport for formatProfile --- table/field/types/array.spec.ts | 18 +++++++-------- table/field/types/geojson.spec.ts | 18 +++++++-------- table/field/types/json.ts | 37 ++++++++++++++++++++++++++----- 3 files changed, 50 insertions(+), 23 deletions(-) diff --git a/table/field/types/array.spec.ts b/table/field/types/array.spec.ts index 6b7bd07d..9fb69f65 100644 --- a/table/field/types/array.spec.ts +++ b/table/field/types/array.spec.ts @@ -77,7 +77,7 @@ describe("validateArrayField", () => { { type: "cell/type", fieldName: "data", - fieldType: "object", + fieldType: "array", rowNumber: 2, cell: '{"key":"value"}', }, @@ -103,14 +103,14 @@ describe("validateArrayField", () => { expect(errors).toContainEqual({ type: "cell/type", fieldName: "data", - fieldType: "object", + fieldType: "array", rowNumber: 2, cell: "invalid json", }) expect(errors).toContainEqual({ type: "cell/type", fieldName: "data", - fieldType: "object", + fieldType: "array", rowNumber: 4, cell: "[broken", }) @@ -153,7 +153,7 @@ describe("validateArrayField", () => { { type: "cell/type", fieldName: "data", - fieldType: "object", + fieldType: "array", rowNumber: 2, cell: "", }, @@ -179,35 +179,35 @@ describe("validateArrayField", () => { { type: "cell/type", fieldName: "data", - fieldType: "object", + fieldType: "array", rowNumber: 1, cell: '"string"', }, { type: "cell/type", fieldName: "data", - fieldType: "object", + fieldType: "array", rowNumber: 2, cell: "123", }, { type: "cell/type", fieldName: "data", - fieldType: "object", + fieldType: "array", rowNumber: 3, cell: "true", }, { type: "cell/type", fieldName: "data", - fieldType: "object", + fieldType: "array", rowNumber: 4, cell: "false", }, { type: "cell/type", fieldName: "data", - fieldType: "object", + fieldType: "array", rowNumber: 5, cell: "null", }, diff --git a/table/field/types/geojson.spec.ts b/table/field/types/geojson.spec.ts index 04bcd4a0..73283a22 100644 --- a/table/field/types/geojson.spec.ts +++ b/table/field/types/geojson.spec.ts @@ -136,7 +136,7 @@ describe("validateGeojsonField", () => { { type: "cell/type", fieldName: "data", - fieldType: "object", + fieldType: "geojson", rowNumber: 2, cell: "[[0,0],[1,1]]", }, @@ -167,14 +167,14 @@ describe("validateGeojsonField", () => { expect(errors).toContainEqual({ type: "cell/type", fieldName: "data", - fieldType: "object", + fieldType: "geojson", rowNumber: 2, cell: "invalid json", }) expect(errors).toContainEqual({ type: "cell/type", fieldName: "data", - fieldType: "object", + fieldType: "geojson", rowNumber: 4, cell: "{broken}", }) @@ -203,7 +203,7 @@ describe("validateGeojsonField", () => { { type: "cell/type", fieldName: "data", - fieldType: "object", + fieldType: "geojson", rowNumber: 2, cell: "", }, @@ -229,35 +229,35 @@ describe("validateGeojsonField", () => { { type: "cell/type", fieldName: "data", - fieldType: "object", + fieldType: "geojson", rowNumber: 1, cell: '"string"', }, { type: "cell/type", fieldName: "data", - fieldType: "object", + fieldType: "geojson", rowNumber: 2, cell: "123", }, { type: "cell/type", fieldName: "data", - fieldType: "object", + fieldType: "geojson", rowNumber: 3, cell: "true", }, { type: "cell/type", fieldName: "data", - fieldType: "object", + fieldType: "geojson", rowNumber: 4, cell: "false", }, { type: "cell/type", fieldName: "data", - fieldType: "object", + fieldType: "geojson", rowNumber: 5, cell: "null", }, diff --git a/table/field/types/json.ts b/table/field/types/json.ts index edc6f34a..80af4aa5 100644 --- a/table/field/types/json.ts +++ b/table/field/types/json.ts @@ -11,9 +11,14 @@ import type { Table } from "../../table/index.ts" export async function validateJsonField( field: ArrayField | GeojsonField | ObjectField, table: Table, + options?: { + formatProfile?: Record + }, ) { const errors: CellError[] = [] - const jsonSchema = field.constraints?.jsonSchema + + const formatProfile = options?.formatProfile + const constraintProfile = field.constraints?.jsonSchema const frame = await table .withRowCount() @@ -38,17 +43,39 @@ export async function validateJsonField( type: "cell/type", cell: String(row.source), fieldName: field.name, - fieldType: "object", + fieldType: field.type, rowNumber: row.number, }) continue } - if (jsonSchema) { + if (formatProfile) { // TODO: Extract more generic function validateJson? // @ts-ignore - const report = await validateDescriptor(target, { profile: jsonSchema }) + const report = await validateDescriptor(target, { + profile: constraintProfile, + }) + + if (!report.valid) { + errors.push({ + type: "cell/type", + cell: String(row.source), + fieldName: field.name, + fieldType: field.type, + rowNumber: row.number, + }) + } + + continue + } + + if (constraintProfile) { + // TODO: Extract more generic function validateJson? + // @ts-ignore + const report = await validateDescriptor(target, { + profile: constraintProfile, + }) if (!report.valid) { errors.push({ @@ -56,7 +83,7 @@ export async function validateJsonField( cell: String(row.source), fieldName: field.name, rowNumber: row.number, - jsonSchema, + jsonSchema: constraintProfile, }) } } From d7d464eb9c13589d179421cee123ea9d981e617d Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 30 Oct 2025 12:24:54 +0000 Subject: [PATCH 11/11] Added geojson and topojson formats --- table/field/types/geojson.spec.ts | 301 +++++++++++++++++++----------- table/field/types/geojson.ts | 7 +- table/field/types/json.ts | 8 +- table/profiles/geojson.json | 1 - 4 files changed, 197 insertions(+), 120 deletions(-) diff --git a/table/field/types/geojson.spec.ts b/table/field/types/geojson.spec.ts index 73283a22..2500fb4d 100644 --- a/table/field/types/geojson.spec.ts +++ b/table/field/types/geojson.spec.ts @@ -148,7 +148,6 @@ describe("validateGeojsonField", () => { data: [ '{"type":"Point","coordinates":[0,0]}', "invalid json", - '{"type":"Feature"}', "{broken}", ], }).lazy() @@ -175,7 +174,7 @@ describe("validateGeojsonField", () => { type: "cell/type", fieldName: "data", fieldType: "geojson", - rowNumber: 4, + rowNumber: 3, cell: "{broken}", }) }) @@ -264,12 +263,12 @@ describe("validateGeojsonField", () => { ]) }) - it("should not report errors for GeoJSON matching jsonSchema", async () => { + it("should report errors for invalid GeoJSON Point coordinates", async () => { const table = DataFrame({ location: [ '{"type":"Point","coordinates":[0,0]}', - '{"type":"Point","coordinates":[12.5,41.9]}', - '{"type":"Point","coordinates":[-73.9,40.7]}', + '{"type":"Point","coordinates":[0]}', + '{"type":"Point","coordinates":[0,0,0,0]}', ], }).lazy() @@ -278,95 +277,70 @@ describe("validateGeojsonField", () => { { name: "location", type: "geojson", - constraints: { - jsonSchema: { - type: "object", - properties: { - type: { type: "string", const: "Point" }, - coordinates: { - type: "array", - items: { type: "number" }, - minItems: 2, - maxItems: 2, - }, - }, - required: ["type", "coordinates"], - }, - }, }, ], } const { errors } = await validateTable(table, { schema }) - expect(errors).toHaveLength(0) + expect(errors).toHaveLength(2) + expect(errors).toContainEqual({ + type: "cell/type", + fieldName: "location", + fieldType: "geojson", + rowNumber: 2, + cell: '{"type":"Point","coordinates":[0]}', + }) + expect(errors).toContainEqual({ + type: "cell/type", + fieldName: "location", + fieldType: "geojson", + rowNumber: 3, + cell: '{"type":"Point","coordinates":[0,0,0,0]}', + }) }) - it("should report errors for GeoJSON not matching jsonSchema", async () => { - const jsonSchema = { - type: "object", - properties: { - type: { type: "string", const: "Point" }, - coordinates: { - type: "array", - items: { type: "number" }, - minItems: 2, - maxItems: 2, - }, - }, - required: ["type", "coordinates"], - } - + it("should report errors for invalid GeoJSON LineString", async () => { const table = DataFrame({ - location: [ - '{"type":"Point","coordinates":[0,0]}', + line: [ '{"type":"LineString","coordinates":[[0,0],[1,1]]}', - '{"type":"Point"}', - '{"type":"Point","coordinates":[0,0,0]}', + '{"type":"LineString","coordinates":[[0,0]]}', + '{"type":"LineString","coordinates":[0,0]}', ], }).lazy() const schema: Schema = { fields: [ { - name: "location", + name: "line", type: "geojson", - constraints: { - jsonSchema, - }, }, ], } const { errors } = await validateTable(table, { schema }) - expect(errors.filter(e => e.type === "cell/jsonSchema")).toHaveLength(3) + expect(errors).toHaveLength(2) expect(errors).toContainEqual({ - type: "cell/jsonSchema", - fieldName: "location", - jsonSchema, + type: "cell/type", + fieldName: "line", + fieldType: "geojson", rowNumber: 2, - cell: '{"type":"LineString","coordinates":[[0,0],[1,1]]}', + cell: '{"type":"LineString","coordinates":[[0,0]]}', }) expect(errors).toContainEqual({ - type: "cell/jsonSchema", - fieldName: "location", - jsonSchema, + type: "cell/type", + fieldName: "line", + fieldType: "geojson", rowNumber: 3, - cell: '{"type":"Point"}', - }) - expect(errors).toContainEqual({ - type: "cell/jsonSchema", - fieldName: "location", - jsonSchema, - rowNumber: 4, - cell: '{"type":"Point","coordinates":[0,0,0]}', + cell: '{"type":"LineString","coordinates":[0,0]}', }) }) - it("should validate complex GeoJSON Feature with jsonSchema", async () => { + it("should report errors for incomplete GeoJSON Feature", async () => { const table = DataFrame({ feature: [ - '{"type":"Feature","geometry":{"type":"Point","coordinates":[0,0]},"properties":{"name":"Valid","category":"test"}}', - '{"type":"Feature","geometry":{"type":"Point","coordinates":[0,0]},"properties":{"name":"Missing category"}}', + '{"type":"Feature","geometry":{"type":"Point","coordinates":[0,0]},"properties":{}}', + '{"type":"Feature","geometry":{"type":"Point","coordinates":[0,0]}}', + '{"type":"Feature","properties":{}}', ], }).lazy() @@ -375,24 +349,41 @@ describe("validateGeojsonField", () => { { name: "feature", type: "geojson", - constraints: { - jsonSchema: { - type: "object", - properties: { - type: { type: "string", const: "Feature" }, - geometry: { type: "object" }, - properties: { - type: "object", - properties: { - name: { type: "string" }, - category: { type: "string" }, - }, - required: ["name", "category"], - }, - }, - required: ["type", "geometry", "properties"], - }, - }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(2) + expect(errors).toContainEqual({ + type: "cell/type", + fieldName: "feature", + fieldType: "geojson", + rowNumber: 2, + cell: '{"type":"Feature","geometry":{"type":"Point","coordinates":[0,0]}}', + }) + expect(errors).toContainEqual({ + type: "cell/type", + fieldName: "feature", + fieldType: "geojson", + rowNumber: 3, + cell: '{"type":"Feature","properties":{}}', + }) + }) + + it("should report errors for invalid GeoJSON FeatureCollection", async () => { + const table = DataFrame({ + collection: [ + '{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[0,0]},"properties":{}}]}', + '{"type":"FeatureCollection"}', + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "collection", + type: "geojson", }, ], } @@ -400,48 +391,37 @@ describe("validateGeojsonField", () => { const { errors } = await validateTable(table, { schema }) expect(errors).toEqual([ { - type: "cell/jsonSchema", - fieldName: "feature", - // @ts-ignore - jsonSchema: schema.fields[0].constraints?.jsonSchema, + type: "cell/type", + fieldName: "collection", + fieldType: "geojson", rowNumber: 2, - cell: '{"type":"Feature","geometry":{"type":"Point","coordinates":[0,0]},"properties":{"name":"Missing category"}}', + cell: '{"type":"FeatureCollection"}', }, ]) }) - it("should validate GeoJSON FeatureCollection with jsonSchema", async () => { + it("should not validate jsonSchema constraints for geojson fields", async () => { const table = DataFrame({ - collection: [ - '{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[0,0]},"properties":{}}]}', - '{"type":"FeatureCollection","features":"invalid"}', + location: [ + '{"type":"Point","coordinates":[0,0]}', + '{"type":"Point","coordinates":[100,200]}', ], }).lazy() const schema: Schema = { fields: [ { - name: "collection", + name: "location", type: "geojson", constraints: { jsonSchema: { type: "object", properties: { - type: { type: "string", const: "FeatureCollection" }, - features: { + coordinates: { type: "array", - items: { - type: "object", - properties: { - type: { type: "string", const: "Feature" }, - geometry: { type: "object" }, - properties: { type: "object" }, - }, - required: ["type", "geometry", "properties"], - }, + items: { type: "number", minimum: -50, maximum: 50 }, }, }, - required: ["type", "features"], }, }, }, @@ -449,15 +429,110 @@ describe("validateGeojsonField", () => { } const { errors } = await validateTable(table, { schema }) - expect(errors).toEqual([ - { - type: "cell/jsonSchema", - fieldName: "collection", - // @ts-ignore - jsonSchema: schema.fields[0].constraints?.jsonSchema, - rowNumber: 2, - cell: '{"type":"FeatureCollection","features":"invalid"}', - }, - ]) + expect(errors).toHaveLength(0) + }) + + it("should not report errors for valid TopoJSON", async () => { + const table = DataFrame({ + topology: [ + '{"type":"Topology","objects":{"example":{"type":"GeometryCollection","geometries":[{"type":"Point","coordinates":[0,0]}]}},"arcs":[]}', + '{"type":"Topology","objects":{"collection":{"type":"GeometryCollection","geometries":[]}},"arcs":[]}', + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "topology", + type: "geojson", + format: "topojson", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(0) + }) + + it("should report errors for invalid TopoJSON structure", async () => { + const table = DataFrame({ + topology: [ + '{"type":"Topology","objects":{"example":{"type":"GeometryCollection","geometries":[]}},"arcs":[]}', + '{"type":"Topology","objects":{}}', + '{"type":"Topology"}', + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "topology", + type: "geojson", + format: "topojson", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(2) + expect(errors).toContainEqual({ + type: "cell/type", + fieldName: "topology", + fieldType: "geojson", + rowNumber: 2, + cell: '{"type":"Topology","objects":{}}', + }) + expect(errors).toContainEqual({ + type: "cell/type", + fieldName: "topology", + fieldType: "geojson", + rowNumber: 3, + cell: '{"type":"Topology"}', + }) + }) + + it("should accept TopoJSON geometry objects", async () => { + const table = DataFrame({ + geometry: [ + '{"type":"Point","coordinates":[0,0]}', + '{"type":"LineString","arcs":[0,1]}', + '{"type":"Polygon","arcs":[[0,1,2]]}', + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "geometry", + type: "geojson", + format: "topojson", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(0) + }) + + it("should handle null values for topojson format", async () => { + const table = DataFrame({ + topology: [ + '{"type":"Topology","objects":{"example":{"type":"GeometryCollection","geometries":[]}},"arcs":[]}', + null, + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "topology", + type: "geojson", + format: "topojson", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(0) }) }) diff --git a/table/field/types/geojson.ts b/table/field/types/geojson.ts index 909a7438..7a455ab3 100644 --- a/table/field/types/geojson.ts +++ b/table/field/types/geojson.ts @@ -1,7 +1,12 @@ import type { GeojsonField } from "@dpkit/core" +import geojsonProfile from "../../profiles/geojson.json" with { type: "json" } +import topojsonProfile from "../../profiles/topojson.json" with { type: "json" } import type { Table } from "../../table/index.ts" import { validateJsonField } from "./json.ts" export async function validateGeojsonField(field: GeojsonField, table: Table) { - return validateJsonField(field, table) + return validateJsonField(field, table, { + formatProfile: + field.format === "topojson" ? topojsonProfile : geojsonProfile, + }) } diff --git a/table/field/types/json.ts b/table/field/types/json.ts index 80af4aa5..2b8fb663 100644 --- a/table/field/types/json.ts +++ b/table/field/types/json.ts @@ -52,9 +52,8 @@ export async function validateJsonField( if (formatProfile) { // TODO: Extract more generic function validateJson? - // @ts-ignore - const report = await validateDescriptor(target, { - profile: constraintProfile, + const report = await validateDescriptor(target as any, { + profile: formatProfile, }) if (!report.valid) { @@ -72,8 +71,7 @@ export async function validateJsonField( if (constraintProfile) { // TODO: Extract more generic function validateJson? - // @ts-ignore - const report = await validateDescriptor(target, { + const report = await validateDescriptor(target as any, { profile: constraintProfile, }) diff --git a/table/profiles/geojson.json b/table/profiles/geojson.json index 4e4ae100..5e9039c5 100644 --- a/table/profiles/geojson.json +++ b/table/profiles/geojson.json @@ -1,6 +1,5 @@ { "$schema": "http://json-schema.org/draft-04/schema#", - "id": "https://raw.githubusercontent.com/fge/sample-json-schemas/master/geojson/geojson.json#", "title": "Geo JSON object", "description": "Schema for a Geo JSON object", "type": "object",