diff --git a/browser/components/Report/Error/Cell.tsx b/browser/components/Report/Error/Cell.tsx index 1a900d8f..7e9b440d 100644 --- a/browser/components/Report/Error/Cell.tsx +++ b/browser/components/Report/Error/Cell.tsx @@ -301,3 +301,28 @@ export function CellEnumError(props: { error: errorTypes.CellEnumError }) { ) } + +export function CellJsonSchemaError(props: { + error: errorTypes.CellJsonSchemaError +}) { + const { t } = useTranslation() + const { error } = props + + return ( + + {t("Value of the cell")}{" "} + + {error.cell} + {" "} + {t("in field")}{" "} + + {error.fieldName} + {" "} + {t("of row")}{" "} + + {error.rowNumber} + {" "} + {t("does not match the")} JSON schema + + ) +} diff --git a/browser/components/Report/Error/Error.tsx b/browser/components/Report/Error/Error.tsx index 3c2a8471..49d85b74 100644 --- a/browser/components/Report/Error/Error.tsx +++ b/browser/components/Report/Error/Error.tsx @@ -3,6 +3,7 @@ import { CellEnumError, CellExclusiveMaximumError, CellExclusiveMinimumError, + CellJsonSchemaError, CellMaxLengthError, CellMaximumError, CellMinLengthError, @@ -65,5 +66,7 @@ export function Error(props: { return case "cell/enum": return + case "cell/jsonSchema": + return } } diff --git a/core/field/types/Geojson.ts b/core/field/types/Geojson.ts index 1151dea0..2da68193 100644 --- a/core/field/types/Geojson.ts +++ b/core/field/types/Geojson.ts @@ -21,6 +21,11 @@ export interface GeojsonField extends BaseField { * GeoJSON-specific constraints */ export interface GeojsonConstraints extends BaseConstraints { + /** + * JSON Schema object for validating the object structure and properties + */ + jsonSchema?: Record + /** * Restrict values to a specified set of GeoJSON objects * Serialized as strings or GeoJSON object literals diff --git a/package.json b/package.json index af581bfc..36e92c12 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "spec": "vitest run", "test": "pnpm lint && pnpm type && pnpm spec", "test:win": "pnpm spec", - "type": "tsc --noEmit && pnpm -F browser type && pnpm -F cloud type", + "type": "tsc --noEmit && pnpm -F browser type", "version:ci": "pnpm --include-workspace-root -r exec pnpm version --no-git-tag-version" }, "devDependencies": { diff --git a/table/error/Cell.ts b/table/error/Cell.ts index 09bc7412..ae179dde 100644 --- a/table/error/Cell.ts +++ b/table/error/Cell.ts @@ -13,6 +13,7 @@ export type CellError = | CellPatternError | CellUniqueError | CellEnumError + | CellJsonSchemaError export interface BaseCellError extends BaseTableError { fieldName: string @@ -73,3 +74,8 @@ export interface CellEnumError extends BaseCellError { type: "cell/enum" enum: string[] } + +export interface CellJsonSchemaError extends BaseCellError { + type: "cell/jsonSchema" + jsonSchema: Record +} diff --git a/table/field/parse.ts b/table/field/parse.ts index 896417fa..c59042b6 100644 --- a/table/field/parse.ts +++ b/table/field/parse.ts @@ -1,17 +1,14 @@ import type { Expr } from "nodejs-polars" import { DataType } from "nodejs-polars" import type { FieldMapping } from "./Mapping.ts" -import { parseArrayField } from "./types/array.ts" import { parseBooleanField } from "./types/boolean.ts" import { parseDateField } from "./types/date.ts" import { parseDatetimeField } from "./types/datetime.ts" import { parseDurationField } from "./types/duration.ts" -import { parseGeojsonField } from "./types/geojson.ts" import { parseGeopointField } from "./types/geopoint.ts" import { parseIntegerField } from "./types/integer.ts" import { parseListField } from "./types/list.ts" import { parseNumberField } from "./types/number.ts" -import { parseObjectField } from "./types/object.ts" import { parseStringField } from "./types/string.ts" import { parseTimeField } from "./types/time.ts" import { parseYearField } from "./types/year.ts" @@ -22,8 +19,6 @@ export function parseField(mapping: FieldMapping, fieldExpr: Expr) { const field = mapping.target switch (field.type) { - case "array": - return parseArrayField(field, fieldExpr) case "boolean": return parseBooleanField(field, fieldExpr) case "date": @@ -32,8 +27,6 @@ export function parseField(mapping: FieldMapping, fieldExpr: Expr) { return parseDatetimeField(field, fieldExpr) case "duration": return parseDurationField(field, fieldExpr) - case "geojson": - return parseGeojsonField(field, fieldExpr) case "geopoint": return parseGeopointField(field, fieldExpr) case "integer": @@ -42,8 +35,6 @@ export function parseField(mapping: FieldMapping, fieldExpr: Expr) { return parseListField(field, fieldExpr) case "number": return parseNumberField(field, fieldExpr) - case "object": - return parseObjectField(field, fieldExpr) case "string": return parseStringField(field, fieldExpr) case "time": diff --git a/table/field/stringify.ts b/table/field/stringify.ts index f03b4dce..185410a3 100644 --- a/table/field/stringify.ts +++ b/table/field/stringify.ts @@ -1,16 +1,13 @@ import type { Field } from "@dpkit/core" import type { Expr } from "nodejs-polars" -import { stringifyArrayField } from "./types/array.ts" import { stringifyBooleanField } from "./types/boolean.ts" import { stringifyDateField } from "./types/date.ts" import { stringifyDatetimeField } from "./types/datetime.ts" import { stringifyDurationField } from "./types/duration.ts" -import { stringifyGeojsonField } from "./types/geojson.ts" import { stringifyGeopointField } from "./types/geopoint.ts" import { stringifyIntegerField } from "./types/integer.ts" import { stringifyListField } from "./types/list.ts" import { stringifyNumberField } from "./types/number.ts" -import { stringifyObjectField } from "./types/object.ts" import { stringifyStringField } from "./types/string.ts" import { stringifyTimeField } from "./types/time.ts" import { stringifyYearField } from "./types/year.ts" @@ -18,8 +15,6 @@ import { stringifyYearmonthField } from "./types/yearmonth.ts" export function stringifyField(field: Field, fieldExpr: Expr) { switch (field.type) { - case "array": - return stringifyArrayField(field, fieldExpr) case "boolean": return stringifyBooleanField(field, fieldExpr) case "date": @@ -28,8 +23,6 @@ export function stringifyField(field: Field, fieldExpr: Expr) { return stringifyDatetimeField(field, fieldExpr) case "duration": return stringifyDurationField(field, fieldExpr) - case "geojson": - return stringifyGeojsonField(field, fieldExpr) case "geopoint": return stringifyGeopointField(field, fieldExpr) case "integer": @@ -38,8 +31,6 @@ export function stringifyField(field: Field, fieldExpr: Expr) { return stringifyListField(field, fieldExpr) case "number": return stringifyNumberField(field, fieldExpr) - case "object": - return stringifyObjectField(field, fieldExpr) case "string": return stringifyStringField(field, fieldExpr) case "time": diff --git a/table/field/types/array.spec.ts b/table/field/types/array.spec.ts index 01d2ec1f..9fb69f65 100644 --- a/table/field/types/array.spec.ts +++ b/table/field/types/array.spec.ts @@ -1,71 +1,360 @@ -import { DataFrame, DataType, Series } from "nodejs-polars" +import type { Schema } from "@dpkit/core" +import { DataFrame } from "nodejs-polars" import { describe, expect, it } from "vitest" -import { denormalizeTable, normalizeTable } from "../../table/index.ts" - -describe("parseArrayField", () => { - it.each([ - // Valid JSON arrays - ["[1,2,3]", [1, 2, 3]], - ['["a","b","c"]', ["a", "b", "c"]], - ['[{"name":"John"},{"name":"Jane"}]', [{ name: "John" }, { name: "Jane" }]], - ["[]", []], - - // JSON but not an array - ['{"name":"John"}', null], - ["{}", null], - - // Trimming whitespace - //[" [1,2,3] ", [1, 2, 3]], - //['\t["a","b","c"]\n', ["a", "b", "c"]], - - // Invalid JSON - skip test that's causing issues - // ["[invalid]", null], - ["not json", null], - ["", null], - ["null", null], - ["undefined", null], - ])("%s -> %s", async (cell, value) => { - const table = DataFrame([Series("name", [cell], DataType.String)]).lazy() - - const schema = { - fields: [{ name: "name", type: "array" as const }], +import { validateTable } from "../../table/index.ts" + +describe("validateArrayField", () => { + it("should not report errors for valid JSON arrays", async () => { + const table = DataFrame({ + tags: ['["tag1","tag2"]', "[1,2,3]", '["a","b","c"]'], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "tags", + type: "array", + }, + ], } - const ldf = await normalizeTable(table, schema) - const df = await ldf.collect() - const res = df.getColumn("name").get(0) - expect(res ? JSON.parse(res) : res).toEqual(value) + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(0) }) -}) -describe("stringifyArrayField", () => { - it.each([ - // JSON array strings should be returned as-is - ["[1,2,3]", "[1,2,3]"], - ['["a","b","c"]', '["a","b","c"]'], - ['[{"name":"John"},{"name":"Jane"}]', '[{"name":"John"},{"name":"Jane"}]'], - ["[]", "[]"], - - // Complex nested arrays - [ - '[{"users":[{"id":1,"name":"Alice"}],"count":2}]', - '[{"users":[{"id":1,"name":"Alice"}],"count":2}]', - ], - ["[[1,2],[3,4]]", "[[1,2],[3,4]]"], - ['[true,false,null,"text",42,3.14]', '[true,false,null,"text",42,3.14]'], - - // Null handling - [null, ""], - ])("%s -> %s", async (value, expected) => { - const table = DataFrame([Series("name", [value], DataType.String)]).lazy() - - const schema = { - fields: [{ name: "name", type: "array" as const }], + it("should not report errors for empty arrays", async () => { + const table = DataFrame({ + items: ["[]", "[]", "[]"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "items", + type: "array", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(0) + }) + + it("should not report errors for null values", async () => { + const table = DataFrame({ + data: ['["value"]', null, "[1,2,3]"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "data", + type: "array", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(0) + }) + + it("should report errors for JSON objects", async () => { + const table = DataFrame({ + data: ["[1,2,3]", '{"key":"value"}', '["a","b"]'], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "data", + type: "array", + }, + ], } - const ldf = await denormalizeTable(table, schema) - const df = await ldf.collect() + const { errors } = await validateTable(table, { schema }) + expect(errors).toEqual([ + { + type: "cell/type", + fieldName: "data", + fieldType: "array", + rowNumber: 2, + cell: '{"key":"value"}', + }, + ]) + }) + + it("should report errors for invalid JSON", async () => { + const table = DataFrame({ + data: ['["valid"]', "invalid json", "[1,2,3]", "[broken"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "data", + type: "array", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors.filter(e => e.type === "cell/type")).toHaveLength(2) + expect(errors).toContainEqual({ + type: "cell/type", + fieldName: "data", + fieldType: "array", + rowNumber: 2, + cell: "invalid json", + }) + expect(errors).toContainEqual({ + type: "cell/type", + fieldName: "data", + fieldType: "array", + rowNumber: 4, + cell: "[broken", + }) + }) + + it("should handle nested arrays", async () => { + const table = DataFrame({ + matrix: ["[[1,2],[3,4]]", "[[5,6],[7,8]]", '[["a","b"],["c","d"]]'], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "matrix", + type: "array", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(0) + }) + + it("should report errors for empty strings", async () => { + const table = DataFrame({ + data: ['["valid"]', "", "[1,2,3]"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "data", + type: "array", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toEqual([ + { + type: "cell/type", + fieldName: "data", + fieldType: "array", + rowNumber: 2, + cell: "", + }, + ]) + }) + + it("should report errors for JSON primitives", async () => { + const table = DataFrame({ + data: ['"string"', "123", "true", "false", "null"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "data", + type: "array", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toEqual([ + { + type: "cell/type", + fieldName: "data", + fieldType: "array", + rowNumber: 1, + cell: '"string"', + }, + { + type: "cell/type", + fieldName: "data", + fieldType: "array", + rowNumber: 2, + cell: "123", + }, + { + type: "cell/type", + fieldName: "data", + fieldType: "array", + rowNumber: 3, + cell: "true", + }, + { + type: "cell/type", + fieldName: "data", + fieldType: "array", + rowNumber: 4, + cell: "false", + }, + { + type: "cell/type", + fieldName: "data", + fieldType: "array", + rowNumber: 5, + cell: "null", + }, + ]) + }) + + it("should not report errors for arrays matching jsonSchema", async () => { + const table = DataFrame({ + scores: ["[80,90,100]", "[75,85,95]", "[90,95,100]"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "scores", + type: "array", + constraints: { + jsonSchema: { + type: "array", + items: { type: "number" }, + minItems: 3, + maxItems: 3, + }, + }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(0) + }) + + it("should report errors for arrays not matching jsonSchema", async () => { + const jsonSchema = { + type: "array", + items: { type: "number" }, + minItems: 2, + } + + const table = DataFrame({ + numbers: ["[1,2,3]", '["not","numbers"]', "[1]", "[4,5,6]"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "numbers", + type: "array", + constraints: { + jsonSchema, + }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors.filter(e => e.type === "cell/jsonSchema")).toHaveLength(2) + expect(errors).toContainEqual({ + type: "cell/jsonSchema", + fieldName: "numbers", + jsonSchema, + rowNumber: 2, + cell: '["not","numbers"]', + }) + expect(errors).toContainEqual({ + type: "cell/jsonSchema", + fieldName: "numbers", + jsonSchema, + rowNumber: 3, + cell: "[1]", + }) + }) + + it("should validate complex jsonSchema with array of objects", async () => { + const table = DataFrame({ + users: [ + '[{"name":"John","age":30},{"name":"Jane","age":25}]', + '[{"name":"Bob","age":"invalid"}]', + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "users", + type: "array", + constraints: { + jsonSchema: { + type: "array", + items: { + type: "object", + properties: { + name: { type: "string" }, + age: { type: "number" }, + }, + required: ["name", "age"], + }, + }, + }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toEqual([ + { + type: "cell/jsonSchema", + fieldName: "users", + // @ts-ignore + jsonSchema: schema.fields[0].constraints?.jsonSchema, + rowNumber: 2, + cell: '[{"name":"Bob","age":"invalid"}]', + }, + ]) + }) + + it("should validate jsonSchema with unique items constraint", async () => { + const table = DataFrame({ + tags: ['["unique","values"]', '["duplicate","duplicate"]'], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "tags", + type: "array", + constraints: { + jsonSchema: { + type: "array", + items: { type: "string" }, + uniqueItems: true, + }, + }, + }, + ], + } - expect(df.toRecords()[0]?.name).toEqual(expected) + const { errors } = await validateTable(table, { schema }) + expect(errors).toEqual([ + { + type: "cell/jsonSchema", + fieldName: "tags", + // @ts-ignore + jsonSchema: schema.fields[0].constraints?.jsonSchema, + rowNumber: 2, + cell: '["duplicate","duplicate"]', + }, + ]) }) }) diff --git a/table/field/types/array.ts b/table/field/types/array.ts index f3c7ddb3..76d672f0 100644 --- a/table/field/types/array.ts +++ b/table/field/types/array.ts @@ -1,17 +1,7 @@ import type { ArrayField } from "@dpkit/core" -import { lit, when } from "nodejs-polars" -import type { Expr } from "nodejs-polars" +import type { Table } from "../../table/index.ts" +import { validateJsonField } from "./json.ts" -// TODO: Is there a better way to do this? -// Polars does not support really support free-form JSON -// So we just make a basic check and return as it is -export function parseArrayField(_field: ArrayField, fieldExpr: Expr) { - return when(fieldExpr.str.contains("^\\[")) - .then(fieldExpr) - .otherwise(lit(null)) -} - -export function stringifyArrayField(_field: ArrayField, fieldExpr: Expr) { - // TODO: implement - return fieldExpr +export async function validateArrayField(field: ArrayField, table: Table) { + return validateJsonField(field, table) } diff --git a/table/field/types/geojson.spec.ts b/table/field/types/geojson.spec.ts index 93da3adb..2500fb4d 100644 --- a/table/field/types/geojson.spec.ts +++ b/table/field/types/geojson.spec.ts @@ -1,91 +1,538 @@ -import { DataFrame, DataType, Series } from "nodejs-polars" +import type { Schema } from "@dpkit/core" +import { DataFrame } from "nodejs-polars" import { describe, expect, it } from "vitest" -import { denormalizeTable, normalizeTable } from "../../table/index.ts" +import { validateTable } from "../../table/index.ts" -describe("parseGeojsonField", () => { - it.each([ - // Valid JSON objects - ['{"name":"John","age":30}', { name: "John", age: 30 }], - ['{"numbers":[1,2,3]}', { numbers: [1, 2, 3] }], - ['{"nested":{"prop":"value"}}', { nested: { prop: "value" } }], - ["{}", {}], +describe("validateGeojsonField", () => { + it("should not report errors for valid GeoJSON Point", async () => { + const table = DataFrame({ + location: [ + '{"type":"Point","coordinates":[0,0]}', + '{"type":"Point","coordinates":[12.5,41.9]}', + '{"type":"Point","coordinates":[-73.9,40.7]}', + ], + }).lazy() - // JSON but not an object - ["[1,2,3]", null], - ['["a","b","c"]', null], + const schema: Schema = { + fields: [ + { + name: "location", + type: "geojson", + }, + ], + } - // Trimming whitespace - //[' {"name":"John"} ', { name: "John" }], - //['\t{"name":"John"}\n', { name: "John" }], + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(0) + }) - // Invalid JSON - //["{invalid}", null], - ["not json", null], - ["", null], - ["null", null], - ["undefined", null], - ])("%s -> %s", async (cell, value) => { - const table = DataFrame([Series("name", [cell], DataType.String)]).lazy() + it("should not report errors for valid GeoJSON geometries", async () => { + const table = DataFrame({ + geometry: [ + '{"type":"LineString","coordinates":[[0,0],[1,1]]}', + '{"type":"Polygon","coordinates":[[[0,0],[1,0],[1,1],[0,1],[0,0]]]}', + '{"type":"MultiPoint","coordinates":[[0,0],[1,1]]}', + ], + }).lazy() - const schema = { - fields: [{ name: "name", type: "geojson" as const }], + const schema: Schema = { + fields: [ + { + name: "geometry", + type: "geojson", + }, + ], } - const ldf = await normalizeTable(table, schema) - const df = await ldf.collect() + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(0) + }) + + it("should not report errors for valid GeoJSON Feature", async () => { + const table = DataFrame({ + feature: [ + '{"type":"Feature","geometry":{"type":"Point","coordinates":[0,0]},"properties":{"name":"Test"}}', + '{"type":"Feature","geometry":{"type":"LineString","coordinates":[[0,0],[1,1]]},"properties":{"id":1}}', + '{"type":"Feature","geometry":null,"properties":{}}', + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "feature", + type: "geojson", + }, + ], + } - const res = df.getColumn("name").get(0) - expect(res ? JSON.parse(res) : res).toEqual(value) + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(0) }) -}) -describe("stringifyGeojsonField", () => { - it.each([ - // GeoJSON Point - [ - '{"type":"Point","coordinates":[125.6,10.1]}', - '{"type":"Point","coordinates":[125.6,10.1]}', - ], - - // GeoJSON LineString - [ - '{"type":"LineString","coordinates":[[125.6,10.1],[125.7,10.2]]}', - '{"type":"LineString","coordinates":[[125.6,10.1],[125.7,10.2]]}', - ], - - // GeoJSON Polygon - [ - '{"type":"Polygon","coordinates":[[[125.6,10.1],[125.7,10.1],[125.7,10.2],[125.6,10.2],[125.6,10.1]]]}', - '{"type":"Polygon","coordinates":[[[125.6,10.1],[125.7,10.1],[125.7,10.2],[125.6,10.2],[125.6,10.1]]]}', - ], - - // GeoJSON Feature - [ - '{"type":"Feature","geometry":{"type":"Point","coordinates":[125.6,10.1]},"properties":{"name":"Sample Point"}}', - '{"type":"Feature","geometry":{"type":"Point","coordinates":[125.6,10.1]},"properties":{"name":"Sample Point"}}', - ], - - // GeoJSON FeatureCollection - [ - '{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[125.6,10.1]},"properties":{"name":"Point 1"}}]}', - '{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[125.6,10.1]},"properties":{"name":"Point 1"}}]}', - ], - - // Empty GeoJSON object - ["{}", "{}"], - - // Null handling - [null, ""], - ])("%s -> %s", async (value, expected) => { - const table = DataFrame([Series("name", [value], DataType.String)]).lazy() - - const schema = { - fields: [{ name: "name", type: "geojson" as const }], - } - - const ldf = await denormalizeTable(table, schema) - const df = await ldf.collect() - - expect(df.toRecords()[0]?.name).toEqual(expected) + it("should not report errors for valid GeoJSON FeatureCollection", async () => { + const table = DataFrame({ + collection: [ + '{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[0,0]},"properties":{}}]}', + '{"type":"FeatureCollection","features":[]}', + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "collection", + type: "geojson", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(0) + }) + + it("should not report errors for null values", async () => { + const table = DataFrame({ + location: [ + '{"type":"Point","coordinates":[0,0]}', + null, + '{"type":"Feature","geometry":null,"properties":{}}', + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "location", + type: "geojson", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(0) + }) + + it("should report errors for JSON arrays", async () => { + const table = DataFrame({ + data: [ + '{"type":"Point","coordinates":[0,0]}', + "[[0,0],[1,1]]", + '{"type":"Feature","geometry":null,"properties":{}}', + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "data", + type: "geojson", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toEqual([ + { + type: "cell/type", + fieldName: "data", + fieldType: "geojson", + rowNumber: 2, + cell: "[[0,0],[1,1]]", + }, + ]) + }) + + it("should report errors for invalid JSON", async () => { + const table = DataFrame({ + data: [ + '{"type":"Point","coordinates":[0,0]}', + "invalid json", + "{broken}", + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "data", + type: "geojson", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors.filter(e => e.type === "cell/type")).toHaveLength(2) + expect(errors).toContainEqual({ + type: "cell/type", + fieldName: "data", + fieldType: "geojson", + rowNumber: 2, + cell: "invalid json", + }) + expect(errors).toContainEqual({ + type: "cell/type", + fieldName: "data", + fieldType: "geojson", + rowNumber: 3, + cell: "{broken}", + }) + }) + + it("should report errors for empty strings", async () => { + const table = DataFrame({ + data: [ + '{"type":"Point","coordinates":[0,0]}', + "", + '{"type":"Feature","geometry":null,"properties":{}}', + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "data", + type: "geojson", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toEqual([ + { + type: "cell/type", + fieldName: "data", + fieldType: "geojson", + rowNumber: 2, + cell: "", + }, + ]) + }) + + it("should report errors for JSON primitives", async () => { + const table = DataFrame({ + data: ['"string"', "123", "true", "false", "null"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "data", + type: "geojson", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toEqual([ + { + type: "cell/type", + fieldName: "data", + fieldType: "geojson", + rowNumber: 1, + cell: '"string"', + }, + { + type: "cell/type", + fieldName: "data", + fieldType: "geojson", + rowNumber: 2, + cell: "123", + }, + { + type: "cell/type", + fieldName: "data", + fieldType: "geojson", + rowNumber: 3, + cell: "true", + }, + { + type: "cell/type", + fieldName: "data", + fieldType: "geojson", + rowNumber: 4, + cell: "false", + }, + { + type: "cell/type", + fieldName: "data", + fieldType: "geojson", + rowNumber: 5, + cell: "null", + }, + ]) + }) + + it("should report errors for invalid GeoJSON Point coordinates", async () => { + const table = DataFrame({ + location: [ + '{"type":"Point","coordinates":[0,0]}', + '{"type":"Point","coordinates":[0]}', + '{"type":"Point","coordinates":[0,0,0,0]}', + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "location", + type: "geojson", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(2) + expect(errors).toContainEqual({ + type: "cell/type", + fieldName: "location", + fieldType: "geojson", + rowNumber: 2, + cell: '{"type":"Point","coordinates":[0]}', + }) + expect(errors).toContainEqual({ + type: "cell/type", + fieldName: "location", + fieldType: "geojson", + rowNumber: 3, + cell: '{"type":"Point","coordinates":[0,0,0,0]}', + }) + }) + + it("should report errors for invalid GeoJSON LineString", async () => { + const table = DataFrame({ + line: [ + '{"type":"LineString","coordinates":[[0,0],[1,1]]}', + '{"type":"LineString","coordinates":[[0,0]]}', + '{"type":"LineString","coordinates":[0,0]}', + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "line", + type: "geojson", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(2) + expect(errors).toContainEqual({ + type: "cell/type", + fieldName: "line", + fieldType: "geojson", + rowNumber: 2, + cell: '{"type":"LineString","coordinates":[[0,0]]}', + }) + expect(errors).toContainEqual({ + type: "cell/type", + fieldName: "line", + fieldType: "geojson", + rowNumber: 3, + cell: '{"type":"LineString","coordinates":[0,0]}', + }) + }) + + it("should report errors for incomplete GeoJSON Feature", async () => { + const table = DataFrame({ + feature: [ + '{"type":"Feature","geometry":{"type":"Point","coordinates":[0,0]},"properties":{}}', + '{"type":"Feature","geometry":{"type":"Point","coordinates":[0,0]}}', + '{"type":"Feature","properties":{}}', + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "feature", + type: "geojson", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(2) + expect(errors).toContainEqual({ + type: "cell/type", + fieldName: "feature", + fieldType: "geojson", + rowNumber: 2, + cell: '{"type":"Feature","geometry":{"type":"Point","coordinates":[0,0]}}', + }) + expect(errors).toContainEqual({ + type: "cell/type", + fieldName: "feature", + fieldType: "geojson", + rowNumber: 3, + cell: '{"type":"Feature","properties":{}}', + }) + }) + + it("should report errors for invalid GeoJSON FeatureCollection", async () => { + const table = DataFrame({ + collection: [ + '{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[0,0]},"properties":{}}]}', + '{"type":"FeatureCollection"}', + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "collection", + type: "geojson", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toEqual([ + { + type: "cell/type", + fieldName: "collection", + fieldType: "geojson", + rowNumber: 2, + cell: '{"type":"FeatureCollection"}', + }, + ]) + }) + + it("should not validate jsonSchema constraints for geojson fields", async () => { + const table = DataFrame({ + location: [ + '{"type":"Point","coordinates":[0,0]}', + '{"type":"Point","coordinates":[100,200]}', + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "location", + type: "geojson", + constraints: { + jsonSchema: { + type: "object", + properties: { + coordinates: { + type: "array", + items: { type: "number", minimum: -50, maximum: 50 }, + }, + }, + }, + }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(0) + }) + + it("should not report errors for valid TopoJSON", async () => { + const table = DataFrame({ + topology: [ + '{"type":"Topology","objects":{"example":{"type":"GeometryCollection","geometries":[{"type":"Point","coordinates":[0,0]}]}},"arcs":[]}', + '{"type":"Topology","objects":{"collection":{"type":"GeometryCollection","geometries":[]}},"arcs":[]}', + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "topology", + type: "geojson", + format: "topojson", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(0) + }) + + it("should report errors for invalid TopoJSON structure", async () => { + const table = DataFrame({ + topology: [ + '{"type":"Topology","objects":{"example":{"type":"GeometryCollection","geometries":[]}},"arcs":[]}', + '{"type":"Topology","objects":{}}', + '{"type":"Topology"}', + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "topology", + type: "geojson", + format: "topojson", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(2) + expect(errors).toContainEqual({ + type: "cell/type", + fieldName: "topology", + fieldType: "geojson", + rowNumber: 2, + cell: '{"type":"Topology","objects":{}}', + }) + expect(errors).toContainEqual({ + type: "cell/type", + fieldName: "topology", + fieldType: "geojson", + rowNumber: 3, + cell: '{"type":"Topology"}', + }) + }) + + it("should accept TopoJSON geometry objects", async () => { + const table = DataFrame({ + geometry: [ + '{"type":"Point","coordinates":[0,0]}', + '{"type":"LineString","arcs":[0,1]}', + '{"type":"Polygon","arcs":[[0,1,2]]}', + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "geometry", + type: "geojson", + format: "topojson", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(0) + }) + + it("should handle null values for topojson format", async () => { + const table = DataFrame({ + topology: [ + '{"type":"Topology","objects":{"example":{"type":"GeometryCollection","geometries":[]}},"arcs":[]}', + null, + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "topology", + type: "geojson", + format: "topojson", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(0) }) }) diff --git a/table/field/types/geojson.ts b/table/field/types/geojson.ts index da7a404f..7a455ab3 100644 --- a/table/field/types/geojson.ts +++ b/table/field/types/geojson.ts @@ -1,16 +1,12 @@ import type { GeojsonField } from "@dpkit/core" -import { lit, when } from "nodejs-polars" -import type { Expr } from "nodejs-polars" +import geojsonProfile from "../../profiles/geojson.json" with { type: "json" } +import topojsonProfile from "../../profiles/topojson.json" with { type: "json" } +import type { Table } from "../../table/index.ts" +import { validateJsonField } from "./json.ts" -// TODO: Is there a better way to do this? -// Polars does not support really support free-form JSON -// So we just make a basic check and return as it is -export function parseGeojsonField(_field: GeojsonField, fieldExpr: Expr) { - return when(fieldExpr.str.contains("^\\{")) - .then(fieldExpr) - .otherwise(lit(null)) -} - -export function stringifyGeojsonField(_field: GeojsonField, fieldExpr: Expr) { - return fieldExpr +export async function validateGeojsonField(field: GeojsonField, table: Table) { + return validateJsonField(field, table, { + formatProfile: + field.format === "topojson" ? topojsonProfile : geojsonProfile, + }) } diff --git a/table/field/types/json.ts b/table/field/types/json.ts new file mode 100644 index 00000000..2b8fb663 --- /dev/null +++ b/table/field/types/json.ts @@ -0,0 +1,91 @@ +import type { ArrayField, GeojsonField, ObjectField } from "@dpkit/core" +import { validateDescriptor } from "@dpkit/core" +import * as pl from "nodejs-polars" +import type { CellError } from "../../error/index.ts" +import { isObject } from "../../helpers.ts" +import type { Table } from "../../table/index.ts" + +// TODO: Improve the implementation +// Make unblocking / handle large data / process in parallel / move processing to Rust? + +export async function validateJsonField( + field: ArrayField | GeojsonField | ObjectField, + table: Table, + options?: { + formatProfile?: Record + }, +) { + const errors: CellError[] = [] + + const formatProfile = options?.formatProfile + const constraintProfile = field.constraints?.jsonSchema + + const frame = await table + .withRowCount() + .select( + pl.col("row_nr").add(1).alias("number"), + pl.col(field.name).alias("source"), + ) + .collect() + + for (const row of frame.toRecords() as any[]) { + if (row.source === null) continue + + let target: Record | undefined + const checkCompat = field.type === "array" ? Array.isArray : isObject + + try { + target = JSON.parse(row.source) + } catch (error) {} + + if (!target || !checkCompat(target)) { + errors.push({ + type: "cell/type", + cell: String(row.source), + fieldName: field.name, + fieldType: field.type, + rowNumber: row.number, + }) + + continue + } + + if (formatProfile) { + // TODO: Extract more generic function validateJson? + const report = await validateDescriptor(target as any, { + profile: formatProfile, + }) + + if (!report.valid) { + errors.push({ + type: "cell/type", + cell: String(row.source), + fieldName: field.name, + fieldType: field.type, + rowNumber: row.number, + }) + } + + continue + } + + if (constraintProfile) { + // TODO: Extract more generic function validateJson? + const report = await validateDescriptor(target as any, { + profile: constraintProfile, + }) + + if (!report.valid) { + errors.push({ + type: "cell/jsonSchema", + cell: String(row.source), + fieldName: field.name, + rowNumber: row.number, + jsonSchema: constraintProfile, + }) + } + } + } + + return errors +} diff --git a/table/field/types/object.spec.ts b/table/field/types/object.spec.ts index 36cdbf30..aa302b1c 100644 --- a/table/field/types/object.spec.ts +++ b/table/field/types/object.spec.ts @@ -1,74 +1,391 @@ -import { DataFrame, DataType, Series } from "nodejs-polars" +import type { Schema } from "@dpkit/core" +import { DataFrame } from "nodejs-polars" import { describe, expect, it } from "vitest" -import { denormalizeTable, normalizeTable } from "../../table/index.ts" - -describe("parseObjectField", () => { - it.each([ - // Valid JSON objects - ['{"name":"John","age":30}', { name: "John", age: 30 }], - ['{"numbers":[1,2,3]}', { numbers: [1, 2, 3] }], - ['{"nested":{"prop":"value"}}', { nested: { prop: "value" } }], - ["{}", {}], - - // JSON but not an object - ["[1,2,3]", null], - ['["a","b","c"]', null], - - // Trimming whitespace - //[' {"name":"John"} ', { name: "John" }], - //['\t{"name":"John"}\n', { name: "John" }], - - // Invalid JSON - //["{invalid}", null], - ["not json", null], - ["", null], - ["null", null], - ["undefined", null], - ])("%s -> %s", async (cell, value) => { - const table = DataFrame([Series("name", [cell], DataType.String)]).lazy() - - const schema = { - fields: [{ name: "name", type: "object" as const }], +import { validateTable } from "../../table/index.ts" + +describe("validateObjectField", () => { + it("should not report errors for valid JSON objects", async () => { + const table = DataFrame({ + metadata: ['{"key":"value"}', '{"num":123}', '{"arr":[1,2,3]}'], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "metadata", + type: "object", + }, + ], } - const ldf = await normalizeTable(table, schema) - const df = await ldf.collect() + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(0) + }) + + it("should report errors for JSON arrays", async () => { + const table = DataFrame({ + data: ["[1,2,3]", '{"key":"value"}', '["a","b","c"]'], + }).lazy() - const res = df.getColumn("name").get(0) - expect(res ? JSON.parse(res) : res).toEqual(value) + const schema: Schema = { + fields: [ + { + name: "data", + type: "object", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toEqual([ + { + type: "cell/type", + fieldName: "data", + fieldType: "object", + rowNumber: 1, + cell: "[1,2,3]", + }, + { + type: "cell/type", + fieldName: "data", + fieldType: "object", + rowNumber: 3, + cell: '["a","b","c"]', + }, + ]) }) -}) -describe("stringifyObjectField", () => { - it.each([ - // JSON strings should be returned as-is - ['{"name":"John","age":30}', '{"name":"John","age":30}'], - ['{"numbers":[1,2,3]}', '{"numbers":[1,2,3]}'], - ['{"nested":{"prop":"value"}}', '{"nested":{"prop":"value"}}'], - ["{}", "{}"], - - // Complex nested objects as JSON strings - [ - '{"users":[{"id":1,"name":"Alice"},{"id":2,"name":"Bob"}]}', - '{"users":[{"id":1,"name":"Alice"},{"id":2,"name":"Bob"}]}', - ], - [ - '{"config":{"debug":true,"timeout":5000}}', - '{"config":{"debug":true,"timeout":5000}}', - ], - - // Null handling - [null, ""], - ])("%s -> %s", async (value, expected) => { - const table = DataFrame([Series("name", [value], DataType.String)]).lazy() - - const schema = { - fields: [{ name: "name", type: "object" as const }], + it("should not report errors for null values", async () => { + const table = DataFrame({ + config: ['{"key":"value"}', null, '{"num":123}'], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "config", + type: "object", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(0) + }) + + it("should report errors for invalid JSON", async () => { + const table = DataFrame({ + data: ['{"valid":true}', "invalid json", '{"key":"value"}', "{broken}"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "data", + type: "object", + }, + ], } - const ldf = await denormalizeTable(table, schema) - const df = await ldf.collect() + const { errors } = await validateTable(table, { schema }) + expect(errors.filter(e => e.type === "cell/type")).toHaveLength(2) + expect(errors).toContainEqual({ + type: "cell/type", + fieldName: "data", + fieldType: "object", + rowNumber: 2, + cell: "invalid json", + }) + expect(errors).toContainEqual({ + type: "cell/type", + fieldName: "data", + fieldType: "object", + rowNumber: 4, + cell: "{broken}", + }) + }) + + it("should handle complex nested JSON structures", async () => { + const table = DataFrame({ + complex: [ + '{"user":{"name":"John","age":30,"tags":["admin","user"]}}', + '{"nested":{"deep":{"value":true}}}', + '{"array":[{"id":1},{"id":2}]}', + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "complex", + type: "object", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(0) + }) + + it("should report errors for empty strings", async () => { + const table = DataFrame({ + data: ['{"valid":true}', "", '{"key":"value"}'], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "data", + type: "object", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toEqual([ + { + type: "cell/type", + fieldName: "data", + fieldType: "object", + rowNumber: 2, + cell: "", + }, + ]) + }) + + it("should report errors for JSON primitives", async () => { + const table = DataFrame({ + data: ['"string"', "123", "true", "false", "null"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "data", + type: "object", + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toEqual([ + { + type: "cell/type", + fieldName: "data", + fieldType: "object", + rowNumber: 1, + cell: '"string"', + }, + { + type: "cell/type", + fieldName: "data", + fieldType: "object", + rowNumber: 2, + cell: "123", + }, + { + type: "cell/type", + fieldName: "data", + fieldType: "object", + rowNumber: 3, + cell: "true", + }, + { + type: "cell/type", + fieldName: "data", + fieldType: "object", + rowNumber: 4, + cell: "false", + }, + { + type: "cell/type", + fieldName: "data", + fieldType: "object", + rowNumber: 5, + cell: "null", + }, + ]) + }) + + it("should not report errors for objects matching jsonSchema", async () => { + const table = DataFrame({ + user: [ + '{"name":"John","age":30}', + '{"name":"Jane","age":25}', + '{"name":"Bob","age":35}', + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "user", + type: "object", + constraints: { + jsonSchema: { + type: "object", + properties: { + name: { type: "string" }, + age: { type: "number" }, + }, + required: ["name", "age"], + }, + }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(0) + }) + + it("should report errors for objects not matching jsonSchema", async () => { + const jsonSchema = { + type: "object", + properties: { + name: { type: "string" }, + age: { type: "number" }, + }, + required: ["name", "age"], + } + + const table = DataFrame({ + user: [ + '{"name":"John","age":30}', + '{"name":"Jane"}', + '{"age":25}', + '{"name":"Bob","age":"invalid"}', + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "user", + type: "object", + constraints: { + jsonSchema, + }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors.filter(e => e.type === "cell/jsonSchema")).toHaveLength(3) + expect(errors).toContainEqual({ + type: "cell/jsonSchema", + fieldName: "user", + jsonSchema, + rowNumber: 2, + cell: '{"name":"Jane"}', + }) + expect(errors).toContainEqual({ + type: "cell/jsonSchema", + fieldName: "user", + jsonSchema, + rowNumber: 3, + cell: '{"age":25}', + }) + expect(errors).toContainEqual({ + type: "cell/jsonSchema", + fieldName: "user", + jsonSchema, + rowNumber: 4, + cell: '{"name":"Bob","age":"invalid"}', + }) + }) + + it("should validate complex jsonSchema with nested objects", async () => { + const table = DataFrame({ + config: [ + '{"database":{"host":"localhost","port":5432},"cache":{"enabled":true}}', + '{"database":{"host":"localhost","port":"invalid"},"cache":{"enabled":true}}', + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "config", + type: "object", + constraints: { + jsonSchema: { + type: "object", + properties: { + database: { + type: "object", + properties: { + host: { type: "string" }, + port: { type: "number" }, + }, + required: ["host", "port"], + }, + cache: { + type: "object", + properties: { + enabled: { type: "boolean" }, + }, + required: ["enabled"], + }, + }, + required: ["database", "cache"], + }, + }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toEqual([ + { + type: "cell/jsonSchema", + fieldName: "config", + jsonSchema: schema.fields[0].constraints?.jsonSchema, + rowNumber: 2, + cell: '{"database":{"host":"localhost","port":"invalid"},"cache":{"enabled":true}}', + }, + ]) + }) + + it("should validate jsonSchema with array properties", async () => { + const table = DataFrame({ + data: [ + '{"items":[1,2,3],"name":"test"}', + '{"items":["not","numbers"],"name":"test"}', + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "data", + type: "object", + constraints: { + jsonSchema: { + type: "object", + properties: { + items: { + type: "array", + items: { type: "number" }, + }, + name: { type: "string" }, + }, + required: ["items", "name"], + }, + }, + }, + ], + } - expect(df.toRecords()[0]?.name).toEqual(expected) + const { errors } = await validateTable(table, { schema }) + expect(errors).toEqual([ + { + type: "cell/jsonSchema", + fieldName: "data", + jsonSchema: schema.fields[0].constraints?.jsonSchema, + rowNumber: 2, + cell: '{"items":["not","numbers"],"name":"test"}', + }, + ]) }) }) diff --git a/table/field/types/object.ts b/table/field/types/object.ts index 02e0c8c5..312f8240 100644 --- a/table/field/types/object.ts +++ b/table/field/types/object.ts @@ -1,16 +1,7 @@ import type { ObjectField } from "@dpkit/core" -import { lit, when } from "nodejs-polars" -import type { Expr } from "nodejs-polars" +import type { Table } from "../../table/index.ts" +import { validateJsonField } from "./json.ts" -// TODO: Is there a better way to do this? -// Polars does not support really support free-form JSON -// So we just make a basic check and return as it is -export function parseObjectField(_field: ObjectField, fieldExpr: Expr) { - return when(fieldExpr.str.contains("^\\{")) - .then(fieldExpr) - .otherwise(lit(null)) -} - -export function stringifyObjectField(_field: ObjectField, fieldExpr: Expr) { - return fieldExpr +export async function validateObjectField(field: ObjectField, table: Table) { + return validateJsonField(field, table) } diff --git a/table/field/validate.ts b/table/field/validate.ts index 963c02b1..b544d227 100644 --- a/table/field/validate.ts +++ b/table/field/validate.ts @@ -13,6 +13,9 @@ import { checkCellRequired } from "./checks/required.ts" import { checkCellType } from "./checks/type.ts" import { checkCellUnique } from "./checks/unique.ts" import { normalizeField } from "./normalize.ts" +import { validateArrayField } from "./types/array.ts" +import { validateGeojsonField } from "./types/geojson.ts" +import { validateObjectField } from "./types/object.ts" export async function validateField( mapping: FieldMapping, @@ -54,9 +57,13 @@ function validateName(mapping: FieldMapping) { function validateType(mapping: FieldMapping) { const errors: FieldError[] = [] + const variant = mapping.source.type.variant + // TODO: Rebase on proper polars type definition when available + // https://github.com/pola-rs/nodejs-polars/issues/372 const compatMapping: Record = { Bool: ["boolean"], + Categorical: ["string"], Date: ["date"], Datetime: ["datetime"], Float32: ["number", "integer"], @@ -66,18 +73,21 @@ function validateType(mapping: FieldMapping) { Int64: ["integer"], Int8: ["integer"], List: ["list"], + String: ["any"], Time: ["time"], UInt16: ["integer"], UInt32: ["integer"], UInt64: ["integer"], UInt8: ["integer"], - Utf8: ["string"], + Utf8: ["any"], } - const compatTypes = compatMapping[mapping.source.type.variant] - if (!compatTypes) return errors + const compatTypes = compatMapping[variant] ?? [] + const isCompat = !!new Set(compatTypes).intersection( + new Set([mapping.target.type, "any"]), + ).size - if (!compatTypes.includes(mapping.target.type)) { + if (!isCompat) { errors.push({ type: "field/type", fieldName: mapping.target.name, @@ -99,6 +109,16 @@ async function validateCells( const { maxErrors } = options const errors: CellError[] = [] + // Types that require non-polars validation + switch (mapping.target.type) { + case "array": + return await validateArrayField(mapping.target, table) + case "geojson": + return await validateGeojsonField(mapping.target, table) + case "object": + return await validateObjectField(mapping.target, table) + } + let fieldCheckTable = table .withRowCount() .select( diff --git a/table/helpers.ts b/table/helpers.ts index 032b745c..f5c85729 100644 --- a/table/helpers.ts +++ b/table/helpers.ts @@ -1,6 +1,10 @@ import type { Expr } from "nodejs-polars" import * as pl from "nodejs-polars" +export function isObject(value: any): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value) +} + export function arrayDiff(a: string[], b: string[]) { return a.filter(x => !b.includes(x)) } diff --git a/table/profiles/geojson.json b/table/profiles/geojson.json new file mode 100644 index 00000000..5e9039c5 --- /dev/null +++ b/table/profiles/geojson.json @@ -0,0 +1,216 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "Geo JSON object", + "description": "Schema for a Geo JSON object", + "type": "object", + "required": ["type"], + "properties": { + "crs": { "$ref": "#/definitions/crs" }, + "bbox": { "$ref": "#/definitions/bbox" } + }, + "oneOf": [ + { "$ref": "#/definitions/geometry" }, + { "$ref": "#/definitions/geometryCollection" }, + { "$ref": "#/definitions/feature" }, + { "$ref": "#/definitions/featureCollection" } + ], + "definitions": { + "geometryCollection": { + "title": "GeometryCollection", + "description": "A collection of geometry objects", + "required": ["geometries"], + "properties": { + "type": { "enum": ["GeometryCollection"] }, + "geometries": { + "type": "array", + "items": { "$ref": "#/definitions/geometry" } + } + } + }, + "feature": { + "title": "Feature", + "description": "A Geo JSON feature object", + "required": ["geometry", "properties"], + "properties": { + "type": { "enum": ["Feature"] }, + "geometry": { + "oneOf": [{ "type": "null" }, { "$ref": "#/definitions/geometry" }] + }, + "properties": { "type": ["object", "null"] }, + "id": { "FIXME": "may be there, type not known (string? number?)" } + } + }, + "featureCollection": { + "title": "FeatureCollection", + "description": "A Geo JSON feature collection", + "required": ["features"], + "properties": { + "type": { "enum": ["FeatureCollection"] }, + "features": { + "type": "array", + "items": { "$ref": "#/definitions/feature" } + } + } + }, + "geometry": { + "title": "geometry", + "description": "One geometry as defined by GeoJSON", + "type": "object", + "required": ["type", "coordinates"], + "oneOf": [ + { + "title": "Point", + "properties": { + "type": { "enum": ["Point"] }, + "coordinates": { + "$ref": "#/definitions/geometry/definitions/position" + } + } + }, + { + "title": "MultiPoint", + "properties": { + "type": { "enum": ["MultiPoint"] }, + "coordinates": { + "$ref": "#/definitions/geometry/definitions/positionArray" + } + } + }, + { + "title": "LineString", + "properties": { + "type": { "enum": ["LineString"] }, + "coordinates": { + "$ref": "#/definitions/geometry/definitions/lineString" + } + } + }, + { + "title": "MultiLineString", + "properties": { + "type": { "enum": ["MultiLineString"] }, + "coordinates": { + "type": "array", + "items": { + "$ref": "#/definitions/geometry/definitions/lineString" + } + } + } + }, + { + "title": "Polygon", + "properties": { + "type": { "enum": ["Polygon"] }, + "coordinates": { + "$ref": "#/definitions/geometry/definitions/polygon" + } + } + }, + { + "title": "MultiPolygon", + "properties": { + "type": { "enum": ["MultiPolygon"] }, + "coordinates": { + "type": "array", + "items": { "$ref": "#/definitions/geometry/definitions/polygon" } + } + } + } + ], + "definitions": { + "position": { + "description": "A single position", + "type": "array", + "minItems": 2, + "items": [{ "type": "number" }, { "type": "number" }], + "additionalItems": false + }, + "positionArray": { + "description": "An array of positions", + "type": "array", + "items": { "$ref": "#/definitions/geometry/definitions/position" } + }, + "lineString": { + "description": "An array of two or more positions", + "allOf": [ + { "$ref": "#/definitions/geometry/definitions/positionArray" }, + { "minItems": 2 } + ] + }, + "linearRing": { + "description": "An array of four positions where the first equals the last", + "allOf": [ + { "$ref": "#/definitions/geometry/definitions/positionArray" }, + { "minItems": 4 } + ] + }, + "polygon": { + "description": "An array of linear rings", + "type": "array", + "items": { "$ref": "#/definitions/geometry/definitions/linearRing" } + } + } + }, + "crs": { + "title": "crs", + "description": "a Coordinate Reference System object", + "type": ["object", "null"], + "required": ["type", "properties"], + "properties": { + "type": { "type": "string" }, + "properties": { "type": "object" } + }, + "additionalProperties": false, + "oneOf": [ + { "$ref": "#/definitions/crs/definitions/namedCrs" }, + { "$ref": "#/definitions/crs/definitions/linkedCrs" } + ], + "definitions": { + "namedCrs": { + "properties": { + "type": { "enum": ["name"] }, + "properties": { + "required": ["name"], + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "FIXME": "semantic validation necessary" + } + } + } + } + }, + "linkedObject": { + "type": "object", + "required": ["href"], + "properties": { + "href": { + "type": "string", + "format": "uri", + "FIXME": "spec says \"dereferenceable\", cannot enforce that" + }, + "type": { + "type": "string", + "description": "Suggested values: proj4, ogjwkt, esriwkt" + } + } + }, + "linkedCrs": { + "properties": { + "type": { "enum": ["link"] }, + "properties": { + "$ref": "#/definitions/crs/definitions/linkedObject" + } + } + } + } + }, + "bbox": { + "description": "A bounding box as defined by GeoJSON", + "FIXME": "unenforceable constraint: even number of elements in array", + "type": "array", + "items": { "type": "number" } + } + } +} diff --git a/table/profiles/topojson.json b/table/profiles/topojson.json new file mode 100644 index 00000000..eea22837 --- /dev/null +++ b/table/profiles/topojson.json @@ -0,0 +1,259 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "title": "TopoJSON object", + "description": "Schema for a TopoJSON object", + "type": "object", + "required": ["type"], + "properties": { + "bbox": { + "$ref": "#/definitions/bbox" + } + }, + "oneOf": [ + { + "$ref": "#/definitions/topology" + }, + { + "$ref": "#/definitions/geometry" + } + ], + "definitions": { + "bbox": { + "title": "TopoJSON bounding box", + "description": "A bounding box as defined by TopoJSON", + "type": "array", + "items": { + "$ref": "#/definitions/bbox/definitions/dimension" + }, + "minItems": 2, + "maxItems": 2, + "definitions": { + "dimension": { + "type": "array", + "description": "This array should have an entry per dimension in the geometries", + "items": { + "type": "number" + } + } + } + }, + "geometry": { + "title": "Geometry objects", + "description": "A Geometry object as defined by TopoJSON", + "type": "object", + "required": ["type"], + "properties": { + "id": { + "type": ["string", "integer"] + }, + "properties": { + "type": "object" + } + }, + "oneOf": [ + { + "title": "Point", + "description": "A Point Geometry object as defined by TopoJSON", + "required": ["type", "coordinates"], + "properties": { + "type": { + "enum": ["Point"] + }, + "coordinates": { + "$ref": "#/definitions/geometry/definitions/position" + } + } + }, + { + "title": "MultiPoint", + "description": "A MultiPoint Geometry object as defined by TopoJSON", + "required": ["type", "coordinates"], + "properties": { + "type": { + "enum": ["MultiPoint"] + }, + "coordinates": { + "type": "array", + "items": { + "$ref": "#/definitions/geometry/definitions/position" + } + } + } + }, + { + "title": "LineString", + "description": "A LineString Geometry object as defined by TopoJSON", + "required": ["type", "arcs"], + "properties": { + "type": { + "enum": ["LineString"] + }, + "arcs": { + "type": "array", + "items": { + "type": "integer" + } + } + } + }, + { + "title": "MultiLineString", + "description": "A MultiLineString Geometry object as defined by TopoJSON", + "required": ["type", "arcs"], + "properties": { + "type": { + "enum": ["MultiLineString"] + }, + "arcs": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "integer" + } + } + } + } + }, + { + "title": "Polygon", + "description": "A Polygon Geometry object as defined by TopoJSON", + "required": ["type", "arcs"], + "properties": { + "type": { + "enum": ["Polygon"] + }, + "arcs": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "integer" + } + } + } + } + }, + { + "title": "MultiPolygon", + "description": "A MultiPolygon Geometry object as defined by TopoJSON", + "required": ["type", "arcs"], + "properties": { + "type": { + "enum": ["MultiPolygon"] + }, + "arcs": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "integer" + } + } + } + } + } + }, + { + "title": "GeometryCollection", + "description": "A MultiPolygon Geometry object as defined by TopoJSON", + "required": ["type", "geometries"], + "properties": { + "type": { + "enum": ["GeometryCollection"] + }, + "geometries": { + "type": "array", + "items": { + "$ref": "#/definitions/geometry" + } + } + } + } + ], + "definitions": { + "position": { + "type": "array", + "items": { + "type": "number" + }, + "minItems": 2 + } + } + }, + "topology": { + "title": "Topology", + "description": "A Topology object as defined by TopoJSON", + "type": "object", + "required": ["objects", "arcs"], + "properties": { + "type": { + "enum": ["Topology"] + }, + "objects": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/geometry" + } + }, + "arcs": { + "$ref": "#/definitions/topology/definitions/arcs" + }, + "transform": { + "$ref": "#/definitions/topology/definitions/transform" + }, + "bbox": { + "$ref": "#/definitions/bbox" + } + }, + "definitions": { + "transform": { + "type": "object", + "required": ["scale", "translate"], + "properties": { + "scale": { + "type": "array", + "items": { + "type": "number" + }, + "minItems": 2 + }, + "translate": { + "type": "array", + "items": { + "type": "number" + }, + "minItems": 2 + } + } + }, + "arcs": { + "type": "array", + "items": { + "type": "array", + "items": { + "oneOf": [ + { + "$ref": "#/definitions/topology/definitions/position" + }, + { + "type": "null" + } + ] + }, + "minItems": 2 + } + }, + "position": { + "type": "array", + "items": { + "type": "number" + }, + "minItems": 2 + } + } + } + } +}