From 54a2c930167fafedf0096f3d48646a7ace8ff12a Mon Sep 17 00:00:00 2001 From: roll Date: Tue, 28 Oct 2025 08:07:04 +0000 Subject: [PATCH 01/42] Rebased on anyHorizontal for errors --- table/field/checks/enum.ts | 6 +++--- table/field/checks/maxLength.ts | 6 +++--- table/field/checks/maximum.ts | 16 ++++++---------- table/field/checks/minLength.ts | 6 +++--- table/field/checks/minimum.ts | 16 ++++++---------- table/field/checks/pattern.ts | 6 +++--- table/field/checks/required.ts | 1 - table/field/checks/type.ts | 8 +++----- table/field/checks/unique.ts | 8 +++----- table/row/checks/unique.ts | 1 - table/table/validate.ts | 19 +++++++++++-------- 11 files changed, 41 insertions(+), 52 deletions(-) diff --git a/table/field/checks/enum.ts b/table/field/checks/enum.ts index 5ad37c52..8e9f6c7f 100644 --- a/table/field/checks/enum.ts +++ b/table/field/checks/enum.ts @@ -10,9 +10,9 @@ export function checkCellEnum(field: Field, errorTable: Table) { const target = col(`target:${field.name}`) const errorName = `error:cell/enum:${field.name}` - errorTable = errorTable - .withColumn(target.isIn(rawEnum).not().alias(errorName)) - .withColumn(col("error").or(col(errorName)).alias("error")) + errorTable = errorTable.withColumn( + target.isIn(rawEnum).not().alias(errorName), + ) } } diff --git a/table/field/checks/maxLength.ts b/table/field/checks/maxLength.ts index d1b6ef2a..3779685d 100644 --- a/table/field/checks/maxLength.ts +++ b/table/field/checks/maxLength.ts @@ -10,9 +10,9 @@ export function checkCellMaxLength(field: Field, errorTable: Table) { const target = col(`target:${field.name}`) const errorName = `error:cell/maxLength:${field.name}` - errorTable = errorTable - .withColumn(target.str.lengths().gt(maxLength).alias(errorName)) - .withColumn(col("error").or(col(errorName)).alias("error")) + errorTable = errorTable.withColumn( + target.str.lengths().gt(maxLength).alias(errorName), + ) } } diff --git a/table/field/checks/maximum.ts b/table/field/checks/maximum.ts index ccb7d8be..b5037075 100644 --- a/table/field/checks/maximum.ts +++ b/table/field/checks/maximum.ts @@ -28,17 +28,13 @@ export function checkCellMaximum( const parsedMaximum = typeof maximum === "string" ? parser(maximum) : maximum - errorTable = errorTable - .withColumn( - options?.isExclusive - ? target.gtEq(parsedMaximum).alias(errorName) - : target.gt(parsedMaximum).alias(errorName), - ) - .withColumn(col("error").or(col(errorName)).alias("error")) + errorTable = errorTable.withColumn( + options?.isExclusive + ? target.gtEq(parsedMaximum).alias(errorName) + : target.gt(parsedMaximum).alias(errorName), + ) } catch (error) { - errorTable = errorTable - .withColumn(lit(true).alias(errorName)) - .withColumn(lit(true).alias("error")) + errorTable = errorTable.withColumn(lit(true).alias(errorName)) } } } diff --git a/table/field/checks/minLength.ts b/table/field/checks/minLength.ts index aa9106a3..3214bc96 100644 --- a/table/field/checks/minLength.ts +++ b/table/field/checks/minLength.ts @@ -10,9 +10,9 @@ export function checkCellMinLength(field: Field, errorTable: Table) { const target = col(`target:${field.name}`) const errorName = `error:cell/minLength:${field.name}` - errorTable = errorTable - .withColumn(target.str.lengths().lt(minLength).alias(errorName)) - .withColumn(col("error").or(col(errorName)).alias("error")) + errorTable = errorTable.withColumn( + target.str.lengths().lt(minLength).alias(errorName), + ) } } diff --git a/table/field/checks/minimum.ts b/table/field/checks/minimum.ts index c193df3c..7cefdcfc 100644 --- a/table/field/checks/minimum.ts +++ b/table/field/checks/minimum.ts @@ -27,17 +27,13 @@ export function checkCellMinimum( const parsedMinimum = typeof minimum === "string" ? parser(minimum) : minimum - errorTable = errorTable - .withColumn( - options?.isExclusive - ? target.ltEq(parsedMinimum).alias(errorName) - : target.lt(parsedMinimum).alias(errorName), - ) - .withColumn(col("error").or(col(errorName)).alias("error")) + errorTable = errorTable.withColumn( + options?.isExclusive + ? target.ltEq(parsedMinimum).alias(errorName) + : target.lt(parsedMinimum).alias(errorName), + ) } catch (error) { - errorTable = errorTable - .withColumn(lit(true).alias(errorName)) - .withColumn(lit(true).alias("error")) + errorTable = errorTable.withColumn(lit(true).alias(errorName)) } } } diff --git a/table/field/checks/pattern.ts b/table/field/checks/pattern.ts index 82acbe3c..14c21b55 100644 --- a/table/field/checks/pattern.ts +++ b/table/field/checks/pattern.ts @@ -10,9 +10,9 @@ export function checkCellPattern(field: Field, errorTable: Table) { const target = col(`target:${field.name}`) const errorName = `error:cell/pattern:${field.name}` - errorTable = errorTable - .withColumn(target.str.contains(pattern).not().alias(errorName)) - .withColumn(col("error").or(col(errorName)).alias("error")) + errorTable = errorTable.withColumn( + target.str.contains(pattern).not().alias(errorName), + ) } } diff --git a/table/field/checks/required.ts b/table/field/checks/required.ts index b7eac84d..5e8d58be 100644 --- a/table/field/checks/required.ts +++ b/table/field/checks/required.ts @@ -9,7 +9,6 @@ export function checkCellRequired(field: Field, errorTable: Table) { errorTable = errorTable .withColumn(target.isNull().alias(errorName)) - .withColumn(col("error").or(col(errorName)).alias("error")) } return errorTable diff --git a/table/field/checks/type.ts b/table/field/checks/type.ts index eb5db3e8..18d89e75 100644 --- a/table/field/checks/type.ts +++ b/table/field/checks/type.ts @@ -7,9 +7,7 @@ export function checkCellType(field: Field, errorTable: Table) { const target = col(`target:${field.name}`) const errorName = `error:cell/type:${field.name}` - errorTable = errorTable - .withColumn(source.isNotNull().and(target.isNull()).alias(errorName)) - .withColumn(col("error").or(col(errorName)).alias("error")) - - return errorTable + return errorTable.withColumn( + source.isNotNull().and(target.isNull()).alias(errorName), + ) } diff --git a/table/field/checks/unique.ts b/table/field/checks/unique.ts index 5078afba..ed648de1 100644 --- a/table/field/checks/unique.ts +++ b/table/field/checks/unique.ts @@ -10,11 +10,9 @@ export function checkCellUnique(field: Field, errorTable: Table) { const target = col(`target:${field.name}`) const errorName = `error:cell/unique:${field.name}` - errorTable = errorTable - .withColumn( - target.isNotNull().and(target.isFirstDistinct().not()).alias(errorName), - ) - .withColumn(col("error").or(col(errorName)).alias("error")) + errorTable = errorTable.withColumn( + target.isNotNull().and(target.isFirstDistinct().not()).alias(errorName), + ) } return errorTable diff --git a/table/row/checks/unique.ts b/table/row/checks/unique.ts index 3ed1e21e..f49e6453 100644 --- a/table/row/checks/unique.ts +++ b/table/row/checks/unique.ts @@ -25,7 +25,6 @@ export function checkRowUnique(schema: Schema, errorTable: Table) { .and(col(errorName).isFirstDistinct().not()) .alias(errorName), ) - .withColumn(col("error").or(col(errorName)).alias("error")) } return errorTable diff --git a/table/table/validate.ts b/table/table/validate.ts index 9ee59b89..91f12dff 100644 --- a/table/table/validate.ts +++ b/table/table/validate.ts @@ -1,5 +1,5 @@ import type { Schema } from "@dpkit/core" -import { col, lit } from "nodejs-polars" +import { anyHorizontal, col } from "nodejs-polars" import type { TableError } from "../error/index.ts" import { matchField } from "../field/index.ts" import { validateField } from "../field/index.ts" @@ -152,12 +152,7 @@ async function validateFields( let errorTable = table .withRowCount() - .select( - col("row_nr").add(1), - lit(false).alias("error"), - ...sources, - ...targets, - ) + .select(col("row_nr").add(1), ...sources, ...targets) for (const [index, field] of schema.fields.entries()) { const polarsField = matchField(index, field, schema, polarsSchema) @@ -172,8 +167,16 @@ async function validateFields( errorTable = rowsResult.errorTable errors.push(...rowsResult.errors) + const erorrColumns = errorTable.columns.filter(name => + name.startsWith("error:"), + ) + + if (!erorrColumns.length) { + return errors + } + const errorFrame = await errorTable - .filter(col("error").eq(true)) + .filter(anyHorizontal(erorrColumns.map(col))) .head(invalidRowsLimit) .drop(targetNames) .collect() From 52d0461878002802a3eed95d8ebd520567ff4626 Mon Sep 17 00:00:00 2001 From: roll Date: Tue, 28 Oct 2025 08:32:52 +0000 Subject: [PATCH 02/42] Drop target names first --- table/table/validate.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/table/table/validate.ts b/table/table/validate.ts index 91f12dff..aecea864 100644 --- a/table/table/validate.ts +++ b/table/table/validate.ts @@ -176,9 +176,9 @@ async function validateFields( } const errorFrame = await errorTable + .drop(targetNames) .filter(anyHorizontal(erorrColumns.map(col))) .head(invalidRowsLimit) - .drop(targetNames) .collect() for (const record of errorFrame.toRecords() as any[]) { From 34bddd69dfa76f2b42ad3d1d46de4977ce5c197f Mon Sep 17 00:00:00 2001 From: roll Date: Tue, 28 Oct 2025 08:51:26 +0000 Subject: [PATCH 03/42] Use limit over head --- table/table/validate.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/table/table/validate.ts b/table/table/validate.ts index aecea864..e8e82f09 100644 --- a/table/table/validate.ts +++ b/table/table/validate.ts @@ -176,9 +176,9 @@ async function validateFields( } const errorFrame = await errorTable - .drop(targetNames) .filter(anyHorizontal(erorrColumns.map(col))) - .head(invalidRowsLimit) + .limit(invalidRowsLimit) + .drop(targetNames) .collect() for (const record of errorFrame.toRecords() as any[]) { From 5daf8a5a1e00d7c3a117aff01eeea9155be24f63 Mon Sep 17 00:00:00 2001 From: roll Date: Tue, 28 Oct 2025 09:10:04 +0000 Subject: [PATCH 04/42] Added streaming --- table/table/validate.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/table/table/validate.ts b/table/table/validate.ts index e8e82f09..0d784b2c 100644 --- a/table/table/validate.ts +++ b/table/table/validate.ts @@ -179,7 +179,7 @@ async function validateFields( .filter(anyHorizontal(erorrColumns.map(col))) .limit(invalidRowsLimit) .drop(targetNames) - .collect() + .collect({ streaming: true }) for (const record of errorFrame.toRecords() as any[]) { const typeErrorInFields: string[] = [] From 5e2021617210c233f33487fa665bb532ea83ca52 Mon Sep 17 00:00:00 2001 From: roll Date: Tue, 28 Oct 2025 10:28:36 +0000 Subject: [PATCH 05/42] Added p-all --- pnpm-lock.yaml | 17 +++++++++++++++++ table/package.json | 3 ++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 87a7534a..a26e3475 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -859,6 +859,9 @@ importers: nodejs-polars: specifier: ^0.22.1 version: 0.22.1 + p-all: + specifier: ^5.0.1 + version: 5.0.1 test: dependencies: @@ -5042,6 +5045,10 @@ packages: openapi-types@12.1.3: resolution: {integrity: sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw==} + p-all@5.0.1: + resolution: {integrity: sha512-LMT7WX9ZSaq3J1zjloApkIVmtz0ZdMFSIqbuiEa3txGYPLjUPOvgOPOx3nFjo+f37ZYL+1aY666I2SG7GVwLOA==} + engines: {node: '>=16'} + p-each-series@3.0.0: resolution: {integrity: sha512-lastgtAdoH9YaLyDa5i5z64q+kzOcQHsQ5SsZJD3q0VEyI8mq872S3geuNbRUQLVAE9siMfgKrpj7MloKFHruw==} engines: {node: '>=12'} @@ -5066,6 +5073,10 @@ packages: resolution: {integrity: sha512-nQja7m7gSKuewoVRen45CtVfODR3crN3goVQ0DDZ9N3yHxgpkuBhZqsaiotSQRrADUrne346peY7kT3TSACykg==} engines: {node: '>=4'} + p-map@6.0.0: + resolution: {integrity: sha512-T8BatKGY+k5rU+Q/GTYgrEf2r4xRMevAN5mtXc2aPc4rS1j3s+vWTaO2Wag94neXuCAUAs8cxBL9EeB5EA6diw==} + engines: {node: '>=16'} + p-map@7.0.3: resolution: {integrity: sha512-VkndIv2fIB99swvQoA65bm+fsmt6UNdGeIB0oxBs+WhAhdh08QA04JXpI7rbB9r08/nkbysKoya9rtDERYOYMA==} engines: {node: '>=18'} @@ -11417,6 +11428,10 @@ snapshots: openapi-types@12.1.3: {} + p-all@5.0.1: + dependencies: + p-map: 6.0.0 + p-each-series@3.0.0: {} p-filter@4.1.0: @@ -11437,6 +11452,8 @@ snapshots: dependencies: p-limit: 1.3.0 + p-map@6.0.0: {} + p-map@7.0.3: {} p-queue@8.1.1: diff --git a/table/package.json b/table/package.json index faebc257..d30f4eb4 100644 --- a/table/package.json +++ b/table/package.json @@ -25,6 +25,7 @@ }, "dependencies": { "@dpkit/core": "workspace:*", - "nodejs-polars": "^0.22.1" + "nodejs-polars": "^0.22.1", + "p-all": "^5.0.1" } } From cdec8d5f4794d1d490c452ab9fc250e534a781f3 Mon Sep 17 00:00:00 2001 From: roll Date: Tue, 28 Oct 2025 12:21:19 +0000 Subject: [PATCH 06/42] Updated validateFields/validateData --- table/field/denormalize.ts | 19 ++---- table/field/desubstitute.ts | 21 +++++++ table/field/normalize.ts | 31 +++------- table/field/parse.ts | 4 +- table/field/stringify.ts | 4 +- table/field/substitute.ts | 22 +++++++ table/field/validate.ts | 42 ++++++++------ table/table/validate.ts | 111 ++++++++++-------------------------- 8 files changed, 111 insertions(+), 143 deletions(-) create mode 100644 table/field/desubstitute.ts create mode 100644 table/field/substitute.ts diff --git a/table/field/denormalize.ts b/table/field/denormalize.ts index e0c3ec2e..6c4cc074 100644 --- a/table/field/denormalize.ts +++ b/table/field/denormalize.ts @@ -1,23 +1,14 @@ import type { Field } from "@dpkit/core" -import { col, lit, when } from "nodejs-polars" +import { col } from "nodejs-polars" import type { Expr } from "nodejs-polars" +import { desubstituteField } from "./desubstitute.ts" import { stringifyField } from "./stringify.ts" -const DEFAULT_MISSING_VALUE = "" +export function denormalizeField(field: Field, fieldExpr?: Expr) { + let expr = fieldExpr ?? col(field.name) -export function denormalizeField(field: Field, expr?: Expr) { - expr = expr ?? col(field.name) expr = stringifyField(field, expr) - - const flattenMissingValues = field.missingValues?.map(it => - typeof it === "string" ? it : it.value, - ) - - const missingValue = flattenMissingValues?.[0] ?? DEFAULT_MISSING_VALUE - expr = when(expr.isNull()) - .then(lit(missingValue)) - .otherwise(expr) - .alias(field.name) + expr = desubstituteField(field, expr) return expr } diff --git a/table/field/desubstitute.ts b/table/field/desubstitute.ts new file mode 100644 index 00000000..3262870e --- /dev/null +++ b/table/field/desubstitute.ts @@ -0,0 +1,21 @@ +import type { Field } from "@dpkit/core" +import { col, lit, when } from "nodejs-polars" +import type { Expr } from "nodejs-polars" + +const DEFAULT_MISSING_VALUE = "" + +export function desubstituteField(field: Field, expr?: Expr) { + expr = expr ?? col(field.name) + + const flattenMissingValues = field.missingValues?.map(it => + typeof it === "string" ? it : it.value, + ) + + const missingValue = flattenMissingValues?.[0] ?? DEFAULT_MISSING_VALUE + expr = when(expr.isNull()) + .then(lit(missingValue)) + .otherwise(expr) + .alias(field.name) + + return expr +} diff --git a/table/field/normalize.ts b/table/field/normalize.ts index f1329b67..d9d7abeb 100644 --- a/table/field/normalize.ts +++ b/table/field/normalize.ts @@ -1,31 +1,14 @@ import type { Field } from "@dpkit/core" -import { col, lit, when } from "nodejs-polars" +import { col } from "nodejs-polars" import type { Expr } from "nodejs-polars" import { parseField } from "./parse.ts" +import { substituteField } from "./substitute.ts" -const DEFAULT_MISSING_VALUES = [""] +export function normalizeField(field: Field, fieldExpr?: Expr) { + let expr = fieldExpr ?? col(field.name) -export function normalizeField( - field: Field, - expr?: Expr, - options?: { dontParse?: boolean }, -) { - expr = expr ?? col(field.name) + expr = substituteField(field, expr) + expr = parseField(field, expr) - const flattenMissingValues = - field.missingValues?.map(it => (typeof it === "string" ? it : it.value)) ?? - DEFAULT_MISSING_VALUES - - if (flattenMissingValues.length) { - expr = when(expr.isIn(flattenMissingValues)) - .then(lit(null)) - .otherwise(expr) - .alias(field.name) - } - - if (options?.dontParse) { - return expr - } - - return parseField(field, expr) + return expr } diff --git a/table/field/parse.ts b/table/field/parse.ts index 64a8eb2d..33c50aac 100644 --- a/table/field/parse.ts +++ b/table/field/parse.ts @@ -17,8 +17,8 @@ import { parseTimeField } from "./types/time.ts" import { parseYearField } from "./types/year.ts" import { parseYearmonthField } from "./types/yearmonth.ts" -export function parseField(field: Field, expr?: Expr) { - expr = expr ?? col(field.name) +export function parseField(field: Field, fieldExpr?: Expr) { + const expr = fieldExpr ?? col(field.name) switch (field.type) { case "array": diff --git a/table/field/stringify.ts b/table/field/stringify.ts index f2d2ad8c..9bf525a2 100644 --- a/table/field/stringify.ts +++ b/table/field/stringify.ts @@ -17,8 +17,8 @@ import { stringifyTimeField } from "./types/time.ts" import { stringifyYearField } from "./types/year.ts" import { stringifyYearmonthField } from "./types/yearmonth.ts" -export function stringifyField(field: Field, expr?: Expr) { - expr = expr ?? col(field.name) +export function stringifyField(field: Field, fieldExpr?: Expr) { + const expr = fieldExpr ?? col(field.name) switch (field.type) { case "array": diff --git a/table/field/substitute.ts b/table/field/substitute.ts new file mode 100644 index 00000000..46e19ab9 --- /dev/null +++ b/table/field/substitute.ts @@ -0,0 +1,22 @@ +import type { Field } from "@dpkit/core" +import { col, lit, when } from "nodejs-polars" +import type { Expr } from "nodejs-polars" + +const DEFAULT_MISSING_VALUES = [""] + +export function substituteField(field: Field, fieldExpr?: Expr) { + let expr = fieldExpr ?? col(field.name) + + const flattenMissingValues = + field.missingValues?.map(it => (typeof it === "string" ? it : it.value)) ?? + DEFAULT_MISSING_VALUES + + if (flattenMissingValues.length) { + expr = when(expr.isIn(flattenMissingValues)) + .then(lit(null)) + .otherwise(expr) + .alias(field.name) + } + + return expr +} diff --git a/table/field/validate.ts b/table/field/validate.ts index 7220a772..f22c6a50 100644 --- a/table/field/validate.ts +++ b/table/field/validate.ts @@ -1,4 +1,5 @@ import type { Field } from "@dpkit/core" +import { col } from "nodejs-polars" import type { TableError } from "../error/index.ts" import type { Table } from "../table/index.ts" import type { PolarsField } from "./Field.ts" @@ -11,11 +12,14 @@ import { checkCellPattern } from "./checks/pattern.ts" import { checkCellRequired } from "./checks/required.ts" import { checkCellType } from "./checks/type.ts" import { checkCellUnique } from "./checks/unique.ts" +import { normalizeField } from "./normalize.ts" +import { parseField } from "./parse.ts" +import { substituteField } from "./substitute.ts" -export function validateField( +export async function validateField( field: Field, options: { - errorTable: Table + table: Table polarsField: PolarsField }, ) { @@ -28,11 +32,12 @@ export function validateField( const typeErrors = validateType(field, polarsField) errors.push(...typeErrors) - const errorTable = !typeErrors.length - ? validateCells(field, options.errorTable) - : options.errorTable + if (!typeErrors.length) { + const dataErorrs = validateData(field, options.polarsField, options.table) + errors.push(...dataErorrs) + } - return { valid: !errors.length, errors, errorTable } + return { errors, valid: !errors.length } } function validateName(field: Field, polarsField: PolarsField) { @@ -86,17 +91,16 @@ function validateType(field: Field, polarsField: PolarsField) { return errors } -function validateCells(field: Field, errorTable: Table) { - errorTable = checkCellType(field, errorTable) - errorTable = checkCellRequired(field, errorTable) - errorTable = checkCellPattern(field, errorTable) - errorTable = checkCellEnum(field, errorTable) - errorTable = checkCellMinimum(field, errorTable) - errorTable = checkCellMaximum(field, errorTable) - errorTable = checkCellMinimum(field, errorTable, { isExclusive: true }) - errorTable = checkCellMaximum(field, errorTable, { isExclusive: true }) - errorTable = checkCellMinLength(field, errorTable) - errorTable = checkCellMaxLength(field, errorTable) - errorTable = checkCellUnique(field, errorTable) - return errorTable +function validateData(field: Field, polarsField: PolarsField, table: Table) { + const fieldExpr = col(polarsField.name) + + const checkTable = table + .withRowCount() + .select( + col("row_nr").add(1).alias("number"), + substituteField(field, fieldExpr).alias("source"), + normalizeField(field, fieldExpr).alias("target"), + ) + + return [] } diff --git a/table/table/validate.ts b/table/table/validate.ts index 0d784b2c..0a5c3bfb 100644 --- a/table/table/validate.ts +++ b/table/table/validate.ts @@ -1,5 +1,7 @@ -import type { Schema } from "@dpkit/core" +import os from "node:os" +import type { Field, Schema } from "@dpkit/core" import { anyHorizontal, col } from "nodejs-polars" +import pAll from "p-all" import type { TableError } from "../error/index.ts" import { matchField } from "../field/index.ts" import { validateField } from "../field/index.ts" @@ -14,10 +16,10 @@ export async function validateTable( options?: { schema?: Schema sampleRows?: number - invalidRowsLimit?: number + maxErorrs?: number }, ) { - const { schema, sampleRows = 100, invalidRowsLimit = 100 } = options ?? {} + const { schema, sampleRows = 100, maxErorrs = 100 } = options ?? {} const errors: TableError[] = [] if (schema) { @@ -31,12 +33,15 @@ export async function validateTable( table, schema, polarsSchema, - invalidRowsLimit, + maxErorrs, ) errors.push(...fieldErrors) } - return { errors, valid: !errors.length } + return { + errors: errors.slice(0, maxErorrs), + valid: !errors.length, + } } function validateFieldsMatch(props: { @@ -131,90 +136,32 @@ async function validateFields( table: Table, schema: Schema, polarsSchema: PolarsSchema, - invalidRowsLimit: number, + maxErorrs: number, ) { const errors: TableError[] = [] - const targetNames: string[] = [] - - const sources = Object.entries( - normalizeFields(schema, polarsSchema, { dontParse: true }), - ).map(([name, expr]) => { - return expr.alias(`source:${name}`) - }) - - const targets = Object.entries( - normalizeFields(schema, polarsSchema, { dontParse: false }), - ).map(([name, expr]) => { - const targetName = `target:${name}` - targetNames.push(targetName) - return expr.alias(targetName) - }) - - let errorTable = table - .withRowCount() - .select(col("row_nr").add(1), ...sources, ...targets) - - for (const [index, field] of schema.fields.entries()) { - const polarsField = matchField(index, field, schema, polarsSchema) - if (polarsField) { - const fieldResult = validateField(field, { errorTable, polarsField }) - errorTable = fieldResult.errorTable - errors.push(...fieldResult.errors) - } - } + const concurrency = os.cpus().length + const abortController = new AbortController() - const rowsResult = validateRows(schema, errorTable) - errorTable = rowsResult.errorTable - errors.push(...rowsResult.errors) + const collectFieldErrors = async (index: number, field: Field) => { + const polarsField = matchField(index, field, schema, polarsSchema) + if (!polarsField) return - const erorrColumns = errorTable.columns.filter(name => - name.startsWith("error:"), - ) + const report = await validateField(field, { table, polarsField }) + errors.push(...report.errors) - if (!erorrColumns.length) { - return errors + if (errors.length > maxErorrs) { + abortController.abort() + } } - const errorFrame = await errorTable - .filter(anyHorizontal(erorrColumns.map(col))) - .limit(invalidRowsLimit) - .drop(targetNames) - .collect({ streaming: true }) - - for (const record of errorFrame.toRecords() as any[]) { - const typeErrorInFields: string[] = [] - for (const [key, value] of Object.entries(record)) { - const [kind, type, name] = key.split(":") - if (kind === "error" && value === true && type && name) { - const rowNumber = record.row_nr - - // Cell-level errors - if (type.startsWith("cell/")) { - if (!typeErrorInFields.includes(name)) { - errors.push({ - rowNumber, - type: type as any, - fieldName: name as any, - cell: (record[`source:${name}`] ?? "").toString(), - }) - } - - // Type error is a terminating error for a cell - if (type === "cell/type") { - typeErrorInFields.push(name) - } - } - - // Row-level errors - if (type.startsWith("row/")) { - errors.push({ - rowNumber, - type: type as any, - fieldNames: name.split(","), - }) - } - } - } + try { + await pAll( + schema.fields.map((field, idx) => () => collectFieldErrors(idx, field)), + { concurrency }, + ) + } catch (error) { + const isAborted = error instanceof Error && error.name === "AbortError" + if (!isAborted) throw error } return errors From 026460492c408de227e265c2a8fe418ddea515d2 Mon Sep 17 00:00:00 2001 From: roll Date: Tue, 28 Oct 2025 12:42:08 +0000 Subject: [PATCH 07/42] Removed dontParse --- table/table/normalize.ts | 25 ++++--------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/table/table/normalize.ts b/table/table/normalize.ts index 39cded33..72d78724 100644 --- a/table/table/normalize.ts +++ b/table/table/normalize.ts @@ -10,31 +10,14 @@ import type { Table } from "./Table.ts" const HEAD_ROWS = 100 -export async function normalizeTable( - table: Table, - schema: Schema, - options?: { - dontParse?: boolean - }, -) { - const { dontParse } = options ?? {} - +export async function normalizeTable(table: Table, schema: Schema) { const head = await table.head(HEAD_ROWS).collect() const polarsSchema = getPolarsSchema(head.schema) - return table.select( - ...Object.values(normalizeFields(schema, polarsSchema, { dontParse })), - ) + return table.select(...Object.values(normalizeFields(schema, polarsSchema))) } -export function normalizeFields( - schema: Schema, - polarsSchema: PolarsSchema, - options?: { - dontParse?: boolean - }, -) { - const { dontParse } = options ?? {} +export function normalizeFields(schema: Schema, polarsSchema: PolarsSchema) { const exprs: Record = {} for (const [index, field] of schema.fields.entries()) { @@ -48,7 +31,7 @@ export function normalizeFields( if (polarsField.type.equals(DataType.String)) { const missingValues = field.missingValues ?? schema.missingValues const mergedField = { ...field, missingValues } - expr = normalizeField(mergedField, expr, { dontParse }) + expr = normalizeField(mergedField, expr) } } From 209e55c2634789f4369f2a76559d89ba32397bd6 Mon Sep 17 00:00:00 2001 From: roll Date: Tue, 28 Oct 2025 12:51:38 +0000 Subject: [PATCH 08/42] Added narrow TODO --- table/field/narrow.ts | 1 + 1 file changed, 1 insertion(+) create mode 100644 table/field/narrow.ts diff --git a/table/field/narrow.ts b/table/field/narrow.ts new file mode 100644 index 00000000..1039d4c5 --- /dev/null +++ b/table/field/narrow.ts @@ -0,0 +1 @@ +// TODO: implement From 1228a3eecab0215f896ff4036d59f6a9aaa9cdac Mon Sep 17 00:00:00 2001 From: roll Date: Tue, 28 Oct 2025 13:01:18 +0000 Subject: [PATCH 09/42] Added error to fieldTable --- table/field/validate.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/table/field/validate.ts b/table/field/validate.ts index f22c6a50..67ce40a3 100644 --- a/table/field/validate.ts +++ b/table/field/validate.ts @@ -1,5 +1,5 @@ import type { Field } from "@dpkit/core" -import { col } from "nodejs-polars" +import { col, lit } from "nodejs-polars" import type { TableError } from "../error/index.ts" import type { Table } from "../table/index.ts" import type { PolarsField } from "./Field.ts" @@ -94,12 +94,13 @@ function validateType(field: Field, polarsField: PolarsField) { function validateData(field: Field, polarsField: PolarsField, table: Table) { const fieldExpr = col(polarsField.name) - const checkTable = table + const fieldTable = table .withRowCount() .select( col("row_nr").add(1).alias("number"), substituteField(field, fieldExpr).alias("source"), normalizeField(field, fieldExpr).alias("target"), + lit(null).alias("error"), ) return [] From 0ea12699ebecc47c56945563a14364233f5aea31 Mon Sep 17 00:00:00 2001 From: roll Date: Tue, 28 Oct 2025 15:35:01 +0000 Subject: [PATCH 10/42] First working version --- table/error/Cell.ts | 13 +++++++++ table/error/Field.ts | 2 ++ table/error/Fields.ts | 2 ++ table/error/Row.ts | 2 ++ table/error/Table.ts | 38 ++++--------------------- table/field/checks/type.ts | 19 +++++++------ table/field/validate.ts | 58 ++++++++++++++++++++++++++++++-------- table/table/validate.ts | 10 +++---- 8 files changed, 86 insertions(+), 58 deletions(-) diff --git a/table/error/Cell.ts b/table/error/Cell.ts index 2eaf5407..0d441a0e 100644 --- a/table/error/Cell.ts +++ b/table/error/Cell.ts @@ -1,5 +1,18 @@ import type { BaseTableError } from "./Base.ts" +export type CellError = + | CellTypeError + | CellRequiredError + | CellMinimumError + | CellMaximumError + | CellExclusiveMinimumError + | CellExclusiveMaximumError + | CellMinLengthError + | CellMaxLengthError + | CellPatternError + | CellUniqueError + | CellEnumError + export interface BaseCellError extends BaseTableError { fieldName: string rowNumber: number diff --git a/table/error/Field.ts b/table/error/Field.ts index 936196d2..df3af39f 100644 --- a/table/error/Field.ts +++ b/table/error/Field.ts @@ -1,6 +1,8 @@ import type { FieldType } from "@dpkit/core" import type { BaseTableError } from "./Base.ts" +export type FieldError = FieldNameError | FieldTypeError + export interface BaseFieldError extends BaseTableError { fieldName: string } diff --git a/table/error/Fields.ts b/table/error/Fields.ts index 1e7da0a7..0a30ca84 100644 --- a/table/error/Fields.ts +++ b/table/error/Fields.ts @@ -1,5 +1,7 @@ import type { BaseTableError } from "./Base.ts" +export type FieldsError = FieldsMissingError | FieldsExtraError + export interface BaseFieldsError extends BaseTableError { fieldNames: string[] } diff --git a/table/error/Row.ts b/table/error/Row.ts index 0569a3e3..aa9833ec 100644 --- a/table/error/Row.ts +++ b/table/error/Row.ts @@ -1,5 +1,7 @@ import type { BaseTableError } from "./Base.ts" +export type RowError = RowUniqueError + export interface BaseRowError extends BaseTableError { rowNumber: number } diff --git a/table/error/Table.ts b/table/error/Table.ts index e6f1d362..b9be6b7f 100644 --- a/table/error/Table.ts +++ b/table/error/Table.ts @@ -1,34 +1,6 @@ -import type { - CellEnumError, - CellExclusiveMaximumError, - CellExclusiveMinimumError, - CellMaxLengthError, - CellMaximumError, - CellMinLengthError, - CellMinimumError, - CellPatternError, - CellRequiredError, - CellTypeError, - CellUniqueError, -} from "./Cell.ts" -import type { FieldNameError, FieldTypeError } from "./Field.ts" -import type { FieldsExtraError, FieldsMissingError } from "./Fields.ts" -import type { RowUniqueError } from "./Row.ts" +import type { CellError } from "./Cell.ts" +import type { FieldError } from "./Field.ts" +import type { FieldsError } from "./Fields.ts" +import type { RowError } from "./Row.ts" -export type TableError = - | FieldsMissingError - | FieldsExtraError - | FieldNameError - | FieldTypeError - | RowUniqueError - | CellTypeError - | CellRequiredError - | CellMinimumError - | CellMaximumError - | CellExclusiveMinimumError - | CellExclusiveMaximumError - | CellMinLengthError - | CellMaxLengthError - | CellPatternError - | CellUniqueError - | CellEnumError +export type TableError = FieldsError | FieldError | RowError | CellError diff --git a/table/field/checks/type.ts b/table/field/checks/type.ts index 18d89e75..479accfd 100644 --- a/table/field/checks/type.ts +++ b/table/field/checks/type.ts @@ -1,13 +1,16 @@ import type { Field } from "@dpkit/core" import { col } from "nodejs-polars" -import type { Table } from "../../table/index.ts" +import type { CellTypeError } from "../../error/index.ts" -export function checkCellType(field: Field, errorTable: Table) { - const source = col(`source:${field.name}`) - const target = col(`target:${field.name}`) - const errorName = `error:cell/type:${field.name}` +export function checkCellType(field: Field) { + const isErrorExpr = col("source").isNotNull().and(col("target").isNull()) - return errorTable.withColumn( - source.isNotNull().and(target.isNull()).alias(errorName), - ) + const errorTemplate: CellTypeError = { + type: "cell/type", + fieldName: field.name, + rowNumber: 0, + cell: "", + } + + return { isErrorExpr, errorTemplate } } diff --git a/table/field/validate.ts b/table/field/validate.ts index 67ce40a3..bb01b718 100644 --- a/table/field/validate.ts +++ b/table/field/validate.ts @@ -1,6 +1,6 @@ import type { Field } from "@dpkit/core" -import { col, lit } from "nodejs-polars" -import type { TableError } from "../error/index.ts" +import { col, lit, when } from "nodejs-polars" +import type { CellError, FieldError, TableError } from "../error/index.ts" import type { Table } from "../table/index.ts" import type { PolarsField } from "./Field.ts" import { checkCellEnum } from "./checks/enum.ts" @@ -13,17 +13,17 @@ import { checkCellRequired } from "./checks/required.ts" import { checkCellType } from "./checks/type.ts" import { checkCellUnique } from "./checks/unique.ts" import { normalizeField } from "./normalize.ts" -import { parseField } from "./parse.ts" import { substituteField } from "./substitute.ts" export async function validateField( field: Field, options: { - table: Table polarsField: PolarsField + maxErrors: number + table: Table }, ) { - const { polarsField } = options + const { polarsField, maxErrors, table } = options const errors: TableError[] = [] const nameErrors = validateName(field, polarsField) @@ -33,7 +33,7 @@ export async function validateField( errors.push(...typeErrors) if (!typeErrors.length) { - const dataErorrs = validateData(field, options.polarsField, options.table) + const dataErorrs = await validateData(field, polarsField, maxErrors, table) errors.push(...dataErorrs) } @@ -41,7 +41,7 @@ export async function validateField( } function validateName(field: Field, polarsField: PolarsField) { - const errors: TableError[] = [] + const errors: FieldError[] = [] if (field.name !== polarsField.name) { errors.push({ @@ -55,7 +55,7 @@ function validateName(field: Field, polarsField: PolarsField) { } function validateType(field: Field, polarsField: PolarsField) { - const errors: TableError[] = [] + const errors: FieldError[] = [] const mapping: Record = { Bool: "boolean", @@ -91,10 +91,16 @@ function validateType(field: Field, polarsField: PolarsField) { return errors } -function validateData(field: Field, polarsField: PolarsField, table: Table) { +async function validateData( + field: Field, + polarsField: PolarsField, + maxErrors: number, + table: Table, +) { + const errors: CellError[] = [] const fieldExpr = col(polarsField.name) - const fieldTable = table + let fieldCheckTable = table .withRowCount() .select( col("row_nr").add(1).alias("number"), @@ -103,5 +109,35 @@ function validateData(field: Field, polarsField: PolarsField, table: Table) { lit(null).alias("error"), ) - return [] + for (const checkCell of [checkCellType]) { + const check = checkCell(field) + + if (check.isErrorExpr) { + fieldCheckTable = fieldCheckTable.withColumn( + when(col("error").isNotNull()) + .then(col("error")) + .when(check.isErrorExpr) + .then(lit(JSON.stringify(check.errorTemplate))) + .otherwise(lit(null)) + .alias("error"), + ) + } + } + + const fieldCheckFrame = await fieldCheckTable + .filter(col("error").isNotNull()) + .drop(["target"]) + .head(maxErrors) + .collect() + + for (const row of fieldCheckFrame.toRecords() as any[]) { + const errorTemplate = JSON.parse(row.error) as CellError + errors.push({ + ...errorTemplate, + rowNumber: row.number, + cell: String(row.source ?? ""), + }) + } + + return errors } diff --git a/table/table/validate.ts b/table/table/validate.ts index 0a5c3bfb..390e0112 100644 --- a/table/table/validate.ts +++ b/table/table/validate.ts @@ -1,6 +1,5 @@ import os from "node:os" import type { Field, Schema } from "@dpkit/core" -import { anyHorizontal, col } from "nodejs-polars" import pAll from "p-all" import type { TableError } from "../error/index.ts" import { matchField } from "../field/index.ts" @@ -9,7 +8,6 @@ import { validateRows } from "../row/index.ts" import { getPolarsSchema } from "../schema/index.ts" import type { PolarsSchema } from "../schema/index.ts" import type { Table } from "./Table.ts" -import { normalizeFields } from "./normalize.ts" export async function validateTable( table: Table, @@ -136,20 +134,20 @@ async function validateFields( table: Table, schema: Schema, polarsSchema: PolarsSchema, - maxErorrs: number, + maxErrors: number, ) { const errors: TableError[] = [] - const concurrency = os.cpus().length + const concurrency = os.cpus().length - 1 const abortController = new AbortController() const collectFieldErrors = async (index: number, field: Field) => { const polarsField = matchField(index, field, schema, polarsSchema) if (!polarsField) return - const report = await validateField(field, { table, polarsField }) + const report = await validateField(field, { table, polarsField, maxErrors }) errors.push(...report.errors) - if (errors.length > maxErorrs) { + if (errors.length > maxErrors) { abortController.abort() } } From 5ccaa165d646791d4b2aef09572ccc99c99a1ba6 Mon Sep 17 00:00:00 2001 From: roll Date: Tue, 28 Oct 2025 15:46:11 +0000 Subject: [PATCH 11/42] Added fielType to type error --- table/error/Cell.ts | 2 ++ table/field/checks/type.ts | 1 + 2 files changed, 3 insertions(+) diff --git a/table/error/Cell.ts b/table/error/Cell.ts index 0d441a0e..50249385 100644 --- a/table/error/Cell.ts +++ b/table/error/Cell.ts @@ -1,3 +1,4 @@ +import type { FieldType } from "@dpkit/core" import type { BaseTableError } from "./Base.ts" export type CellError = @@ -21,6 +22,7 @@ export interface BaseCellError extends BaseTableError { export interface CellTypeError extends BaseCellError { type: "cell/type" + fieldType: FieldType } export interface CellRequiredError extends BaseCellError { diff --git a/table/field/checks/type.ts b/table/field/checks/type.ts index 479accfd..1166d92c 100644 --- a/table/field/checks/type.ts +++ b/table/field/checks/type.ts @@ -8,6 +8,7 @@ export function checkCellType(field: Field) { const errorTemplate: CellTypeError = { type: "cell/type", fieldName: field.name, + fieldType: field.type ?? "any", rowNumber: 0, cell: "", } From 22dac80a817e386de828b618e02f54cc80eaed25 Mon Sep 17 00:00:00 2001 From: roll Date: Tue, 28 Oct 2025 16:10:31 +0000 Subject: [PATCH 12/42] Added maxFieldErorrs --- table/table/validate.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/table/table/validate.ts b/table/table/validate.ts index 390e0112..8a510b69 100644 --- a/table/table/validate.ts +++ b/table/table/validate.ts @@ -17,7 +17,7 @@ export async function validateTable( maxErorrs?: number }, ) { - const { schema, sampleRows = 100, maxErorrs = 100 } = options ?? {} + const { schema, sampleRows = 100, maxErorrs = 1000 } = options ?? {} const errors: TableError[] = [] if (schema) { @@ -139,14 +139,19 @@ async function validateFields( const errors: TableError[] = [] const concurrency = os.cpus().length - 1 const abortController = new AbortController() + const maxFieldErrors = Math.ceil(maxErrors / schema.fields.length) const collectFieldErrors = async (index: number, field: Field) => { const polarsField = matchField(index, field, schema, polarsSchema) if (!polarsField) return - const report = await validateField(field, { table, polarsField, maxErrors }) - errors.push(...report.errors) + const report = await validateField(field, { + table, + polarsField, + maxErrors: maxFieldErrors, + }) + errors.push(...report.errors) if (errors.length > maxErrors) { abortController.abort() } From 92f877cd87f8d40e72513dac69ccd5bf55d1dfa3 Mon Sep 17 00:00:00 2001 From: roll Date: Tue, 28 Oct 2025 18:59:47 +0000 Subject: [PATCH 13/42] Updated unique check --- table/error/Cell.ts | 1 + table/field/checks/type.ts | 8 +++++--- table/field/checks/unique.ts | 21 +++++++++++---------- table/field/desubstitute.ts | 4 ++-- table/field/validate.ts | 8 ++++---- 5 files changed, 23 insertions(+), 19 deletions(-) diff --git a/table/error/Cell.ts b/table/error/Cell.ts index 50249385..92377c9e 100644 --- a/table/error/Cell.ts +++ b/table/error/Cell.ts @@ -23,6 +23,7 @@ export interface BaseCellError extends BaseTableError { export interface CellTypeError extends BaseCellError { type: "cell/type" fieldType: FieldType + fieldFormat?: string } export interface CellRequiredError extends BaseCellError { diff --git a/table/field/checks/type.ts b/table/field/checks/type.ts index 1166d92c..e4551de5 100644 --- a/table/field/checks/type.ts +++ b/table/field/checks/type.ts @@ -1,14 +1,16 @@ import type { Field } from "@dpkit/core" -import { col } from "nodejs-polars" +import type { Expr } from "nodejs-polars" import type { CellTypeError } from "../../error/index.ts" -export function checkCellType(field: Field) { - const isErrorExpr = col("source").isNotNull().and(col("target").isNull()) +export function checkCellType(field: Field, target: Expr, source: Expr) { + const isErrorExpr = source.isNotNull().and(target.isNull()) const errorTemplate: CellTypeError = { type: "cell/type", fieldName: field.name, fieldType: field.type ?? "any", + // @ts-ignore + fieldFormat: field.format, rowNumber: 0, cell: "", } diff --git a/table/field/checks/unique.ts b/table/field/checks/unique.ts index ed648de1..463104d5 100644 --- a/table/field/checks/unique.ts +++ b/table/field/checks/unique.ts @@ -1,19 +1,20 @@ import type { Field } from "@dpkit/core" -import { col } from "nodejs-polars" -import type { Table } from "../../table/index.ts" +import type { Expr } from "nodejs-polars" +import type { CellUniqueError } from "../../error/index.ts" // TODO: Support schema.primaryKey and schema.uniqueKeys -export function checkCellUnique(field: Field, errorTable: Table) { +export function checkCellUnique(field: Field, target: Expr) { const unique = field.constraints?.unique + if (!unique) return undefined - if (unique) { - const target = col(`target:${field.name}`) - const errorName = `error:cell/unique:${field.name}` + const isErrorExpr = target.isNotNull().and(target.isFirstDistinct().not()) - errorTable = errorTable.withColumn( - target.isNotNull().and(target.isFirstDistinct().not()).alias(errorName), - ) + const errorTemplate: CellUniqueError = { + type: "cell/unique", + fieldName: field.name, + rowNumber: 0, + cell: "", } - return errorTable + return { isErrorExpr, errorTemplate } } diff --git a/table/field/desubstitute.ts b/table/field/desubstitute.ts index 3262870e..285931e3 100644 --- a/table/field/desubstitute.ts +++ b/table/field/desubstitute.ts @@ -4,8 +4,8 @@ import type { Expr } from "nodejs-polars" const DEFAULT_MISSING_VALUE = "" -export function desubstituteField(field: Field, expr?: Expr) { - expr = expr ?? col(field.name) +export function desubstituteField(field: Field, fieldExpr?: Expr) { + let expr = fieldExpr ?? col(field.name) const flattenMissingValues = field.missingValues?.map(it => typeof it === "string" ? it : it.value, diff --git a/table/field/validate.ts b/table/field/validate.ts index bb01b718..814dd42c 100644 --- a/table/field/validate.ts +++ b/table/field/validate.ts @@ -104,15 +104,15 @@ async function validateData( .withRowCount() .select( col("row_nr").add(1).alias("number"), - substituteField(field, fieldExpr).alias("source"), normalizeField(field, fieldExpr).alias("target"), + substituteField(field, fieldExpr).alias("source"), lit(null).alias("error"), ) - for (const checkCell of [checkCellType]) { - const check = checkCell(field) + for (const checkCell of [checkCellType, checkCellUnique]) { + const check = checkCell(field, col("target"), col("source")) - if (check.isErrorExpr) { + if (check) { fieldCheckTable = fieldCheckTable.withColumn( when(col("error").isNotNull()) .then(col("error")) From 2e36936e11aa6907445eefcda5cdbeda2075a7ab Mon Sep 17 00:00:00 2001 From: roll Date: Tue, 28 Oct 2025 19:01:26 +0000 Subject: [PATCH 14/42] Updated required check --- table/field/checks/required.ts | 22 +++++++++++++--------- table/field/validate.ts | 2 +- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/table/field/checks/required.ts b/table/field/checks/required.ts index 5e8d58be..e0b1e232 100644 --- a/table/field/checks/required.ts +++ b/table/field/checks/required.ts @@ -1,15 +1,19 @@ import type { Field } from "@dpkit/core" -import { col } from "nodejs-polars" -import type { Table } from "../../table/index.ts" +import type { CellRequiredError } from "../../error/index.ts" +import type { Expr } from "nodejs-polars" -export function checkCellRequired(field: Field, errorTable: Table) { - if (field.constraints?.required) { - const target = col(`target:${field.name}`) - const errorName = `error:cell/required:${field.name}` +export function checkCellRequired(field: Field, target: Expr) { + const required = field.constraints?.required + if (!required) return undefined - errorTable = errorTable - .withColumn(target.isNull().alias(errorName)) + const isErrorExpr = target.isNull() + + const errorTemplate: CellRequiredError = { + type: "cell/required", + fieldName: field.name, + rowNumber: 0, + cell: "", } - return errorTable + return { isErrorExpr, errorTemplate } } diff --git a/table/field/validate.ts b/table/field/validate.ts index 814dd42c..8135635c 100644 --- a/table/field/validate.ts +++ b/table/field/validate.ts @@ -109,7 +109,7 @@ async function validateData( lit(null).alias("error"), ) - for (const checkCell of [checkCellType, checkCellUnique]) { + for (const checkCell of [checkCellType, checkCellRequired, checkCellUnique]) { const check = checkCell(field, col("target"), col("source")) if (check) { From 0045d4562908e7a08eb70df73071616b8075630f Mon Sep 17 00:00:00 2001 From: roll Date: Tue, 28 Oct 2025 19:02:49 +0000 Subject: [PATCH 15/42] Updated pattern check --- table/field/checks/pattern.ts | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/table/field/checks/pattern.ts b/table/field/checks/pattern.ts index 14c21b55..ad0d9588 100644 --- a/table/field/checks/pattern.ts +++ b/table/field/checks/pattern.ts @@ -1,20 +1,21 @@ import type { Field } from "@dpkit/core" -import { col } from "nodejs-polars" -import type { Table } from "../../table/index.ts" +import type { Expr } from "nodejs-polars" +import type { CellPatternError } from "../../error/index.ts" -export function checkCellPattern(field: Field, errorTable: Table) { - if (field.type === "string") { - const pattern = field.constraints?.pattern +export function checkCellPattern(field: Field, target: Expr) { + if (field.type !== "string") return undefined - if (pattern) { - const target = col(`target:${field.name}`) - const errorName = `error:cell/pattern:${field.name}` + const pattern = field.constraints?.pattern + if (!pattern) return undefined - errorTable = errorTable.withColumn( - target.str.contains(pattern).not().alias(errorName), - ) - } + const isErrorExpr = target.str.contains(pattern).not() + + const errorTemplate: CellPatternError = { + type: "cell/pattern", + fieldName: field.name, + rowNumber: 0, + cell: "", } - return errorTable + return { isErrorExpr, errorTemplate } } From 8f7f8ec85ae71181afe69986bdfce38abeade83d Mon Sep 17 00:00:00 2001 From: roll Date: Tue, 28 Oct 2025 19:08:20 +0000 Subject: [PATCH 16/42] Updated minLength check --- table/field/checks/minLength.ts | 27 ++++++++++++++------------- table/field/validate.ts | 7 ++++++- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/table/field/checks/minLength.ts b/table/field/checks/minLength.ts index 3214bc96..996e82ce 100644 --- a/table/field/checks/minLength.ts +++ b/table/field/checks/minLength.ts @@ -1,20 +1,21 @@ import type { Field } from "@dpkit/core" -import { col } from "nodejs-polars" -import type { Table } from "../../table/index.ts" +import type { Expr } from "nodejs-polars" +import type { CellMinLengthError } from "../../error/index.ts" -export function checkCellMinLength(field: Field, errorTable: Table) { - if (field.type === "string") { - const minLength = field.constraints?.minLength +export function checkCellMinLength(field: Field, target: Expr) { + if (field.type !== "string") return undefined - if (minLength !== undefined) { - const target = col(`target:${field.name}`) - const errorName = `error:cell/minLength:${field.name}` + const minLength = field.constraints?.minLength + if (!minLength) return undefined - errorTable = errorTable.withColumn( - target.str.lengths().lt(minLength).alias(errorName), - ) - } + const isErrorExpr = target.str.lengths().lt(minLength) + + const errorTemplate: CellMinLengthError = { + type: "cell/minLength", + fieldName: field.name, + rowNumber: 0, + cell: "", } - return errorTable + return { isErrorExpr, errorTemplate } } diff --git a/table/field/validate.ts b/table/field/validate.ts index 8135635c..7e3d4786 100644 --- a/table/field/validate.ts +++ b/table/field/validate.ts @@ -109,7 +109,12 @@ async function validateData( lit(null).alias("error"), ) - for (const checkCell of [checkCellType, checkCellRequired, checkCellUnique]) { + for (const checkCell of [ + checkCellType, + checkCellRequired, + checkCellPattern, + checkCellUnique, + ]) { const check = checkCell(field, col("target"), col("source")) if (check) { From f1994e6ff9d2e19667caf0d7274576868a9967c2 Mon Sep 17 00:00:00 2001 From: roll Date: Tue, 28 Oct 2025 19:09:20 +0000 Subject: [PATCH 17/42] Updated maxLength check --- table/field/checks/maxLength.ts | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/table/field/checks/maxLength.ts b/table/field/checks/maxLength.ts index 3779685d..07ac654c 100644 --- a/table/field/checks/maxLength.ts +++ b/table/field/checks/maxLength.ts @@ -1,20 +1,21 @@ import type { Field } from "@dpkit/core" -import { col } from "nodejs-polars" -import type { Table } from "../../table/index.ts" +import type { Expr } from "nodejs-polars" +import type { CellMaxLengthError } from "../../error/index.ts" -export function checkCellMaxLength(field: Field, errorTable: Table) { - if (field.type === "string") { - const maxLength = field.constraints?.maxLength +export function checkCellMaxLength(field: Field, target: Expr) { + if (field.type !== "string") return undefined - if (maxLength !== undefined) { - const target = col(`target:${field.name}`) - const errorName = `error:cell/maxLength:${field.name}` + const maxLength = field.constraints?.maxLength + if (!maxLength) return undefined - errorTable = errorTable.withColumn( - target.str.lengths().gt(maxLength).alias(errorName), - ) - } + const isErrorExpr = target.str.lengths().gt(maxLength) + + const errorTemplate: CellMaxLengthError = { + type: "cell/maxLength", + fieldName: field.name, + rowNumber: 0, + cell: "", } - return errorTable + return { isErrorExpr, errorTemplate } } From c48e6f3362d02ddf0d38e44fa44f06522d0cd2bc Mon Sep 17 00:00:00 2001 From: roll Date: Tue, 28 Oct 2025 19:10:18 +0000 Subject: [PATCH 18/42] Updated enum --- table/field/checks/enum.ts | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/table/field/checks/enum.ts b/table/field/checks/enum.ts index 8e9f6c7f..8556c133 100644 --- a/table/field/checks/enum.ts +++ b/table/field/checks/enum.ts @@ -1,20 +1,21 @@ import type { Field } from "@dpkit/core" -import { col } from "nodejs-polars" -import type { Table } from "../../table/index.ts" +import type { Expr } from "nodejs-polars" +import type { CellEnumError } from "../../error/index.ts" -export function checkCellEnum(field: Field, errorTable: Table) { - if (field.type === "string") { - const rawEnum = field.constraints?.enum +export function checkCellEnum(field: Field, target: Expr) { + if (field.type !== "string") return undefined - if (rawEnum) { - const target = col(`target:${field.name}`) - const errorName = `error:cell/enum:${field.name}` + const rawEnum = field.constraints?.enum + if (!rawEnum) return undefined - errorTable = errorTable.withColumn( - target.isIn(rawEnum).not().alias(errorName), - ) - } + const isErrorExpr = target.isIn(rawEnum).not() + + const errorTemplate: CellEnumError = { + type: "cell/enum", + fieldName: field.name, + rowNumber: 0, + cell: "", } - return errorTable + return { isErrorExpr, errorTemplate } } From 508e0cdb6ed5deab9416a9437d6c3ae20a2ded77 Mon Sep 17 00:00:00 2001 From: roll Date: Tue, 28 Oct 2025 19:17:36 +0000 Subject: [PATCH 19/42] Updated maximum/minimum --- table/field/checks/maximum.ts | 59 +++++++++++++++++------------------ table/field/checks/minimum.ts | 58 +++++++++++++++++----------------- table/field/validate.ts | 15 ++++++--- 3 files changed, 67 insertions(+), 65 deletions(-) diff --git a/table/field/checks/maximum.ts b/table/field/checks/maximum.ts index b5037075..bd04d90d 100644 --- a/table/field/checks/maximum.ts +++ b/table/field/checks/maximum.ts @@ -1,43 +1,40 @@ import type { Field } from "@dpkit/core" -import { col, lit } from "nodejs-polars" -import type { Table } from "../../table/index.ts" +import { lit } from "nodejs-polars" +import type { Expr } from "nodejs-polars" +import type { CellExclusiveMaximumError } from "../../error/index.ts" +import type { CellMaximumError } from "../../error/index.ts" + +export function createCheckCellMaximum(options?: { isExclusive?: boolean }) { + return (field: Field, target: Expr) => { + if (field.type !== "integer" && field.type !== "number") return undefined -export function checkCellMaximum( - field: Field, - errorTable: Table, - options?: { - isExclusive?: boolean - }, -) { - if (field.type === "integer" || field.type === "number") { const maximum = options?.isExclusive ? field.constraints?.exclusiveMaximum : field.constraints?.maximum + if (maximum === undefined) return undefined - if (maximum !== undefined) { - const target = col(`target:${field.name}`) - const errorName = options?.isExclusive - ? `error:cell/exclusiveMaximum:${field.name}` - : `error:cell/maximum:${field.name}` + let isErrorExpr: Expr + const parser = + field.type === "integer" ? Number.parseInt : Number.parseFloat - // TODO: Support numeric options (decimalChar, groupChar, etc) - const parser = - field.type === "integer" ? Number.parseInt : Number.parseFloat + try { + const parsedMaximum = + typeof maximum === "string" ? parser(maximum) : maximum - try { - const parsedMaximum = - typeof maximum === "string" ? parser(maximum) : maximum + isErrorExpr = options?.isExclusive + ? target.gtEq(parsedMaximum) + : target.gt(parsedMaximum) + } catch (error) { + isErrorExpr = lit(true) + } - errorTable = errorTable.withColumn( - options?.isExclusive - ? target.gtEq(parsedMaximum).alias(errorName) - : target.gt(parsedMaximum).alias(errorName), - ) - } catch (error) { - errorTable = errorTable.withColumn(lit(true).alias(errorName)) - } + const errorTemplate: CellMaximumError | CellExclusiveMaximumError = { + type: options?.isExclusive ? "cell/exclusiveMaximum" : "cell/maximum", + fieldName: field.name, + rowNumber: 0, + cell: "", } - } - return errorTable + return { isErrorExpr, errorTemplate } + } } diff --git a/table/field/checks/minimum.ts b/table/field/checks/minimum.ts index 7cefdcfc..a992807a 100644 --- a/table/field/checks/minimum.ts +++ b/table/field/checks/minimum.ts @@ -1,42 +1,40 @@ import type { Field } from "@dpkit/core" -import { col, lit } from "nodejs-polars" -import type { Table } from "../../table/index.ts" +import { lit } from "nodejs-polars" +import type { Expr } from "nodejs-polars" +import type { CellExclusiveMinimumError } from "../../error/index.ts" +import type { CellMinimumError } from "../../error/index.ts" + +export function createCheckCellMinimum(options?: { isExclusive?: boolean }) { + return (field: Field, target: Expr) => { + if (field.type !== "integer" && field.type !== "number") return undefined -export function checkCellMinimum( - field: Field, - errorTable: Table, - options?: { - isExclusive?: boolean - }, -) { - if (field.type === "integer" || field.type === "number") { const minimum = options?.isExclusive ? field.constraints?.exclusiveMinimum : field.constraints?.minimum + if (minimum === undefined) return undefined - if (minimum !== undefined) { - const target = col(`target:${field.name}`) - const errorName = options?.isExclusive - ? `error:cell/exclusiveMinimum:${field.name}` - : `error:cell/minimum:${field.name}` + let isErrorExpr: Expr + const parser = + field.type === "integer" ? Number.parseInt : Number.parseFloat - const parser = - field.type === "integer" ? Number.parseInt : Number.parseFloat + try { + const parsedMinimum = + typeof minimum === "string" ? parser(minimum) : minimum - try { - const parsedMinimum = - typeof minimum === "string" ? parser(minimum) : minimum + isErrorExpr = options?.isExclusive + ? target.ltEq(parsedMinimum) + : target.lt(parsedMinimum) + } catch (error) { + isErrorExpr = lit(true) + } - errorTable = errorTable.withColumn( - options?.isExclusive - ? target.ltEq(parsedMinimum).alias(errorName) - : target.lt(parsedMinimum).alias(errorName), - ) - } catch (error) { - errorTable = errorTable.withColumn(lit(true).alias(errorName)) - } + const errorTemplate: CellMinimumError | CellExclusiveMinimumError = { + type: options?.isExclusive ? "cell/exclusiveMinimum" : "cell/minimum", + fieldName: field.name, + rowNumber: 0, + cell: "", } - } - return errorTable + return { isErrorExpr, errorTemplate } + } } diff --git a/table/field/validate.ts b/table/field/validate.ts index 7e3d4786..266a1cf0 100644 --- a/table/field/validate.ts +++ b/table/field/validate.ts @@ -5,9 +5,9 @@ import type { Table } from "../table/index.ts" import type { PolarsField } from "./Field.ts" import { checkCellEnum } from "./checks/enum.ts" import { checkCellMaxLength } from "./checks/maxLength.ts" -import { checkCellMaximum } from "./checks/maximum.ts" +import { createCheckCellMaximum } from "./checks/maximum.ts" import { checkCellMinLength } from "./checks/minLength.ts" -import { checkCellMinimum } from "./checks/minimum.ts" +import { createCheckCellMinimum } from "./checks/minimum.ts" import { checkCellPattern } from "./checks/pattern.ts" import { checkCellRequired } from "./checks/required.ts" import { checkCellType } from "./checks/type.ts" @@ -33,7 +33,7 @@ export async function validateField( errors.push(...typeErrors) if (!typeErrors.length) { - const dataErorrs = await validateData(field, polarsField, maxErrors, table) + const dataErorrs = await validateCells(field, polarsField, maxErrors, table) errors.push(...dataErorrs) } @@ -91,7 +91,7 @@ function validateType(field: Field, polarsField: PolarsField) { return errors } -async function validateData( +async function validateCells( field: Field, polarsField: PolarsField, maxErrors: number, @@ -113,6 +113,13 @@ async function validateData( checkCellType, checkCellRequired, checkCellPattern, + checkCellEnum, + createCheckCellMinimum(), + createCheckCellMaximum(), + createCheckCellMinimum({ isExclusive: true }), + createCheckCellMaximum({ isExclusive: true }), + checkCellMinLength, + checkCellMaxLength, checkCellUnique, ]) { const check = checkCell(field, col("target"), col("source")) From 5b0b387b1b6d22014f0660c0a1ad576a66426c5f Mon Sep 17 00:00:00 2001 From: roll Date: Wed, 29 Oct 2025 07:55:12 +0000 Subject: [PATCH 20/42] Extended cell errors metadata --- table/error/Cell.ts | 8 ++++++++ table/field/checks/enum.ts | 1 + table/field/checks/maxLength.ts | 1 + table/field/checks/maximum.ts | 1 + table/field/checks/minLength.ts | 1 + table/field/checks/minimum.ts | 1 + table/field/checks/pattern.ts | 1 + table/table/validate.ts | 2 +- 8 files changed, 15 insertions(+), 1 deletion(-) diff --git a/table/error/Cell.ts b/table/error/Cell.ts index 92377c9e..09bc7412 100644 --- a/table/error/Cell.ts +++ b/table/error/Cell.ts @@ -32,30 +32,37 @@ export interface CellRequiredError extends BaseCellError { export interface CellMinimumError extends BaseCellError { type: "cell/minimum" + minimum: string } export interface CellMaximumError extends BaseCellError { type: "cell/maximum" + maximum: string } export interface CellExclusiveMinimumError extends BaseCellError { type: "cell/exclusiveMinimum" + minimum: string } export interface CellExclusiveMaximumError extends BaseCellError { type: "cell/exclusiveMaximum" + maximum: string } export interface CellMinLengthError extends BaseCellError { type: "cell/minLength" + minLength: number } export interface CellMaxLengthError extends BaseCellError { type: "cell/maxLength" + maxLength: number } export interface CellPatternError extends BaseCellError { type: "cell/pattern" + pattern: string } export interface CellUniqueError extends BaseCellError { @@ -64,4 +71,5 @@ export interface CellUniqueError extends BaseCellError { export interface CellEnumError extends BaseCellError { type: "cell/enum" + enum: string[] } diff --git a/table/field/checks/enum.ts b/table/field/checks/enum.ts index 8556c133..cdddbd18 100644 --- a/table/field/checks/enum.ts +++ b/table/field/checks/enum.ts @@ -13,6 +13,7 @@ export function checkCellEnum(field: Field, target: Expr) { const errorTemplate: CellEnumError = { type: "cell/enum", fieldName: field.name, + enum: rawEnum.map(String), rowNumber: 0, cell: "", } diff --git a/table/field/checks/maxLength.ts b/table/field/checks/maxLength.ts index 07ac654c..41eb4f99 100644 --- a/table/field/checks/maxLength.ts +++ b/table/field/checks/maxLength.ts @@ -13,6 +13,7 @@ export function checkCellMaxLength(field: Field, target: Expr) { const errorTemplate: CellMaxLengthError = { type: "cell/maxLength", fieldName: field.name, + maxLength: maxLength, rowNumber: 0, cell: "", } diff --git a/table/field/checks/maximum.ts b/table/field/checks/maximum.ts index bd04d90d..0f5789e2 100644 --- a/table/field/checks/maximum.ts +++ b/table/field/checks/maximum.ts @@ -31,6 +31,7 @@ export function createCheckCellMaximum(options?: { isExclusive?: boolean }) { const errorTemplate: CellMaximumError | CellExclusiveMaximumError = { type: options?.isExclusive ? "cell/exclusiveMaximum" : "cell/maximum", fieldName: field.name, + maximum: String(maximum), rowNumber: 0, cell: "", } diff --git a/table/field/checks/minLength.ts b/table/field/checks/minLength.ts index 996e82ce..4f0fb9ff 100644 --- a/table/field/checks/minLength.ts +++ b/table/field/checks/minLength.ts @@ -13,6 +13,7 @@ export function checkCellMinLength(field: Field, target: Expr) { const errorTemplate: CellMinLengthError = { type: "cell/minLength", fieldName: field.name, + minLength: minLength, rowNumber: 0, cell: "", } diff --git a/table/field/checks/minimum.ts b/table/field/checks/minimum.ts index a992807a..23f6800f 100644 --- a/table/field/checks/minimum.ts +++ b/table/field/checks/minimum.ts @@ -31,6 +31,7 @@ export function createCheckCellMinimum(options?: { isExclusive?: boolean }) { const errorTemplate: CellMinimumError | CellExclusiveMinimumError = { type: options?.isExclusive ? "cell/exclusiveMinimum" : "cell/minimum", fieldName: field.name, + minimum: String(minimum), rowNumber: 0, cell: "", } diff --git a/table/field/checks/pattern.ts b/table/field/checks/pattern.ts index ad0d9588..44f76447 100644 --- a/table/field/checks/pattern.ts +++ b/table/field/checks/pattern.ts @@ -13,6 +13,7 @@ export function checkCellPattern(field: Field, target: Expr) { const errorTemplate: CellPatternError = { type: "cell/pattern", fieldName: field.name, + pattern: pattern, rowNumber: 0, cell: "", } diff --git a/table/table/validate.ts b/table/table/validate.ts index 8a510b69..8ca7aef5 100644 --- a/table/table/validate.ts +++ b/table/table/validate.ts @@ -4,7 +4,7 @@ import pAll from "p-all" import type { TableError } from "../error/index.ts" import { matchField } from "../field/index.ts" import { validateField } from "../field/index.ts" -import { validateRows } from "../row/index.ts" +//import { validateRows } from "../row/index.ts" import { getPolarsSchema } from "../schema/index.ts" import type { PolarsSchema } from "../schema/index.ts" import type { Table } from "./Table.ts" From c4970f2af04cd860281a7e9bdcda3a0300f7f8bd Mon Sep 17 00:00:00 2001 From: roll Date: Wed, 29 Oct 2025 08:09:26 +0000 Subject: [PATCH 21/42] Updated browser errors --- browser/components/Report/Error/Cell.tsx | 139 ++++++++++++++++------- browser/locales/de.json | 14 +++ browser/locales/en.json | 14 +++ browser/locales/es.json | 14 +++ browser/locales/fr.json | 14 +++ browser/locales/it.json | 14 +++ browser/locales/pt.json | 14 +++ browser/locales/ru.json | 14 +++ browser/locales/uk.json | 14 +++ 9 files changed, 210 insertions(+), 41 deletions(-) diff --git a/browser/components/Report/Error/Cell.tsx b/browser/components/Report/Error/Cell.tsx index 0cbb5cbb..1a900d8f 100644 --- a/browser/components/Report/Error/Cell.tsx +++ b/browser/components/Report/Error/Cell.tsx @@ -4,21 +4,27 @@ import { useTranslation } from "react-i18next" export function CellTypeError(props: { error: errorTypes.CellTypeError }) { const { t } = useTranslation() + const { error } = props + return ( {t("Value of the cell")}{" "} - {props.error.cell} + {error.cell} {" "} {t("in field")}{" "} - {props.error.fieldName} + {error.fieldName} {" "} {t("of row")}{" "} - {props.error.rowNumber} + {error.rowNumber} + {" "} + {t("is not")} + + {[error.fieldType, error.fieldFormat].filter(Boolean).join("/")} {" "} - {t("has a wrong type")} + {"type"} ) } @@ -27,15 +33,17 @@ export function CellRequiredError(props: { error: errorTypes.CellRequiredError }) { const { t } = useTranslation() + const { error } = props + return ( {t("A required cell in field")}{" "} - {props.error.fieldName} + {error.fieldName} {" "} {t("of row")}{" "} - {props.error.rowNumber} + {error.rowNumber} {" "} {t("is missing")} @@ -46,21 +54,27 @@ export function CellMinimumError(props: { error: errorTypes.CellMinimumError }) { const { t } = useTranslation() + const { error } = props + return ( {t("Value of the cell")}{" "} - {props.error.cell} + {error.cell} {" "} {t("in field")}{" "} - {props.error.fieldName} + {error.fieldName} {" "} {t("of row")}{" "} - {props.error.rowNumber} + {error.rowNumber} + {" "} + {t("is less than")} + + {error.minimum} {" "} - {t("is less than minimum")} + {t("minimum")} ) } @@ -69,21 +83,27 @@ export function CellMaximumError(props: { error: errorTypes.CellMaximumError }) { const { t } = useTranslation() + const { error } = props + return ( {t("Value of the cell")}{" "} - {props.error.cell} + {error.cell} {" "} {t("in field")}{" "} - {props.error.fieldName} + {error.fieldName} {" "} {t("of row")}{" "} - {props.error.rowNumber} + {error.rowNumber} {" "} - {t("is more than maximum")} + {t("is more than")} + + {error.maximum} + {" "} + {t("maximum")} ) } @@ -92,21 +112,27 @@ export function CellExclusiveMinimumError(props: { error: errorTypes.CellExclusiveMinimumError }) { const { t } = useTranslation() + const { error } = props + return ( {t("Value of the cell")}{" "} - {props.error.cell} + {error.cell} {" "} {t("in field")}{" "} - {props.error.fieldName} + {error.fieldName} {" "} {t("of row")}{" "} - {props.error.rowNumber} + {error.rowNumber} {" "} - {t("is less or equal to exclusive minimum")} + {t("is less or equal to")} + + {error.minimum} + {" "} + {t("exclusive minimum")} ) } @@ -115,21 +141,27 @@ export function CellExclusiveMaximumError(props: { error: errorTypes.CellExclusiveMaximumError }) { const { t } = useTranslation() + const { error } = props + return ( {t("Value of the cell")}{" "} - {props.error.cell} + {error.cell} {" "} {t("in field")}{" "} - {props.error.fieldName} + {error.fieldName} {" "} {t("of row")}{" "} - {props.error.rowNumber} + {error.rowNumber} {" "} - {t("is more or equal to exclusive maximum")} + {t("is less or equal to")} + + {error.maximum} + {" "} + {t("exclusive maximum")} ) } @@ -138,21 +170,27 @@ export function CellMinLengthError(props: { error: errorTypes.CellMinLengthError }) { const { t } = useTranslation() + const { error } = props + return ( {t("Length of the cell")}{" "} - {props.error.cell} + {error.cell} {" "} {t("in field")}{" "} - {props.error.fieldName} + {error.fieldName} {" "} {t("of row")}{" "} - {props.error.rowNumber} + {error.rowNumber} + {" "} + {t("is less than")} + + {error.minLength} {" "} - {t("is less than minimum")} + {t("minimum")} ) } @@ -161,21 +199,26 @@ export function CellMaxLengthError(props: { error: errorTypes.CellMaxLengthError }) { const { t } = useTranslation() + const { error } = props + return ( {t("Length of the cell")}{" "} - {props.error.cell} + {error.cell} {" "} {t("in field")}{" "} - {props.error.fieldName} + {error.fieldName} {" "} {t("of row")}{" "} - {props.error.rowNumber} + {error.rowNumber} + {" "} + + {error.maxLength} {" "} - {t("is more than maximum")} + {t("maximum")} ) } @@ -184,40 +227,48 @@ export function CellPatternError(props: { error: errorTypes.CellPatternError }) { const { t } = useTranslation() + const { error } = props + return ( {t("Value of the cell")}{" "} - {props.error.cell} + {error.cell} {" "} {t("in field")}{" "} - {props.error.fieldName} + {error.fieldName} {" "} {t("of row")}{" "} - {props.error.rowNumber} + {error.rowNumber} + {" "} + {t("does not match the")} + + {error.pattern} {" "} - {t("does not match the pattern")} + {t("pattern")} ) } export function CellUniqueError(props: { error: errorTypes.CellUniqueError }) { const { t } = useTranslation() + const { error } = props + return ( {t("Value of the cell")}{" "} - {props.error.cell} + {error.cell} {" "} {t("in field")}{" "} - {props.error.fieldName} + {error.fieldName} {" "} {t("of row")}{" "} - {props.error.rowNumber} + {error.rowNumber} {" "} {t("is not unique")} @@ -226,21 +277,27 @@ export function CellUniqueError(props: { error: errorTypes.CellUniqueError }) { export function CellEnumError(props: { error: errorTypes.CellEnumError }) { const { t } = useTranslation() + const { error } = props + return ( {t("Value of the cell")}{" "} - {props.error.cell} + {error.cell} {" "} {t("in field")}{" "} - {props.error.fieldName} + {error.fieldName} {" "} {t("of row")}{" "} - {props.error.rowNumber} + {error.rowNumber} + {" "} + {t("is not in the allowed")} + + {error.enum.join(", ")} {" "} - {t("is not in allowed values")} + {t("values")} ) } diff --git a/browser/locales/de.json b/browser/locales/de.json index 14b4b813..329d656a 100644 --- a/browser/locales/de.json +++ b/browser/locales/de.json @@ -34,13 +34,27 @@ "has a wrong type": "hat einen falschen Typ", "A required cell in field": "Eine erforderliche Zelle im Feld", "is missing": "fehlt", + "is not": "ist nicht", + "is not of the": "ist nicht vom", + "is less than": "ist kleiner als", + "minimum": "Minimum", + "is more than": "ist größer als", + "maximum": "Maximum", + "is less or equal to": "ist kleiner oder gleich", + "exclusive minimum": "exklusives Minimum", + "is more or equal to": "ist größer oder gleich", + "exclusive maximum": "exklusives Maximum", "is less than minimum": "ist kleiner als das Minimum", "is more than maximum": "ist größer als das Maximum", "is less or equal to exclusive minimum": "ist kleiner oder gleich dem exklusiven Minimum", "is more or equal to exclusive maximum": "ist größer oder gleich dem exklusiven Maximum", "Length of the cell": "Länge der Zelle", + "does not match the": "stimmt nicht überein mit", + "pattern": "Muster", "does not match the pattern": "stimmt nicht mit dem Muster überein", "is not unique": "ist nicht eindeutig", + "is not in the allowed": "ist nicht in den erlaubten", + "values": "Werten", "is not in allowed values": "ist nicht in den erlaubten Werten", "Field name is expected to be": "Feldname soll sein", "but it is actually": "aber es ist tatsächlich", diff --git a/browser/locales/en.json b/browser/locales/en.json index 72ea6d2d..bc65af93 100644 --- a/browser/locales/en.json +++ b/browser/locales/en.json @@ -34,13 +34,27 @@ "has a wrong type": "has a wrong type", "A required cell in field": "A required cell in field", "is missing": "is missing", + "is not": "is not", + "is not of the": "is not of the", + "is less than": "is less than", + "minimum": "minimum", + "is more than": "is more than", + "maximum": "maximum", + "is less or equal to": "is less or equal to", + "exclusive minimum": "exclusive minimum", + "is more or equal to": "is more or equal to", + "exclusive maximum": "exclusive maximum", "is less than minimum": "is less than minimum", "is more than maximum": "is more than maximum", "is less or equal to exclusive minimum": "is less or equal to exclusive minimum", "is more or equal to exclusive maximum": "is more or equal to exclusive maximum", "Length of the cell": "Length of the cell", + "does not match the": "does not match the", + "pattern": "pattern", "does not match the pattern": "does not match the pattern", "is not unique": "is not unique", + "is not in the allowed": "is not in the allowed", + "values": "values", "is not in allowed values": "is not in allowed values", "Field name is expected to be": "Field name is expected to be", "but it is actually": "but it is actually", diff --git a/browser/locales/es.json b/browser/locales/es.json index 9810ec87..8ab2f48d 100644 --- a/browser/locales/es.json +++ b/browser/locales/es.json @@ -34,13 +34,27 @@ "has a wrong type": "tiene un tipo incorrecto", "A required cell in field": "Una celda requerida en el campo", "is missing": "falta", + "is not": "no es", + "is not of the": "no es del", + "is less than": "es menor que", + "minimum": "mínimo", + "is more than": "es mayor que", + "maximum": "máximo", + "is less or equal to": "es menor o igual a", + "exclusive minimum": "mínimo exclusivo", + "is more or equal to": "es mayor o igual a", + "exclusive maximum": "máximo exclusivo", "is less than minimum": "es menor que el mínimo", "is more than maximum": "es mayor que el máximo", "is less or equal to exclusive minimum": "es menor o igual al mínimo exclusivo", "is more or equal to exclusive maximum": "es mayor o igual al máximo exclusivo", "Length of the cell": "Longitud de la celda", + "does not match the": "no coincide con", + "pattern": "patrón", "does not match the pattern": "no coincide con el patrón", "is not unique": "no es único", + "is not in the allowed": "no está en los", + "values": "valores permitidos", "is not in allowed values": "no está en los valores permitidos", "Field name is expected to be": "Se espera que el nombre del campo sea", "but it is actually": "pero en realidad es", diff --git a/browser/locales/fr.json b/browser/locales/fr.json index 81f2078d..4fdb5125 100644 --- a/browser/locales/fr.json +++ b/browser/locales/fr.json @@ -34,13 +34,27 @@ "has a wrong type": "a un type incorrect", "A required cell in field": "Une cellule requise dans le champ", "is missing": "est manquante", + "is not": "n'est pas", + "is not of the": "n'est pas du", + "is less than": "est inférieur à", + "minimum": "minimum", + "is more than": "est supérieur à", + "maximum": "maximum", + "is less or equal to": "est inférieur ou égal à", + "exclusive minimum": "minimum exclusif", + "is more or equal to": "est supérieur ou égal à", + "exclusive maximum": "maximum exclusif", "is less than minimum": "est inférieur au minimum", "is more than maximum": "est supérieur au maximum", "is less or equal to exclusive minimum": "est inférieur ou égal au minimum exclusif", "is more or equal to exclusive maximum": "est supérieur ou égal au maximum exclusif", "Length of the cell": "Longueur de la cellule", + "does not match the": "ne correspond pas à", + "pattern": "modèle", "does not match the pattern": "ne correspond pas au modèle", "is not unique": "n'est pas unique", + "is not in the allowed": "n'est pas dans les", + "values": "valeurs autorisées", "is not in allowed values": "n'est pas dans les valeurs autorisées", "Field name is expected to be": "Le nom du champ devrait être", "but it is actually": "mais il est en fait", diff --git a/browser/locales/it.json b/browser/locales/it.json index 150a4122..38fdd254 100644 --- a/browser/locales/it.json +++ b/browser/locales/it.json @@ -34,13 +34,27 @@ "has a wrong type": "ha un tipo errato", "A required cell in field": "Una cella richiesta nel campo", "is missing": "manca", + "is not": "non è", + "is not of the": "non è del", + "is less than": "è inferiore a", + "minimum": "minimo", + "is more than": "è superiore a", + "maximum": "massimo", + "is less or equal to": "è inferiore o uguale a", + "exclusive minimum": "minimo esclusivo", + "is more or equal to": "è superiore o uguale a", + "exclusive maximum": "massimo esclusivo", "is less than minimum": "è inferiore al minimo", "is more than maximum": "è superiore al massimo", "is less or equal to exclusive minimum": "è inferiore o uguale al minimo esclusivo", "is more or equal to exclusive maximum": "è superiore o uguale al massimo esclusivo", "Length of the cell": "Lunghezza della cella", + "does not match the": "non corrisponde al", + "pattern": "pattern", "does not match the pattern": "non corrisponde al pattern", "is not unique": "non è univoco", + "is not in the allowed": "non è tra i", + "values": "valori consentiti", "is not in allowed values": "non è tra i valori consentiti", "Field name is expected to be": "Il nome del campo dovrebbe essere", "but it is actually": "ma in realtà è", diff --git a/browser/locales/pt.json b/browser/locales/pt.json index f80054a9..fbafa1c8 100644 --- a/browser/locales/pt.json +++ b/browser/locales/pt.json @@ -34,13 +34,27 @@ "has a wrong type": "tem um tipo errado", "A required cell in field": "Uma célula obrigatória no campo", "is missing": "está faltando", + "is not": "não é", + "is not of the": "não é do", + "is less than": "é menor que", + "minimum": "mínimo", + "is more than": "é maior que", + "maximum": "máximo", + "is less or equal to": "é menor ou igual a", + "exclusive minimum": "mínimo exclusivo", + "is more or equal to": "é maior ou igual a", + "exclusive maximum": "máximo exclusivo", "is less than minimum": "é menor que o mínimo", "is more than maximum": "é maior que o máximo", "is less or equal to exclusive minimum": "é menor ou igual ao mínimo exclusivo", "is more or equal to exclusive maximum": "é maior ou igual ao máximo exclusivo", "Length of the cell": "Comprimento da célula", + "does not match the": "não corresponde ao", + "pattern": "padrão", "does not match the pattern": "não corresponde ao padrão", "is not unique": "não é único", + "is not in the allowed": "não está nos", + "values": "valores permitidos", "is not in allowed values": "não está nos valores permitidos", "Field name is expected to be": "O nome do campo deveria ser", "but it is actually": "mas na verdade é", diff --git a/browser/locales/ru.json b/browser/locales/ru.json index fac2a1fa..12e47c9e 100644 --- a/browser/locales/ru.json +++ b/browser/locales/ru.json @@ -34,13 +34,27 @@ "has a wrong type": "имеет неправильный тип", "A required cell in field": "Обязательная ячейка в поле", "is missing": "отсутствует", + "is not": "не является", + "is not of the": "не имеет", + "is less than": "меньше", + "minimum": "минимума", + "is more than": "больше", + "maximum": "максимума", + "is less or equal to": "меньше или равно", + "exclusive minimum": "эксклюзивному минимуму", + "is more or equal to": "больше или равно", + "exclusive maximum": "эксклюзивному максимуму", "is less than minimum": "меньше минимума", "is more than maximum": "больше максимума", "is less or equal to exclusive minimum": "меньше или равно эксклюзивному минимуму", "is more or equal to exclusive maximum": "больше или равно эксклюзивному максимуму", "Length of the cell": "Длина ячейки", + "does not match the": "не соответствует", + "pattern": "шаблону", "does not match the pattern": "не соответствует шаблону", "is not unique": "не уникально", + "is not in the allowed": "не входит в допустимые", + "values": "значения", "is not in allowed values": "не входит в допустимые значения", "Field name is expected to be": "Ожидается, что имя поля будет", "but it is actually": "но на самом деле", diff --git a/browser/locales/uk.json b/browser/locales/uk.json index 9bd3e518..805c0c70 100644 --- a/browser/locales/uk.json +++ b/browser/locales/uk.json @@ -34,13 +34,27 @@ "has a wrong type": "має неправильний тип", "A required cell in field": "Обов'язкова комірка в полі", "is missing": "відсутня", + "is not": "не є", + "is not of the": "не має", + "is less than": "менше", + "minimum": "мінімуму", + "is more than": "більше", + "maximum": "максимуму", + "is less or equal to": "менше або дорівнює", + "exclusive minimum": "ексклюзивному мінімуму", + "is more or equal to": "більше або дорівнює", + "exclusive maximum": "ексклюзивному максимуму", "is less than minimum": "менше мінімуму", "is more than maximum": "більше максимуму", "is less or equal to exclusive minimum": "менше або дорівнює ексклюзивному мінімуму", "is more or equal to exclusive maximum": "більше або дорівнює ексклюзивному максимуму", "Length of the cell": "Довжина комірки", + "does not match the": "не відповідає", + "pattern": "шаблону", "does not match the pattern": "не відповідає шаблону", "is not unique": "не унікальне", + "is not in the allowed": "не входить до допустимих", + "values": "значень", "is not in allowed values": "не входить до допустимих значень", "Field name is expected to be": "Очікується, що ім'я поля буде", "but it is actually": "але насправді", From e8d0540efe58d4b0b6d3dd7c0bbff10b82c8ea85 Mon Sep 17 00:00:00 2001 From: roll Date: Wed, 29 Oct 2025 10:24:21 +0000 Subject: [PATCH 22/42] Rebased on mappings --- cli/commands/package/validate.tsx | 2 ++ table/field/Mapping.ts | 13 ++++++++ table/field/denormalize.ts | 19 ++++++++--- table/field/desubstitute.ts | 6 ++-- table/field/index.ts | 3 +- table/field/match.ts | 14 -------- table/field/normalize.ts | 16 +++++---- table/field/parse.ts | 5 +-- table/field/stringify.ts | 5 +-- table/field/substitute.ts | 6 ++-- table/field/validate.ts | 50 ++++++++++++++-------------- table/schema/Mapping.ts | 7 ++++ table/schema/index.ts | 2 ++ table/schema/match.ts | 17 ++++++++++ table/table/denormalize.ts | 41 +++++------------------ table/table/normalize.ts | 29 ++++++++--------- table/table/validate.ts | 54 +++++++++++++------------------ 17 files changed, 141 insertions(+), 148 deletions(-) create mode 100644 table/field/Mapping.ts delete mode 100644 table/field/match.ts create mode 100644 table/schema/Mapping.ts create mode 100644 table/schema/match.ts diff --git a/cli/commands/package/validate.tsx b/cli/commands/package/validate.tsx index 16d2be83..b7617d03 100644 --- a/cli/commands/package/validate.tsx +++ b/cli/commands/package/validate.tsx @@ -37,6 +37,7 @@ export const validatePackageCommand = new Command("validate") // @ts-ignore if (name) report.errors = report.errors.filter(e => e.resource === name) + // @ts-ignore const type = await selectErrorType(session, report.errors) // @ts-ignore if (type) report.errors = report.errors.filter(e => e.type === type) @@ -49,6 +50,7 @@ export const validatePackageCommand = new Command("validate") session.render( report, + // @ts-ignore , ) }) diff --git a/table/field/Mapping.ts b/table/field/Mapping.ts new file mode 100644 index 00000000..a6899dc2 --- /dev/null +++ b/table/field/Mapping.ts @@ -0,0 +1,13 @@ +import type { Field } from "@dpkit/core" +import type { Expr } from "nodejs-polars" +import type { PolarsField } from "./Field.ts" + +export interface FieldMapping { + source: PolarsField + target: Field +} + +export interface CellMapping { + source: Expr + target: Expr +} diff --git a/table/field/denormalize.ts b/table/field/denormalize.ts index 6c4cc074..3a45539e 100644 --- a/table/field/denormalize.ts +++ b/table/field/denormalize.ts @@ -1,14 +1,23 @@ import type { Field } from "@dpkit/core" import { col } from "nodejs-polars" -import type { Expr } from "nodejs-polars" import { desubstituteField } from "./desubstitute.ts" import { stringifyField } from "./stringify.ts" -export function denormalizeField(field: Field, fieldExpr?: Expr) { - let expr = fieldExpr ?? col(field.name) +export type DenormalizeFieldOptions = { + nativeTypes?: Exclude[] +} + +export function denormalizeField( + field: Field, + options?: DenormalizeFieldOptions, +) { + let expr = col(field.name) + const { nativeTypes } = options ?? {} - expr = stringifyField(field, expr) - expr = desubstituteField(field, expr) + if (!nativeTypes?.includes(field.type ?? "any")) { + expr = stringifyField(field, expr) + expr = desubstituteField(field, expr) + } return expr } diff --git a/table/field/desubstitute.ts b/table/field/desubstitute.ts index 285931e3..a30f93a4 100644 --- a/table/field/desubstitute.ts +++ b/table/field/desubstitute.ts @@ -1,12 +1,10 @@ import type { Field } from "@dpkit/core" -import { col, lit, when } from "nodejs-polars" +import { lit, when } from "nodejs-polars" import type { Expr } from "nodejs-polars" const DEFAULT_MISSING_VALUE = "" -export function desubstituteField(field: Field, fieldExpr?: Expr) { - let expr = fieldExpr ?? col(field.name) - +export function desubstituteField(field: Field, expr: Expr) { const flattenMissingValues = field.missingValues?.map(it => typeof it === "string" ? it : it.value, ) diff --git a/table/field/index.ts b/table/field/index.ts index 990d8eb1..902f73d4 100644 --- a/table/field/index.ts +++ b/table/field/index.ts @@ -1,7 +1,8 @@ export { denormalizeField } from "./denormalize.ts" export { parseField } from "./parse.ts" export { validateField } from "./validate.ts" -export { matchField } from "./match.ts" export { normalizeField } from "./normalize.ts" export { stringifyField } from "./stringify.ts" export type { PolarsField } from "./Field.ts" +export type { FieldMapping } from "./Mapping.ts" +export type { DenormalizeFieldOptions } from "./denormalize.ts" diff --git a/table/field/match.ts b/table/field/match.ts deleted file mode 100644 index ce9b33e9..00000000 --- a/table/field/match.ts +++ /dev/null @@ -1,14 +0,0 @@ -import type { Field, Schema } from "@dpkit/core" -import type { PolarsSchema } from "../schema/index.ts" - -export function matchField( - index: number, - field: Field, - schema: Schema, - polarsSchema: PolarsSchema, -) { - const fieldsMatch = schema.fieldsMatch ?? "exact" - return fieldsMatch !== "exact" - ? polarsSchema.fields.find(polarsField => polarsField.name === field.name) - : polarsSchema.fields[index] -} diff --git a/table/field/normalize.ts b/table/field/normalize.ts index d9d7abeb..a32b9fe1 100644 --- a/table/field/normalize.ts +++ b/table/field/normalize.ts @@ -1,14 +1,16 @@ -import type { Field } from "@dpkit/core" +import { DataType } from "nodejs-polars" import { col } from "nodejs-polars" -import type { Expr } from "nodejs-polars" +import type { FieldMapping } from "./Mapping.ts" import { parseField } from "./parse.ts" import { substituteField } from "./substitute.ts" -export function normalizeField(field: Field, fieldExpr?: Expr) { - let expr = fieldExpr ?? col(field.name) +export function normalizeField(mapping: FieldMapping) { + let expr = col(mapping.source.name) - expr = substituteField(field, expr) - expr = parseField(field, expr) + if (mapping.source.type.equals(DataType.String)) { + expr = substituteField(mapping.target, expr) + expr = parseField(mapping.target, expr) + } - return expr + return expr.alias(mapping.target.name) } diff --git a/table/field/parse.ts b/table/field/parse.ts index 33c50aac..68b366e3 100644 --- a/table/field/parse.ts +++ b/table/field/parse.ts @@ -1,5 +1,4 @@ import type { Field } from "@dpkit/core" -import { col } from "nodejs-polars" import type { Expr } from "nodejs-polars" import { parseArrayField } from "./types/array.ts" import { parseBooleanField } from "./types/boolean.ts" @@ -17,9 +16,7 @@ import { parseTimeField } from "./types/time.ts" import { parseYearField } from "./types/year.ts" import { parseYearmonthField } from "./types/yearmonth.ts" -export function parseField(field: Field, fieldExpr?: Expr) { - const expr = fieldExpr ?? col(field.name) - +export function parseField(field: Field, expr: Expr) { switch (field.type) { case "array": return parseArrayField(field, expr) diff --git a/table/field/stringify.ts b/table/field/stringify.ts index 9bf525a2..bfd88901 100644 --- a/table/field/stringify.ts +++ b/table/field/stringify.ts @@ -1,5 +1,4 @@ import type { Field } from "@dpkit/core" -import { col } from "nodejs-polars" import type { Expr } from "nodejs-polars" import { stringifyArrayField } from "./types/array.ts" import { stringifyBooleanField } from "./types/boolean.ts" @@ -17,9 +16,7 @@ import { stringifyTimeField } from "./types/time.ts" import { stringifyYearField } from "./types/year.ts" import { stringifyYearmonthField } from "./types/yearmonth.ts" -export function stringifyField(field: Field, fieldExpr?: Expr) { - const expr = fieldExpr ?? col(field.name) - +export function stringifyField(field: Field, expr: Expr) { switch (field.type) { case "array": return stringifyArrayField(field, expr) diff --git a/table/field/substitute.ts b/table/field/substitute.ts index 46e19ab9..8d23d8f7 100644 --- a/table/field/substitute.ts +++ b/table/field/substitute.ts @@ -1,12 +1,10 @@ import type { Field } from "@dpkit/core" -import { col, lit, when } from "nodejs-polars" +import { lit, when } from "nodejs-polars" import type { Expr } from "nodejs-polars" const DEFAULT_MISSING_VALUES = [""] -export function substituteField(field: Field, fieldExpr?: Expr) { - let expr = fieldExpr ?? col(field.name) - +export function substituteField(field: Field, expr: Expr) { const flattenMissingValues = field.missingValues?.map(it => (typeof it === "string" ? it : it.value)) ?? DEFAULT_MISSING_VALUES diff --git a/table/field/validate.ts b/table/field/validate.ts index 266a1cf0..86f46887 100644 --- a/table/field/validate.ts +++ b/table/field/validate.ts @@ -2,7 +2,7 @@ import type { Field } from "@dpkit/core" import { col, lit, when } from "nodejs-polars" import type { CellError, FieldError, TableError } from "../error/index.ts" import type { Table } from "../table/index.ts" -import type { PolarsField } from "./Field.ts" +import type { FieldMapping } from "./Mapping.ts" import { checkCellEnum } from "./checks/enum.ts" import { checkCellMaxLength } from "./checks/maxLength.ts" import { createCheckCellMaximum } from "./checks/maximum.ts" @@ -16,48 +16,47 @@ import { normalizeField } from "./normalize.ts" import { substituteField } from "./substitute.ts" export async function validateField( - field: Field, + mapping: FieldMapping, + table: Table, options: { - polarsField: PolarsField maxErrors: number - table: Table }, ) { - const { polarsField, maxErrors, table } = options + const { maxErrors } = options const errors: TableError[] = [] - const nameErrors = validateName(field, polarsField) + const nameErrors = validateName(mapping) errors.push(...nameErrors) - const typeErrors = validateType(field, polarsField) + const typeErrors = validateType(mapping) errors.push(...typeErrors) if (!typeErrors.length) { - const dataErorrs = await validateCells(field, polarsField, maxErrors, table) + const dataErorrs = await validateCells(mapping, table, { maxErrors }) errors.push(...dataErorrs) } return { errors, valid: !errors.length } } -function validateName(field: Field, polarsField: PolarsField) { +function validateName(mapping: FieldMapping) { const errors: FieldError[] = [] - if (field.name !== polarsField.name) { + if (mapping.source.name !== mapping.target.name) { errors.push({ type: "field/name", - fieldName: field.name, - actualFieldName: polarsField.name, + fieldName: mapping.target.name, + actualFieldName: mapping.source.name, }) } return errors } -function validateType(field: Field, polarsField: PolarsField) { +function validateType(mapping: FieldMapping) { const errors: FieldError[] = [] - const mapping: Record = { + const types: Record = { Bool: "boolean", Date: "date", Datetime: "datetime", @@ -77,13 +76,13 @@ function validateType(field: Field, polarsField: PolarsField) { Utf8: "string", } - const actualFieldType = mapping[polarsField.type.variant] ?? "any" + const actualFieldType = types[mapping.source.type.variant] ?? "any" - if (actualFieldType !== field.type && actualFieldType !== "string") { + if (actualFieldType !== mapping.target.type && actualFieldType !== "string") { errors.push({ type: "field/type", - fieldName: field.name, - fieldType: field.type ?? "any", + fieldName: mapping.target.name, + fieldType: mapping.target.type ?? "any", actualFieldType, }) } @@ -92,20 +91,21 @@ function validateType(field: Field, polarsField: PolarsField) { } async function validateCells( - field: Field, - polarsField: PolarsField, - maxErrors: number, + mapping: FieldMapping, table: Table, + options: { + maxErrors: number + }, ) { + const { maxErrors } = options const errors: CellError[] = [] - const fieldExpr = col(polarsField.name) let fieldCheckTable = table .withRowCount() .select( col("row_nr").add(1).alias("number"), - normalizeField(field, fieldExpr).alias("target"), - substituteField(field, fieldExpr).alias("source"), + normalizeField(mapping).alias("target"), + substituteField(mapping.target, col(mapping.source.name)).alias("source"), lit(null).alias("error"), ) @@ -122,7 +122,7 @@ async function validateCells( checkCellMaxLength, checkCellUnique, ]) { - const check = checkCell(field, col("target"), col("source")) + const check = checkCell(mapping.target, col("target"), col("source")) if (check) { fieldCheckTable = fieldCheckTable.withColumn( diff --git a/table/schema/Mapping.ts b/table/schema/Mapping.ts new file mode 100644 index 00000000..baaee2cc --- /dev/null +++ b/table/schema/Mapping.ts @@ -0,0 +1,7 @@ +import type { Schema } from "@dpkit/core" +import type { PolarsSchema } from "./Schema.ts" + +export interface SchemaMapping { + source: PolarsSchema + target: Schema +} diff --git a/table/schema/index.ts b/table/schema/index.ts index d888123b..8cd4331e 100644 --- a/table/schema/index.ts +++ b/table/schema/index.ts @@ -4,3 +4,5 @@ export type { SchemaOptions } from "./Options.ts" export { inferSchemaFromTable } from "./infer.ts" export { inferSchemaFromSample } from "./infer.ts" export type { InferSchemaOptions } from "./infer.ts" +export type { SchemaMapping } from "./Mapping.ts" +export { matchSchemaField } from "./match.ts" diff --git a/table/schema/match.ts b/table/schema/match.ts new file mode 100644 index 00000000..24583b0f --- /dev/null +++ b/table/schema/match.ts @@ -0,0 +1,17 @@ +import type { Field } from "@dpkit/core" +import type { SchemaMapping } from "./Mapping.ts" + +export function matchSchemaField( + mapping: SchemaMapping, + field: Field, + index: number, +) { + const fieldsMatch = mapping.target.fieldsMatch ?? "exact" + + const polarsField = + fieldsMatch !== "exact" + ? mapping.source.fields.find(it => it.name === field.name) + : mapping.source.fields[index] + + return polarsField ? { source: polarsField, target: field } : undefined +} diff --git a/table/table/denormalize.ts b/table/table/denormalize.ts index f39be815..37b7bfae 100644 --- a/table/table/denormalize.ts +++ b/table/table/denormalize.ts @@ -1,53 +1,28 @@ -import type { Field, Schema } from "@dpkit/core" -import { col, lit } from "nodejs-polars" +import type { Schema } from "@dpkit/core" import type { Expr } from "nodejs-polars" import { denormalizeField } from "../field/index.ts" -import type { PolarsSchema } from "../schema/index.ts" -import { getPolarsSchema } from "../schema/index.ts" +import type { DenormalizeFieldOptions } from "../field/index.ts" import type { Table } from "./Table.ts" -const HEAD_ROWS = 100 - -type DenormalizeTableOptions = { - nativeTypes?: Exclude[] -} - export async function denormalizeTable( table: Table, schema: Schema, - options?: DenormalizeTableOptions, + options?: DenormalizeFieldOptions, ) { - const head = await table.head(HEAD_ROWS).collect() - const polarsSchema = getPolarsSchema(head.schema) - - return table.select( - ...Object.values(denormalizeFields(schema, polarsSchema, options)), - ) + return table.select(...Object.values(denormalizeFields(schema, options))) } export function denormalizeFields( schema: Schema, - polarsSchema: PolarsSchema, - options?: DenormalizeTableOptions, + options?: DenormalizeFieldOptions, ) { - const { nativeTypes } = options ?? {} const exprs: Record = {} for (const field of schema.fields) { - const polarsField = polarsSchema.fields.find(f => f.name === field.name) - let expr = lit(null).alias(field.name) - - if (polarsField) { - expr = col(polarsField.name).alias(field.name) - - // TODO: Move this logic to denormalizeField? - if (!nativeTypes?.includes(field.type ?? "any")) { - const missingValues = field.missingValues ?? schema.missingValues - const mergedField = { ...field, missingValues } - expr = denormalizeField(mergedField, expr) - } - } + const missingValues = field.missingValues ?? schema.missingValues + const mergedField = { ...field, missingValues } + const expr = denormalizeField(mergedField, options) exprs[field.name] = expr } diff --git a/table/table/normalize.ts b/table/table/normalize.ts index 72d78724..8d23b81e 100644 --- a/table/table/normalize.ts +++ b/table/table/normalize.ts @@ -1,11 +1,10 @@ import type { Schema } from "@dpkit/core" import type { Expr } from "nodejs-polars" -import { DataType } from "nodejs-polars" -import { col, lit } from "nodejs-polars" -import { matchField } from "../field/index.ts" +import { lit } from "nodejs-polars" import { normalizeField } from "../field/index.ts" +import { matchSchemaField } from "../schema/index.ts" import { getPolarsSchema } from "../schema/index.ts" -import type { PolarsSchema } from "../schema/index.ts" +import type { SchemaMapping } from "../schema/index.ts" import type { Table } from "./Table.ts" const HEAD_ROWS = 100 @@ -14,25 +13,23 @@ export async function normalizeTable(table: Table, schema: Schema) { const head = await table.head(HEAD_ROWS).collect() const polarsSchema = getPolarsSchema(head.schema) - return table.select(...Object.values(normalizeFields(schema, polarsSchema))) + const mapping = { source: polarsSchema, target: schema } + return table.select(...Object.values(normalizeFields(mapping))) } -export function normalizeFields(schema: Schema, polarsSchema: PolarsSchema) { +export function normalizeFields(mapping: SchemaMapping) { const exprs: Record = {} - for (const [index, field] of schema.fields.entries()) { - const polarsField = matchField(index, field, schema, polarsSchema) + for (const [index, field] of mapping.target.fields.entries()) { + const fieldMapping = matchSchemaField(mapping, field, index) let expr = lit(null).alias(field.name) - if (polarsField) { - expr = col(polarsField.name).alias(field.name) + if (fieldMapping) { + const missingValues = field.missingValues ?? mapping.target.missingValues + const mergedField = { ...field, missingValues } - // TODO: Move this logic to normalizeField? - if (polarsField.type.equals(DataType.String)) { - const missingValues = field.missingValues ?? schema.missingValues - const mergedField = { ...field, missingValues } - expr = normalizeField(mergedField, expr) - } + const column = { source: fieldMapping.source, target: mergedField } + expr = normalizeField(column) } exprs[field.name] = expr diff --git a/table/table/validate.ts b/table/table/validate.ts index 8ca7aef5..226fd65d 100644 --- a/table/table/validate.ts +++ b/table/table/validate.ts @@ -2,11 +2,11 @@ import os from "node:os" import type { Field, Schema } from "@dpkit/core" import pAll from "p-all" import type { TableError } from "../error/index.ts" -import { matchField } from "../field/index.ts" import { validateField } from "../field/index.ts" +import { matchSchemaField } from "../schema/index.ts" //import { validateRows } from "../row/index.ts" import { getPolarsSchema } from "../schema/index.ts" -import type { PolarsSchema } from "../schema/index.ts" +import type { SchemaMapping } from "../schema/index.ts" import type { Table } from "./Table.ts" export async function validateTable( @@ -14,45 +14,36 @@ export async function validateTable( options?: { schema?: Schema sampleRows?: number - maxErorrs?: number + maxErrors?: number }, ) { - const { schema, sampleRows = 100, maxErorrs = 1000 } = options ?? {} + const { schema, sampleRows = 100, maxErrors = 1000 } = options ?? {} const errors: TableError[] = [] if (schema) { const sample = await table.head(sampleRows).collect() const polarsSchema = getPolarsSchema(sample.schema) + const mapping = { source: polarsSchema, target: schema } - const matchErrors = validateFieldsMatch({ schema, polarsSchema }) + const matchErrors = validateFieldsMatch(mapping) errors.push(...matchErrors) - const fieldErrors = await validateFields( - table, - schema, - polarsSchema, - maxErorrs, - ) + const fieldErrors = await validateFields(mapping, table, { maxErrors }) errors.push(...fieldErrors) } return { - errors: errors.slice(0, maxErorrs), + errors: errors.slice(0, maxErrors), valid: !errors.length, } } -function validateFieldsMatch(props: { - schema: Schema - polarsSchema: PolarsSchema -}) { - const { schema, polarsSchema } = props - +function validateFieldsMatch(mapping: SchemaMapping) { const errors: TableError[] = [] - const fieldsMatch = schema.fieldsMatch ?? "exact" + const fieldsMatch = mapping.target.fieldsMatch ?? "exact" - const fields = schema.fields - const polarsFields = polarsSchema.fields + const fields = mapping.target.fields + const polarsFields = mapping.source.fields const names = fields.map(field => field.name) const polarsNames = polarsFields.map(field => field.name) @@ -131,23 +122,24 @@ function validateFieldsMatch(props: { } async function validateFields( + mapping: SchemaMapping, table: Table, - schema: Schema, - polarsSchema: PolarsSchema, - maxErrors: number, + options: { + maxErrors: number + }, ) { + const { maxErrors } = options const errors: TableError[] = [] + const fields = mapping.target.fields const concurrency = os.cpus().length - 1 const abortController = new AbortController() - const maxFieldErrors = Math.ceil(maxErrors / schema.fields.length) + const maxFieldErrors = Math.ceil(maxErrors / mapping.target.fields.length) const collectFieldErrors = async (index: number, field: Field) => { - const polarsField = matchField(index, field, schema, polarsSchema) - if (!polarsField) return + const fieldMapping = matchSchemaField(mapping, field, index) + if (!fieldMapping) return - const report = await validateField(field, { - table, - polarsField, + const report = await validateField(fieldMapping, table, { maxErrors: maxFieldErrors, }) @@ -159,7 +151,7 @@ async function validateFields( try { await pAll( - schema.fields.map((field, idx) => () => collectFieldErrors(idx, field)), + fields.map((field, index) => () => collectFieldErrors(index, field)), { concurrency }, ) } catch (error) { From a56fcd089b96c2209cd4f05ae9cd003831f463e7 Mon Sep 17 00:00:00 2001 From: roll Date: Wed, 29 Oct 2025 10:28:27 +0000 Subject: [PATCH 23/42] Fixed enum tests --- table/field/checks/enum.spec.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/table/field/checks/enum.spec.ts b/table/field/checks/enum.spec.ts index 576b6450..bd657609 100644 --- a/table/field/checks/enum.spec.ts +++ b/table/field/checks/enum.spec.ts @@ -26,6 +26,8 @@ describe("validateTable (cell/enum)", () => { }) it("should report errors for values not in the enum", async () => { + const allowedValues = ["pending", "approved", "rejected"] + const table = DataFrame({ status: ["pending", "approved", "unknown", "cancelled", "rejected"], }).lazy() @@ -36,7 +38,7 @@ describe("validateTable (cell/enum)", () => { name: "status", type: "string", constraints: { - enum: ["pending", "approved", "rejected"], + enum: allowedValues, }, }, ], @@ -47,12 +49,14 @@ describe("validateTable (cell/enum)", () => { expect(errors).toContainEqual({ type: "cell/enum", fieldName: "status", + enum: allowedValues, rowNumber: 3, cell: "unknown", }) expect(errors).toContainEqual({ type: "cell/enum", fieldName: "status", + enum: allowedValues, rowNumber: 4, cell: "cancelled", }) @@ -80,6 +84,8 @@ describe("validateTable (cell/enum)", () => { }) it("should handle case sensitivity correctly", async () => { + const allowedValues = ["pending", "approved", "rejected"] + const table = DataFrame({ status: ["Pending", "APPROVED", "rejected"], }).lazy() @@ -90,7 +96,7 @@ describe("validateTable (cell/enum)", () => { name: "status", type: "string", constraints: { - enum: ["pending", "approved", "rejected"], + enum: allowedValues, }, }, ], @@ -101,12 +107,14 @@ describe("validateTable (cell/enum)", () => { expect(errors).toContainEqual({ type: "cell/enum", fieldName: "status", + enum: allowedValues, rowNumber: 1, cell: "Pending", }) expect(errors).toContainEqual({ type: "cell/enum", fieldName: "status", + enum: allowedValues, rowNumber: 2, cell: "APPROVED", }) From 719325b856abcfce3c41bfe7756c531055d256fc Mon Sep 17 00:00:00 2001 From: roll Date: Wed, 29 Oct 2025 10:57:54 +0000 Subject: [PATCH 24/42] Improved expr naming --- table/field/checks/enum.ts | 6 +++--- table/field/checks/maxLength.ts | 6 +++--- table/field/checks/maximum.ts | 7 ++++--- table/field/checks/minLength.ts | 6 +++--- table/field/checks/minimum.ts | 7 ++++--- table/field/checks/pattern.ts | 6 +++--- table/field/checks/required.ts | 6 +++--- table/field/checks/type.ts | 6 +++--- table/field/checks/unique.ts | 8 +++++--- table/field/desubstitute.ts | 8 ++++---- table/field/normalize.ts | 8 ++++---- table/field/parse.ts | 34 ++++++++++++++++----------------- table/field/stringify.ts | 34 ++++++++++++++++----------------- table/field/substitute.ts | 8 ++++---- table/field/types/array.ts | 16 +++++++--------- table/field/types/boolean.ts | 22 +++++++++------------ table/field/types/date.ts | 13 ++++--------- table/field/types/datetime.ts | 13 ++++--------- table/field/types/duration.ts | 13 ++++--------- table/field/types/geojson.ts | 16 +++++++--------- table/field/types/geopoint.ts | 34 +++++++++++++++------------------ table/field/types/integer.ts | 28 ++++++++++++--------------- table/field/types/list.ts | 16 ++++++---------- table/field/types/number.ts | 34 ++++++++++++++++----------------- table/field/types/object.ts | 16 +++++++--------- table/field/types/string.ts | 22 +++++++++------------ table/field/types/time.ts | 14 +++++--------- table/field/types/year.ts | 19 +++++++----------- table/field/types/yearmonth.ts | 29 +++++++++++++++++----------- table/field/validate.ts | 3 ++- table/table/normalize.ts | 13 +++++++------ table/table/validate.ts | 27 +++++++++++++++----------- 32 files changed, 232 insertions(+), 266 deletions(-) diff --git a/table/field/checks/enum.ts b/table/field/checks/enum.ts index cdddbd18..626b653b 100644 --- a/table/field/checks/enum.ts +++ b/table/field/checks/enum.ts @@ -1,14 +1,14 @@ import type { Field } from "@dpkit/core" -import type { Expr } from "nodejs-polars" import type { CellEnumError } from "../../error/index.ts" +import type { CellMapping } from "../Mapping.ts" -export function checkCellEnum(field: Field, target: Expr) { +export function checkCellEnum(field: Field, mapping: CellMapping) { if (field.type !== "string") return undefined const rawEnum = field.constraints?.enum if (!rawEnum) return undefined - const isErrorExpr = target.isIn(rawEnum).not() + const isErrorExpr = mapping.target.isIn(rawEnum).not() const errorTemplate: CellEnumError = { type: "cell/enum", diff --git a/table/field/checks/maxLength.ts b/table/field/checks/maxLength.ts index 41eb4f99..84306b21 100644 --- a/table/field/checks/maxLength.ts +++ b/table/field/checks/maxLength.ts @@ -1,14 +1,14 @@ import type { Field } from "@dpkit/core" -import type { Expr } from "nodejs-polars" import type { CellMaxLengthError } from "../../error/index.ts" +import type { CellMapping } from "../Mapping.ts" -export function checkCellMaxLength(field: Field, target: Expr) { +export function checkCellMaxLength(field: Field, mapping: CellMapping) { if (field.type !== "string") return undefined const maxLength = field.constraints?.maxLength if (!maxLength) return undefined - const isErrorExpr = target.str.lengths().gt(maxLength) + const isErrorExpr = mapping.target.str.lengths().gt(maxLength) const errorTemplate: CellMaxLengthError = { type: "cell/maxLength", diff --git a/table/field/checks/maximum.ts b/table/field/checks/maximum.ts index 0f5789e2..7a70dd28 100644 --- a/table/field/checks/maximum.ts +++ b/table/field/checks/maximum.ts @@ -3,9 +3,10 @@ import { lit } from "nodejs-polars" import type { Expr } from "nodejs-polars" import type { CellExclusiveMaximumError } from "../../error/index.ts" import type { CellMaximumError } from "../../error/index.ts" +import type { CellMapping } from "../Mapping.ts" export function createCheckCellMaximum(options?: { isExclusive?: boolean }) { - return (field: Field, target: Expr) => { + return (field: Field, mapping: CellMapping) => { if (field.type !== "integer" && field.type !== "number") return undefined const maximum = options?.isExclusive @@ -22,8 +23,8 @@ export function createCheckCellMaximum(options?: { isExclusive?: boolean }) { typeof maximum === "string" ? parser(maximum) : maximum isErrorExpr = options?.isExclusive - ? target.gtEq(parsedMaximum) - : target.gt(parsedMaximum) + ? mapping.target.gtEq(parsedMaximum) + : mapping.target.gt(parsedMaximum) } catch (error) { isErrorExpr = lit(true) } diff --git a/table/field/checks/minLength.ts b/table/field/checks/minLength.ts index 4f0fb9ff..b9bfce53 100644 --- a/table/field/checks/minLength.ts +++ b/table/field/checks/minLength.ts @@ -1,14 +1,14 @@ import type { Field } from "@dpkit/core" -import type { Expr } from "nodejs-polars" import type { CellMinLengthError } from "../../error/index.ts" +import type { CellMapping } from "../Mapping.ts" -export function checkCellMinLength(field: Field, target: Expr) { +export function checkCellMinLength(field: Field, mapping: CellMapping) { if (field.type !== "string") return undefined const minLength = field.constraints?.minLength if (!minLength) return undefined - const isErrorExpr = target.str.lengths().lt(minLength) + const isErrorExpr = mapping.target.str.lengths().lt(minLength) const errorTemplate: CellMinLengthError = { type: "cell/minLength", diff --git a/table/field/checks/minimum.ts b/table/field/checks/minimum.ts index 23f6800f..da7fae2a 100644 --- a/table/field/checks/minimum.ts +++ b/table/field/checks/minimum.ts @@ -3,9 +3,10 @@ import { lit } from "nodejs-polars" import type { Expr } from "nodejs-polars" import type { CellExclusiveMinimumError } from "../../error/index.ts" import type { CellMinimumError } from "../../error/index.ts" +import type { CellMapping } from "../Mapping.ts" export function createCheckCellMinimum(options?: { isExclusive?: boolean }) { - return (field: Field, target: Expr) => { + return (field: Field, mapping: CellMapping) => { if (field.type !== "integer" && field.type !== "number") return undefined const minimum = options?.isExclusive @@ -22,8 +23,8 @@ export function createCheckCellMinimum(options?: { isExclusive?: boolean }) { typeof minimum === "string" ? parser(minimum) : minimum isErrorExpr = options?.isExclusive - ? target.ltEq(parsedMinimum) - : target.lt(parsedMinimum) + ? mapping.target.ltEq(parsedMinimum) + : mapping.target.lt(parsedMinimum) } catch (error) { isErrorExpr = lit(true) } diff --git a/table/field/checks/pattern.ts b/table/field/checks/pattern.ts index 44f76447..e4c31d0c 100644 --- a/table/field/checks/pattern.ts +++ b/table/field/checks/pattern.ts @@ -1,14 +1,14 @@ import type { Field } from "@dpkit/core" -import type { Expr } from "nodejs-polars" import type { CellPatternError } from "../../error/index.ts" +import type { CellMapping } from "../Mapping.ts" -export function checkCellPattern(field: Field, target: Expr) { +export function checkCellPattern(field: Field, mapping: CellMapping) { if (field.type !== "string") return undefined const pattern = field.constraints?.pattern if (!pattern) return undefined - const isErrorExpr = target.str.contains(pattern).not() + const isErrorExpr = mapping.target.str.contains(pattern).not() const errorTemplate: CellPatternError = { type: "cell/pattern", diff --git a/table/field/checks/required.ts b/table/field/checks/required.ts index e0b1e232..08602f26 100644 --- a/table/field/checks/required.ts +++ b/table/field/checks/required.ts @@ -1,12 +1,12 @@ import type { Field } from "@dpkit/core" import type { CellRequiredError } from "../../error/index.ts" -import type { Expr } from "nodejs-polars" +import type { CellMapping } from "../Mapping.ts" -export function checkCellRequired(field: Field, target: Expr) { +export function checkCellRequired(field: Field, mapping: CellMapping) { const required = field.constraints?.required if (!required) return undefined - const isErrorExpr = target.isNull() + const isErrorExpr = mapping.target.isNull() const errorTemplate: CellRequiredError = { type: "cell/required", diff --git a/table/field/checks/type.ts b/table/field/checks/type.ts index e4551de5..0e88dd62 100644 --- a/table/field/checks/type.ts +++ b/table/field/checks/type.ts @@ -1,9 +1,9 @@ import type { Field } from "@dpkit/core" -import type { Expr } from "nodejs-polars" import type { CellTypeError } from "../../error/index.ts" +import type { CellMapping } from "../Mapping.ts" -export function checkCellType(field: Field, target: Expr, source: Expr) { - const isErrorExpr = source.isNotNull().and(target.isNull()) +export function checkCellType(field: Field, mapping: CellMapping) { + const isErrorExpr = mapping.source.isNotNull().and(mapping.target.isNull()) const errorTemplate: CellTypeError = { type: "cell/type", diff --git a/table/field/checks/unique.ts b/table/field/checks/unique.ts index 463104d5..b6f3078d 100644 --- a/table/field/checks/unique.ts +++ b/table/field/checks/unique.ts @@ -1,13 +1,15 @@ import type { Field } from "@dpkit/core" -import type { Expr } from "nodejs-polars" import type { CellUniqueError } from "../../error/index.ts" +import type { CellMapping } from "../Mapping.ts" // TODO: Support schema.primaryKey and schema.uniqueKeys -export function checkCellUnique(field: Field, target: Expr) { +export function checkCellUnique(field: Field, mapping: CellMapping) { const unique = field.constraints?.unique if (!unique) return undefined - const isErrorExpr = target.isNotNull().and(target.isFirstDistinct().not()) + const isErrorExpr = mapping.target + .isNotNull() + .and(mapping.target.isFirstDistinct().not()) const errorTemplate: CellUniqueError = { type: "cell/unique", diff --git a/table/field/desubstitute.ts b/table/field/desubstitute.ts index a30f93a4..e14396db 100644 --- a/table/field/desubstitute.ts +++ b/table/field/desubstitute.ts @@ -4,16 +4,16 @@ import type { Expr } from "nodejs-polars" const DEFAULT_MISSING_VALUE = "" -export function desubstituteField(field: Field, expr: Expr) { +export function desubstituteField(field: Field, fieldExpr: Expr) { const flattenMissingValues = field.missingValues?.map(it => typeof it === "string" ? it : it.value, ) const missingValue = flattenMissingValues?.[0] ?? DEFAULT_MISSING_VALUE - expr = when(expr.isNull()) + fieldExpr = when(fieldExpr.isNull()) .then(lit(missingValue)) - .otherwise(expr) + .otherwise(fieldExpr) .alias(field.name) - return expr + return fieldExpr } diff --git a/table/field/normalize.ts b/table/field/normalize.ts index a32b9fe1..63117ec1 100644 --- a/table/field/normalize.ts +++ b/table/field/normalize.ts @@ -5,12 +5,12 @@ import { parseField } from "./parse.ts" import { substituteField } from "./substitute.ts" export function normalizeField(mapping: FieldMapping) { - let expr = col(mapping.source.name) + let fieldExpr = col(mapping.source.name) if (mapping.source.type.equals(DataType.String)) { - expr = substituteField(mapping.target, expr) - expr = parseField(mapping.target, expr) + fieldExpr = substituteField(mapping.target, fieldExpr) + fieldExpr = parseField(mapping.target, fieldExpr) } - return expr.alias(mapping.target.name) + return fieldExpr.alias(mapping.target.name) } diff --git a/table/field/parse.ts b/table/field/parse.ts index 68b366e3..8e3113ac 100644 --- a/table/field/parse.ts +++ b/table/field/parse.ts @@ -16,39 +16,39 @@ import { parseTimeField } from "./types/time.ts" import { parseYearField } from "./types/year.ts" import { parseYearmonthField } from "./types/yearmonth.ts" -export function parseField(field: Field, expr: Expr) { +export function parseField(field: Field, fieldExpr: Expr) { switch (field.type) { case "array": - return parseArrayField(field, expr) + return parseArrayField(field, fieldExpr) case "boolean": - return parseBooleanField(field, expr) + return parseBooleanField(field, fieldExpr) case "date": - return parseDateField(field, expr) + return parseDateField(field, fieldExpr) case "datetime": - return parseDatetimeField(field, expr) + return parseDatetimeField(field, fieldExpr) case "duration": - return parseDurationField(field, expr) + return parseDurationField(field, fieldExpr) case "geojson": - return parseGeojsonField(field, expr) + return parseGeojsonField(field, fieldExpr) case "geopoint": - return parseGeopointField(field, expr) + return parseGeopointField(field, fieldExpr) case "integer": - return parseIntegerField(field, expr) + return parseIntegerField(field, fieldExpr) case "list": - return parseListField(field, expr) + return parseListField(field, fieldExpr) case "number": - return parseNumberField(field, expr) + return parseNumberField(field, fieldExpr) case "object": - return parseObjectField(field, expr) + return parseObjectField(field, fieldExpr) case "string": - return parseStringField(field, expr) + return parseStringField(field, fieldExpr) case "time": - return parseTimeField(field, expr) + return parseTimeField(field, fieldExpr) case "year": - return parseYearField(field, expr) + return parseYearField(field, fieldExpr) case "yearmonth": - return parseYearmonthField(field, expr) + return parseYearmonthField(field, fieldExpr) default: - return expr + return fieldExpr } } diff --git a/table/field/stringify.ts b/table/field/stringify.ts index bfd88901..f03b4dce 100644 --- a/table/field/stringify.ts +++ b/table/field/stringify.ts @@ -16,39 +16,39 @@ import { stringifyTimeField } from "./types/time.ts" import { stringifyYearField } from "./types/year.ts" import { stringifyYearmonthField } from "./types/yearmonth.ts" -export function stringifyField(field: Field, expr: Expr) { +export function stringifyField(field: Field, fieldExpr: Expr) { switch (field.type) { case "array": - return stringifyArrayField(field, expr) + return stringifyArrayField(field, fieldExpr) case "boolean": - return stringifyBooleanField(field, expr) + return stringifyBooleanField(field, fieldExpr) case "date": - return stringifyDateField(field, expr) + return stringifyDateField(field, fieldExpr) case "datetime": - return stringifyDatetimeField(field, expr) + return stringifyDatetimeField(field, fieldExpr) case "duration": - return stringifyDurationField(field, expr) + return stringifyDurationField(field, fieldExpr) case "geojson": - return stringifyGeojsonField(field, expr) + return stringifyGeojsonField(field, fieldExpr) case "geopoint": - return stringifyGeopointField(field, expr) + return stringifyGeopointField(field, fieldExpr) case "integer": - return stringifyIntegerField(field, expr) + return stringifyIntegerField(field, fieldExpr) case "list": - return stringifyListField(field, expr) + return stringifyListField(field, fieldExpr) case "number": - return stringifyNumberField(field, expr) + return stringifyNumberField(field, fieldExpr) case "object": - return stringifyObjectField(field, expr) + return stringifyObjectField(field, fieldExpr) case "string": - return stringifyStringField(field, expr) + return stringifyStringField(field, fieldExpr) case "time": - return stringifyTimeField(field, expr) + return stringifyTimeField(field, fieldExpr) case "year": - return stringifyYearField(field, expr) + return stringifyYearField(field, fieldExpr) case "yearmonth": - return stringifyYearmonthField(field, expr) + return stringifyYearmonthField(field, fieldExpr) default: - return expr + return fieldExpr } } diff --git a/table/field/substitute.ts b/table/field/substitute.ts index 8d23d8f7..5f024a3d 100644 --- a/table/field/substitute.ts +++ b/table/field/substitute.ts @@ -4,17 +4,17 @@ import type { Expr } from "nodejs-polars" const DEFAULT_MISSING_VALUES = [""] -export function substituteField(field: Field, expr: Expr) { +export function substituteField(field: Field, fieldExpr: Expr) { const flattenMissingValues = field.missingValues?.map(it => (typeof it === "string" ? it : it.value)) ?? DEFAULT_MISSING_VALUES if (flattenMissingValues.length) { - expr = when(expr.isIn(flattenMissingValues)) + fieldExpr = when(fieldExpr.isIn(flattenMissingValues)) .then(lit(null)) - .otherwise(expr) + .otherwise(fieldExpr) .alias(field.name) } - return expr + return fieldExpr } diff --git a/table/field/types/array.ts b/table/field/types/array.ts index 7c21ec25..f3c7ddb3 100644 --- a/table/field/types/array.ts +++ b/table/field/types/array.ts @@ -1,19 +1,17 @@ import type { ArrayField } from "@dpkit/core" -import { col, lit, when } from "nodejs-polars" +import { lit, when } from "nodejs-polars" import type { Expr } from "nodejs-polars" // TODO: Is there a better way to do this? // Polars does not support really support free-form JSON // So we just make a basic check and return as it is -export function parseArrayField(field: ArrayField, expr?: Expr) { - expr = expr ?? col(field.name) - - return when(expr.str.contains("^\\[")).then(expr).otherwise(lit(null)) +export function parseArrayField(_field: ArrayField, fieldExpr: Expr) { + return when(fieldExpr.str.contains("^\\[")) + .then(fieldExpr) + .otherwise(lit(null)) } -export function stringifyArrayField(field: ArrayField, expr?: Expr) { - expr = expr ?? col(field.name) - +export function stringifyArrayField(_field: ArrayField, fieldExpr: Expr) { // TODO: implement - return expr + return fieldExpr } diff --git a/table/field/types/boolean.ts b/table/field/types/boolean.ts index 054cae60..0183da39 100644 --- a/table/field/types/boolean.ts +++ b/table/field/types/boolean.ts @@ -1,25 +1,23 @@ import type { BooleanField } from "@dpkit/core" import { DataType } from "nodejs-polars" -import { col, lit, when } from "nodejs-polars" +import { lit, when } from "nodejs-polars" import type { Expr } from "nodejs-polars" const DEFAULT_TRUE_VALUES = ["true", "True", "TRUE", "1"] const DEFAULT_FALSE_VALUES = ["false", "False", "FALSE", "0"] -export function parseBooleanField(field: BooleanField, expr?: Expr) { - expr = expr ?? col(field.name) - +export function parseBooleanField(field: BooleanField, fieldExpr: Expr) { const trueValues = field.trueValues ?? DEFAULT_TRUE_VALUES const falseValues = field.falseValues ?? DEFAULT_FALSE_VALUES - for (const value of trueValues) expr = expr.replace(value, "1") - for (const value of falseValues) expr = expr.replace(value, "0") + for (const value of trueValues) fieldExpr = fieldExpr.replace(value, "1") + for (const value of falseValues) fieldExpr = fieldExpr.replace(value, "0") - expr = expr.cast(DataType.Int8) + fieldExpr = fieldExpr.cast(DataType.Int8) - return when(expr.eq(1)) + return when(fieldExpr.eq(1)) .then(lit(true)) - .when(expr.eq(0)) + .when(fieldExpr.eq(0)) .then(lit(false)) .otherwise(lit(null)) .alias(field.name) @@ -28,13 +26,11 @@ export function parseBooleanField(field: BooleanField, expr?: Expr) { const DEFAULT_TRUE_VALUE = "true" const DEFAULT_FALSE_VALUE = "false" -export function stringifyBooleanField(field: BooleanField, expr?: Expr) { - expr = expr ?? col(field.name) - +export function stringifyBooleanField(field: BooleanField, fieldExpr: Expr) { const trueValue = field.trueValues?.[0] ?? DEFAULT_TRUE_VALUE const falseValue = field.falseValues?.[0] ?? DEFAULT_FALSE_VALUE - return when(expr.eq(lit(true))) + return when(fieldExpr.eq(lit(true))) .then(lit(trueValue)) .otherwise(lit(falseValue)) .alias(field.name) diff --git a/table/field/types/date.ts b/table/field/types/date.ts index 3199a413..bedf0dfd 100644 --- a/table/field/types/date.ts +++ b/table/field/types/date.ts @@ -1,25 +1,20 @@ import type { DateField } from "@dpkit/core" import { DataType } from "nodejs-polars" -import { col } from "nodejs-polars" import type { Expr } from "nodejs-polars" const DEFAULT_FORMAT = "%Y-%m-%d" -export function parseDateField(field: DateField, expr?: Expr) { - expr = expr ?? col(field.name) - +export function parseDateField(field: DateField, fieldExpr: Expr) { let format = DEFAULT_FORMAT if (field.format && field.format !== "default" && field.format !== "any") { format = field.format } - return expr.str.strptime(DataType.Date, format) + return fieldExpr.str.strptime(DataType.Date, format) } -export function stringifyDateField(field: DateField, expr?: Expr) { - expr = expr ?? col(field.name) - +export function stringifyDateField(field: DateField, fieldExpr: Expr) { const format = field.format ?? DEFAULT_FORMAT - return expr.date.strftime(format) + return fieldExpr.date.strftime(format) } diff --git a/table/field/types/datetime.ts b/table/field/types/datetime.ts index c12ab8a4..bee920ac 100644 --- a/table/field/types/datetime.ts +++ b/table/field/types/datetime.ts @@ -1,26 +1,21 @@ import type { DatetimeField } from "@dpkit/core" import { DataType } from "nodejs-polars" -import { col } from "nodejs-polars" import type { Expr } from "nodejs-polars" const DEFAULT_FORMAT = "%Y-%m-%dT%H:%M:%S" // TODO: Add support for timezone handling -export function parseDatetimeField(field: DatetimeField, expr?: Expr) { - expr = expr ?? col(field.name) - +export function parseDatetimeField(field: DatetimeField, fieldExpr: Expr) { let format = DEFAULT_FORMAT if (field.format && field.format !== "default" && field.format !== "any") { format = field.format } - return expr.str.strptime(DataType.Datetime, format) + return fieldExpr.str.strptime(DataType.Datetime, format) } -export function stringifyDatetimeField(field: DatetimeField, expr?: Expr) { - expr = expr ?? col(field.name) - +export function stringifyDatetimeField(field: DatetimeField, fieldExpr: Expr) { const format = field.format ?? DEFAULT_FORMAT - return expr.date.strftime(format) + return fieldExpr.date.strftime(format) } diff --git a/table/field/types/duration.ts b/table/field/types/duration.ts index cca2d21e..8b16d4e5 100644 --- a/table/field/types/duration.ts +++ b/table/field/types/duration.ts @@ -1,17 +1,12 @@ import type { DurationField } from "@dpkit/core" -import { col } from "nodejs-polars" import type { Expr } from "nodejs-polars" // TODO: raise an issue on nodejs-polars repo as this is not supported yet // So we do nothing on this column type for now -export function parseDurationField(field: DurationField, expr?: Expr) { - expr = expr ?? col(field.name) - - return expr +export function parseDurationField(_field: DurationField, fieldExpr: Expr) { + return fieldExpr } -export function stringifyDurationField(field: DurationField, expr?: Expr) { - expr = expr ?? col(field.name) - - return expr +export function stringifyDurationField(_field: DurationField, fieldExpr: Expr) { + return fieldExpr } diff --git a/table/field/types/geojson.ts b/table/field/types/geojson.ts index d4986074..da7a404f 100644 --- a/table/field/types/geojson.ts +++ b/table/field/types/geojson.ts @@ -1,18 +1,16 @@ import type { GeojsonField } from "@dpkit/core" -import { col, lit, when } from "nodejs-polars" +import { lit, when } from "nodejs-polars" import type { Expr } from "nodejs-polars" // TODO: Is there a better way to do this? // Polars does not support really support free-form JSON // So we just make a basic check and return as it is -export function parseGeojsonField(field: GeojsonField, expr?: Expr) { - expr = expr ?? col(field.name) - - return when(expr.str.contains("^\\{")).then(expr).otherwise(lit(null)) +export function parseGeojsonField(_field: GeojsonField, fieldExpr: Expr) { + return when(fieldExpr.str.contains("^\\{")) + .then(fieldExpr) + .otherwise(lit(null)) } -export function stringifyGeojsonField(field: GeojsonField, expr?: Expr) { - expr = expr ?? col(field.name) - - return expr +export function stringifyGeojsonField(_field: GeojsonField, fieldExpr: Expr) { + return fieldExpr } diff --git a/table/field/types/geopoint.ts b/table/field/types/geopoint.ts index 601bd5d7..7ab063cd 100644 --- a/table/field/types/geopoint.ts +++ b/table/field/types/geopoint.ts @@ -1,5 +1,5 @@ import type { GeopointField } from "@dpkit/core" -import { DataType, col, concatList, concatString, lit } from "nodejs-polars" +import { DataType, concatList, concatString, lit } from "nodejs-polars" import type { Expr } from "nodejs-polars" // TODO: @@ -8,50 +8,46 @@ import type { Expr } from "nodejs-polars" // - Check the values are within -180..180 and -90..90 // - Return null instead of list if any of the values are out of range -export function parseGeopointField(field: GeopointField, expr?: Expr) { - expr = expr ?? col(field.name) - +export function parseGeopointField(field: GeopointField, fieldExpr: Expr) { // Default format is "lon,lat" string const format = field.format ?? "default" if (format === "default") { - expr = expr.str.split(",").cast(DataType.List(DataType.Float64)) + fieldExpr = fieldExpr.str.split(",").cast(DataType.List(DataType.Float64)) } if (format === "array") { - expr = expr.str + fieldExpr = fieldExpr.str .replaceAll("[\\[\\]\\s]", "") .str.split(",") .cast(DataType.List(DataType.Float64)) } if (format === "object") { - expr = concatList([ - expr.str.jsonPathMatch("$.lon").cast(DataType.Float64), - expr.str.jsonPathMatch("$.lat").cast(DataType.Float64), + fieldExpr = concatList([ + fieldExpr.str.jsonPathMatch("$.lon").cast(DataType.Float64), + fieldExpr.str.jsonPathMatch("$.lat").cast(DataType.Float64), ]).alias(field.name) } - return expr + return fieldExpr } -export function stringifyGeopointField(field: GeopointField, expr?: Expr) { - expr = expr ?? col(field.name) - +export function stringifyGeopointField(field: GeopointField, fieldExpr: Expr) { // Default format is "lon,lat" string const format = field.format ?? "default" if (format === "default") { - return expr.cast(DataType.List(DataType.String)).lst.join(",") + return fieldExpr.cast(DataType.List(DataType.String)).lst.join(",") } if (format === "array") { return concatString( [ lit("["), - expr.lst.get(0).cast(DataType.String), + fieldExpr.lst.get(0).cast(DataType.String), lit(","), - expr.lst.get(1).cast(DataType.String), + fieldExpr.lst.get(1).cast(DataType.String), lit("]"), ], "", @@ -62,14 +58,14 @@ export function stringifyGeopointField(field: GeopointField, expr?: Expr) { return concatString( [ lit('{"lon":'), - expr.lst.get(0).cast(DataType.String), + fieldExpr.lst.get(0).cast(DataType.String), lit(',"lat":'), - expr.lst.get(1).cast(DataType.String), + fieldExpr.lst.get(1).cast(DataType.String), lit("}"), ], "", ).alias(field.name) as Expr } - return expr + return fieldExpr } diff --git a/table/field/types/integer.ts b/table/field/types/integer.ts index 0765a912..0a1d04c3 100644 --- a/table/field/types/integer.ts +++ b/table/field/types/integer.ts @@ -1,12 +1,10 @@ import type { IntegerField } from "@dpkit/core" -import { DataType, col, lit, when } from "nodejs-polars" +import { DataType, lit, when } from "nodejs-polars" import type { Expr } from "nodejs-polars" // TODO: support categories // TODO: support categoriesOrder -export function parseIntegerField(field: IntegerField, expr?: Expr) { - expr = expr ?? col(field.name) - +export function parseIntegerField(field: IntegerField, fieldExpr: Expr) { const groupChar = field.groupChar const bareNumber = field.bareNumber const flattenCategories = field.categories?.map(it => @@ -16,36 +14,34 @@ export function parseIntegerField(field: IntegerField, expr?: Expr) { // Handle non-bare numbers (with currency symbols, percent signs, etc.) if (bareNumber === false) { // Preserve the minus sign when removing leading characters - expr = expr.str.replaceAll("^[^\\d\\-]+", "") - expr = expr.str.replaceAll("[^\\d\\-]+$", "") + fieldExpr = fieldExpr.str.replaceAll("^[^\\d\\-]+", "") + fieldExpr = fieldExpr.str.replaceAll("[^\\d\\-]+$", "") } // Handle group character (thousands separator) if (groupChar) { // Escape special characters for regex const escapedGroupChar = groupChar.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") - expr = expr.str.replaceAll(escapedGroupChar, "") + fieldExpr = fieldExpr.str.replaceAll(escapedGroupChar, "") } // Cast to int64 (will handle values up to 2^63-1) - expr = expr.cast(DataType.Int64) + fieldExpr = fieldExpr.cast(DataType.Int64) // Currently, only string categories are supported if (flattenCategories) { - return when(expr.isIn(flattenCategories)) - .then(expr) + return when(fieldExpr.isIn(flattenCategories)) + .then(fieldExpr) .otherwise(lit(null)) .alias(field.name) } - return expr + return fieldExpr } -export function stringifyIntegerField(field: IntegerField, expr?: Expr) { - expr = expr ?? col(field.name) - +export function stringifyIntegerField(_field: IntegerField, fieldExpr: Expr) { // Convert to string - expr = expr.cast(DataType.String) + fieldExpr = fieldExpr.cast(DataType.String) //const groupChar = field.groupChar //const bareNumber = field.bareNumber @@ -53,5 +49,5 @@ export function stringifyIntegerField(field: IntegerField, expr?: Expr) { // TODO: Add group character formatting (thousands separator) when needed // TODO: Add non-bare number formatting (currency symbols, etc.) when needed - return expr + return fieldExpr } diff --git a/table/field/types/list.ts b/table/field/types/list.ts index 09cad5c3..926954f3 100644 --- a/table/field/types/list.ts +++ b/table/field/types/list.ts @@ -1,13 +1,11 @@ import type { ListField } from "@dpkit/core" -import { DataType, col } from "nodejs-polars" +import { DataType } from "nodejs-polars" import type { Expr } from "nodejs-polars" // TODO: // Add more validation: // - Return null instead of list if all array values are nulls? -export function parseListField(field: ListField, expr?: Expr) { - expr = expr ?? col(field.name) - +export function parseListField(field: ListField, fieldExpr: Expr) { const delimiter = field.delimiter ?? "," const itemType = field.itemType @@ -19,17 +17,15 @@ export function parseListField(field: ListField, expr?: Expr) { if (itemType === "date") dtype = DataType.Date if (itemType === "time") dtype = DataType.Time - expr = expr.str.split(delimiter).cast(DataType.List(dtype)) + fieldExpr = fieldExpr.str.split(delimiter).cast(DataType.List(dtype)) - return expr + return fieldExpr } -export function stringifyListField(field: ListField, expr?: Expr) { - expr = expr ?? col(field.name) - +export function stringifyListField(field: ListField, fieldExpr: Expr) { const delimiter = field.delimiter ?? "," - return expr + return fieldExpr .cast(DataType.List(DataType.String)) .lst.join({ separator: delimiter, ignoreNulls: true }) } diff --git a/table/field/types/number.ts b/table/field/types/number.ts index c5791117..de55a395 100644 --- a/table/field/types/number.ts +++ b/table/field/types/number.ts @@ -1,11 +1,8 @@ import type { NumberField } from "@dpkit/core" import { DataType } from "nodejs-polars" -import { col } from "nodejs-polars" import type { Expr } from "nodejs-polars" -export function parseNumberField(field: NumberField, expr?: Expr) { - expr = expr ?? col(field.name) - +export function parseNumberField(field: NumberField, fieldExpr: Expr) { // Extract the decimal and group characters const decimalChar = field.decimalChar ?? "." const groupChar = field.groupChar ?? "" @@ -16,43 +13,44 @@ export function parseNumberField(field: NumberField, expr?: Expr) { // Remove leading non-digit characters (except minus sign and allowed decimal points) const allowedDecimalChars = decimalChar === "." ? "\\." : `\\.${decimalChar}` - expr = expr.str.replaceAll(`^[^\\d\\-${allowedDecimalChars}]+`, "") + fieldExpr = fieldExpr.str.replaceAll( + `^[^\\d\\-${allowedDecimalChars}]+`, + "", + ) // Remove trailing non-digit characters - expr = expr.str.replaceAll(`[^\\d${allowedDecimalChars}]+$`, "") + fieldExpr = fieldExpr.str.replaceAll(`[^\\d${allowedDecimalChars}]+$`, "") } // Special case handling for European number format where "." is group and "," is decimal if (groupChar === "." && decimalChar === ",") { // First temporarily replace the decimal comma with a placeholder - expr = expr.str.replaceAll(",", "###DECIMAL###") + fieldExpr = fieldExpr.str.replaceAll(",", "###DECIMAL###") // Remove the group dots - expr = expr.str.replaceAll("\\.", "") + fieldExpr = fieldExpr.str.replaceAll("\\.", "") // Replace the placeholder with an actual decimal point - expr = expr.str.replaceAll("###DECIMAL###", ".") + fieldExpr = fieldExpr.str.replaceAll("###DECIMAL###", ".") } else { // Standard case: first remove group characters if (groupChar) { // Escape special characters for regex const escapedGroupChar = groupChar.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") - expr = expr.str.replaceAll(escapedGroupChar, "") + fieldExpr = fieldExpr.str.replaceAll(escapedGroupChar, "") } // Then handle decimal character if (decimalChar && decimalChar !== ".") { - expr = expr.str.replaceAll(decimalChar, ".") + fieldExpr = fieldExpr.str.replaceAll(decimalChar, ".") } } // Cast to float64 - expr = expr.cast(DataType.Float64) - return expr + fieldExpr = fieldExpr.cast(DataType.Float64) + return fieldExpr } -export function stringifyNumberField(field: NumberField, expr?: Expr) { - expr = expr ?? col(field.name) - +export function stringifyNumberField(_field: NumberField, fieldExpr: Expr) { // Convert to string - expr = expr.cast(DataType.String) + fieldExpr = fieldExpr.cast(DataType.String) //const decimalChar = field.decimalChar ?? "." //const groupChar = field.groupChar ?? "" @@ -61,5 +59,5 @@ export function stringifyNumberField(field: NumberField, expr?: Expr) { // TODO: Add group character formatting (thousands separator) when needed // TODO: Add non-bare number formatting (currency symbols, etc.) when needed - return expr + return fieldExpr } diff --git a/table/field/types/object.ts b/table/field/types/object.ts index 164d7f94..02e0c8c5 100644 --- a/table/field/types/object.ts +++ b/table/field/types/object.ts @@ -1,18 +1,16 @@ import type { ObjectField } from "@dpkit/core" -import { col, lit, when } from "nodejs-polars" +import { lit, when } from "nodejs-polars" import type { Expr } from "nodejs-polars" // TODO: Is there a better way to do this? // Polars does not support really support free-form JSON // So we just make a basic check and return as it is -export function parseObjectField(field: ObjectField, expr?: Expr) { - expr = expr ?? col(field.name) - - return when(expr.str.contains("^\\{")).then(expr).otherwise(lit(null)) +export function parseObjectField(_field: ObjectField, fieldExpr: Expr) { + return when(fieldExpr.str.contains("^\\{")) + .then(fieldExpr) + .otherwise(lit(null)) } -export function stringifyObjectField(field: ObjectField, expr?: Expr) { - expr = expr ?? col(field.name) - - return expr +export function stringifyObjectField(_field: ObjectField, fieldExpr: Expr) { + return fieldExpr } diff --git a/table/field/types/string.ts b/table/field/types/string.ts index 69bc6871..996303aa 100644 --- a/table/field/types/string.ts +++ b/table/field/types/string.ts @@ -1,5 +1,5 @@ import type { StringField } from "@dpkit/core" -import { DataType, col, lit, when } from "nodejs-polars" +import { DataType, lit, when } from "nodejs-polars" import type { Expr } from "nodejs-polars" const FORMAT_REGEX = { @@ -11,34 +11,30 @@ const FORMAT_REGEX = { } as const // TODO: support categoriesOrder? -export function parseStringField(field: StringField, expr?: Expr) { - expr = expr ?? col(field.name) - +export function parseStringField(field: StringField, fieldExpr: Expr) { const format = field.format const flattenCategories = field.categories?.map(it => typeof it === "string" ? it : it.value, ) if (flattenCategories) { - return when(expr.isIn(flattenCategories)) - .then(expr.cast(DataType.Categorical)) + return when(fieldExpr.isIn(flattenCategories)) + .then(fieldExpr.cast(DataType.Categorical)) .otherwise(lit(null)) .alias(field.name) } if (format) { const regex = FORMAT_REGEX[format] - return when(expr.str.contains(regex)) - .then(expr) + return when(fieldExpr.str.contains(regex)) + .then(fieldExpr) .otherwise(lit(null)) .alias(field.name) } - return expr + return fieldExpr } -export function stringifyStringField(field: StringField, expr?: Expr) { - expr = expr ?? col(field.name) - - return expr +export function stringifyStringField(_field: StringField, fieldExpr: Expr) { + return fieldExpr } diff --git a/table/field/types/time.ts b/table/field/types/time.ts index 13ebe1c8..578d2507 100644 --- a/table/field/types/time.ts +++ b/table/field/types/time.ts @@ -1,28 +1,24 @@ import type { TimeField } from "@dpkit/core" import { DataType } from "nodejs-polars" -import { col, concatString, lit } from "nodejs-polars" +import { concatString, lit } from "nodejs-polars" import type { Expr } from "nodejs-polars" const DEFAULT_FORMAT = "%H:%M:%S" -export function parseTimeField(field: TimeField, expr?: Expr) { - expr = expr ?? col(field.name) - +export function parseTimeField(field: TimeField, fieldExpr: Expr) { let format = DEFAULT_FORMAT if (field.format && field.format !== "default" && field.format !== "any") { format = field.format } - return concatString([lit("1970-01-01T"), expr], "") + return concatString([lit("1970-01-01T"), fieldExpr], "") .str.strptime(DataType.Datetime, `%Y-%m-%dT${format}`) .cast(DataType.Time) .alias(field.name) } -export function stringifyTimeField(field: TimeField, expr?: Expr) { - expr = expr ?? col(field.name) - +export function stringifyTimeField(field: TimeField, fieldExpr: Expr) { const format = field.format ?? DEFAULT_FORMAT - return expr.date.strftime(format) + return fieldExpr.date.strftime(format) } diff --git a/table/field/types/year.ts b/table/field/types/year.ts index 380e0631..48c85750 100644 --- a/table/field/types/year.ts +++ b/table/field/types/year.ts @@ -1,23 +1,18 @@ import type { YearField } from "@dpkit/core" import { DataType, lit, when } from "nodejs-polars" -import { col } from "nodejs-polars" import type { Expr } from "nodejs-polars" -export function parseYearField(field: YearField, expr?: Expr) { - expr = expr ?? col(field.name) - - expr = when(expr.str.lengths().eq(4)) - .then(expr) +export function parseYearField(_field: YearField, fieldExpr: Expr) { + fieldExpr = when(fieldExpr.str.lengths().eq(4)) + .then(fieldExpr) .otherwise(lit(null)) .cast(DataType.Int16) - return when(expr.gtEq(0).and(expr.ltEq(9999))) - .then(expr) + return when(fieldExpr.gtEq(0).and(fieldExpr.ltEq(9999))) + .then(fieldExpr) .otherwise(lit(null)) } -export function stringifyYearField(field: YearField, expr?: Expr) { - expr = expr ?? col(field.name) - - return expr.cast(DataType.String).str.zFill(4) +export function stringifyYearField(_field: YearField, fieldExpr: Expr) { + return fieldExpr.cast(DataType.String).str.zFill(4) } diff --git a/table/field/types/yearmonth.ts b/table/field/types/yearmonth.ts index b284b87c..4146a76a 100644 --- a/table/field/types/yearmonth.ts +++ b/table/field/types/yearmonth.ts @@ -1,5 +1,5 @@ import type { YearmonthField } from "@dpkit/core" -import { DataType, col, concatString } from "nodejs-polars" +import { DataType, concatString } from "nodejs-polars" import type { Expr } from "nodejs-polars" // TODO: @@ -9,23 +9,30 @@ import type { Expr } from "nodejs-polars" // - Return null instead of list if any of the values are out of range // - Rebase on Struct when lst.toStruct() is available? -export function parseYearmonthField(field: YearmonthField, expr?: Expr) { - expr = expr ?? col(field.name) +export function parseYearmonthField(_field: YearmonthField, fieldExpr: Expr) { + fieldExpr = fieldExpr.str.split("-").cast(DataType.List(DataType.Int16)) - expr = expr.str.split("-").cast(DataType.List(DataType.Int16)) - - return expr + return fieldExpr } -export function stringifyYearmonthField(field: YearmonthField, expr?: Expr) { - expr = expr ?? col(field.name) - +export function stringifyYearmonthField( + field: YearmonthField, + fieldExpr: Expr, +) { // TODO: remove int casting when resolved: // https://github.com/pola-rs/nodejs-polars/issues/362 return concatString( [ - expr.lst.get(0).cast(DataType.Int16).cast(DataType.String).str.zFill(4), - expr.lst.get(1).cast(DataType.Int16).cast(DataType.String).str.zFill(2), + fieldExpr.lst + .get(0) + .cast(DataType.Int16) + .cast(DataType.String) + .str.zFill(4), + fieldExpr.lst + .get(1) + .cast(DataType.Int16) + .cast(DataType.String) + .str.zFill(2), ], "-", ).alias(field.name) as Expr diff --git a/table/field/validate.ts b/table/field/validate.ts index 86f46887..01544234 100644 --- a/table/field/validate.ts +++ b/table/field/validate.ts @@ -122,7 +122,8 @@ async function validateCells( checkCellMaxLength, checkCellUnique, ]) { - const check = checkCell(mapping.target, col("target"), col("source")) + const cellMapping = { source: col("source"), target: col("target") } + const check = checkCell(mapping.target, cellMapping) if (check) { fieldCheckTable = fieldCheckTable.withColumn( diff --git a/table/table/normalize.ts b/table/table/normalize.ts index 8d23b81e..14b22ae4 100644 --- a/table/table/normalize.ts +++ b/table/table/normalize.ts @@ -13,19 +13,20 @@ export async function normalizeTable(table: Table, schema: Schema) { const head = await table.head(HEAD_ROWS).collect() const polarsSchema = getPolarsSchema(head.schema) - const mapping = { source: polarsSchema, target: schema } - return table.select(...Object.values(normalizeFields(mapping))) + const schemaMapping = { source: polarsSchema, target: schema } + return table.select(...Object.values(normalizeFields(schemaMapping))) } -export function normalizeFields(mapping: SchemaMapping) { +export function normalizeFields(schemaMapping: SchemaMapping) { const exprs: Record = {} - for (const [index, field] of mapping.target.fields.entries()) { - const fieldMapping = matchSchemaField(mapping, field, index) + for (const [index, field] of schemaMapping.target.fields.entries()) { + const fieldMapping = matchSchemaField(schemaMapping, field, index) let expr = lit(null).alias(field.name) if (fieldMapping) { - const missingValues = field.missingValues ?? mapping.target.missingValues + const missingValues = + field.missingValues ?? schemaMapping.target.missingValues const mergedField = { ...field, missingValues } const column = { source: fieldMapping.source, target: mergedField } diff --git a/table/table/validate.ts b/table/table/validate.ts index 226fd65d..37231c60 100644 --- a/table/table/validate.ts +++ b/table/table/validate.ts @@ -23,12 +23,14 @@ export async function validateTable( if (schema) { const sample = await table.head(sampleRows).collect() const polarsSchema = getPolarsSchema(sample.schema) - const mapping = { source: polarsSchema, target: schema } + const schemaMapping = { source: polarsSchema, target: schema } - const matchErrors = validateFieldsMatch(mapping) + const matchErrors = validateFieldsMatch(schemaMapping) errors.push(...matchErrors) - const fieldErrors = await validateFields(mapping, table, { maxErrors }) + const fieldErrors = await validateFields(schemaMapping, table, { + maxErrors, + }) errors.push(...fieldErrors) } @@ -38,12 +40,12 @@ export async function validateTable( } } -function validateFieldsMatch(mapping: SchemaMapping) { +function validateFieldsMatch(schemaMapping: SchemaMapping) { const errors: TableError[] = [] - const fieldsMatch = mapping.target.fieldsMatch ?? "exact" + const fieldsMatch = schemaMapping.target.fieldsMatch ?? "exact" - const fields = mapping.target.fields - const polarsFields = mapping.source.fields + const fields = schemaMapping.target.fields + const polarsFields = schemaMapping.source.fields const names = fields.map(field => field.name) const polarsNames = polarsFields.map(field => field.name) @@ -122,7 +124,7 @@ function validateFieldsMatch(mapping: SchemaMapping) { } async function validateFields( - mapping: SchemaMapping, + schemaMapping: SchemaMapping, table: Table, options: { maxErrors: number @@ -130,13 +132,16 @@ async function validateFields( ) { const { maxErrors } = options const errors: TableError[] = [] - const fields = mapping.target.fields + const fields = schemaMapping.target.fields const concurrency = os.cpus().length - 1 const abortController = new AbortController() - const maxFieldErrors = Math.ceil(maxErrors / mapping.target.fields.length) + + const maxFieldErrors = Math.ceil( + maxErrors / schemaMapping.target.fields.length, + ) const collectFieldErrors = async (index: number, field: Field) => { - const fieldMapping = matchSchemaField(mapping, field, index) + const fieldMapping = matchSchemaField(schemaMapping, field, index) if (!fieldMapping) return const report = await validateField(fieldMapping, table, { From 0b7cf27f39260a997df8029eb1da313803e04481 Mon Sep 17 00:00:00 2001 From: roll Date: Wed, 29 Oct 2025 11:07:37 +0000 Subject: [PATCH 25/42] Fixed maximum tests --- table/field/checks/maximum.spec.ts | 3 +++ table/field/normalize.ts | 10 ++++++++-- table/field/validate.ts | 3 +-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/table/field/checks/maximum.spec.ts b/table/field/checks/maximum.spec.ts index e68e47ca..55e9cffa 100644 --- a/table/field/checks/maximum.spec.ts +++ b/table/field/checks/maximum.spec.ts @@ -43,6 +43,7 @@ describe("validateTable (cell/maximum)", () => { expect(errors).toContainEqual({ type: "cell/maximum", fieldName: "temperature", + maximum: "40", rowNumber: 4, cell: "50.5", }) @@ -70,12 +71,14 @@ describe("validateTable (cell/maximum)", () => { expect(errors).toContainEqual({ type: "cell/exclusiveMaximum", fieldName: "temperature", + maximum: "40", rowNumber: 3, cell: "40", }) expect(errors).toContainEqual({ type: "cell/exclusiveMaximum", fieldName: "temperature", + maximum: "40", rowNumber: 4, cell: "50.5", }) diff --git a/table/field/normalize.ts b/table/field/normalize.ts index 63117ec1..2324e092 100644 --- a/table/field/normalize.ts +++ b/table/field/normalize.ts @@ -4,12 +4,18 @@ import type { FieldMapping } from "./Mapping.ts" import { parseField } from "./parse.ts" import { substituteField } from "./substitute.ts" -export function normalizeField(mapping: FieldMapping) { +export function normalizeField( + mapping: FieldMapping, + options?: { keepType?: boolean }, +) { let fieldExpr = col(mapping.source.name) if (mapping.source.type.equals(DataType.String)) { fieldExpr = substituteField(mapping.target, fieldExpr) - fieldExpr = parseField(mapping.target, fieldExpr) + + if (!options?.keepType) { + fieldExpr = parseField(mapping.target, fieldExpr) + } } return fieldExpr.alias(mapping.target.name) diff --git a/table/field/validate.ts b/table/field/validate.ts index 01544234..a24354e6 100644 --- a/table/field/validate.ts +++ b/table/field/validate.ts @@ -13,7 +13,6 @@ import { checkCellRequired } from "./checks/required.ts" import { checkCellType } from "./checks/type.ts" import { checkCellUnique } from "./checks/unique.ts" import { normalizeField } from "./normalize.ts" -import { substituteField } from "./substitute.ts" export async function validateField( mapping: FieldMapping, @@ -105,7 +104,7 @@ async function validateCells( .select( col("row_nr").add(1).alias("number"), normalizeField(mapping).alias("target"), - substituteField(mapping.target, col(mapping.source.name)).alias("source"), + normalizeField(mapping, { keepType: true }).alias("source"), lit(null).alias("error"), ) From ff11f37a90922baf45f0acd9cb40d201b84c0792 Mon Sep 17 00:00:00 2001 From: roll Date: Wed, 29 Oct 2025 11:23:19 +0000 Subject: [PATCH 26/42] Fixed tests --- lib/package/validate.spec.ts | 1 + table/error/Cell.ts | 4 +- table/field/checks/maxLength.spec.ts | 1 + table/field/checks/maxLength.ts | 2 +- table/field/checks/minLength.spec.ts | 2 + table/field/checks/minLength.ts | 2 +- table/field/checks/minimum.spec.ts | 3 + table/field/checks/pattern.spec.ts | 10 +- table/field/checks/type.spec.ts | 46 ++++++- table/field/validate.spec.ts | 16 +++ table/table/checks/unique.spec.ts | 194 +++++++++++++++++++++++++++ table/table/checks/unique.ts | 31 +++++ table/table/normalize.ts | 13 +- table/table/validate.ts | 26 ++-- 14 files changed, 319 insertions(+), 32 deletions(-) create mode 100644 table/table/checks/unique.spec.ts create mode 100644 table/table/checks/unique.ts diff --git a/lib/package/validate.spec.ts b/lib/package/validate.spec.ts index ef097e63..ade13e8a 100644 --- a/lib/package/validate.spec.ts +++ b/lib/package/validate.spec.ts @@ -182,6 +182,7 @@ describe("validatePackage", () => { rowNumber: 3, type: "cell/type", fieldName: "longitude", + fieldType: "number", cell: "bad", resource: "deployments", }, diff --git a/table/error/Cell.ts b/table/error/Cell.ts index 09bc7412..d5ef6b56 100644 --- a/table/error/Cell.ts +++ b/table/error/Cell.ts @@ -52,12 +52,12 @@ export interface CellExclusiveMaximumError extends BaseCellError { export interface CellMinLengthError extends BaseCellError { type: "cell/minLength" - minLength: number + minLength: string } export interface CellMaxLengthError extends BaseCellError { type: "cell/maxLength" - maxLength: number + maxLength: string } export interface CellPatternError extends BaseCellError { diff --git a/table/field/checks/maxLength.spec.ts b/table/field/checks/maxLength.spec.ts index b8dd585f..95cc6796 100644 --- a/table/field/checks/maxLength.spec.ts +++ b/table/field/checks/maxLength.spec.ts @@ -43,6 +43,7 @@ describe("validateTable (cell/maxLength)", () => { expect(errors).toContainEqual({ type: "cell/maxLength", fieldName: "username", + maxLength: "8", rowNumber: 3, cell: "christopher", }) diff --git a/table/field/checks/maxLength.ts b/table/field/checks/maxLength.ts index 84306b21..d7ad2716 100644 --- a/table/field/checks/maxLength.ts +++ b/table/field/checks/maxLength.ts @@ -13,7 +13,7 @@ export function checkCellMaxLength(field: Field, mapping: CellMapping) { const errorTemplate: CellMaxLengthError = { type: "cell/maxLength", fieldName: field.name, - maxLength: maxLength, + maxLength: String(maxLength), rowNumber: 0, cell: "", } diff --git a/table/field/checks/minLength.spec.ts b/table/field/checks/minLength.spec.ts index 7c34c7bf..97c25369 100644 --- a/table/field/checks/minLength.spec.ts +++ b/table/field/checks/minLength.spec.ts @@ -43,12 +43,14 @@ describe("validateTable (cell/minLength)", () => { expect(errors).toContainEqual({ type: "cell/minLength", fieldName: "username", + minLength: "3", rowNumber: 2, cell: "a", }) expect(errors).toContainEqual({ type: "cell/minLength", fieldName: "username", + minLength: "3", rowNumber: 4, cell: "ab", }) diff --git a/table/field/checks/minLength.ts b/table/field/checks/minLength.ts index b9bfce53..733717f0 100644 --- a/table/field/checks/minLength.ts +++ b/table/field/checks/minLength.ts @@ -13,7 +13,7 @@ export function checkCellMinLength(field: Field, mapping: CellMapping) { const errorTemplate: CellMinLengthError = { type: "cell/minLength", fieldName: field.name, - minLength: minLength, + minLength: String(minLength), rowNumber: 0, cell: "", } diff --git a/table/field/checks/minimum.spec.ts b/table/field/checks/minimum.spec.ts index 1e44e6c1..487cbd15 100644 --- a/table/field/checks/minimum.spec.ts +++ b/table/field/checks/minimum.spec.ts @@ -43,6 +43,7 @@ describe("validateTable (cell/minimum)", () => { expect(errors).toContainEqual({ type: "cell/minimum", fieldName: "temperature", + minimum: "10", rowNumber: 4, cell: "3.5", }) @@ -70,12 +71,14 @@ describe("validateTable (cell/minimum)", () => { expect(errors).toContainEqual({ type: "cell/exclusiveMinimum", fieldName: "temperature", + minimum: "10", rowNumber: 3, cell: "10", }) expect(errors).toContainEqual({ type: "cell/exclusiveMinimum", fieldName: "temperature", + minimum: "10", rowNumber: 4, cell: "5.5", }) diff --git a/table/field/checks/pattern.spec.ts b/table/field/checks/pattern.spec.ts index 39eb1ba6..afc8a255 100644 --- a/table/field/checks/pattern.spec.ts +++ b/table/field/checks/pattern.spec.ts @@ -26,6 +26,8 @@ describe("validateTable (cell/pattern)", () => { }) it("should report an error for strings that don't match the pattern", async () => { + const pattern = "^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$" + const table = DataFrame({ email: ["john@example.com", "alice@domain", "test.io", "valid@email.com"], }).lazy() @@ -36,7 +38,7 @@ describe("validateTable (cell/pattern)", () => { name: "email", type: "string", constraints: { - pattern: "^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$", + pattern, }, }, ], @@ -47,13 +49,15 @@ describe("validateTable (cell/pattern)", () => { expect(errors).toContainEqual({ type: "cell/pattern", fieldName: "email", - rowNumber: 2, // Second row (alice@domain) + pattern, + rowNumber: 2, cell: "alice@domain", }) expect(errors).toContainEqual({ type: "cell/pattern", fieldName: "email", - rowNumber: 3, // Third row (test.io) + pattern, + rowNumber: 3, cell: "test.io", }) }) diff --git a/table/field/checks/type.spec.ts b/table/field/checks/type.spec.ts index 641f5a89..3b0304e2 100644 --- a/table/field/checks/type.spec.ts +++ b/table/field/checks/type.spec.ts @@ -20,12 +20,14 @@ describe("validateTable", () => { type: "cell/type", cell: "bad", fieldName: "id", + fieldType: "integer", rowNumber: 2, }) expect(errors).toContainEqual({ type: "cell/type", cell: "4x", fieldName: "id", + fieldType: "integer", rowNumber: 4, }) }) @@ -46,12 +48,14 @@ describe("validateTable", () => { type: "cell/type", cell: "twenty", fieldName: "price", + fieldType: "number", rowNumber: 2, }) expect(errors).toContainEqual({ type: "cell/type", cell: "$40", fieldName: "price", + fieldType: "number", rowNumber: 4, }) }) @@ -72,6 +76,7 @@ describe("validateTable", () => { type: "cell/type", cell: "yes", fieldName: "active", + fieldType: "boolean", rowNumber: 2, }) }) @@ -92,18 +97,21 @@ describe("validateTable", () => { type: "cell/type", cell: "Jan 15, 2023", fieldName: "created", + fieldType: "date", rowNumber: 2, }) expect(errors).toContainEqual({ type: "cell/type", cell: "20230115", fieldName: "created", + fieldType: "date", rowNumber: 3, }) expect(errors).toContainEqual({ type: "cell/type", cell: "not-a-date", fieldName: "created", + fieldType: "date", rowNumber: 4, }) }) @@ -124,22 +132,49 @@ describe("validateTable", () => { type: "cell/type", cell: "2:30pm", fieldName: "time", + fieldType: "time", rowNumber: 2, }) expect(errors).toContainEqual({ type: "cell/type", cell: "invalid", fieldName: "time", + fieldType: "time", rowNumber: 3, }) expect(errors).toContainEqual({ type: "cell/type", cell: "14h30", fieldName: "time", + fieldType: "time", rowNumber: 4, }) }) + it("should validate string to time conversion errors with custom format", async () => { + const table = DataFrame({ + time: ["14:30", "invalid"], + }).lazy() + + const schema: Schema = { + fields: [{ name: "time", type: "time", format: "%H:%M" }], + } + + const { errors } = await validateTable(table, { schema }) + + console.log(errors) + + expect(errors).toHaveLength(1) + expect(errors).toContainEqual({ + type: "cell/type", + cell: "invalid", + fieldName: "time", + fieldType: "time", + fieldFormat: "%H:%M", + rowNumber: 2, + }) + }) + it("should validate string to year conversion errors", async () => { const table = DataFrame({ year: ["2023", "23", "MMXXIII", "two-thousand-twenty-three"], @@ -156,18 +191,21 @@ describe("validateTable", () => { type: "cell/type", cell: "23", fieldName: "year", + fieldType: "year", rowNumber: 2, }) expect(errors).toContainEqual({ type: "cell/type", cell: "MMXXIII", fieldName: "year", + fieldType: "year", rowNumber: 3, }) expect(errors).toContainEqual({ type: "cell/type", cell: "two-thousand-twenty-three", fieldName: "year", + fieldType: "year", rowNumber: 4, }) }) @@ -183,7 +221,7 @@ describe("validateTable", () => { }).lazy() const schema: Schema = { - fields: [{ name: "timestamp", type: "datetime" }], + fields: [{ name: "datetime", type: "datetime" }], } const { errors } = await validateTable(table, { schema }) @@ -195,14 +233,16 @@ describe("validateTable", () => { expect(errors).toContainEqual({ type: "cell/type", cell: "January 15, 2023 2:30 PM", - fieldName: "timestamp", + fieldName: "datetime", + fieldType: "datetime", rowNumber: 2, }) expect(errors).toContainEqual({ type: "cell/type", cell: "not-a-datetime", - fieldName: "timestamp", + fieldName: "datetime", + fieldType: "datetime", rowNumber: 4, }) }) diff --git a/table/field/validate.spec.ts b/table/field/validate.spec.ts index 7ea3d0f3..4c9369d4 100644 --- a/table/field/validate.spec.ts +++ b/table/field/validate.spec.ts @@ -138,12 +138,14 @@ describe("validateField", () => { type: "cell/type", cell: "bad", fieldName: "id", + fieldType: "integer", rowNumber: 2, }) expect(errors).toContainEqual({ type: "cell/type", cell: "4x", fieldName: "id", + fieldType: "integer", rowNumber: 4, }) }) @@ -169,12 +171,14 @@ describe("validateField", () => { type: "cell/type", cell: "twenty", fieldName: "price", + fieldType: "number", rowNumber: 2, }) expect(errors).toContainEqual({ type: "cell/type", cell: "$40", fieldName: "price", + fieldType: "number", rowNumber: 4, }) }) @@ -200,6 +204,7 @@ describe("validateField", () => { type: "cell/type", cell: "yes", fieldName: "active", + fieldType: "boolean", rowNumber: 2, }) }) @@ -225,18 +230,21 @@ describe("validateField", () => { type: "cell/type", cell: "Jan 15, 2023", fieldName: "created", + fieldType: "date", rowNumber: 2, }) expect(errors).toContainEqual({ type: "cell/type", cell: "20230115", fieldName: "created", + fieldType: "date", rowNumber: 3, }) expect(errors).toContainEqual({ type: "cell/type", cell: "not-a-date", fieldName: "created", + fieldType: "date", rowNumber: 4, }) }) @@ -262,18 +270,21 @@ describe("validateField", () => { type: "cell/type", cell: "2:30pm", fieldName: "time", + fieldType: "time", rowNumber: 2, }) expect(errors).toContainEqual({ type: "cell/type", cell: "invalid", fieldName: "time", + fieldType: "time", rowNumber: 3, }) expect(errors).toContainEqual({ type: "cell/type", cell: "14h30", fieldName: "time", + fieldType: "time", rowNumber: 4, }) }) @@ -299,18 +310,21 @@ describe("validateField", () => { type: "cell/type", cell: "23", fieldName: "year", + fieldType: "year", rowNumber: 2, }) expect(errors).toContainEqual({ type: "cell/type", cell: "MMXXIII", fieldName: "year", + fieldType: "year", rowNumber: 3, }) expect(errors).toContainEqual({ type: "cell/type", cell: "two-thousand-twenty-three", fieldName: "year", + fieldType: "year", rowNumber: 4, }) }) @@ -344,6 +358,7 @@ describe("validateField", () => { type: "cell/type", cell: "January 15, 2023 2:30 PM", fieldName: "timestamp", + fieldType: "datetime", rowNumber: 2, }) @@ -351,6 +366,7 @@ describe("validateField", () => { type: "cell/type", cell: "not-a-datetime", fieldName: "timestamp", + fieldType: "datetime", rowNumber: 4, }) }) diff --git a/table/table/checks/unique.spec.ts b/table/table/checks/unique.spec.ts new file mode 100644 index 00000000..22f39af4 --- /dev/null +++ b/table/table/checks/unique.spec.ts @@ -0,0 +1,194 @@ +import type { Schema } from "@dpkit/core" +import { DataFrame } from "nodejs-polars" +import { describe, expect, it } from "vitest" +import { validateTable } from "../../table/index.ts" + +describe("validateTable (row/unique)", () => { + it("should not report errors when all rows are unique for primary key", async () => { + const table = DataFrame({ + id: [1, 2, 3, 4, 5], + name: ["Alice", "Bob", "Charlie", "David", "Eve"], + }).lazy() + + const schema: Schema = { + fields: [ + { name: "id", type: "number" }, + { name: "name", type: "string" }, + ], + primaryKey: ["id"], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(0) + }) + + it("should report errors for duplicate primary key rows", async () => { + const table = DataFrame({ + id: [1, 2, 3, 2, 5], + name: ["Alice", "Bob", "Charlie", "Bob2", "Eve"], + }).lazy() + + const schema: Schema = { + fields: [ + { name: "id", type: "number" }, + { name: "name", type: "string" }, + ], + primaryKey: ["id"], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors.filter(e => e.type === "row/unique")).toHaveLength(1) + expect(errors).toContainEqual({ + type: "row/unique", + rowNumber: 4, + fieldNames: ["id"], + }) + }) + + it("should not report errors when all rows are unique for unique key", async () => { + const table = DataFrame({ + id: [1, 2, 3, 4, 5], + email: [ + "a@test.com", + "b@test.com", + "c@test.com", + "d@test.com", + "e@test.com", + ], + }).lazy() + + const schema: Schema = { + fields: [ + { name: "id", type: "number" }, + { name: "email", type: "string" }, + ], + uniqueKeys: [["email"]], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors).toHaveLength(0) + }) + + it("should report errors for duplicate unique key rows", async () => { + const table = DataFrame({ + id: [1, 2, 3, 4, 5], + email: [ + "a@test.com", + "b@test.com", + "a@test.com", + "d@test.com", + "b@test.com", + ], + }).lazy() + + const schema: Schema = { + fields: [ + { name: "id", type: "number" }, + { name: "email", type: "string" }, + ], + uniqueKeys: [["email"]], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors.filter(e => e.type === "row/unique")).toHaveLength(2) + expect(errors).toContainEqual({ + type: "row/unique", + rowNumber: 3, + fieldNames: ["email"], + }) + expect(errors).toContainEqual({ + type: "row/unique", + rowNumber: 5, + fieldNames: ["email"], + }) + }) + + it("should handle composite unique keys", async () => { + const table = DataFrame({ + category: ["A", "A", "B", "A", "B"], + subcategory: ["X", "Y", "X", "X", "Y"], + value: [1, 2, 3, 4, 5], + }).lazy() + + const schema: Schema = { + fields: [ + { name: "category", type: "string" }, + { name: "subcategory", type: "string" }, + { name: "value", type: "number" }, + ], + uniqueKeys: [["category", "subcategory"]], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors.filter(e => e.type === "row/unique")).toHaveLength(1) + expect(errors).toContainEqual({ + type: "row/unique", + rowNumber: 4, + fieldNames: ["category", "subcategory"], + }) + }) + + it("should handle both primary key and unique keys", async () => { + const table = DataFrame({ + id: [1, 2, 3, 2, 5], + email: [ + "a@test.com", + "b@test.com", + "c@test.com", + "d@test.com", + "a@test.com", + ], + }).lazy() + + const schema: Schema = { + fields: [ + { name: "id", type: "number" }, + { name: "email", type: "string" }, + ], + primaryKey: ["id"], + uniqueKeys: [["email"]], + } + + const { errors } = await validateTable(table, { schema }) + expect(errors.filter(e => e.type === "row/unique")).toHaveLength(2) + expect(errors).toContainEqual({ + type: "row/unique", + rowNumber: 4, + fieldNames: ["id"], + }) + expect(errors).toContainEqual({ + type: "row/unique", + rowNumber: 5, + fieldNames: ["email"], + }) + }) + + it("should handle null values in unique keys correctly", async () => { + const table = DataFrame({ + id: [1, 2, null, 4, null, 2], + name: ["Alice", "Bob", "Charlie", "David", "Eve", "Bob"], + }).lazy() + + const schema: Schema = { + fields: [ + { name: "id", type: "number" }, + { name: "name", type: "string" }, + ], + uniqueKeys: [["id"], ["id", "name"]], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toHaveLength(2) + expect(errors).toContainEqual({ + type: "row/unique", + rowNumber: 6, + fieldNames: ["id"], + }) + expect(errors).toContainEqual({ + type: "row/unique", + rowNumber: 6, + fieldNames: ["id", "name"], + }) + }) +}) diff --git a/table/table/checks/unique.ts b/table/table/checks/unique.ts new file mode 100644 index 00000000..f49e6453 --- /dev/null +++ b/table/table/checks/unique.ts @@ -0,0 +1,31 @@ +import type { Schema } from "@dpkit/core" +import { col, concatList } from "nodejs-polars" +import type { Table } from "../../table/Table.ts" + +// TODO: fold is not available so we use a tricky way to eliminate list nulls +// TODO: Using comma as separator might rarely clash with comma in field names +export function checkRowUnique(schema: Schema, errorTable: Table) { + const uniqueKeys = schema.uniqueKeys ?? [] + + if (schema.primaryKey) { + uniqueKeys.push(schema.primaryKey) + } + + for (const uniqueKey of uniqueKeys) { + const targetNames = uniqueKey.map(field => `target:${field}`) + const errorName = `error:row/unique:${uniqueKey.join(",")}` + + errorTable = errorTable + .withColumn(concatList(targetNames).alias(errorName)) + .withColumn( + col(errorName) + .lst.min() + .isNull() + .not() + .and(col(errorName).isFirstDistinct().not()) + .alias(errorName), + ) + } + + return errorTable +} diff --git a/table/table/normalize.ts b/table/table/normalize.ts index 14b22ae4..8d23b81e 100644 --- a/table/table/normalize.ts +++ b/table/table/normalize.ts @@ -13,20 +13,19 @@ export async function normalizeTable(table: Table, schema: Schema) { const head = await table.head(HEAD_ROWS).collect() const polarsSchema = getPolarsSchema(head.schema) - const schemaMapping = { source: polarsSchema, target: schema } - return table.select(...Object.values(normalizeFields(schemaMapping))) + const mapping = { source: polarsSchema, target: schema } + return table.select(...Object.values(normalizeFields(mapping))) } -export function normalizeFields(schemaMapping: SchemaMapping) { +export function normalizeFields(mapping: SchemaMapping) { const exprs: Record = {} - for (const [index, field] of schemaMapping.target.fields.entries()) { - const fieldMapping = matchSchemaField(schemaMapping, field, index) + for (const [index, field] of mapping.target.fields.entries()) { + const fieldMapping = matchSchemaField(mapping, field, index) let expr = lit(null).alias(field.name) if (fieldMapping) { - const missingValues = - field.missingValues ?? schemaMapping.target.missingValues + const missingValues = field.missingValues ?? mapping.target.missingValues const mergedField = { ...field, missingValues } const column = { source: fieldMapping.source, target: mergedField } diff --git a/table/table/validate.ts b/table/table/validate.ts index 37231c60..19d2c819 100644 --- a/table/table/validate.ts +++ b/table/table/validate.ts @@ -23,14 +23,12 @@ export async function validateTable( if (schema) { const sample = await table.head(sampleRows).collect() const polarsSchema = getPolarsSchema(sample.schema) - const schemaMapping = { source: polarsSchema, target: schema } + const mapping = { source: polarsSchema, target: schema } - const matchErrors = validateFieldsMatch(schemaMapping) + const matchErrors = validateFieldsMatch(mapping) errors.push(...matchErrors) - const fieldErrors = await validateFields(schemaMapping, table, { - maxErrors, - }) + const fieldErrors = await validateFields(mapping, table, { maxErrors }) errors.push(...fieldErrors) } @@ -40,12 +38,12 @@ export async function validateTable( } } -function validateFieldsMatch(schemaMapping: SchemaMapping) { +function validateFieldsMatch(mapping: SchemaMapping) { const errors: TableError[] = [] - const fieldsMatch = schemaMapping.target.fieldsMatch ?? "exact" + const fieldsMatch = mapping.target.fieldsMatch ?? "exact" - const fields = schemaMapping.target.fields - const polarsFields = schemaMapping.source.fields + const fields = mapping.target.fields + const polarsFields = mapping.source.fields const names = fields.map(field => field.name) const polarsNames = polarsFields.map(field => field.name) @@ -124,7 +122,7 @@ function validateFieldsMatch(schemaMapping: SchemaMapping) { } async function validateFields( - schemaMapping: SchemaMapping, + mapping: SchemaMapping, table: Table, options: { maxErrors: number @@ -132,16 +130,14 @@ async function validateFields( ) { const { maxErrors } = options const errors: TableError[] = [] - const fields = schemaMapping.target.fields + const fields = mapping.target.fields const concurrency = os.cpus().length - 1 const abortController = new AbortController() - const maxFieldErrors = Math.ceil( - maxErrors / schemaMapping.target.fields.length, - ) + const maxFieldErrors = Math.ceil(maxErrors / mapping.target.fields.length) const collectFieldErrors = async (index: number, field: Field) => { - const fieldMapping = matchSchemaField(schemaMapping, field, index) + const fieldMapping = matchSchemaField(mapping, field, index) if (!fieldMapping) return const report = await validateField(fieldMapping, table, { From 5dbede8bf90764ea0317f7c5223461bbda3d7bae Mon Sep 17 00:00:00 2001 From: roll Date: Wed, 29 Oct 2025 13:02:51 +0000 Subject: [PATCH 27/42] Fixed unique row checks --- table/field/validate.ts | 22 ++-- table/row/checks/unique.spec.ts | 194 ------------------------------ table/row/checks/unique.ts | 31 ----- table/row/index.ts | 1 - table/row/validate.ts | 12 -- table/table/checks/unique.spec.ts | 2 + table/table/checks/unique.ts | 44 ++++--- table/table/validate.ts | 65 +++++++++- 8 files changed, 96 insertions(+), 275 deletions(-) delete mode 100644 table/row/checks/unique.spec.ts delete mode 100644 table/row/checks/unique.ts delete mode 100644 table/row/index.ts delete mode 100644 table/row/validate.ts diff --git a/table/field/validate.ts b/table/field/validate.ts index a24354e6..4ce7a869 100644 --- a/table/field/validate.ts +++ b/table/field/validate.ts @@ -122,18 +122,18 @@ async function validateCells( checkCellUnique, ]) { const cellMapping = { source: col("source"), target: col("target") } - const check = checkCell(mapping.target, cellMapping) - if (check) { - fieldCheckTable = fieldCheckTable.withColumn( - when(col("error").isNotNull()) - .then(col("error")) - .when(check.isErrorExpr) - .then(lit(JSON.stringify(check.errorTemplate))) - .otherwise(lit(null)) - .alias("error"), - ) - } + const check = checkCell(mapping.target, cellMapping) + if (!check) continue + + fieldCheckTable = fieldCheckTable.withColumn( + when(col("error").isNotNull()) + .then(col("error")) + .when(check.isErrorExpr) + .then(lit(JSON.stringify(check.errorTemplate))) + .otherwise(lit(null)) + .alias("error"), + ) } const fieldCheckFrame = await fieldCheckTable diff --git a/table/row/checks/unique.spec.ts b/table/row/checks/unique.spec.ts deleted file mode 100644 index 22f39af4..00000000 --- a/table/row/checks/unique.spec.ts +++ /dev/null @@ -1,194 +0,0 @@ -import type { Schema } from "@dpkit/core" -import { DataFrame } from "nodejs-polars" -import { describe, expect, it } from "vitest" -import { validateTable } from "../../table/index.ts" - -describe("validateTable (row/unique)", () => { - it("should not report errors when all rows are unique for primary key", async () => { - const table = DataFrame({ - id: [1, 2, 3, 4, 5], - name: ["Alice", "Bob", "Charlie", "David", "Eve"], - }).lazy() - - const schema: Schema = { - fields: [ - { name: "id", type: "number" }, - { name: "name", type: "string" }, - ], - primaryKey: ["id"], - } - - const { errors } = await validateTable(table, { schema }) - expect(errors).toHaveLength(0) - }) - - it("should report errors for duplicate primary key rows", async () => { - const table = DataFrame({ - id: [1, 2, 3, 2, 5], - name: ["Alice", "Bob", "Charlie", "Bob2", "Eve"], - }).lazy() - - const schema: Schema = { - fields: [ - { name: "id", type: "number" }, - { name: "name", type: "string" }, - ], - primaryKey: ["id"], - } - - const { errors } = await validateTable(table, { schema }) - expect(errors.filter(e => e.type === "row/unique")).toHaveLength(1) - expect(errors).toContainEqual({ - type: "row/unique", - rowNumber: 4, - fieldNames: ["id"], - }) - }) - - it("should not report errors when all rows are unique for unique key", async () => { - const table = DataFrame({ - id: [1, 2, 3, 4, 5], - email: [ - "a@test.com", - "b@test.com", - "c@test.com", - "d@test.com", - "e@test.com", - ], - }).lazy() - - const schema: Schema = { - fields: [ - { name: "id", type: "number" }, - { name: "email", type: "string" }, - ], - uniqueKeys: [["email"]], - } - - const { errors } = await validateTable(table, { schema }) - expect(errors).toHaveLength(0) - }) - - it("should report errors for duplicate unique key rows", async () => { - const table = DataFrame({ - id: [1, 2, 3, 4, 5], - email: [ - "a@test.com", - "b@test.com", - "a@test.com", - "d@test.com", - "b@test.com", - ], - }).lazy() - - const schema: Schema = { - fields: [ - { name: "id", type: "number" }, - { name: "email", type: "string" }, - ], - uniqueKeys: [["email"]], - } - - const { errors } = await validateTable(table, { schema }) - expect(errors.filter(e => e.type === "row/unique")).toHaveLength(2) - expect(errors).toContainEqual({ - type: "row/unique", - rowNumber: 3, - fieldNames: ["email"], - }) - expect(errors).toContainEqual({ - type: "row/unique", - rowNumber: 5, - fieldNames: ["email"], - }) - }) - - it("should handle composite unique keys", async () => { - const table = DataFrame({ - category: ["A", "A", "B", "A", "B"], - subcategory: ["X", "Y", "X", "X", "Y"], - value: [1, 2, 3, 4, 5], - }).lazy() - - const schema: Schema = { - fields: [ - { name: "category", type: "string" }, - { name: "subcategory", type: "string" }, - { name: "value", type: "number" }, - ], - uniqueKeys: [["category", "subcategory"]], - } - - const { errors } = await validateTable(table, { schema }) - expect(errors.filter(e => e.type === "row/unique")).toHaveLength(1) - expect(errors).toContainEqual({ - type: "row/unique", - rowNumber: 4, - fieldNames: ["category", "subcategory"], - }) - }) - - it("should handle both primary key and unique keys", async () => { - const table = DataFrame({ - id: [1, 2, 3, 2, 5], - email: [ - "a@test.com", - "b@test.com", - "c@test.com", - "d@test.com", - "a@test.com", - ], - }).lazy() - - const schema: Schema = { - fields: [ - { name: "id", type: "number" }, - { name: "email", type: "string" }, - ], - primaryKey: ["id"], - uniqueKeys: [["email"]], - } - - const { errors } = await validateTable(table, { schema }) - expect(errors.filter(e => e.type === "row/unique")).toHaveLength(2) - expect(errors).toContainEqual({ - type: "row/unique", - rowNumber: 4, - fieldNames: ["id"], - }) - expect(errors).toContainEqual({ - type: "row/unique", - rowNumber: 5, - fieldNames: ["email"], - }) - }) - - it("should handle null values in unique keys correctly", async () => { - const table = DataFrame({ - id: [1, 2, null, 4, null, 2], - name: ["Alice", "Bob", "Charlie", "David", "Eve", "Bob"], - }).lazy() - - const schema: Schema = { - fields: [ - { name: "id", type: "number" }, - { name: "name", type: "string" }, - ], - uniqueKeys: [["id"], ["id", "name"]], - } - - const { errors } = await validateTable(table, { schema }) - - expect(errors).toHaveLength(2) - expect(errors).toContainEqual({ - type: "row/unique", - rowNumber: 6, - fieldNames: ["id"], - }) - expect(errors).toContainEqual({ - type: "row/unique", - rowNumber: 6, - fieldNames: ["id", "name"], - }) - }) -}) diff --git a/table/row/checks/unique.ts b/table/row/checks/unique.ts deleted file mode 100644 index f49e6453..00000000 --- a/table/row/checks/unique.ts +++ /dev/null @@ -1,31 +0,0 @@ -import type { Schema } from "@dpkit/core" -import { col, concatList } from "nodejs-polars" -import type { Table } from "../../table/Table.ts" - -// TODO: fold is not available so we use a tricky way to eliminate list nulls -// TODO: Using comma as separator might rarely clash with comma in field names -export function checkRowUnique(schema: Schema, errorTable: Table) { - const uniqueKeys = schema.uniqueKeys ?? [] - - if (schema.primaryKey) { - uniqueKeys.push(schema.primaryKey) - } - - for (const uniqueKey of uniqueKeys) { - const targetNames = uniqueKey.map(field => `target:${field}`) - const errorName = `error:row/unique:${uniqueKey.join(",")}` - - errorTable = errorTable - .withColumn(concatList(targetNames).alias(errorName)) - .withColumn( - col(errorName) - .lst.min() - .isNull() - .not() - .and(col(errorName).isFirstDistinct().not()) - .alias(errorName), - ) - } - - return errorTable -} diff --git a/table/row/index.ts b/table/row/index.ts deleted file mode 100644 index 0628f872..00000000 --- a/table/row/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { validateRows } from "./validate.ts" diff --git a/table/row/validate.ts b/table/row/validate.ts deleted file mode 100644 index 6e4f33cd..00000000 --- a/table/row/validate.ts +++ /dev/null @@ -1,12 +0,0 @@ -import type { Schema } from "@dpkit/core" -import type { TableError } from "../error/Table.ts" -import type { Table } from "../table/Table.ts" -import { checkRowUnique } from "./checks/unique.ts" - -export function validateRows(schema: Schema, errorTable: Table) { - const errors: TableError[] = [] - - errorTable = checkRowUnique(schema, errorTable) - - return { valid: !errors.length, errors, errorTable } -} diff --git a/table/table/checks/unique.spec.ts b/table/table/checks/unique.spec.ts index 22f39af4..c3365a80 100644 --- a/table/table/checks/unique.spec.ts +++ b/table/table/checks/unique.spec.ts @@ -37,6 +37,7 @@ describe("validateTable (row/unique)", () => { } const { errors } = await validateTable(table, { schema }) + expect(errors.filter(e => e.type === "row/unique")).toHaveLength(1) expect(errors).toContainEqual({ type: "row/unique", @@ -178,6 +179,7 @@ describe("validateTable (row/unique)", () => { } const { errors } = await validateTable(table, { schema }) + console.log(errors) expect(errors).toHaveLength(2) expect(errors).toContainEqual({ diff --git a/table/table/checks/unique.ts b/table/table/checks/unique.ts index f49e6453..77d085fd 100644 --- a/table/table/checks/unique.ts +++ b/table/table/checks/unique.ts @@ -1,31 +1,29 @@ -import type { Schema } from "@dpkit/core" -import { col, concatList } from "nodejs-polars" -import type { Table } from "../../table/Table.ts" +import { concatList } from "nodejs-polars" +import type { RowUniqueError } from "../../error/index.ts" +import type { SchemaMapping } from "../../schema/index.ts" -// TODO: fold is not available so we use a tricky way to eliminate list nulls -// TODO: Using comma as separator might rarely clash with comma in field names -export function checkRowUnique(schema: Schema, errorTable: Table) { - const uniqueKeys = schema.uniqueKeys ?? [] +export function createChecksRowUnique(mapping: SchemaMapping) { + const uniqueKeys = mapping.target.uniqueKeys ?? [] - if (schema.primaryKey) { - uniqueKeys.push(schema.primaryKey) + if (mapping.target.primaryKey) { + uniqueKeys.push(mapping.target.primaryKey) } - for (const uniqueKey of uniqueKeys) { - const targetNames = uniqueKey.map(field => `target:${field}`) - const errorName = `error:row/unique:${uniqueKey.join(",")}` + return uniqueKeys.map(createCheckRowUnique) +} + +function createCheckRowUnique(uniqueKey: string[]) { + const isErrorExpr = concatList(uniqueKey) + .isFirstDistinct() + .not() + // Fold is not available so we use a tricky way to eliminate nulls + .and(concatList(uniqueKey).lst.min().isNotNull()) - errorTable = errorTable - .withColumn(concatList(targetNames).alias(errorName)) - .withColumn( - col(errorName) - .lst.min() - .isNull() - .not() - .and(col(errorName).isFirstDistinct().not()) - .alias(errorName), - ) + const errorTemplate: RowUniqueError = { + type: "row/unique", + fieldNames: uniqueKey, + rowNumber: 0, } - return errorTable + return { isErrorExpr, errorTemplate } } diff --git a/table/table/validate.ts b/table/table/validate.ts index 19d2c819..fdd6634d 100644 --- a/table/table/validate.ts +++ b/table/table/validate.ts @@ -1,13 +1,15 @@ import os from "node:os" import type { Field, Schema } from "@dpkit/core" +import { col, lit, when } from "nodejs-polars" import pAll from "p-all" +import type { RowError } from "../error/index.ts" import type { TableError } from "../error/index.ts" import { validateField } from "../field/index.ts" import { matchSchemaField } from "../schema/index.ts" -//import { validateRows } from "../row/index.ts" import { getPolarsSchema } from "../schema/index.ts" import type { SchemaMapping } from "../schema/index.ts" import type { Table } from "./Table.ts" +import { createChecksRowUnique } from "./checks/unique.ts" export async function validateTable( table: Table, @@ -30,6 +32,9 @@ export async function validateTable( const fieldErrors = await validateFields(mapping, table, { maxErrors }) errors.push(...fieldErrors) + + const rowErrors = await validateRows(mapping, table, { maxErrors }) + errors.push(...rowErrors) } return { @@ -133,8 +138,7 @@ async function validateFields( const fields = mapping.target.fields const concurrency = os.cpus().length - 1 const abortController = new AbortController() - - const maxFieldErrors = Math.ceil(maxErrors / mapping.target.fields.length) + const maxFieldErrors = Math.ceil(maxErrors / fields.length) const collectFieldErrors = async (index: number, field: Field) => { const fieldMapping = matchSchemaField(mapping, field, index) @@ -163,6 +167,61 @@ async function validateFields( return errors } +async function validateRows( + mapping: SchemaMapping, + table: Table, + options: { maxErrors: number }, +) { + const { maxErrors } = options + const errors: TableError[] = [] + const fields = mapping.target.fields + const concurrency = os.cpus().length - 1 + const abortController = new AbortController() + const maxRowErrors = Math.ceil(maxErrors / fields.length) + + const collectRowErrors = async (check: any) => { + const rowCheckTable = table + .withRowCount() + .withColumn(col("row_nr").add(1)) + .rename({ row_nr: "dpkit:number" }) + .withColumn( + when(check.isErrorExpr) + .then(lit(JSON.stringify(check.errorTemplate))) + .otherwise(lit(null)) + .alias("dpkit:error"), + ) + + const rowCheckFrame = await rowCheckTable + .filter(col("dpkit:error").isNotNull()) + .head(maxRowErrors) + .collect() + + for (const row of rowCheckFrame.toRecords() as any[]) { + const errorTemplate = JSON.parse(row["dpkit:error"]) as RowError + errors.push({ + ...errorTemplate, + rowNumber: row["dpkit:number"], + }) + } + + if (errors.length > maxErrors) { + abortController.abort() + } + } + + try { + await pAll( + [...createChecksRowUnique(mapping)].map(it => () => collectRowErrors(it)), + { concurrency }, + ) + } catch (error) { + const isAborted = error instanceof Error && error.name === "AbortError" + if (!isAborted) throw error + } + + return errors +} + function arrayDiff(a: string[], b: string[]) { return a.filter(x => !b.includes(x)) } From 82fc22329164f59345774bb416c6688072b438e9 Mon Sep 17 00:00:00 2001 From: roll Date: Wed, 29 Oct 2025 13:24:39 +0000 Subject: [PATCH 28/42] Enabled support for `dialect.lineTerminator` --- csv/table/load.spec.ts | 2 +- csv/table/load.ts | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/csv/table/load.spec.ts b/csv/table/load.spec.ts index c74c98c3..93031c16 100644 --- a/csv/table/load.spec.ts +++ b/csv/table/load.spec.ts @@ -94,7 +94,7 @@ describe("loadCsvTable", () => { ]) }) - it.skip("should handle custom line terminator", async () => { + it("should handle custom line terminator", async () => { const path = await writeTempFile("id,name|1,alice|2,bob") const table = await loadCsvTable({ path, diff --git a/csv/table/load.ts b/csv/table/load.ts index b451dc66..6657b0ca 100644 --- a/csv/table/load.ts +++ b/csv/table/load.ts @@ -84,12 +84,9 @@ function getScanOptions(resource: Partial, dialect?: Dialect) { options.skipRows = getRowsToSkip(dialect) options.hasHeader = dialect?.header !== false + options.eolChar = dialect?.lineTerminator ?? "\n" options.sep = dialect?.delimiter ?? "," - // TODO: enable after this polars issues is fixed - // https://github.com/pola-rs/nodejs-polars/issues/333 - //options.eolChar = dialect?.lineTerminator ?? "\n" - // TODO: try convincing nodejs-polars to support escapeChar // https://github.com/pola-rs/polars/issues/3074 //options.escapeChar = dialect?.escapeChar From 80ae048160e6c93c7ab43060cf25fa531b161314 Mon Sep 17 00:00:00 2001 From: roll Date: Wed, 29 Oct 2025 15:37:53 +0000 Subject: [PATCH 29/42] Improved nomalizeField --- table/field/normalize.ts | 4 ++-- table/field/parse.ts | 8 ++++++-- table/field/substitute.ts | 14 +++++++++----- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/table/field/normalize.ts b/table/field/normalize.ts index 2324e092..e32c60aa 100644 --- a/table/field/normalize.ts +++ b/table/field/normalize.ts @@ -11,10 +11,10 @@ export function normalizeField( let fieldExpr = col(mapping.source.name) if (mapping.source.type.equals(DataType.String)) { - fieldExpr = substituteField(mapping.target, fieldExpr) + fieldExpr = substituteField(mapping, fieldExpr) if (!options?.keepType) { - fieldExpr = parseField(mapping.target, fieldExpr) + fieldExpr = parseField(mapping, fieldExpr) } } diff --git a/table/field/parse.ts b/table/field/parse.ts index 8e3113ac..896417fa 100644 --- a/table/field/parse.ts +++ b/table/field/parse.ts @@ -1,5 +1,6 @@ -import type { Field } from "@dpkit/core" import type { Expr } from "nodejs-polars" +import { DataType } from "nodejs-polars" +import type { FieldMapping } from "./Mapping.ts" import { parseArrayField } from "./types/array.ts" import { parseBooleanField } from "./types/boolean.ts" import { parseDateField } from "./types/date.ts" @@ -16,7 +17,10 @@ import { parseTimeField } from "./types/time.ts" import { parseYearField } from "./types/year.ts" import { parseYearmonthField } from "./types/yearmonth.ts" -export function parseField(field: Field, fieldExpr: Expr) { +export function parseField(mapping: FieldMapping, fieldExpr: Expr) { + if (!mapping.source.type.equals(DataType.String)) return fieldExpr + + const field = mapping.target switch (field.type) { case "array": return parseArrayField(field, fieldExpr) diff --git a/table/field/substitute.ts b/table/field/substitute.ts index 5f024a3d..7234e0d4 100644 --- a/table/field/substitute.ts +++ b/table/field/substitute.ts @@ -1,19 +1,23 @@ -import type { Field } from "@dpkit/core" +import { DataType } from "nodejs-polars" import { lit, when } from "nodejs-polars" import type { Expr } from "nodejs-polars" +import type { FieldMapping } from "./Mapping.ts" const DEFAULT_MISSING_VALUES = [""] -export function substituteField(field: Field, fieldExpr: Expr) { +export function substituteField(mapping: FieldMapping, fieldExpr: Expr) { + if (!mapping.source.type.equals(DataType.String)) return fieldExpr + const flattenMissingValues = - field.missingValues?.map(it => (typeof it === "string" ? it : it.value)) ?? - DEFAULT_MISSING_VALUES + mapping.target.missingValues?.map(it => + typeof it === "string" ? it : it.value, + ) ?? DEFAULT_MISSING_VALUES if (flattenMissingValues.length) { fieldExpr = when(fieldExpr.isIn(flattenMissingValues)) .then(lit(null)) .otherwise(fieldExpr) - .alias(field.name) + .alias(mapping.target.name) } return fieldExpr From 56b46018924bfd16ba1a971b0f4ad7c596223264 Mon Sep 17 00:00:00 2001 From: roll Date: Wed, 29 Oct 2025 15:46:30 +0000 Subject: [PATCH 30/42] Implementd type narrowing from numbers to integers --- lib/resource/validate.spec.ts | 18 +++++------ table/field/narrow.spec.ts | 56 +++++++++++++++++++++++++++++++++++ table/field/narrow.ts | 19 +++++++++++- table/field/normalize.ts | 12 ++++---- table/field/validate.ts | 42 +++++++++++++------------- 5 files changed, 109 insertions(+), 38 deletions(-) create mode 100644 table/field/narrow.spec.ts diff --git a/lib/resource/validate.spec.ts b/lib/resource/validate.spec.ts index 366796f9..9d397ec3 100644 --- a/lib/resource/validate.spec.ts +++ b/lib/resource/validate.spec.ts @@ -19,10 +19,10 @@ describe("validateResource", () => { }, } - const result = await validateResource(resource) + const report = await validateResource(resource) - expect(result.valid).toBe(false) - expect(result.errors.length).toEqual(2) + expect(report.valid).toBe(false) + expect(report.errors.length).toEqual(1) }) it("should validate correct tabular data", async () => { @@ -41,10 +41,10 @@ describe("validateResource", () => { }, } - const result = await validateResource(resource) + const report = await validateResource(resource) - expect(result.valid).toBe(true) - expect(result.errors).toEqual([]) + expect(report.valid).toBe(true) + expect(report.errors).toEqual([]) }) it("should catch multiple validation errors", async () => { @@ -69,9 +69,9 @@ describe("validateResource", () => { }, } - const result = await validateResource(resource) + const report = await validateResource(resource) - expect(result.valid).toBe(false) - expect(result.errors.length).toEqual(3) + expect(report.valid).toBe(false) + expect(report.errors.length).toEqual(3) }) }) diff --git a/table/field/narrow.spec.ts b/table/field/narrow.spec.ts new file mode 100644 index 00000000..8a7f9628 --- /dev/null +++ b/table/field/narrow.spec.ts @@ -0,0 +1,56 @@ +import type { Schema } from "@dpkit/core" +import { DataFrame } from "nodejs-polars" +import { describe, expect, it } from "vitest" +import { normalizeTable } from "../table/normalize.ts" +import { validateTable } from "../table/validate.ts" + +describe("narrowField", () => { + it("should narrow float to integer", async () => { + const table = DataFrame({ + id: [1.0, 2.0, 3.0], + name: ["a", "b", "c"], + }).lazy() + + const schema: Schema = { + fields: [ + { name: "id", type: "integer" }, + { name: "name", type: "string" }, + ], + } + + const ldf = await normalizeTable(table, schema) + const df = await ldf.collect() + + expect(df.toRecords()).toEqual([ + { id: 1, name: "a" }, + { id: 2, name: "b" }, + { id: 3, name: "c" }, + ]) + }) + + it("should detect error when float cannot be narrowed to integer", async () => { + const table = DataFrame({ + id: [1.0, 2.0, 3.5], + name: ["a", "b", "c"], + }).lazy() + + const schema: Schema = { + fields: [ + { name: "id", type: "integer" }, + { name: "name", type: "string" }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/type", + fieldName: "id", + fieldType: "integer", + rowNumber: 3, + cell: "3.5", + }, + ]) + }) +}) diff --git a/table/field/narrow.ts b/table/field/narrow.ts index 1039d4c5..9918e38a 100644 --- a/table/field/narrow.ts +++ b/table/field/narrow.ts @@ -1 +1,18 @@ -// TODO: implement +import { DataType } from "nodejs-polars" +import { lit, when } from "nodejs-polars" +import type { Expr } from "nodejs-polars" +import type { FieldMapping } from "./Mapping.ts" + +export function narrowField(mapping: FieldMapping, fieldExpr: Expr) { + const variant = mapping.source.type.variant + + if (mapping.target.type === "integer") { + if (["Float32", "Float64"].includes(variant)) { + fieldExpr = when(fieldExpr.eq(fieldExpr.round(0))) + .then(fieldExpr.cast(DataType.Int64)) + .otherwise(lit(null)) + } + } + + return fieldExpr +} diff --git a/table/field/normalize.ts b/table/field/normalize.ts index e32c60aa..838f68e4 100644 --- a/table/field/normalize.ts +++ b/table/field/normalize.ts @@ -1,6 +1,6 @@ -import { DataType } from "nodejs-polars" import { col } from "nodejs-polars" import type { FieldMapping } from "./Mapping.ts" +import { narrowField } from "./narrow.ts" import { parseField } from "./parse.ts" import { substituteField } from "./substitute.ts" @@ -9,13 +9,11 @@ export function normalizeField( options?: { keepType?: boolean }, ) { let fieldExpr = col(mapping.source.name) + fieldExpr = substituteField(mapping, fieldExpr) - if (mapping.source.type.equals(DataType.String)) { - fieldExpr = substituteField(mapping, fieldExpr) - - if (!options?.keepType) { - fieldExpr = parseField(mapping, fieldExpr) - } + if (!options?.keepType) { + fieldExpr = parseField(mapping, fieldExpr) + fieldExpr = narrowField(mapping, fieldExpr) } return fieldExpr.alias(mapping.target.name) diff --git a/table/field/validate.ts b/table/field/validate.ts index 4ce7a869..963c02b1 100644 --- a/table/field/validate.ts +++ b/table/field/validate.ts @@ -55,34 +55,34 @@ function validateName(mapping: FieldMapping) { function validateType(mapping: FieldMapping) { const errors: FieldError[] = [] - const types: Record = { - Bool: "boolean", - Date: "date", - Datetime: "datetime", - Float32: "number", - Float64: "number", - Int16: "integer", - Int32: "integer", - Int64: "integer", - Int8: "integer", - List: "list", - String: "string", - Time: "time", - UInt16: "integer", - UInt32: "integer", - UInt64: "integer", - UInt8: "integer", - Utf8: "string", + const compatMapping: Record = { + Bool: ["boolean"], + Date: ["date"], + Datetime: ["datetime"], + Float32: ["number", "integer"], + Float64: ["number", "integer"], + Int16: ["integer"], + Int32: ["integer"], + Int64: ["integer"], + Int8: ["integer"], + List: ["list"], + Time: ["time"], + UInt16: ["integer"], + UInt32: ["integer"], + UInt64: ["integer"], + UInt8: ["integer"], + Utf8: ["string"], } - const actualFieldType = types[mapping.source.type.variant] ?? "any" + const compatTypes = compatMapping[mapping.source.type.variant] + if (!compatTypes) return errors - if (actualFieldType !== mapping.target.type && actualFieldType !== "string") { + if (!compatTypes.includes(mapping.target.type)) { errors.push({ type: "field/type", fieldName: mapping.target.name, fieldType: mapping.target.type ?? "any", - actualFieldType, + actualFieldType: compatTypes[0] ?? "any", }) } From 61752367402c7ade6c6e9827a7f1dab87ca1eb33 Mon Sep 17 00:00:00 2001 From: roll Date: Wed, 29 Oct 2025 16:42:20 +0000 Subject: [PATCH 31/42] Renamed to resolveDialect/Schema --- arrow/table/load.ts | 4 ++-- cli/commands/dialect/explore.tsx | 4 ++-- cli/commands/dialect/script.tsx | 4 ++-- cli/commands/dialect/validate.tsx | 7 ++----- cli/commands/schema/explore.tsx | 4 ++-- cli/commands/schema/script.tsx | 4 ++-- cli/commands/schema/validate.tsx | 7 ++----- cli/commands/table/explore.tsx | 4 ++-- cli/commands/table/validate.tsx | 4 ++-- core/dialect/index.ts | 1 + core/{resource/dialect.ts => dialect/resolve.ts} | 4 ++-- core/resource/index.ts | 7 +------ core/schema/index.ts | 1 + core/{resource/schema.ts => schema/resolve.ts} | 4 ++-- csv/table/load.ts | 6 +++--- database/package/save.ts | 4 ++-- database/schema/infer.ts | 4 ++-- database/table/load.ts | 6 +++--- inline/table/load.ts | 8 ++++---- json/table/load.ts | 8 ++++---- lib/resource/validate.ts | 4 ++-- lib/table/infer.ts | 6 +++--- ods/table/load.ts | 8 ++++---- ods/table/save.ts | 4 ++-- parquet/table/load.ts | 4 ++-- xlsx/table/load.ts | 8 ++++---- xlsx/table/save.ts | 4 ++-- 27 files changed, 62 insertions(+), 71 deletions(-) rename core/{resource/dialect.ts => dialect/resolve.ts} (61%) rename core/{resource/schema.ts => schema/resolve.ts} (61%) diff --git a/arrow/table/load.ts b/arrow/table/load.ts index da0ed071..6fa5c2e9 100644 --- a/arrow/table/load.ts +++ b/arrow/table/load.ts @@ -1,5 +1,5 @@ import type { Resource } from "@dpkit/core" -import { loadResourceSchema } from "@dpkit/core" +import { resolveSchema } from "@dpkit/core" import { prefetchFiles } from "@dpkit/file" import type { LoadTableOptions } from "@dpkit/table" import { inferSchemaFromTable, normalizeTable } from "@dpkit/table" @@ -21,7 +21,7 @@ export async function loadArrowTable( } if (!options?.denormalized) { - let schema = await loadResourceSchema(resource.schema) + let schema = await resolveSchema(resource.schema) if (!schema) schema = await inferSchemaFromTable(table, options) table = await normalizeTable(table, schema) } diff --git a/cli/commands/dialect/explore.tsx b/cli/commands/dialect/explore.tsx index 4b4779bf..8f5924a5 100644 --- a/cli/commands/dialect/explore.tsx +++ b/cli/commands/dialect/explore.tsx @@ -1,6 +1,6 @@ import { loadDialect } from "@dpkit/lib" import type { Resource } from "@dpkit/lib" -import { loadResourceDialect } from "@dpkit/lib" +import { resolveDialect } from "@dpkit/lib" import { Command } from "commander" import React from "react" import { DialectGrid } from "../../components/DialectGrid.tsx" @@ -33,7 +33,7 @@ export const exploreDialectCommand = new Command("explore") const dialect = await session.task( "Loading dialect", - path ? loadDialect(path) : loadResourceDialect(resource?.dialect), + path ? loadDialect(path) : resolveDialect(resource?.dialect), ) if (!dialect || isEmptyObject(dialect)) { diff --git a/cli/commands/dialect/script.tsx b/cli/commands/dialect/script.tsx index 97edd40a..8765171f 100644 --- a/cli/commands/dialect/script.tsx +++ b/cli/commands/dialect/script.tsx @@ -2,7 +2,7 @@ import repl from "node:repl" import * as dpkit from "@dpkit/lib" import { loadDialect } from "@dpkit/lib" import type { Resource } from "@dpkit/lib" -import { loadResourceDialect } from "@dpkit/lib" +import { resolveDialect } from "@dpkit/lib" import { Command } from "commander" import pc from "picocolors" import { helpConfiguration } from "../../helpers/help.ts" @@ -34,7 +34,7 @@ export const scriptDialectCommand = new Command("script") const dialect = await session.task( "Loading dialect", - path ? loadDialect(path) : loadResourceDialect(resource?.dialect), + path ? loadDialect(path) : resolveDialect(resource?.dialect), ) if (!dialect || isEmptyObject(dialect)) { diff --git a/cli/commands/dialect/validate.tsx b/cli/commands/dialect/validate.tsx index 6d0e4afa..19f63932 100644 --- a/cli/commands/dialect/validate.tsx +++ b/cli/commands/dialect/validate.tsx @@ -1,6 +1,6 @@ import { loadDescriptor, validateDialect } from "@dpkit/lib" import type { Resource } from "@dpkit/lib" -import { loadResourceDialect } from "@dpkit/lib" +import { resolveDialect } from "@dpkit/lib" import { Command } from "commander" import React from "react" import { ErrorGrid } from "../../components/ErrorGrid.tsx" @@ -36,10 +36,7 @@ export const validateDialectCommand = new Command("validate") : undefined const dialect = resource?.dialect - ? await session.task( - "Loading dialect", - loadResourceDialect(resource.dialect), - ) + ? await session.task("Loading dialect", resolveDialect(resource.dialect)) : undefined const { descriptor } = path diff --git a/cli/commands/schema/explore.tsx b/cli/commands/schema/explore.tsx index b5801b5f..a6ba0632 100644 --- a/cli/commands/schema/explore.tsx +++ b/cli/commands/schema/explore.tsx @@ -1,6 +1,6 @@ import { loadSchema } from "@dpkit/lib" import type { Resource } from "@dpkit/lib" -import { loadResourceSchema } from "@dpkit/lib" +import { resolveSchema } from "@dpkit/lib" import { Command } from "commander" import React from "react" import { SchemaGrid } from "../../components/SchemaGrid.tsx" @@ -33,7 +33,7 @@ export const exploreSchemaCommand = new Command("explore") const schema = await session.task( "Loading schema", - path ? loadSchema(path) : loadResourceSchema(resource?.schema), + path ? loadSchema(path) : resolveSchema(resource?.schema), ) if (!schema || isEmptyObject(schema)) { diff --git a/cli/commands/schema/script.tsx b/cli/commands/schema/script.tsx index cad34f9d..68f766e4 100644 --- a/cli/commands/schema/script.tsx +++ b/cli/commands/schema/script.tsx @@ -1,7 +1,7 @@ import repl from "node:repl" import { loadSchema } from "@dpkit/lib" import type { Resource } from "@dpkit/lib" -import { loadResourceSchema } from "@dpkit/lib" +import { resolveSchema } from "@dpkit/lib" import * as dpkit from "@dpkit/lib" import { Command } from "commander" import pc from "picocolors" @@ -33,7 +33,7 @@ export const scriptSchemaCommand = new Command("script") const schema = await session.task( "Loading schema", - path ? loadSchema(path) : loadResourceSchema(resource?.schema), + path ? loadSchema(path) : resolveSchema(resource?.schema), ) console.log( diff --git a/cli/commands/schema/validate.tsx b/cli/commands/schema/validate.tsx index 5c7281fd..2ea00a21 100644 --- a/cli/commands/schema/validate.tsx +++ b/cli/commands/schema/validate.tsx @@ -1,5 +1,5 @@ import { loadDescriptor, validateSchema } from "@dpkit/lib" -import { loadResourceSchema } from "@dpkit/lib" +import { resolveSchema } from "@dpkit/lib" import type { Resource } from "@dpkit/lib" import { Command } from "commander" import React from "react" @@ -36,10 +36,7 @@ export const validateSchemaCommand = new Command("validate") : undefined const schema = resource?.schema - ? await session.task( - "Loading schema", - loadResourceSchema(resource.schema), - ) + ? await session.task("Loading schema", resolveSchema(resource.schema)) : undefined const { descriptor } = path diff --git a/cli/commands/table/explore.tsx b/cli/commands/table/explore.tsx index ee4db088..4a8cfb5d 100644 --- a/cli/commands/table/explore.tsx +++ b/cli/commands/table/explore.tsx @@ -1,4 +1,4 @@ -import { inferSchemaFromTable, loadResourceSchema } from "@dpkit/lib" +import { inferSchemaFromTable, resolveSchema } from "@dpkit/lib" import { queryTable } from "@dpkit/lib" import { loadSchema } from "@dpkit/lib" import { loadDialect, loadTable, normalizeTable } from "@dpkit/lib" @@ -100,7 +100,7 @@ export const exploreTableCommand = new Command("explore") if (!schema && resource.schema) { schema = await session.task( "Loading schema", - loadResourceSchema(resource.schema), + resolveSchema(resource.schema), ) } diff --git a/cli/commands/table/validate.tsx b/cli/commands/table/validate.tsx index 386f5dd6..79a9bc72 100644 --- a/cli/commands/table/validate.tsx +++ b/cli/commands/table/validate.tsx @@ -1,6 +1,6 @@ import { loadTable, validateTable } from "@dpkit/lib" import { loadSchema } from "@dpkit/lib" -import { inferSchemaFromTable, loadResourceSchema } from "@dpkit/lib" +import { inferSchemaFromTable, resolveSchema } from "@dpkit/lib" import { loadDialect } from "@dpkit/lib" import type { Resource } from "@dpkit/lib" import { Command } from "commander" @@ -107,7 +107,7 @@ export const validateTableCommand = new Command("validate") if (!schema && resource.schema) { schema = await session.task( "Loading schema", - loadResourceSchema(options.schema ?? resource.schema), + resolveSchema(options.schema ?? resource.schema), ) } diff --git a/core/dialect/index.ts b/core/dialect/index.ts index 442b6a8b..b0fc40e2 100644 --- a/core/dialect/index.ts +++ b/core/dialect/index.ts @@ -5,3 +5,4 @@ export { loadDialect } from "./load.ts" export { saveDialect } from "./save.ts" export { convertDialectFromDescriptor } from "./convert/fromDescriptor.ts" export { convertDialectToDescriptor } from "./convert/toDescriptor.ts" +export { resolveDialect } from "./resolve.ts" diff --git a/core/resource/dialect.ts b/core/dialect/resolve.ts similarity index 61% rename from core/resource/dialect.ts rename to core/dialect/resolve.ts index a29e6fa1..9a347d4c 100644 --- a/core/resource/dialect.ts +++ b/core/dialect/resolve.ts @@ -1,7 +1,7 @@ import { loadDialect } from "../dialect/index.ts" -import type { Resource } from "./Resource.ts" +import type { Dialect } from "./Dialect.ts" -export async function loadResourceDialect(dialect: Resource["dialect"]) { +export async function resolveDialect(dialect?: Dialect | string) { if (!dialect) { return undefined } diff --git a/core/resource/index.ts b/core/resource/index.ts index 085b83e2..095f3d1f 100644 --- a/core/resource/index.ts +++ b/core/resource/index.ts @@ -6,11 +6,6 @@ export { saveResourceDescriptor } from "./save.ts" export { validateResourceMetadata } from "./validate.ts" export { convertResourceFromDescriptor } from "./convert/fromDescriptor.ts" export { convertResourceToDescriptor } from "./convert/toDescriptor.ts" +export { isRemoteResource } from "./helpers.ts" export type { Source } from "./Source.ts" export type { License } from "./License.ts" -export { loadResourceDialect } from "./dialect.ts" -export { loadResourceSchema } from "./schema.ts" -export { isRemoteResource } from "./helpers.ts" - -// TODO: Remove in v2 -export { validateResourceMetadata as validateResourceDescriptor } from "./validate.ts" diff --git a/core/schema/index.ts b/core/schema/index.ts index d78cb87d..fcca68a9 100644 --- a/core/schema/index.ts +++ b/core/schema/index.ts @@ -8,3 +8,4 @@ export { convertSchemaFromDescriptor } from "./convert/fromDescriptor.ts" export { convertSchemaToDescriptor } from "./convert/toDescriptor.ts" export { convertSchemaFromJsonSchema } from "./convert/fromJsonSchema.ts" export { convertSchemaToJsonSchema } from "./convert/toJsonSchema.ts" +export { resolveSchema } from "./resolve.ts" diff --git a/core/resource/schema.ts b/core/schema/resolve.ts similarity index 61% rename from core/resource/schema.ts rename to core/schema/resolve.ts index 499543f6..33102eb7 100644 --- a/core/resource/schema.ts +++ b/core/schema/resolve.ts @@ -1,7 +1,7 @@ import { loadSchema } from "../schema/index.ts" -import type { Resource } from "./Resource.ts" +import type { Schema } from "./Schema.ts" -export async function loadResourceSchema(schema: Resource["schema"]) { +export async function resolveSchema(schema?: Schema | string) { if (!schema) { return undefined } diff --git a/csv/table/load.ts b/csv/table/load.ts index 6657b0ca..bfd0e560 100644 --- a/csv/table/load.ts +++ b/csv/table/load.ts @@ -1,5 +1,5 @@ import type { Dialect, Resource } from "@dpkit/core" -import { loadResourceDialect, loadResourceSchema } from "@dpkit/core" +import { resolveDialect, resolveSchema } from "@dpkit/core" import { prefetchFiles } from "@dpkit/file" import type { Table } from "@dpkit/table" import { inferSchemaFromTable, normalizeTable } from "@dpkit/table" @@ -25,7 +25,7 @@ export async function loadCsvTable( throw new Error("Resource path is not defined") } - let dialect = await loadResourceDialect(resource.dialect) + let dialect = await resolveDialect(resource.dialect) if (!dialect) { dialect = await inferCsvDialect({ ...resource, path: paths[0] }, options) } @@ -55,7 +55,7 @@ export async function loadCsvTable( } if (!options?.denormalized) { - let schema = await loadResourceSchema(resource.schema) + let schema = await resolveSchema(resource.schema) if (!schema) schema = await inferSchemaFromTable(table, options) table = await normalizeTable(table, schema) } diff --git a/database/package/save.ts b/database/package/save.ts index 12a9a393..c74bc06b 100644 --- a/database/package/save.ts +++ b/database/package/save.ts @@ -1,5 +1,5 @@ import type { Package } from "@dpkit/core" -import { loadResourceSchema } from "@dpkit/core" +import { resolveSchema } from "@dpkit/core" import { isRemoteResource } from "@dpkit/core" import type { SavePackageOptions } from "@dpkit/core" import type { TablePlugin } from "@dpkit/table" @@ -24,7 +24,7 @@ export async function savePackageToDatabase( if (table) { const dialect = { table: resource.name } - const schema = await loadResourceSchema(resource.schema) + const schema = await resolveSchema(resource.schema) // TODO: support parallel saving? await saveDatabaseTable(table, { diff --git a/database/schema/infer.ts b/database/schema/infer.ts index 7105b5b2..7bf0dbd8 100644 --- a/database/schema/infer.ts +++ b/database/schema/infer.ts @@ -1,5 +1,5 @@ import type { Resource } from "@dpkit/core" -import { loadResourceDialect } from "@dpkit/core" +import { resolveDialect } from "@dpkit/core" import { createAdapter } from "../adapters/create.ts" export async function inferDatabaseSchema( @@ -10,7 +10,7 @@ export async function inferDatabaseSchema( throw new Error("Supported database format is not defined") } - const dialect = await loadResourceDialect(resource.dialect) + const dialect = await resolveDialect(resource.dialect) if (!dialect?.table) { throw new Error("Table name is not defined in dialect") } diff --git a/database/table/load.ts b/database/table/load.ts index 02121f0b..57317889 100644 --- a/database/table/load.ts +++ b/database/table/load.ts @@ -1,4 +1,4 @@ -import { loadResourceDialect, loadResourceSchema } from "@dpkit/core" +import { resolveDialect, resolveSchema } from "@dpkit/core" import type { Resource } from "@dpkit/core" import { normalizeTable } from "@dpkit/table" import type { LoadTableOptions } from "@dpkit/table" @@ -13,7 +13,7 @@ export async function loadDatabaseTable( resource: Partial & { format: "postgresql" | "mysql" | "sqlite" }, options?: LoadTableOptions, ) { - const dialect = await loadResourceDialect(resource.dialect) + const dialect = await resolveDialect(resource.dialect) if (!dialect?.table) { throw new Error("Table name is not defined in dialect") } @@ -31,7 +31,7 @@ export async function loadDatabaseTable( let table = DataFrame(records).lazy() if (!options?.denormalized) { - let schema = await loadResourceSchema(resource.schema) + let schema = await resolveSchema(resource.schema) if (!schema) schema = await inferDatabaseSchema(resource) table = await normalizeTable(table, schema) } diff --git a/inline/table/load.ts b/inline/table/load.ts index 4cb75937..7ef5e51c 100644 --- a/inline/table/load.ts +++ b/inline/table/load.ts @@ -1,6 +1,6 @@ import type { Resource } from "@dpkit/core" -import { loadResourceDialect } from "@dpkit/core" -import { loadResourceSchema } from "@dpkit/core" +import { resolveDialect } from "@dpkit/core" +import { resolveSchema } from "@dpkit/core" import { getRecordsFromRows } from "@dpkit/table" import type { LoadTableOptions } from "@dpkit/table" import { inferSchemaFromTable, normalizeTable } from "@dpkit/table" @@ -15,14 +15,14 @@ export async function loadInlineTable( throw new Error("Resource data is not defined or tabular") } - const dialect = await loadResourceDialect(resource.dialect) + const dialect = await resolveDialect(resource.dialect) const isRows = data.every(row => Array.isArray(row)) const records = isRows ? getRecordsFromRows(data, dialect) : data let table = DataFrame(records).lazy() if (!options?.denormalized) { - let schema = await loadResourceSchema(resource.schema) + let schema = await resolveSchema(resource.schema) if (!schema) schema = await inferSchemaFromTable(table, options) table = await normalizeTable(table, schema) } diff --git a/json/table/load.ts b/json/table/load.ts index 03b7a9c3..9b866c93 100644 --- a/json/table/load.ts +++ b/json/table/load.ts @@ -1,6 +1,6 @@ import type { Dialect, Resource } from "@dpkit/core" -import { loadResourceDialect } from "@dpkit/core" -import { loadResourceSchema } from "@dpkit/core" +import { resolveDialect } from "@dpkit/core" +import { resolveSchema } from "@dpkit/core" import { loadFile, prefetchFiles } from "@dpkit/file" import type { LoadTableOptions } from "@dpkit/table" import { inferSchemaFromTable, normalizeTable } from "@dpkit/table" @@ -20,7 +20,7 @@ export async function loadJsonTable( throw new Error("Resource path is not defined") } - const dialect = await loadResourceDialect(resource.dialect) + const dialect = await resolveDialect(resource.dialect) const tables: Table[] = [] for (const path of paths) { @@ -43,7 +43,7 @@ export async function loadJsonTable( let table = concat(tables) if (!options?.denormalized) { - let schema = await loadResourceSchema(resource.schema) + let schema = await resolveSchema(resource.schema) if (!schema) schema = await inferSchemaFromTable(table, options) table = await normalizeTable(table, schema) } diff --git a/lib/resource/validate.ts b/lib/resource/validate.ts index ec74b4e4..725863e7 100644 --- a/lib/resource/validate.ts +++ b/lib/resource/validate.ts @@ -1,5 +1,5 @@ import type { Descriptor, Resource } from "@dpkit/core" -import { loadResourceSchema } from "@dpkit/core" +import { resolveSchema } from "@dpkit/core" import { loadDescriptor, validateResourceMetadata } from "@dpkit/core" import { validateFile } from "@dpkit/file" import { validateTable } from "@dpkit/table" @@ -48,7 +48,7 @@ export async function validateResourceData( const table = await loadTable(resource, { denormalized: true }) if (table) { - let schema = await loadResourceSchema(resource.schema) + let schema = await resolveSchema(resource.schema) if (!schema) schema = await inferSchema(resource, options) const tableReport = await validateTable(table, { schema }) diff --git a/lib/table/infer.ts b/lib/table/infer.ts index dc25105e..ad282128 100644 --- a/lib/table/infer.ts +++ b/lib/table/infer.ts @@ -1,11 +1,11 @@ import type { Resource } from "@dpkit/core" -import { loadResourceDialect, loadResourceSchema } from "@dpkit/core" +import { resolveDialect, resolveSchema } from "@dpkit/core" import { inferSchemaFromTable } from "@dpkit/table" import { inferDialect } from "../dialect/index.ts" import { loadTable } from "./load.ts" export async function inferTable(resource: Partial) { - let dialect = await loadResourceDialect(resource.dialect) + let dialect = await resolveDialect(resource.dialect) if (!dialect) { dialect = await inferDialect(resource) } @@ -19,7 +19,7 @@ export async function inferTable(resource: Partial) { return undefined } - let schema = await loadResourceSchema(resource.schema) + let schema = await resolveSchema(resource.schema) if (!schema) { schema = await inferSchemaFromTable(table) } diff --git a/ods/table/load.ts b/ods/table/load.ts index 38ef5a04..4d4461a3 100644 --- a/ods/table/load.ts +++ b/ods/table/load.ts @@ -1,6 +1,6 @@ -import { loadResourceDialect } from "@dpkit/core" +import { resolveDialect } from "@dpkit/core" import type { Resource } from "@dpkit/core" -import { loadResourceSchema } from "@dpkit/core" +import { resolveSchema } from "@dpkit/core" import { loadFile, prefetchFiles } from "@dpkit/file" import type { LoadTableOptions } from "@dpkit/table" import { inferSchemaFromTable, normalizeTable } from "@dpkit/table" @@ -18,7 +18,7 @@ export async function loadOdsTable( throw new Error("Resource path is not defined") } - const dialect = await loadResourceDialect(resource.dialect) + const dialect = await resolveDialect(resource.dialect) const tables: Table[] = [] for (const path of paths) { @@ -45,7 +45,7 @@ export async function loadOdsTable( let table = concat(tables) if (!options?.denormalized) { - let schema = await loadResourceSchema(resource.schema) + let schema = await resolveSchema(resource.schema) if (!schema) schema = await inferSchemaFromTable(table, options) table = await normalizeTable(table, schema) } diff --git a/ods/table/save.ts b/ods/table/save.ts index 162d0c9d..cacaea77 100644 --- a/ods/table/save.ts +++ b/ods/table/save.ts @@ -1,4 +1,4 @@ -import { loadResourceDialect } from "@dpkit/core" +import { resolveDialect } from "@dpkit/core" import { saveFile } from "@dpkit/file" import { denormalizeTable, inferSchemaFromTable } from "@dpkit/table" import type { SaveTableOptions, Table } from "@dpkit/table" @@ -19,7 +19,7 @@ export async function saveOdsTable(table: Table, options: SaveTableOptions) { }) const df = await table.collect() - const dialect = await loadResourceDialect(options.dialect) + const dialect = await resolveDialect(options.dialect) const sheetName = dialect?.sheetName ?? "Sheet1" const sheet = utils.json_to_sheet(df.toRecords()) diff --git a/parquet/table/load.ts b/parquet/table/load.ts index 3c8e0df8..bb83e7bb 100644 --- a/parquet/table/load.ts +++ b/parquet/table/load.ts @@ -1,5 +1,5 @@ import type { Resource } from "@dpkit/core" -import { loadResourceSchema } from "@dpkit/core" +import { resolveSchema } from "@dpkit/core" import { prefetchFiles } from "@dpkit/file" import type { LoadTableOptions } from "@dpkit/table" import { inferSchemaFromTable, normalizeTable } from "@dpkit/table" @@ -21,7 +21,7 @@ export async function loadParquetTable( } if (!options?.denormalized) { - let schema = await loadResourceSchema(resource.schema) + let schema = await resolveSchema(resource.schema) if (!schema) schema = await inferSchemaFromTable(table, options) table = await normalizeTable(table, schema) } diff --git a/xlsx/table/load.ts b/xlsx/table/load.ts index 786515c9..ce59d5f9 100644 --- a/xlsx/table/load.ts +++ b/xlsx/table/load.ts @@ -1,6 +1,6 @@ -import { loadResourceDialect } from "@dpkit/core" +import { resolveDialect } from "@dpkit/core" import type { Resource } from "@dpkit/core" -import { loadResourceSchema } from "@dpkit/core" +import { resolveSchema } from "@dpkit/core" import { loadFile, prefetchFiles } from "@dpkit/file" import type { LoadTableOptions } from "@dpkit/table" import { inferSchemaFromTable, normalizeTable } from "@dpkit/table" @@ -21,7 +21,7 @@ export async function loadXlsxTable( throw new Error("Resource path is not defined") } - const dialect = await loadResourceDialect(resource.dialect) + const dialect = await resolveDialect(resource.dialect) const tables: Table[] = [] for (const path of paths) { @@ -48,7 +48,7 @@ export async function loadXlsxTable( let table = concat(tables) if (!options?.denormalized) { - let schema = await loadResourceSchema(resource.schema) + let schema = await resolveSchema(resource.schema) if (!schema) schema = await inferSchemaFromTable(table, options) table = await normalizeTable(table, schema) } diff --git a/xlsx/table/save.ts b/xlsx/table/save.ts index 849918eb..9c5420d6 100644 --- a/xlsx/table/save.ts +++ b/xlsx/table/save.ts @@ -1,4 +1,4 @@ -import { loadResourceDialect } from "@dpkit/core" +import { resolveDialect } from "@dpkit/core" import { saveFile } from "@dpkit/file" import { denormalizeTable, inferSchemaFromTable } from "@dpkit/table" import type { SaveTableOptions, Table } from "@dpkit/table" @@ -22,7 +22,7 @@ export async function saveXlsxTable(table: Table, options: SaveTableOptions) { }) const df = await table.collect() - const dialect = await loadResourceDialect(options.dialect) + const dialect = await resolveDialect(options.dialect) const sheetName = dialect?.sheetName ?? "Sheet1" const sheet = utils.json_to_sheet(df.toRecords()) From cf9c35bc2f704680129c6168d01417792d341de8 Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 30 Oct 2025 06:20:57 +0000 Subject: [PATCH 32/42] Infer integers from floats --- table/schema/infer.spec.ts | 16 ++++++++++++++++ table/schema/infer.ts | 37 +++++++++++++++++++++++++++---------- 2 files changed, 43 insertions(+), 10 deletions(-) diff --git a/table/schema/infer.spec.ts b/table/schema/infer.spec.ts index 62208283..141081ea 100644 --- a/table/schema/infer.spec.ts +++ b/table/schema/infer.spec.ts @@ -20,6 +20,22 @@ describe("inferSchemaFromTable", () => { expect(await inferSchemaFromTable(table)).toEqual(schema) }) + it("should infer integers from floats", async () => { + const table = DataFrame({ + id: [1.0, 2.0, 3.0], + count: [10.0, 20.0, 30.0], + }).lazy() + + const schema = { + fields: [ + { name: "id", type: "integer" }, + { name: "count", type: "integer" }, + ], + } + + expect(await inferSchemaFromTable(table)).toEqual(schema) + }) + it("should infer numeric", async () => { const table = DataFrame({ name1: ["1", "2", "3"], diff --git a/table/schema/infer.ts b/table/schema/infer.ts index 4c5b54ae..d5b7232c 100644 --- a/table/schema/infer.ts +++ b/table/schema/infer.ts @@ -58,21 +58,38 @@ export function inferSchemaFromSample( variant = variant.slice(0, -1) } - let type = fieldTypes?.[name] ?? typeMapping[variant] ?? "any" + const type = fieldTypes?.[name] ?? typeMapping[variant] ?? "any" + let field = { name, type } - if (type === "array" && options?.arrayType === "list") { - type = "list" - } + if (!fieldTypes?.[name]) { + if (type === "array") { + if (options?.arrayType === "list") { + field.type = "list" + } + } - let field = { name, type } - if (!keepStrings && type === "string" && !fieldTypes?.[name]) { - for (const [regex, patch] of Object.entries(regexMapping)) { + if (type === "string") { + if (!keepStrings) { + for (const [regex, patch] of Object.entries(regexMapping)) { + const failures = sample + .filter(col(name).str.contains(regex).not()) + .head(failureThreshold).height + + if (failures < failureThreshold) { + field = { ...field, ...patch } + break + } + } + } + } + + if (type === "number") { const failures = sample - .filter(col(name).str.contains(regex).not()) + .filter(col(name).eq(col(name).round(0)).not()) .head(failureThreshold).height + if (failures < failureThreshold) { - field = { ...field, ...patch } - break + field.type = "integer" } } } From f381f15de8d2a19cfc2ddea72fc1e0f690b03425 Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 30 Oct 2025 06:31:41 +0000 Subject: [PATCH 33/42] Fixed tests --- lib/schema/infer.spec.ts | 68 ++++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/lib/schema/infer.spec.ts b/lib/schema/infer.spec.ts index 2435c302..b51607aa 100644 --- a/lib/schema/infer.spec.ts +++ b/lib/schema/infer.spec.ts @@ -10,11 +10,11 @@ describe("inferSchema", () => { const schema = await inferSchema(resource) expect(schema).toBeDefined() - expect(schema.fields).toBeDefined() - expect(schema.fields?.length).toBe(3) - expect(schema.fields?.[0]?.name).toBe("id") - expect(schema.fields?.[1]?.name).toBe("name") - expect(schema.fields?.[2]?.name).toBe("age") + expect(schema?.fields).toBeDefined() + expect(schema?.fields?.length).toBe(3) + expect(schema?.fields?.[0]?.name).toBe("id") + expect(schema?.fields?.[1]?.name).toBe("name") + expect(schema?.fields?.[2]?.name).toBe("age") }) it("should infer field types correctly", async () => { @@ -25,9 +25,9 @@ describe("inferSchema", () => { const schema = await inferSchema(resource) - expect(schema.fields?.[0]?.type).toBe("integer") - expect(schema.fields?.[1]?.type).toBe("string") - expect(schema.fields?.[2]?.type).toBe("number") + expect(schema?.fields?.[0]?.type).toBe("integer") + expect(schema?.fields?.[1]?.type).toBe("string") + expect(schema?.fields?.[2]?.type).toBe("number") }) it("should infer schema from inline data", async () => { @@ -42,10 +42,10 @@ describe("inferSchema", () => { const schema = await inferSchema(resource) expect(schema).toBeDefined() - expect(schema.fields).toBeDefined() - expect(schema.fields?.length).toBe(2) - expect(schema.fields?.[0]?.name).toBe("id") - expect(schema.fields?.[1]?.name).toBe("name") + expect(schema?.fields).toBeDefined() + expect(schema?.fields?.length).toBe(2) + expect(schema?.fields?.[0]?.name).toBe("id") + expect(schema?.fields?.[1]?.name).toBe("name") }) it("should infer schema with custom delimiter", async () => { @@ -59,11 +59,11 @@ describe("inferSchema", () => { const schema = await inferSchema(resource) expect(schema).toBeDefined() - expect(schema.fields).toBeDefined() - expect(schema.fields?.length).toBe(3) - expect(schema.fields?.[0]?.name).toBe("id") - expect(schema.fields?.[1]?.name).toBe("name") - expect(schema.fields?.[2]?.name).toBe("value") + expect(schema?.fields).toBeDefined() + expect(schema?.fields?.length).toBe(3) + expect(schema?.fields?.[0]?.name).toBe("id") + expect(schema?.fields?.[1]?.name).toBe("name") + expect(schema?.fields?.[2]?.name).toBe("value") }) it("should handle boolean fields", async () => { @@ -72,7 +72,7 @@ describe("inferSchema", () => { const schema = await inferSchema(resource) - expect(schema.fields?.[1]?.type).toBe("boolean") + expect(schema?.fields?.[1]?.type).toBe("boolean") }) it("should handle date fields", async () => { @@ -83,7 +83,7 @@ describe("inferSchema", () => { const schema = await inferSchema(resource) - expect(schema.fields?.[1]?.type).toBe("date") + expect(schema?.fields?.[1]?.type).toBe("date") }) it("should handle mixed numeric types", async () => { @@ -92,7 +92,7 @@ describe("inferSchema", () => { const schema = await inferSchema(resource) - expect(schema.fields?.[1]?.type).toBe("string") + expect(schema?.fields?.[1]?.type).toBe("string") }) it("should infer schema from single row", async () => { @@ -102,8 +102,8 @@ describe("inferSchema", () => { const schema = await inferSchema(resource) expect(schema).toBeDefined() - expect(schema.fields).toBeDefined() - expect(schema.fields?.length).toBe(2) + expect(schema?.fields).toBeDefined() + expect(schema?.fields?.length).toBe(2) }) it("should handle empty string values", async () => { @@ -115,8 +115,8 @@ describe("inferSchema", () => { const schema = await inferSchema(resource) expect(schema).toBeDefined() - expect(schema.fields?.length).toBe(3) - expect(schema.fields?.[2]?.name).toBe("email") + expect(schema?.fields?.length).toBe(3) + expect(schema?.fields?.[2]?.name).toBe("email") }) it("should infer schema with sampleRows option", async () => { @@ -128,8 +128,8 @@ describe("inferSchema", () => { const schema = await inferSchema(resource, { sampleRows: 2 }) expect(schema).toBeDefined() - expect(schema.fields).toBeDefined() - expect(schema.fields?.length).toBe(2) + expect(schema?.fields).toBeDefined() + expect(schema?.fields?.length).toBe(2) }) it("should handle resources with headers only", async () => { @@ -139,8 +139,8 @@ describe("inferSchema", () => { const schema = await inferSchema(resource) expect(schema).toBeDefined() - expect(schema.fields).toBeDefined() - expect(schema.fields?.length).toBe(3) + expect(schema?.fields).toBeDefined() + expect(schema?.fields?.length).toBe(3) }) it("should infer schema from complex inline data", async () => { @@ -167,11 +167,11 @@ describe("inferSchema", () => { const schema = await inferSchema(resource) expect(schema).toBeDefined() - expect(schema.fields?.length).toBe(5) - expect(schema.fields?.find(f => f.name === "id")?.type).toBe("number") - expect(schema.fields?.find(f => f.name === "name")?.type).toBe("string") - expect(schema.fields?.find(f => f.name === "score")?.type).toBe("number") - expect(schema.fields?.find(f => f.name === "active")?.type).toBe("boolean") - expect(schema.fields?.find(f => f.name === "created")?.type).toBe("date") + expect(schema?.fields?.length).toBe(5) + expect(schema?.fields?.find(f => f.name === "id")?.type).toBe("integer") + expect(schema?.fields?.find(f => f.name === "name")?.type).toBe("string") + expect(schema?.fields?.find(f => f.name === "score")?.type).toBe("number") + expect(schema?.fields?.find(f => f.name === "active")?.type).toBe("boolean") + expect(schema?.fields?.find(f => f.name === "created")?.type).toBe("date") }) }) From c33470990d5d607e6b1ffb0693bbe7deca296fba Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 30 Oct 2025 07:01:01 +0000 Subject: [PATCH 34/42] Full minimum support --- table/field/checks/minimum.spec.ts | 63 ++++++++++++++++++++++++++++++ table/field/checks/minimum.ts | 26 ++++++++---- table/helpers.ts | 11 ++++++ table/table/validate.ts | 5 +-- 4 files changed, 93 insertions(+), 12 deletions(-) create mode 100644 table/helpers.ts diff --git a/table/field/checks/minimum.spec.ts b/table/field/checks/minimum.spec.ts index 487cbd15..1a440eb1 100644 --- a/table/field/checks/minimum.spec.ts +++ b/table/field/checks/minimum.spec.ts @@ -83,4 +83,67 @@ describe("validateTable (cell/minimum)", () => { cell: "5.5", }) }) + + it("should handle minimum as string", async () => { + const table = DataFrame({ + price: [10.5, 20.75, 3.0], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "price", + type: "number", + constraints: { minimum: "5" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/minimum", + fieldName: "price", + minimum: "5", + rowNumber: 3, + cell: "3", + }, + ]) + }) + + it("should handle exclusiveMinimum as string", async () => { + const table = DataFrame({ + temperature: [20.5, 10.0, 5.5], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "temperature", + type: "number", + constraints: { exclusiveMinimum: "10" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/exclusiveMinimum", + fieldName: "temperature", + minimum: "10", + rowNumber: 2, + cell: "10", + }, + { + type: "cell/exclusiveMinimum", + fieldName: "temperature", + minimum: "10", + rowNumber: 3, + cell: "5.5", + }, + ]) + }) }) diff --git a/table/field/checks/minimum.ts b/table/field/checks/minimum.ts index da7fae2a..451262ea 100644 --- a/table/field/checks/minimum.ts +++ b/table/field/checks/minimum.ts @@ -1,9 +1,12 @@ import type { Field } from "@dpkit/core" -import { lit } from "nodejs-polars" +import * as pl from "nodejs-polars" import type { Expr } from "nodejs-polars" import type { CellExclusiveMinimumError } from "../../error/index.ts" import type { CellMinimumError } from "../../error/index.ts" +import { evaluateExpression } from "../../helpers.ts" import type { CellMapping } from "../Mapping.ts" +import { parseIntegerField } from "../types/integer.ts" +import { parseNumberField } from "../types/number.ts" export function createCheckCellMinimum(options?: { isExclusive?: boolean }) { return (field: Field, mapping: CellMapping) => { @@ -15,18 +18,13 @@ export function createCheckCellMinimum(options?: { isExclusive?: boolean }) { if (minimum === undefined) return undefined let isErrorExpr: Expr - const parser = - field.type === "integer" ? Number.parseInt : Number.parseFloat - try { - const parsedMinimum = - typeof minimum === "string" ? parser(minimum) : minimum - + const parsedMinimum = parseConstraint(field, minimum) isErrorExpr = options?.isExclusive ? mapping.target.ltEq(parsedMinimum) : mapping.target.lt(parsedMinimum) } catch (error) { - isErrorExpr = lit(true) + isErrorExpr = pl.lit(true) } const errorTemplate: CellMinimumError | CellExclusiveMinimumError = { @@ -40,3 +38,15 @@ export function createCheckCellMinimum(options?: { isExclusive?: boolean }) { return { isErrorExpr, errorTemplate } } } + +function parseConstraint(field: Field, constraint: number | string) { + let expr = pl.lit(constraint) + + if (field.type === "integer") { + expr = parseIntegerField(field, expr) + } else if (field.type === "number") { + expr = parseNumberField(field, expr) + } + + return evaluateExpression(expr) +} diff --git a/table/helpers.ts b/table/helpers.ts new file mode 100644 index 00000000..032b745c --- /dev/null +++ b/table/helpers.ts @@ -0,0 +1,11 @@ +import type { Expr } from "nodejs-polars" +import * as pl from "nodejs-polars" + +export function arrayDiff(a: string[], b: string[]) { + return a.filter(x => !b.includes(x)) +} + +export function evaluateExpression(expr: Expr) { + // @ts-ignore + return pl.select(expr.alias("value")).toRecords()[0].value +} diff --git a/table/table/validate.ts b/table/table/validate.ts index fdd6634d..8ec28107 100644 --- a/table/table/validate.ts +++ b/table/table/validate.ts @@ -5,6 +5,7 @@ import pAll from "p-all" import type { RowError } from "../error/index.ts" import type { TableError } from "../error/index.ts" import { validateField } from "../field/index.ts" +import { arrayDiff } from "../helpers.ts" import { matchSchemaField } from "../schema/index.ts" import { getPolarsSchema } from "../schema/index.ts" import type { SchemaMapping } from "../schema/index.ts" @@ -221,7 +222,3 @@ async function validateRows( return errors } - -function arrayDiff(a: string[], b: string[]) { - return a.filter(x => !b.includes(x)) -} From 8456e05127bdf216a69a21411764f9a7ccc9e595 Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 30 Oct 2025 07:08:18 +0000 Subject: [PATCH 35/42] Full minimum support --- table/field/checks/maximum.spec.ts | 180 +++++++++++++++++++++++++++++ table/field/checks/maximum.ts | 26 +++-- table/field/checks/minimum.spec.ts | 117 +++++++++++++++++++ 3 files changed, 315 insertions(+), 8 deletions(-) diff --git a/table/field/checks/maximum.spec.ts b/table/field/checks/maximum.spec.ts index 55e9cffa..f0cd96b5 100644 --- a/table/field/checks/maximum.spec.ts +++ b/table/field/checks/maximum.spec.ts @@ -83,4 +83,184 @@ describe("validateTable (cell/maximum)", () => { cell: "50.5", }) }) + + it("should handle maximum as string", async () => { + const table = DataFrame({ + price: [10.5, 20.75, 55.0], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "price", + type: "number", + constraints: { maximum: "50" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/maximum", + fieldName: "price", + maximum: "50", + rowNumber: 3, + cell: "55", + }, + ]) + }) + + it("should handle exclusiveMaximum as string", async () => { + const table = DataFrame({ + temperature: [20.5, 40.0, 50.5], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "temperature", + type: "number", + constraints: { exclusiveMaximum: "40" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/exclusiveMaximum", + fieldName: "temperature", + maximum: "40", + rowNumber: 2, + cell: "40", + }, + { + type: "cell/exclusiveMaximum", + fieldName: "temperature", + maximum: "40", + rowNumber: 3, + cell: "50.5", + }, + ]) + }) + + it("should handle maximum as string with groupChar", async () => { + const table = DataFrame({ + price: ["5,000", "10,500", "15,000"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "price", + type: "integer", + groupChar: ",", + constraints: { maximum: "12,000" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/maximum", + fieldName: "price", + maximum: "12,000", + rowNumber: 3, + cell: "15,000", + }, + ]) + }) + + it("should handle maximum as string with decimalChar", async () => { + const table = DataFrame({ + price: ["5,5", "10,75", "15,3"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "price", + type: "number", + decimalChar: ",", + constraints: { maximum: "12,0" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/maximum", + fieldName: "price", + maximum: "12,0", + rowNumber: 3, + cell: "15,3", + }, + ]) + }) + + it("should handle maximum as string with groupChar and decimalChar", async () => { + const table = DataFrame({ + price: ["5.000,50", "10.500,75", "15.000,30"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "price", + type: "number", + groupChar: ".", + decimalChar: ",", + constraints: { maximum: "12.000,00" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/maximum", + fieldName: "price", + maximum: "12.000,00", + rowNumber: 3, + cell: "15.000,30", + }, + ]) + }) + + it("should handle maximum as string with bareNumber false", async () => { + const table = DataFrame({ + price: ["$5.00", "$10.50", "$15.50"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "price", + type: "number", + bareNumber: false, + constraints: { maximum: "$12.00" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/maximum", + fieldName: "price", + maximum: "$12.00", + rowNumber: 3, + cell: "$15.50", + }, + ]) + }) }) diff --git a/table/field/checks/maximum.ts b/table/field/checks/maximum.ts index 7a70dd28..6a330f48 100644 --- a/table/field/checks/maximum.ts +++ b/table/field/checks/maximum.ts @@ -1,9 +1,12 @@ import type { Field } from "@dpkit/core" -import { lit } from "nodejs-polars" +import * as pl from "nodejs-polars" import type { Expr } from "nodejs-polars" import type { CellExclusiveMaximumError } from "../../error/index.ts" import type { CellMaximumError } from "../../error/index.ts" +import { evaluateExpression } from "../../helpers.ts" import type { CellMapping } from "../Mapping.ts" +import { parseIntegerField } from "../types/integer.ts" +import { parseNumberField } from "../types/number.ts" export function createCheckCellMaximum(options?: { isExclusive?: boolean }) { return (field: Field, mapping: CellMapping) => { @@ -15,18 +18,13 @@ export function createCheckCellMaximum(options?: { isExclusive?: boolean }) { if (maximum === undefined) return undefined let isErrorExpr: Expr - const parser = - field.type === "integer" ? Number.parseInt : Number.parseFloat - try { - const parsedMaximum = - typeof maximum === "string" ? parser(maximum) : maximum - + const parsedMaximum = parseConstraint(field, maximum) isErrorExpr = options?.isExclusive ? mapping.target.gtEq(parsedMaximum) : mapping.target.gt(parsedMaximum) } catch (error) { - isErrorExpr = lit(true) + isErrorExpr = pl.lit(true) } const errorTemplate: CellMaximumError | CellExclusiveMaximumError = { @@ -40,3 +38,15 @@ export function createCheckCellMaximum(options?: { isExclusive?: boolean }) { return { isErrorExpr, errorTemplate } } } + +function parseConstraint(field: Field, constraint: number | string) { + let expr = pl.lit(constraint) + + if (field.type === "integer") { + expr = parseIntegerField(field, expr) + } else if (field.type === "number") { + expr = parseNumberField(field, expr) + } + + return evaluateExpression(expr) +} diff --git a/table/field/checks/minimum.spec.ts b/table/field/checks/minimum.spec.ts index 1a440eb1..646096bd 100644 --- a/table/field/checks/minimum.spec.ts +++ b/table/field/checks/minimum.spec.ts @@ -146,4 +146,121 @@ describe("validateTable (cell/minimum)", () => { }, ]) }) + + it("should handle minimum as string with groupChar", async () => { + const table = DataFrame({ + price: ["5,000", "10,500", "2,500"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "price", + type: "integer", + groupChar: ",", + constraints: { minimum: "3,000" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/minimum", + fieldName: "price", + minimum: "3,000", + rowNumber: 3, + cell: "2,500", + }, + ]) + }) + + it("should handle minimum as string with decimalChar", async () => { + const table = DataFrame({ + price: ["5,5", "10,75", "2,3"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "price", + type: "number", + decimalChar: ",", + constraints: { minimum: "3,0" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/minimum", + fieldName: "price", + minimum: "3,0", + rowNumber: 3, + cell: "2,3", + }, + ]) + }) + + it("should handle minimum as string with groupChar and decimalChar", async () => { + const table = DataFrame({ + price: ["5.000,50", "10.500,75", "2.500,30"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "price", + type: "number", + groupChar: ".", + decimalChar: ",", + constraints: { minimum: "3.000,00" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/minimum", + fieldName: "price", + minimum: "3.000,00", + rowNumber: 3, + cell: "2.500,30", + }, + ]) + }) + + it("should handle minimum as string with bareNumber false", async () => { + const table = DataFrame({ + price: ["$5.00", "$10.50", "$2.50"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "price", + type: "number", + bareNumber: false, + constraints: { minimum: "$3.00" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/minimum", + fieldName: "price", + minimum: "$3.00", + rowNumber: 3, + cell: "$2.50", + }, + ]) + }) }) From 5f9a0159a32f14c40925f711c60796cf4000c994 Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 30 Oct 2025 07:11:51 +0000 Subject: [PATCH 36/42] Fixed maxLength and minLength error types --- table/error/Cell.ts | 4 ++-- table/field/checks/maxLength.spec.ts | 2 +- table/field/checks/maxLength.ts | 2 +- table/field/checks/minLength.spec.ts | 4 ++-- table/field/checks/minLength.ts | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/table/error/Cell.ts b/table/error/Cell.ts index d5ef6b56..09bc7412 100644 --- a/table/error/Cell.ts +++ b/table/error/Cell.ts @@ -52,12 +52,12 @@ export interface CellExclusiveMaximumError extends BaseCellError { export interface CellMinLengthError extends BaseCellError { type: "cell/minLength" - minLength: string + minLength: number } export interface CellMaxLengthError extends BaseCellError { type: "cell/maxLength" - maxLength: string + maxLength: number } export interface CellPatternError extends BaseCellError { diff --git a/table/field/checks/maxLength.spec.ts b/table/field/checks/maxLength.spec.ts index 95cc6796..1916d13c 100644 --- a/table/field/checks/maxLength.spec.ts +++ b/table/field/checks/maxLength.spec.ts @@ -43,7 +43,7 @@ describe("validateTable (cell/maxLength)", () => { expect(errors).toContainEqual({ type: "cell/maxLength", fieldName: "username", - maxLength: "8", + maxLength: 8, rowNumber: 3, cell: "christopher", }) diff --git a/table/field/checks/maxLength.ts b/table/field/checks/maxLength.ts index d7ad2716..84306b21 100644 --- a/table/field/checks/maxLength.ts +++ b/table/field/checks/maxLength.ts @@ -13,7 +13,7 @@ export function checkCellMaxLength(field: Field, mapping: CellMapping) { const errorTemplate: CellMaxLengthError = { type: "cell/maxLength", fieldName: field.name, - maxLength: String(maxLength), + maxLength: maxLength, rowNumber: 0, cell: "", } diff --git a/table/field/checks/minLength.spec.ts b/table/field/checks/minLength.spec.ts index 97c25369..d4ac46f3 100644 --- a/table/field/checks/minLength.spec.ts +++ b/table/field/checks/minLength.spec.ts @@ -43,14 +43,14 @@ describe("validateTable (cell/minLength)", () => { expect(errors).toContainEqual({ type: "cell/minLength", fieldName: "username", - minLength: "3", + minLength: 3, rowNumber: 2, cell: "a", }) expect(errors).toContainEqual({ type: "cell/minLength", fieldName: "username", - minLength: "3", + minLength: 3, rowNumber: 4, cell: "ab", }) diff --git a/table/field/checks/minLength.ts b/table/field/checks/minLength.ts index 733717f0..b9bfce53 100644 --- a/table/field/checks/minLength.ts +++ b/table/field/checks/minLength.ts @@ -13,7 +13,7 @@ export function checkCellMinLength(field: Field, mapping: CellMapping) { const errorTemplate: CellMinLengthError = { type: "cell/minLength", fieldName: field.name, - minLength: String(minLength), + minLength: minLength, rowNumber: 0, cell: "", } From 3f701287365492981b399ea83f6893a27965ceb5 Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 30 Oct 2025 07:26:46 +0000 Subject: [PATCH 37/42] Suport date/time in minimum/maximum checks --- core/field/types/Date.ts | 10 +++ core/field/types/Datetime.ts | 10 +++ core/field/types/Time.ts | 10 +++ table/field/checks/maximum.spec.ts | 117 +++++++++++++++++++++++++++++ table/field/checks/maximum.ts | 19 ++++- table/field/checks/minimum.spec.ts | 117 +++++++++++++++++++++++++++++ table/field/checks/minimum.ts | 19 ++++- table/field/types/time.ts | 5 +- 8 files changed, 303 insertions(+), 4 deletions(-) diff --git a/core/field/types/Date.ts b/core/field/types/Date.ts index 99438926..b30630e4 100644 --- a/core/field/types/Date.ts +++ b/core/field/types/Date.ts @@ -32,6 +32,16 @@ export interface DateConstraints extends BaseConstraints { */ maximum?: string + /** + * Exclusive minimum date value + */ + exclusiveMinimum?: string + + /** + * Exclusive maximum date value + */ + exclusiveMaximum?: string + /** * Restrict values to a specified set of dates * Should be in string date format (e.g., "YYYY-MM-DD") diff --git a/core/field/types/Datetime.ts b/core/field/types/Datetime.ts index af37faf2..5e50fbd6 100644 --- a/core/field/types/Datetime.ts +++ b/core/field/types/Datetime.ts @@ -32,6 +32,16 @@ export interface DatetimeConstraints extends BaseConstraints { */ maximum?: string + /** + * Exclusive minimum datetime value + */ + exclusiveMinimum?: string + + /** + * Exclusive maximum datetime value + */ + exclusiveMaximum?: string + /** * Restrict values to a specified set of datetimes * Should be in string datetime format (e.g., ISO8601) diff --git a/core/field/types/Time.ts b/core/field/types/Time.ts index 8b9128ea..b739d844 100644 --- a/core/field/types/Time.ts +++ b/core/field/types/Time.ts @@ -32,6 +32,16 @@ export interface TimeConstraints extends BaseConstraints { */ maximum?: string + /** + * Exclusive minimum time value + */ + exclusiveMinimum?: string + + /** + * Exclusive maximum time value + */ + exclusiveMaximum?: string + /** * Restrict values to a specified set of times * Should be in string time format (e.g., "HH:MM:SS") diff --git a/table/field/checks/maximum.spec.ts b/table/field/checks/maximum.spec.ts index f0cd96b5..409d4808 100644 --- a/table/field/checks/maximum.spec.ts +++ b/table/field/checks/maximum.spec.ts @@ -263,4 +263,121 @@ describe("validateTable (cell/maximum)", () => { }, ]) }) + + it("should handle maximum for date fields", async () => { + const table = DataFrame({ + date: ["2024-01-15", "2024-02-20", "2024-03-25"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "date", + type: "date", + constraints: { maximum: "2024-02-28" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/maximum", + fieldName: "date", + maximum: "2024-02-28", + rowNumber: 3, + cell: "2024-03-25", + }, + ]) + }) + + it.skip("should handle maximum for time fields", async () => { + const table = DataFrame({ + time: ["14:30:00", "16:45:00", "18:00:00"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "time", + type: "time", + constraints: { maximum: "17:00:00" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/maximum", + fieldName: "time", + maximum: "17:00:00", + rowNumber: 3, + cell: "18:00:00", + }, + ]) + }) + + it("should handle maximum for datetime fields", async () => { + const table = DataFrame({ + timestamp: [ + "2024-01-15T14:30:00", + "2024-02-20T08:15:00", + "2024-03-25T10:00:00", + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "timestamp", + type: "datetime", + constraints: { maximum: "2024-02-28T23:59:59" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/maximum", + fieldName: "timestamp", + maximum: "2024-02-28T23:59:59", + rowNumber: 3, + cell: "2024-03-25T10:00:00", + }, + ]) + }) + + it("should handle maximum for date fields with custom format", async () => { + const table = DataFrame({ + date: ["15/01/2024", "20/02/2024", "25/03/2024"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "date", + type: "date", + format: "%d/%m/%Y", + constraints: { maximum: "28/02/2024" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/maximum", + fieldName: "date", + maximum: "28/02/2024", + rowNumber: 3, + cell: "25/03/2024", + }, + ]) + }) }) diff --git a/table/field/checks/maximum.ts b/table/field/checks/maximum.ts index 6a330f48..16db66be 100644 --- a/table/field/checks/maximum.ts +++ b/table/field/checks/maximum.ts @@ -5,12 +5,23 @@ import type { CellExclusiveMaximumError } from "../../error/index.ts" import type { CellMaximumError } from "../../error/index.ts" import { evaluateExpression } from "../../helpers.ts" import type { CellMapping } from "../Mapping.ts" +import { parseDateField } from "../types/date.ts" +import { parseDatetimeField } from "../types/datetime.ts" import { parseIntegerField } from "../types/integer.ts" import { parseNumberField } from "../types/number.ts" +import { parseTimeField } from "../types/time.ts" export function createCheckCellMaximum(options?: { isExclusive?: boolean }) { return (field: Field, mapping: CellMapping) => { - if (field.type !== "integer" && field.type !== "number") return undefined + if ( + field.type !== "integer" && + field.type !== "number" && + field.type !== "date" && + field.type !== "time" && + field.type !== "datetime" + ) { + return undefined + } const maximum = options?.isExclusive ? field.constraints?.exclusiveMaximum @@ -46,6 +57,12 @@ function parseConstraint(field: Field, constraint: number | string) { expr = parseIntegerField(field, expr) } else if (field.type === "number") { expr = parseNumberField(field, expr) + } else if (field.type === "date") { + expr = parseDateField(field, expr) + } else if (field.type === "time") { + expr = parseTimeField(field, expr) + } else if (field.type === "datetime") { + expr = parseDatetimeField(field, expr) } return evaluateExpression(expr) diff --git a/table/field/checks/minimum.spec.ts b/table/field/checks/minimum.spec.ts index 646096bd..33cce8a5 100644 --- a/table/field/checks/minimum.spec.ts +++ b/table/field/checks/minimum.spec.ts @@ -263,4 +263,121 @@ describe("validateTable (cell/minimum)", () => { }, ]) }) + + it("should handle minimum for date fields", async () => { + const table = DataFrame({ + date: ["2024-01-15", "2024-02-20", "2024-01-05"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "date", + type: "date", + constraints: { minimum: "2024-01-10" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/minimum", + fieldName: "date", + minimum: "2024-01-10", + rowNumber: 3, + cell: "2024-01-05", + }, + ]) + }) + + it.skip("should handle minimum for time fields", async () => { + const table = DataFrame({ + time: ["14:30:00", "16:45:00", "12:15:00"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "time", + type: "time", + constraints: { minimum: "13:00:00" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/minimum", + fieldName: "time", + minimum: "13:00:00", + rowNumber: 3, + cell: "12:15:00", + }, + ]) + }) + + it("should handle minimum for datetime fields", async () => { + const table = DataFrame({ + timestamp: [ + "2024-01-15T14:30:00", + "2024-02-20T08:15:00", + "2024-01-10T10:00:00", + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "timestamp", + type: "datetime", + constraints: { minimum: "2024-01-15T00:00:00" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/minimum", + fieldName: "timestamp", + minimum: "2024-01-15T00:00:00", + rowNumber: 3, + cell: "2024-01-10T10:00:00", + }, + ]) + }) + + it("should handle minimum for date fields with custom format", async () => { + const table = DataFrame({ + date: ["15/01/2024", "20/02/2024", "05/01/2024"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "date", + type: "date", + format: "%d/%m/%Y", + constraints: { minimum: "10/01/2024" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/minimum", + fieldName: "date", + minimum: "10/01/2024", + rowNumber: 3, + cell: "05/01/2024", + }, + ]) + }) }) diff --git a/table/field/checks/minimum.ts b/table/field/checks/minimum.ts index 451262ea..2860c0db 100644 --- a/table/field/checks/minimum.ts +++ b/table/field/checks/minimum.ts @@ -5,12 +5,23 @@ import type { CellExclusiveMinimumError } from "../../error/index.ts" import type { CellMinimumError } from "../../error/index.ts" import { evaluateExpression } from "../../helpers.ts" import type { CellMapping } from "../Mapping.ts" +import { parseDateField } from "../types/date.ts" +import { parseDatetimeField } from "../types/datetime.ts" import { parseIntegerField } from "../types/integer.ts" import { parseNumberField } from "../types/number.ts" +import { parseTimeField } from "../types/time.ts" export function createCheckCellMinimum(options?: { isExclusive?: boolean }) { return (field: Field, mapping: CellMapping) => { - if (field.type !== "integer" && field.type !== "number") return undefined + if ( + field.type !== "integer" && + field.type !== "number" && + field.type !== "date" && + field.type !== "time" && + field.type !== "datetime" + ) { + return undefined + } const minimum = options?.isExclusive ? field.constraints?.exclusiveMinimum @@ -46,6 +57,12 @@ function parseConstraint(field: Field, constraint: number | string) { expr = parseIntegerField(field, expr) } else if (field.type === "number") { expr = parseNumberField(field, expr) + } else if (field.type === "date") { + expr = parseDateField(field, expr) + } else if (field.type === "time") { + expr = parseTimeField(field, expr) + } else if (field.type === "datetime") { + expr = parseDatetimeField(field, expr) } return evaluateExpression(expr) diff --git a/table/field/types/time.ts b/table/field/types/time.ts index 578d2507..23ad2370 100644 --- a/table/field/types/time.ts +++ b/table/field/types/time.ts @@ -1,6 +1,6 @@ import type { TimeField } from "@dpkit/core" import { DataType } from "nodejs-polars" -import { concatString, lit } from "nodejs-polars" +import * as pl from "nodejs-polars" import type { Expr } from "nodejs-polars" const DEFAULT_FORMAT = "%H:%M:%S" @@ -11,7 +11,8 @@ export function parseTimeField(field: TimeField, fieldExpr: Expr) { format = field.format } - return concatString([lit("1970-01-01T"), fieldExpr], "") + return pl + .concatString([pl.lit("1970-01-01T"), fieldExpr], "") .str.strptime(DataType.Datetime, `%Y-%m-%dT${format}`) .cast(DataType.Time) .alias(field.name) From c1f7cd41eb5ecf70ae3e8ef79f6ebdbb5b3e8753 Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 30 Oct 2025 07:55:02 +0000 Subject: [PATCH 38/42] Support year min/max --- core/field/types/Year.ts | 14 ++++++- table/field/checks/maximum.spec.ts | 63 ++++++++++++++++++++++++++++++ table/field/checks/maximum.ts | 6 ++- table/field/checks/minimum.spec.ts | 63 ++++++++++++++++++++++++++++++ table/field/checks/minimum.ts | 6 ++- 5 files changed, 148 insertions(+), 4 deletions(-) diff --git a/core/field/types/Year.ts b/core/field/types/Year.ts index ed664d5b..94dda6df 100644 --- a/core/field/types/Year.ts +++ b/core/field/types/Year.ts @@ -17,12 +17,22 @@ export interface YearConstraints extends BaseConstraints { /** * Minimum allowed year */ - minimum?: number + minimum?: number | string /** * Maximum allowed year */ - maximum?: number + maximum?: number | string + + /** + * Exclusive minimum year value + */ + exclusiveMinimum?: number | string + + /** + * Exclusive maximum year value + */ + exclusiveMaximum?: number | string /** * Restrict values to a specified set of years diff --git a/table/field/checks/maximum.spec.ts b/table/field/checks/maximum.spec.ts index 409d4808..2878b65b 100644 --- a/table/field/checks/maximum.spec.ts +++ b/table/field/checks/maximum.spec.ts @@ -380,4 +380,67 @@ describe("validateTable (cell/maximum)", () => { }, ]) }) + + it("should handle maximum for year fields", async () => { + const table = DataFrame({ + year: ["2020", "2021", "2023"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "year", + type: "year", + constraints: { maximum: "2022" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/maximum", + fieldName: "year", + maximum: "2022", + rowNumber: 3, + cell: "2023", + }, + ]) + }) + + it("should handle exclusiveMaximum for year fields", async () => { + const table = DataFrame({ + year: ["2020", "2021", "2022", "2023"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "year", + type: "year", + constraints: { exclusiveMaximum: "2022" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/exclusiveMaximum", + fieldName: "year", + maximum: "2022", + rowNumber: 3, + cell: "2022", + }, + { + type: "cell/exclusiveMaximum", + fieldName: "year", + maximum: "2022", + rowNumber: 4, + cell: "2023", + }, + ]) + }) }) diff --git a/table/field/checks/maximum.ts b/table/field/checks/maximum.ts index 16db66be..891d8ef8 100644 --- a/table/field/checks/maximum.ts +++ b/table/field/checks/maximum.ts @@ -10,6 +10,7 @@ import { parseDatetimeField } from "../types/datetime.ts" import { parseIntegerField } from "../types/integer.ts" import { parseNumberField } from "../types/number.ts" import { parseTimeField } from "../types/time.ts" +import { parseYearField } from "../types/year.ts" export function createCheckCellMaximum(options?: { isExclusive?: boolean }) { return (field: Field, mapping: CellMapping) => { @@ -18,7 +19,8 @@ export function createCheckCellMaximum(options?: { isExclusive?: boolean }) { field.type !== "number" && field.type !== "date" && field.type !== "time" && - field.type !== "datetime" + field.type !== "datetime" && + field.type !== "year" ) { return undefined } @@ -63,6 +65,8 @@ function parseConstraint(field: Field, constraint: number | string) { expr = parseTimeField(field, expr) } else if (field.type === "datetime") { expr = parseDatetimeField(field, expr) + } else if (field.type === "year") { + expr = parseYearField(field, expr) } return evaluateExpression(expr) diff --git a/table/field/checks/minimum.spec.ts b/table/field/checks/minimum.spec.ts index 33cce8a5..477e4fb5 100644 --- a/table/field/checks/minimum.spec.ts +++ b/table/field/checks/minimum.spec.ts @@ -380,4 +380,67 @@ describe("validateTable (cell/minimum)", () => { }, ]) }) + + it("should handle minimum for year fields", async () => { + const table = DataFrame({ + year: ["2020", "2021", "2018"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "year", + type: "year", + constraints: { minimum: "2019" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/minimum", + fieldName: "year", + minimum: "2019", + rowNumber: 3, + cell: "2018", + }, + ]) + }) + + it("should handle exclusiveMinimum for year fields", async () => { + const table = DataFrame({ + year: ["2020", "2021", "2019", "2018"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "year", + type: "year", + constraints: { exclusiveMinimum: "2019" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/exclusiveMinimum", + fieldName: "year", + minimum: "2019", + rowNumber: 3, + cell: "2019", + }, + { + type: "cell/exclusiveMinimum", + fieldName: "year", + minimum: "2019", + rowNumber: 4, + cell: "2018", + }, + ]) + }) }) diff --git a/table/field/checks/minimum.ts b/table/field/checks/minimum.ts index 2860c0db..1da2cf3d 100644 --- a/table/field/checks/minimum.ts +++ b/table/field/checks/minimum.ts @@ -10,6 +10,7 @@ import { parseDatetimeField } from "../types/datetime.ts" import { parseIntegerField } from "../types/integer.ts" import { parseNumberField } from "../types/number.ts" import { parseTimeField } from "../types/time.ts" +import { parseYearField } from "../types/year.ts" export function createCheckCellMinimum(options?: { isExclusive?: boolean }) { return (field: Field, mapping: CellMapping) => { @@ -18,7 +19,8 @@ export function createCheckCellMinimum(options?: { isExclusive?: boolean }) { field.type !== "number" && field.type !== "date" && field.type !== "time" && - field.type !== "datetime" + field.type !== "datetime" && + field.type !== "year" ) { return undefined } @@ -63,6 +65,8 @@ function parseConstraint(field: Field, constraint: number | string) { expr = parseTimeField(field, expr) } else if (field.type === "datetime") { expr = parseDatetimeField(field, expr) + } else if (field.type === "year") { + expr = parseYearField(field, expr) } return evaluateExpression(expr) From d9bce967e36df52669133faf3af5d9c810916c25 Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 30 Oct 2025 08:05:46 +0000 Subject: [PATCH 39/42] Removed TODO --- table/field/types/yearmonth.ts | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/table/field/types/yearmonth.ts b/table/field/types/yearmonth.ts index 4146a76a..d9644723 100644 --- a/table/field/types/yearmonth.ts +++ b/table/field/types/yearmonth.ts @@ -2,13 +2,6 @@ import type { YearmonthField } from "@dpkit/core" import { DataType, concatString } from "nodejs-polars" import type { Expr } from "nodejs-polars" -// TODO: -// Add more validation: -// - Check the length of the list is 2 (no list.lenghts in polars currently) -// - Check the values are year and month limits -// - Return null instead of list if any of the values are out of range -// - Rebase on Struct when lst.toStruct() is available? - export function parseYearmonthField(_field: YearmonthField, fieldExpr: Expr) { fieldExpr = fieldExpr.str.split("-").cast(DataType.List(DataType.Int16)) @@ -19,20 +12,10 @@ export function stringifyYearmonthField( field: YearmonthField, fieldExpr: Expr, ) { - // TODO: remove int casting when resolved: - // https://github.com/pola-rs/nodejs-polars/issues/362 return concatString( [ - fieldExpr.lst - .get(0) - .cast(DataType.Int16) - .cast(DataType.String) - .str.zFill(4), - fieldExpr.lst - .get(1) - .cast(DataType.Int16) - .cast(DataType.String) - .str.zFill(2), + fieldExpr.lst.get(0).cast(DataType.String).str.zFill(4), + fieldExpr.lst.get(1).cast(DataType.String).str.zFill(2), ], "-", ).alias(field.name) as Expr From 39b7c8d67f9a43e7f4f9e07d4e65c3869b2879e4 Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 30 Oct 2025 08:09:32 +0000 Subject: [PATCH 40/42] Bootstrapped yearmonth min/max support --- core/field/types/Yearmonth.ts | 10 +++++ table/field/checks/maximum.spec.ts | 63 ++++++++++++++++++++++++++++++ table/field/checks/maximum.ts | 6 ++- table/field/checks/minimum.spec.ts | 63 ++++++++++++++++++++++++++++++ table/field/checks/minimum.ts | 6 ++- 5 files changed, 146 insertions(+), 2 deletions(-) diff --git a/core/field/types/Yearmonth.ts b/core/field/types/Yearmonth.ts index 6e06c9c6..12fba903 100644 --- a/core/field/types/Yearmonth.ts +++ b/core/field/types/Yearmonth.ts @@ -24,6 +24,16 @@ export interface YearmonthConstraints extends BaseConstraints { */ maximum?: string + /** + * Exclusive minimum yearmonth value + */ + exclusiveMinimum?: string + + /** + * Exclusive maximum yearmonth value + */ + exclusiveMaximum?: string + /** * Restrict values to a specified set of yearmonths * Should be in string format (e.g., "YYYY-MM") diff --git a/table/field/checks/maximum.spec.ts b/table/field/checks/maximum.spec.ts index 2878b65b..3929b184 100644 --- a/table/field/checks/maximum.spec.ts +++ b/table/field/checks/maximum.spec.ts @@ -443,4 +443,67 @@ describe("validateTable (cell/maximum)", () => { }, ]) }) + + it.skip("should handle maximum for yearmonth fields", async () => { + const table = DataFrame({ + yearmonth: ["2024-01", "2024-03", "2024-06"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "yearmonth", + type: "yearmonth", + constraints: { maximum: "2024-05" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/maximum", + fieldName: "yearmonth", + maximum: "2024-05", + rowNumber: 3, + cell: "2024-06", + }, + ]) + }) + + it.skip("should handle exclusiveMaximum for yearmonth fields", async () => { + const table = DataFrame({ + yearmonth: ["2024-01", "2024-03", "2024-05", "2024-06"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "yearmonth", + type: "yearmonth", + constraints: { exclusiveMaximum: "2024-05" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/exclusiveMaximum", + fieldName: "yearmonth", + maximum: "2024-05", + rowNumber: 3, + cell: "2024-05", + }, + { + type: "cell/exclusiveMaximum", + fieldName: "yearmonth", + maximum: "2024-05", + rowNumber: 4, + cell: "2024-06", + }, + ]) + }) }) diff --git a/table/field/checks/maximum.ts b/table/field/checks/maximum.ts index 891d8ef8..830fbf6d 100644 --- a/table/field/checks/maximum.ts +++ b/table/field/checks/maximum.ts @@ -11,6 +11,7 @@ import { parseIntegerField } from "../types/integer.ts" import { parseNumberField } from "../types/number.ts" import { parseTimeField } from "../types/time.ts" import { parseYearField } from "../types/year.ts" +import { parseYearmonthField } from "../types/yearmonth.ts" export function createCheckCellMaximum(options?: { isExclusive?: boolean }) { return (field: Field, mapping: CellMapping) => { @@ -20,7 +21,8 @@ export function createCheckCellMaximum(options?: { isExclusive?: boolean }) { field.type !== "date" && field.type !== "time" && field.type !== "datetime" && - field.type !== "year" + field.type !== "year" && + field.type !== "yearmonth" ) { return undefined } @@ -67,6 +69,8 @@ function parseConstraint(field: Field, constraint: number | string) { expr = parseDatetimeField(field, expr) } else if (field.type === "year") { expr = parseYearField(field, expr) + } else if (field.type === "yearmonth") { + expr = parseYearmonthField(field, expr) } return evaluateExpression(expr) diff --git a/table/field/checks/minimum.spec.ts b/table/field/checks/minimum.spec.ts index 477e4fb5..9ce9b35a 100644 --- a/table/field/checks/minimum.spec.ts +++ b/table/field/checks/minimum.spec.ts @@ -443,4 +443,67 @@ describe("validateTable (cell/minimum)", () => { }, ]) }) + + it.skip("should handle minimum for yearmonth fields", async () => { + const table = DataFrame({ + yearmonth: ["2024-03", "2024-05", "2024-01"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "yearmonth", + type: "yearmonth", + constraints: { minimum: "2024-02" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/minimum", + fieldName: "yearmonth", + minimum: "2024-02", + rowNumber: 3, + cell: "2024-01", + }, + ]) + }) + + it.skip("should handle exclusiveMinimum for yearmonth fields", async () => { + const table = DataFrame({ + yearmonth: ["2024-03", "2024-05", "2024-02", "2024-01"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "yearmonth", + type: "yearmonth", + constraints: { exclusiveMinimum: "2024-02" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/exclusiveMinimum", + fieldName: "yearmonth", + minimum: "2024-02", + rowNumber: 3, + cell: "2024-02", + }, + { + type: "cell/exclusiveMinimum", + fieldName: "yearmonth", + minimum: "2024-02", + rowNumber: 4, + cell: "2024-01", + }, + ]) + }) }) diff --git a/table/field/checks/minimum.ts b/table/field/checks/minimum.ts index 1da2cf3d..d0e3dbaa 100644 --- a/table/field/checks/minimum.ts +++ b/table/field/checks/minimum.ts @@ -11,6 +11,7 @@ import { parseIntegerField } from "../types/integer.ts" import { parseNumberField } from "../types/number.ts" import { parseTimeField } from "../types/time.ts" import { parseYearField } from "../types/year.ts" +import { parseYearmonthField } from "../types/yearmonth.ts" export function createCheckCellMinimum(options?: { isExclusive?: boolean }) { return (field: Field, mapping: CellMapping) => { @@ -20,7 +21,8 @@ export function createCheckCellMinimum(options?: { isExclusive?: boolean }) { field.type !== "date" && field.type !== "time" && field.type !== "datetime" && - field.type !== "year" + field.type !== "year" && + field.type !== "yearmonth" ) { return undefined } @@ -67,6 +69,8 @@ function parseConstraint(field: Field, constraint: number | string) { expr = parseDatetimeField(field, expr) } else if (field.type === "year") { expr = parseYearField(field, expr) + } else if (field.type === "yearmonth") { + expr = parseYearmonthField(field, expr) } return evaluateExpression(expr) From 89b298ee77be26c4dfa2efc2ab36b04a0e3af8bf Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 30 Oct 2025 08:24:26 +0000 Subject: [PATCH 41/42] Support enum parsing --- table/field/checks/enum.ts | 39 ++++++++++++++++++++++++++++++++++- table/field/checks/maximum.ts | 5 +++-- table/field/checks/minimum.ts | 5 +++-- 3 files changed, 44 insertions(+), 5 deletions(-) diff --git a/table/field/checks/enum.ts b/table/field/checks/enum.ts index 626b653b..7ce5023c 100644 --- a/table/field/checks/enum.ts +++ b/table/field/checks/enum.ts @@ -1,6 +1,15 @@ import type { Field } from "@dpkit/core" +import * as pl from "nodejs-polars" import type { CellEnumError } from "../../error/index.ts" +import { evaluateExpression } from "../../helpers.ts" import type { CellMapping } from "../Mapping.ts" +import { parseDateField } from "../types/date.ts" +import { parseDatetimeField } from "../types/datetime.ts" +import { parseIntegerField } from "../types/integer.ts" +import { parseNumberField } from "../types/number.ts" +import { parseTimeField } from "../types/time.ts" +import { parseYearField } from "../types/year.ts" +import { parseYearmonthField } from "../types/yearmonth.ts" export function checkCellEnum(field: Field, mapping: CellMapping) { if (field.type !== "string") return undefined @@ -8,7 +17,8 @@ export function checkCellEnum(field: Field, mapping: CellMapping) { const rawEnum = field.constraints?.enum if (!rawEnum) return undefined - const isErrorExpr = mapping.target.isIn(rawEnum).not() + const parsedEnum = parseConstraint(field, rawEnum) + const isErrorExpr = mapping.target.isIn(parsedEnum).not() const errorTemplate: CellEnumError = { type: "cell/enum", @@ -20,3 +30,30 @@ export function checkCellEnum(field: Field, mapping: CellMapping) { return { isErrorExpr, errorTemplate } } + +function parseConstraint(field: Field, value: string[]) { + return value.map(it => parseConstraintItem(field, it)) +} + +function parseConstraintItem(field: Field, value: number | string) { + if (typeof value !== "string") return value + + let expr = pl.lit(value) + if (field.type === "integer") { + expr = parseIntegerField(field, expr) + } else if (field.type === "number") { + expr = parseNumberField(field, expr) + } else if (field.type === "date") { + expr = parseDateField(field, expr) + } else if (field.type === "time") { + expr = parseTimeField(field, expr) + } else if (field.type === "datetime") { + expr = parseDatetimeField(field, expr) + } else if (field.type === "year") { + expr = parseYearField(field, expr) + } else if (field.type === "yearmonth") { + expr = parseYearmonthField(field, expr) + } + + return evaluateExpression(expr) +} diff --git a/table/field/checks/maximum.ts b/table/field/checks/maximum.ts index 830fbf6d..5f9f06a4 100644 --- a/table/field/checks/maximum.ts +++ b/table/field/checks/maximum.ts @@ -54,9 +54,10 @@ export function createCheckCellMaximum(options?: { isExclusive?: boolean }) { } } -function parseConstraint(field: Field, constraint: number | string) { - let expr = pl.lit(constraint) +function parseConstraint(field: Field, value: number | string) { + if (typeof value !== "string") return value + let expr = pl.lit(value) if (field.type === "integer") { expr = parseIntegerField(field, expr) } else if (field.type === "number") { diff --git a/table/field/checks/minimum.ts b/table/field/checks/minimum.ts index d0e3dbaa..1e33434d 100644 --- a/table/field/checks/minimum.ts +++ b/table/field/checks/minimum.ts @@ -54,9 +54,10 @@ export function createCheckCellMinimum(options?: { isExclusive?: boolean }) { } } -function parseConstraint(field: Field, constraint: number | string) { - let expr = pl.lit(constraint) +function parseConstraint(field: Field, value: number | string) { + if (typeof value !== "string") return value + let expr = pl.lit(value) if (field.type === "integer") { expr = parseIntegerField(field, expr) } else if (field.type === "number") { From 034b02103015532a21f0e99ee9f21dd90a02070b Mon Sep 17 00:00:00 2001 From: roll Date: Thu, 30 Oct 2025 08:30:32 +0000 Subject: [PATCH 42/42] Added partial enum parsing --- table/field/checks/enum.spec.ts | 232 ++++++++++++++++++++++++++++++++ table/field/checks/enum.ts | 25 +++- 2 files changed, 253 insertions(+), 4 deletions(-) diff --git a/table/field/checks/enum.spec.ts b/table/field/checks/enum.spec.ts index bd657609..58b504ea 100644 --- a/table/field/checks/enum.spec.ts +++ b/table/field/checks/enum.spec.ts @@ -119,4 +119,236 @@ describe("validateTable (cell/enum)", () => { cell: "APPROVED", }) }) + + it("should handle integer enum with string values", async () => { + const allowedValues = ["1", "2", "3"] + + const table = DataFrame({ + priority: ["1", "2", "5"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "priority", + type: "integer", + constraints: { + enum: allowedValues, + }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/enum", + fieldName: "priority", + enum: allowedValues, + rowNumber: 3, + cell: "5", + }, + ]) + }) + + it("should handle number enum with string values", async () => { + const allowedValues = ["1.5", "2.5", "3.5"] + + const table = DataFrame({ + rating: ["1.5", "2.5", "4.5"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "rating", + type: "number", + constraints: { + enum: allowedValues, + }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/enum", + fieldName: "rating", + enum: allowedValues, + rowNumber: 3, + cell: "4.5", + }, + ]) + }) + + it.skip("should handle date enum with string values", async () => { + const allowedValues = ["2024-01-01", "2024-02-01", "2024-03-01"] + + const table = DataFrame({ + date: ["2024-01-01", "2024-02-01", "2024-05-01"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "date", + type: "date", + constraints: { + enum: allowedValues, + }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/enum", + fieldName: "date", + enum: allowedValues, + rowNumber: 3, + cell: "2024-05-01", + }, + ]) + }) + + it.skip("should handle datetime enum with string values", async () => { + const allowedValues = [ + "2024-01-01T10:00:00", + "2024-01-01T14:00:00", + "2024-01-01T18:00:00", + ] + + const table = DataFrame({ + timestamp: [ + "2024-01-01T10:00:00", + "2024-01-01T14:00:00", + "2024-01-01T20:00:00", + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "timestamp", + type: "datetime", + constraints: { + enum: allowedValues, + }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/enum", + fieldName: "timestamp", + enum: allowedValues, + rowNumber: 3, + cell: "2024-01-01T20:00:00", + }, + ]) + }) + + it("should handle year enum with string values", async () => { + const allowedValues = ["2020", "2021", "2022"] + + const table = DataFrame({ + year: ["2020", "2021", "2023"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "year", + type: "year", + constraints: { + enum: allowedValues, + }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/enum", + fieldName: "year", + enum: allowedValues, + rowNumber: 3, + cell: "2023", + }, + ]) + }) + + it.skip("should handle time enum with string values", async () => { + const allowedValues = ["10:00:00", "14:00:00", "18:00:00"] + + const table = DataFrame({ + time: ["10:00:00", "14:00:00", "20:00:00"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "time", + type: "time", + constraints: { + enum: allowedValues, + }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/enum", + fieldName: "time", + enum: allowedValues, + rowNumber: 3, + cell: "20:00:00", + }, + ]) + }) + + it.skip("should handle yearmonth enum with string values", async () => { + const allowedValues = ["2024-01", "2024-02", "2024-03"] + + const table = DataFrame({ + yearmonth: ["2024-01", "2024-02", "2024-05"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "yearmonth", + type: "yearmonth", + constraints: { + enum: allowedValues, + }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/enum", + fieldName: "yearmonth", + enum: allowedValues, + rowNumber: 3, + cell: "2024-05", + }, + ]) + }) }) diff --git a/table/field/checks/enum.ts b/table/field/checks/enum.ts index 7ce5023c..52341ffe 100644 --- a/table/field/checks/enum.ts +++ b/table/field/checks/enum.ts @@ -1,4 +1,5 @@ import type { Field } from "@dpkit/core" +import type { Expr } from "nodejs-polars" import * as pl from "nodejs-polars" import type { CellEnumError } from "../../error/index.ts" import { evaluateExpression } from "../../helpers.ts" @@ -12,13 +13,29 @@ import { parseYearField } from "../types/year.ts" import { parseYearmonthField } from "../types/yearmonth.ts" export function checkCellEnum(field: Field, mapping: CellMapping) { - if (field.type !== "string") return undefined + if ( + field.type !== "string" && + field.type !== "integer" && + field.type !== "number" && + field.type !== "date" && + field.type !== "time" && + field.type !== "datetime" && + field.type !== "year" && + field.type !== "yearmonth" + ) { + return undefined + } const rawEnum = field.constraints?.enum if (!rawEnum) return undefined - const parsedEnum = parseConstraint(field, rawEnum) - const isErrorExpr = mapping.target.isIn(parsedEnum).not() + let isErrorExpr: Expr + try { + const parsedEnum = parseConstraint(field, rawEnum) + isErrorExpr = mapping.target.isIn(parsedEnum).not() + } catch (error) { + isErrorExpr = pl.lit(true) + } const errorTemplate: CellEnumError = { type: "cell/enum", @@ -31,7 +48,7 @@ export function checkCellEnum(field: Field, mapping: CellMapping) { return { isErrorExpr, errorTemplate } } -function parseConstraint(field: Field, value: string[]) { +function parseConstraint(field: Field, value: number[] | string[]) { return value.map(it => parseConstraintItem(field, it)) }