diff --git a/arrow/table/load.ts b/arrow/table/load.ts index da0ed071..6fa5c2e9 100644 --- a/arrow/table/load.ts +++ b/arrow/table/load.ts @@ -1,5 +1,5 @@ import type { Resource } from "@dpkit/core" -import { loadResourceSchema } from "@dpkit/core" +import { resolveSchema } from "@dpkit/core" import { prefetchFiles } from "@dpkit/file" import type { LoadTableOptions } from "@dpkit/table" import { inferSchemaFromTable, normalizeTable } from "@dpkit/table" @@ -21,7 +21,7 @@ export async function loadArrowTable( } if (!options?.denormalized) { - let schema = await loadResourceSchema(resource.schema) + let schema = await resolveSchema(resource.schema) if (!schema) schema = await inferSchemaFromTable(table, options) table = await normalizeTable(table, schema) } diff --git a/browser/components/Report/Error/Cell.tsx b/browser/components/Report/Error/Cell.tsx index 0cbb5cbb..1a900d8f 100644 --- a/browser/components/Report/Error/Cell.tsx +++ b/browser/components/Report/Error/Cell.tsx @@ -4,21 +4,27 @@ import { useTranslation } from "react-i18next" export function CellTypeError(props: { error: errorTypes.CellTypeError }) { const { t } = useTranslation() + const { error } = props + return ( {t("Value of the cell")}{" "} - {props.error.cell} + {error.cell} {" "} {t("in field")}{" "} - {props.error.fieldName} + {error.fieldName} {" "} {t("of row")}{" "} - {props.error.rowNumber} + {error.rowNumber} + {" "} + {t("is not")} + + {[error.fieldType, error.fieldFormat].filter(Boolean).join("/")} {" "} - {t("has a wrong type")} + {"type"} ) } @@ -27,15 +33,17 @@ export function CellRequiredError(props: { error: errorTypes.CellRequiredError }) { const { t } = useTranslation() + const { error } = props + return ( {t("A required cell in field")}{" "} - {props.error.fieldName} + {error.fieldName} {" "} {t("of row")}{" "} - {props.error.rowNumber} + {error.rowNumber} {" "} {t("is missing")} @@ -46,21 +54,27 @@ export function CellMinimumError(props: { error: errorTypes.CellMinimumError }) { const { t } = useTranslation() + const { error } = props + return ( {t("Value of the cell")}{" "} - {props.error.cell} + {error.cell} {" "} {t("in field")}{" "} - {props.error.fieldName} + {error.fieldName} {" "} {t("of row")}{" "} - {props.error.rowNumber} + {error.rowNumber} + {" "} + {t("is less than")} + + {error.minimum} {" "} - {t("is less than minimum")} + {t("minimum")} ) } @@ -69,21 +83,27 @@ export function CellMaximumError(props: { error: errorTypes.CellMaximumError }) { const { t } = useTranslation() + const { error } = props + return ( {t("Value of the cell")}{" "} - {props.error.cell} + {error.cell} {" "} {t("in field")}{" "} - {props.error.fieldName} + {error.fieldName} {" "} {t("of row")}{" "} - {props.error.rowNumber} + {error.rowNumber} {" "} - {t("is more than maximum")} + {t("is more than")} + + {error.maximum} + {" "} + {t("maximum")} ) } @@ -92,21 +112,27 @@ export function CellExclusiveMinimumError(props: { error: errorTypes.CellExclusiveMinimumError }) { const { t } = useTranslation() + const { error } = props + return ( {t("Value of the cell")}{" "} - {props.error.cell} + {error.cell} {" "} {t("in field")}{" "} - {props.error.fieldName} + {error.fieldName} {" "} {t("of row")}{" "} - {props.error.rowNumber} + {error.rowNumber} {" "} - {t("is less or equal to exclusive minimum")} + {t("is less or equal to")} + + {error.minimum} + {" "} + {t("exclusive minimum")} ) } @@ -115,21 +141,27 @@ export function CellExclusiveMaximumError(props: { error: errorTypes.CellExclusiveMaximumError }) { const { t } = useTranslation() + const { error } = props + return ( {t("Value of the cell")}{" "} - {props.error.cell} + {error.cell} {" "} {t("in field")}{" "} - {props.error.fieldName} + {error.fieldName} {" "} {t("of row")}{" "} - {props.error.rowNumber} + {error.rowNumber} {" "} - {t("is more or equal to exclusive maximum")} + {t("is less or equal to")} + + {error.maximum} + {" "} + {t("exclusive maximum")} ) } @@ -138,21 +170,27 @@ export function CellMinLengthError(props: { error: errorTypes.CellMinLengthError }) { const { t } = useTranslation() + const { error } = props + return ( {t("Length of the cell")}{" "} - {props.error.cell} + {error.cell} {" "} {t("in field")}{" "} - {props.error.fieldName} + {error.fieldName} {" "} {t("of row")}{" "} - {props.error.rowNumber} + {error.rowNumber} + {" "} + {t("is less than")} + + {error.minLength} {" "} - {t("is less than minimum")} + {t("minimum")} ) } @@ -161,21 +199,26 @@ export function CellMaxLengthError(props: { error: errorTypes.CellMaxLengthError }) { const { t } = useTranslation() + const { error } = props + return ( {t("Length of the cell")}{" "} - {props.error.cell} + {error.cell} {" "} {t("in field")}{" "} - {props.error.fieldName} + {error.fieldName} {" "} {t("of row")}{" "} - {props.error.rowNumber} + {error.rowNumber} + {" "} + + {error.maxLength} {" "} - {t("is more than maximum")} + {t("maximum")} ) } @@ -184,40 +227,48 @@ export function CellPatternError(props: { error: errorTypes.CellPatternError }) { const { t } = useTranslation() + const { error } = props + return ( {t("Value of the cell")}{" "} - {props.error.cell} + {error.cell} {" "} {t("in field")}{" "} - {props.error.fieldName} + {error.fieldName} {" "} {t("of row")}{" "} - {props.error.rowNumber} + {error.rowNumber} + {" "} + {t("does not match the")} + + {error.pattern} {" "} - {t("does not match the pattern")} + {t("pattern")} ) } export function CellUniqueError(props: { error: errorTypes.CellUniqueError }) { const { t } = useTranslation() + const { error } = props + return ( {t("Value of the cell")}{" "} - {props.error.cell} + {error.cell} {" "} {t("in field")}{" "} - {props.error.fieldName} + {error.fieldName} {" "} {t("of row")}{" "} - {props.error.rowNumber} + {error.rowNumber} {" "} {t("is not unique")} @@ -226,21 +277,27 @@ export function CellUniqueError(props: { error: errorTypes.CellUniqueError }) { export function CellEnumError(props: { error: errorTypes.CellEnumError }) { const { t } = useTranslation() + const { error } = props + return ( {t("Value of the cell")}{" "} - {props.error.cell} + {error.cell} {" "} {t("in field")}{" "} - {props.error.fieldName} + {error.fieldName} {" "} {t("of row")}{" "} - {props.error.rowNumber} + {error.rowNumber} + {" "} + {t("is not in the allowed")} + + {error.enum.join(", ")} {" "} - {t("is not in allowed values")} + {t("values")} ) } diff --git a/browser/locales/de.json b/browser/locales/de.json index 14b4b813..329d656a 100644 --- a/browser/locales/de.json +++ b/browser/locales/de.json @@ -34,13 +34,27 @@ "has a wrong type": "hat einen falschen Typ", "A required cell in field": "Eine erforderliche Zelle im Feld", "is missing": "fehlt", + "is not": "ist nicht", + "is not of the": "ist nicht vom", + "is less than": "ist kleiner als", + "minimum": "Minimum", + "is more than": "ist größer als", + "maximum": "Maximum", + "is less or equal to": "ist kleiner oder gleich", + "exclusive minimum": "exklusives Minimum", + "is more or equal to": "ist größer oder gleich", + "exclusive maximum": "exklusives Maximum", "is less than minimum": "ist kleiner als das Minimum", "is more than maximum": "ist größer als das Maximum", "is less or equal to exclusive minimum": "ist kleiner oder gleich dem exklusiven Minimum", "is more or equal to exclusive maximum": "ist größer oder gleich dem exklusiven Maximum", "Length of the cell": "Länge der Zelle", + "does not match the": "stimmt nicht überein mit", + "pattern": "Muster", "does not match the pattern": "stimmt nicht mit dem Muster überein", "is not unique": "ist nicht eindeutig", + "is not in the allowed": "ist nicht in den erlaubten", + "values": "Werten", "is not in allowed values": "ist nicht in den erlaubten Werten", "Field name is expected to be": "Feldname soll sein", "but it is actually": "aber es ist tatsächlich", diff --git a/browser/locales/en.json b/browser/locales/en.json index 72ea6d2d..bc65af93 100644 --- a/browser/locales/en.json +++ b/browser/locales/en.json @@ -34,13 +34,27 @@ "has a wrong type": "has a wrong type", "A required cell in field": "A required cell in field", "is missing": "is missing", + "is not": "is not", + "is not of the": "is not of the", + "is less than": "is less than", + "minimum": "minimum", + "is more than": "is more than", + "maximum": "maximum", + "is less or equal to": "is less or equal to", + "exclusive minimum": "exclusive minimum", + "is more or equal to": "is more or equal to", + "exclusive maximum": "exclusive maximum", "is less than minimum": "is less than minimum", "is more than maximum": "is more than maximum", "is less or equal to exclusive minimum": "is less or equal to exclusive minimum", "is more or equal to exclusive maximum": "is more or equal to exclusive maximum", "Length of the cell": "Length of the cell", + "does not match the": "does not match the", + "pattern": "pattern", "does not match the pattern": "does not match the pattern", "is not unique": "is not unique", + "is not in the allowed": "is not in the allowed", + "values": "values", "is not in allowed values": "is not in allowed values", "Field name is expected to be": "Field name is expected to be", "but it is actually": "but it is actually", diff --git a/browser/locales/es.json b/browser/locales/es.json index 9810ec87..8ab2f48d 100644 --- a/browser/locales/es.json +++ b/browser/locales/es.json @@ -34,13 +34,27 @@ "has a wrong type": "tiene un tipo incorrecto", "A required cell in field": "Una celda requerida en el campo", "is missing": "falta", + "is not": "no es", + "is not of the": "no es del", + "is less than": "es menor que", + "minimum": "mínimo", + "is more than": "es mayor que", + "maximum": "máximo", + "is less or equal to": "es menor o igual a", + "exclusive minimum": "mínimo exclusivo", + "is more or equal to": "es mayor o igual a", + "exclusive maximum": "máximo exclusivo", "is less than minimum": "es menor que el mínimo", "is more than maximum": "es mayor que el máximo", "is less or equal to exclusive minimum": "es menor o igual al mínimo exclusivo", "is more or equal to exclusive maximum": "es mayor o igual al máximo exclusivo", "Length of the cell": "Longitud de la celda", + "does not match the": "no coincide con", + "pattern": "patrón", "does not match the pattern": "no coincide con el patrón", "is not unique": "no es único", + "is not in the allowed": "no está en los", + "values": "valores permitidos", "is not in allowed values": "no está en los valores permitidos", "Field name is expected to be": "Se espera que el nombre del campo sea", "but it is actually": "pero en realidad es", diff --git a/browser/locales/fr.json b/browser/locales/fr.json index 81f2078d..4fdb5125 100644 --- a/browser/locales/fr.json +++ b/browser/locales/fr.json @@ -34,13 +34,27 @@ "has a wrong type": "a un type incorrect", "A required cell in field": "Une cellule requise dans le champ", "is missing": "est manquante", + "is not": "n'est pas", + "is not of the": "n'est pas du", + "is less than": "est inférieur à", + "minimum": "minimum", + "is more than": "est supérieur à", + "maximum": "maximum", + "is less or equal to": "est inférieur ou égal à", + "exclusive minimum": "minimum exclusif", + "is more or equal to": "est supérieur ou égal à", + "exclusive maximum": "maximum exclusif", "is less than minimum": "est inférieur au minimum", "is more than maximum": "est supérieur au maximum", "is less or equal to exclusive minimum": "est inférieur ou égal au minimum exclusif", "is more or equal to exclusive maximum": "est supérieur ou égal au maximum exclusif", "Length of the cell": "Longueur de la cellule", + "does not match the": "ne correspond pas à", + "pattern": "modèle", "does not match the pattern": "ne correspond pas au modèle", "is not unique": "n'est pas unique", + "is not in the allowed": "n'est pas dans les", + "values": "valeurs autorisées", "is not in allowed values": "n'est pas dans les valeurs autorisées", "Field name is expected to be": "Le nom du champ devrait être", "but it is actually": "mais il est en fait", diff --git a/browser/locales/it.json b/browser/locales/it.json index 150a4122..38fdd254 100644 --- a/browser/locales/it.json +++ b/browser/locales/it.json @@ -34,13 +34,27 @@ "has a wrong type": "ha un tipo errato", "A required cell in field": "Una cella richiesta nel campo", "is missing": "manca", + "is not": "non è", + "is not of the": "non è del", + "is less than": "è inferiore a", + "minimum": "minimo", + "is more than": "è superiore a", + "maximum": "massimo", + "is less or equal to": "è inferiore o uguale a", + "exclusive minimum": "minimo esclusivo", + "is more or equal to": "è superiore o uguale a", + "exclusive maximum": "massimo esclusivo", "is less than minimum": "è inferiore al minimo", "is more than maximum": "è superiore al massimo", "is less or equal to exclusive minimum": "è inferiore o uguale al minimo esclusivo", "is more or equal to exclusive maximum": "è superiore o uguale al massimo esclusivo", "Length of the cell": "Lunghezza della cella", + "does not match the": "non corrisponde al", + "pattern": "pattern", "does not match the pattern": "non corrisponde al pattern", "is not unique": "non è univoco", + "is not in the allowed": "non è tra i", + "values": "valori consentiti", "is not in allowed values": "non è tra i valori consentiti", "Field name is expected to be": "Il nome del campo dovrebbe essere", "but it is actually": "ma in realtà è", diff --git a/browser/locales/pt.json b/browser/locales/pt.json index f80054a9..fbafa1c8 100644 --- a/browser/locales/pt.json +++ b/browser/locales/pt.json @@ -34,13 +34,27 @@ "has a wrong type": "tem um tipo errado", "A required cell in field": "Uma célula obrigatória no campo", "is missing": "está faltando", + "is not": "não é", + "is not of the": "não é do", + "is less than": "é menor que", + "minimum": "mínimo", + "is more than": "é maior que", + "maximum": "máximo", + "is less or equal to": "é menor ou igual a", + "exclusive minimum": "mínimo exclusivo", + "is more or equal to": "é maior ou igual a", + "exclusive maximum": "máximo exclusivo", "is less than minimum": "é menor que o mínimo", "is more than maximum": "é maior que o máximo", "is less or equal to exclusive minimum": "é menor ou igual ao mínimo exclusivo", "is more or equal to exclusive maximum": "é maior ou igual ao máximo exclusivo", "Length of the cell": "Comprimento da célula", + "does not match the": "não corresponde ao", + "pattern": "padrão", "does not match the pattern": "não corresponde ao padrão", "is not unique": "não é único", + "is not in the allowed": "não está nos", + "values": "valores permitidos", "is not in allowed values": "não está nos valores permitidos", "Field name is expected to be": "O nome do campo deveria ser", "but it is actually": "mas na verdade é", diff --git a/browser/locales/ru.json b/browser/locales/ru.json index fac2a1fa..12e47c9e 100644 --- a/browser/locales/ru.json +++ b/browser/locales/ru.json @@ -34,13 +34,27 @@ "has a wrong type": "имеет неправильный тип", "A required cell in field": "Обязательная ячейка в поле", "is missing": "отсутствует", + "is not": "не является", + "is not of the": "не имеет", + "is less than": "меньше", + "minimum": "минимума", + "is more than": "больше", + "maximum": "максимума", + "is less or equal to": "меньше или равно", + "exclusive minimum": "эксклюзивному минимуму", + "is more or equal to": "больше или равно", + "exclusive maximum": "эксклюзивному максимуму", "is less than minimum": "меньше минимума", "is more than maximum": "больше максимума", "is less or equal to exclusive minimum": "меньше или равно эксклюзивному минимуму", "is more or equal to exclusive maximum": "больше или равно эксклюзивному максимуму", "Length of the cell": "Длина ячейки", + "does not match the": "не соответствует", + "pattern": "шаблону", "does not match the pattern": "не соответствует шаблону", "is not unique": "не уникально", + "is not in the allowed": "не входит в допустимые", + "values": "значения", "is not in allowed values": "не входит в допустимые значения", "Field name is expected to be": "Ожидается, что имя поля будет", "but it is actually": "но на самом деле", diff --git a/browser/locales/uk.json b/browser/locales/uk.json index 9bd3e518..805c0c70 100644 --- a/browser/locales/uk.json +++ b/browser/locales/uk.json @@ -34,13 +34,27 @@ "has a wrong type": "має неправильний тип", "A required cell in field": "Обов'язкова комірка в полі", "is missing": "відсутня", + "is not": "не є", + "is not of the": "не має", + "is less than": "менше", + "minimum": "мінімуму", + "is more than": "більше", + "maximum": "максимуму", + "is less or equal to": "менше або дорівнює", + "exclusive minimum": "ексклюзивному мінімуму", + "is more or equal to": "більше або дорівнює", + "exclusive maximum": "ексклюзивному максимуму", "is less than minimum": "менше мінімуму", "is more than maximum": "більше максимуму", "is less or equal to exclusive minimum": "менше або дорівнює ексклюзивному мінімуму", "is more or equal to exclusive maximum": "більше або дорівнює ексклюзивному максимуму", "Length of the cell": "Довжина комірки", + "does not match the": "не відповідає", + "pattern": "шаблону", "does not match the pattern": "не відповідає шаблону", "is not unique": "не унікальне", + "is not in the allowed": "не входить до допустимих", + "values": "значень", "is not in allowed values": "не входить до допустимих значень", "Field name is expected to be": "Очікується, що ім'я поля буде", "but it is actually": "але насправді", diff --git a/cli/commands/dialect/explore.tsx b/cli/commands/dialect/explore.tsx index 4b4779bf..8f5924a5 100644 --- a/cli/commands/dialect/explore.tsx +++ b/cli/commands/dialect/explore.tsx @@ -1,6 +1,6 @@ import { loadDialect } from "@dpkit/lib" import type { Resource } from "@dpkit/lib" -import { loadResourceDialect } from "@dpkit/lib" +import { resolveDialect } from "@dpkit/lib" import { Command } from "commander" import React from "react" import { DialectGrid } from "../../components/DialectGrid.tsx" @@ -33,7 +33,7 @@ export const exploreDialectCommand = new Command("explore") const dialect = await session.task( "Loading dialect", - path ? loadDialect(path) : loadResourceDialect(resource?.dialect), + path ? loadDialect(path) : resolveDialect(resource?.dialect), ) if (!dialect || isEmptyObject(dialect)) { diff --git a/cli/commands/dialect/script.tsx b/cli/commands/dialect/script.tsx index 97edd40a..8765171f 100644 --- a/cli/commands/dialect/script.tsx +++ b/cli/commands/dialect/script.tsx @@ -2,7 +2,7 @@ import repl from "node:repl" import * as dpkit from "@dpkit/lib" import { loadDialect } from "@dpkit/lib" import type { Resource } from "@dpkit/lib" -import { loadResourceDialect } from "@dpkit/lib" +import { resolveDialect } from "@dpkit/lib" import { Command } from "commander" import pc from "picocolors" import { helpConfiguration } from "../../helpers/help.ts" @@ -34,7 +34,7 @@ export const scriptDialectCommand = new Command("script") const dialect = await session.task( "Loading dialect", - path ? loadDialect(path) : loadResourceDialect(resource?.dialect), + path ? loadDialect(path) : resolveDialect(resource?.dialect), ) if (!dialect || isEmptyObject(dialect)) { diff --git a/cli/commands/dialect/validate.tsx b/cli/commands/dialect/validate.tsx index 6d0e4afa..19f63932 100644 --- a/cli/commands/dialect/validate.tsx +++ b/cli/commands/dialect/validate.tsx @@ -1,6 +1,6 @@ import { loadDescriptor, validateDialect } from "@dpkit/lib" import type { Resource } from "@dpkit/lib" -import { loadResourceDialect } from "@dpkit/lib" +import { resolveDialect } from "@dpkit/lib" import { Command } from "commander" import React from "react" import { ErrorGrid } from "../../components/ErrorGrid.tsx" @@ -36,10 +36,7 @@ export const validateDialectCommand = new Command("validate") : undefined const dialect = resource?.dialect - ? await session.task( - "Loading dialect", - loadResourceDialect(resource.dialect), - ) + ? await session.task("Loading dialect", resolveDialect(resource.dialect)) : undefined const { descriptor } = path diff --git a/cli/commands/package/validate.tsx b/cli/commands/package/validate.tsx index 16d2be83..b7617d03 100644 --- a/cli/commands/package/validate.tsx +++ b/cli/commands/package/validate.tsx @@ -37,6 +37,7 @@ export const validatePackageCommand = new Command("validate") // @ts-ignore if (name) report.errors = report.errors.filter(e => e.resource === name) + // @ts-ignore const type = await selectErrorType(session, report.errors) // @ts-ignore if (type) report.errors = report.errors.filter(e => e.type === type) @@ -49,6 +50,7 @@ export const validatePackageCommand = new Command("validate") session.render( report, + // @ts-ignore , ) }) diff --git a/cli/commands/schema/explore.tsx b/cli/commands/schema/explore.tsx index b5801b5f..a6ba0632 100644 --- a/cli/commands/schema/explore.tsx +++ b/cli/commands/schema/explore.tsx @@ -1,6 +1,6 @@ import { loadSchema } from "@dpkit/lib" import type { Resource } from "@dpkit/lib" -import { loadResourceSchema } from "@dpkit/lib" +import { resolveSchema } from "@dpkit/lib" import { Command } from "commander" import React from "react" import { SchemaGrid } from "../../components/SchemaGrid.tsx" @@ -33,7 +33,7 @@ export const exploreSchemaCommand = new Command("explore") const schema = await session.task( "Loading schema", - path ? loadSchema(path) : loadResourceSchema(resource?.schema), + path ? loadSchema(path) : resolveSchema(resource?.schema), ) if (!schema || isEmptyObject(schema)) { diff --git a/cli/commands/schema/script.tsx b/cli/commands/schema/script.tsx index cad34f9d..68f766e4 100644 --- a/cli/commands/schema/script.tsx +++ b/cli/commands/schema/script.tsx @@ -1,7 +1,7 @@ import repl from "node:repl" import { loadSchema } from "@dpkit/lib" import type { Resource } from "@dpkit/lib" -import { loadResourceSchema } from "@dpkit/lib" +import { resolveSchema } from "@dpkit/lib" import * as dpkit from "@dpkit/lib" import { Command } from "commander" import pc from "picocolors" @@ -33,7 +33,7 @@ export const scriptSchemaCommand = new Command("script") const schema = await session.task( "Loading schema", - path ? loadSchema(path) : loadResourceSchema(resource?.schema), + path ? loadSchema(path) : resolveSchema(resource?.schema), ) console.log( diff --git a/cli/commands/schema/validate.tsx b/cli/commands/schema/validate.tsx index 5c7281fd..2ea00a21 100644 --- a/cli/commands/schema/validate.tsx +++ b/cli/commands/schema/validate.tsx @@ -1,5 +1,5 @@ import { loadDescriptor, validateSchema } from "@dpkit/lib" -import { loadResourceSchema } from "@dpkit/lib" +import { resolveSchema } from "@dpkit/lib" import type { Resource } from "@dpkit/lib" import { Command } from "commander" import React from "react" @@ -36,10 +36,7 @@ export const validateSchemaCommand = new Command("validate") : undefined const schema = resource?.schema - ? await session.task( - "Loading schema", - loadResourceSchema(resource.schema), - ) + ? await session.task("Loading schema", resolveSchema(resource.schema)) : undefined const { descriptor } = path diff --git a/cli/commands/table/explore.tsx b/cli/commands/table/explore.tsx index ee4db088..4a8cfb5d 100644 --- a/cli/commands/table/explore.tsx +++ b/cli/commands/table/explore.tsx @@ -1,4 +1,4 @@ -import { inferSchemaFromTable, loadResourceSchema } from "@dpkit/lib" +import { inferSchemaFromTable, resolveSchema } from "@dpkit/lib" import { queryTable } from "@dpkit/lib" import { loadSchema } from "@dpkit/lib" import { loadDialect, loadTable, normalizeTable } from "@dpkit/lib" @@ -100,7 +100,7 @@ export const exploreTableCommand = new Command("explore") if (!schema && resource.schema) { schema = await session.task( "Loading schema", - loadResourceSchema(resource.schema), + resolveSchema(resource.schema), ) } diff --git a/cli/commands/table/validate.tsx b/cli/commands/table/validate.tsx index 386f5dd6..79a9bc72 100644 --- a/cli/commands/table/validate.tsx +++ b/cli/commands/table/validate.tsx @@ -1,6 +1,6 @@ import { loadTable, validateTable } from "@dpkit/lib" import { loadSchema } from "@dpkit/lib" -import { inferSchemaFromTable, loadResourceSchema } from "@dpkit/lib" +import { inferSchemaFromTable, resolveSchema } from "@dpkit/lib" import { loadDialect } from "@dpkit/lib" import type { Resource } from "@dpkit/lib" import { Command } from "commander" @@ -107,7 +107,7 @@ export const validateTableCommand = new Command("validate") if (!schema && resource.schema) { schema = await session.task( "Loading schema", - loadResourceSchema(options.schema ?? resource.schema), + resolveSchema(options.schema ?? resource.schema), ) } diff --git a/core/dialect/index.ts b/core/dialect/index.ts index 442b6a8b..b0fc40e2 100644 --- a/core/dialect/index.ts +++ b/core/dialect/index.ts @@ -5,3 +5,4 @@ export { loadDialect } from "./load.ts" export { saveDialect } from "./save.ts" export { convertDialectFromDescriptor } from "./convert/fromDescriptor.ts" export { convertDialectToDescriptor } from "./convert/toDescriptor.ts" +export { resolveDialect } from "./resolve.ts" diff --git a/core/resource/dialect.ts b/core/dialect/resolve.ts similarity index 61% rename from core/resource/dialect.ts rename to core/dialect/resolve.ts index a29e6fa1..9a347d4c 100644 --- a/core/resource/dialect.ts +++ b/core/dialect/resolve.ts @@ -1,7 +1,7 @@ import { loadDialect } from "../dialect/index.ts" -import type { Resource } from "./Resource.ts" +import type { Dialect } from "./Dialect.ts" -export async function loadResourceDialect(dialect: Resource["dialect"]) { +export async function resolveDialect(dialect?: Dialect | string) { if (!dialect) { return undefined } diff --git a/core/field/types/Date.ts b/core/field/types/Date.ts index 99438926..b30630e4 100644 --- a/core/field/types/Date.ts +++ b/core/field/types/Date.ts @@ -32,6 +32,16 @@ export interface DateConstraints extends BaseConstraints { */ maximum?: string + /** + * Exclusive minimum date value + */ + exclusiveMinimum?: string + + /** + * Exclusive maximum date value + */ + exclusiveMaximum?: string + /** * Restrict values to a specified set of dates * Should be in string date format (e.g., "YYYY-MM-DD") diff --git a/core/field/types/Datetime.ts b/core/field/types/Datetime.ts index af37faf2..5e50fbd6 100644 --- a/core/field/types/Datetime.ts +++ b/core/field/types/Datetime.ts @@ -32,6 +32,16 @@ export interface DatetimeConstraints extends BaseConstraints { */ maximum?: string + /** + * Exclusive minimum datetime value + */ + exclusiveMinimum?: string + + /** + * Exclusive maximum datetime value + */ + exclusiveMaximum?: string + /** * Restrict values to a specified set of datetimes * Should be in string datetime format (e.g., ISO8601) diff --git a/core/field/types/Time.ts b/core/field/types/Time.ts index 8b9128ea..b739d844 100644 --- a/core/field/types/Time.ts +++ b/core/field/types/Time.ts @@ -32,6 +32,16 @@ export interface TimeConstraints extends BaseConstraints { */ maximum?: string + /** + * Exclusive minimum time value + */ + exclusiveMinimum?: string + + /** + * Exclusive maximum time value + */ + exclusiveMaximum?: string + /** * Restrict values to a specified set of times * Should be in string time format (e.g., "HH:MM:SS") diff --git a/core/field/types/Year.ts b/core/field/types/Year.ts index ed664d5b..94dda6df 100644 --- a/core/field/types/Year.ts +++ b/core/field/types/Year.ts @@ -17,12 +17,22 @@ export interface YearConstraints extends BaseConstraints { /** * Minimum allowed year */ - minimum?: number + minimum?: number | string /** * Maximum allowed year */ - maximum?: number + maximum?: number | string + + /** + * Exclusive minimum year value + */ + exclusiveMinimum?: number | string + + /** + * Exclusive maximum year value + */ + exclusiveMaximum?: number | string /** * Restrict values to a specified set of years diff --git a/core/field/types/Yearmonth.ts b/core/field/types/Yearmonth.ts index 6e06c9c6..12fba903 100644 --- a/core/field/types/Yearmonth.ts +++ b/core/field/types/Yearmonth.ts @@ -24,6 +24,16 @@ export interface YearmonthConstraints extends BaseConstraints { */ maximum?: string + /** + * Exclusive minimum yearmonth value + */ + exclusiveMinimum?: string + + /** + * Exclusive maximum yearmonth value + */ + exclusiveMaximum?: string + /** * Restrict values to a specified set of yearmonths * Should be in string format (e.g., "YYYY-MM") diff --git a/core/resource/index.ts b/core/resource/index.ts index 085b83e2..095f3d1f 100644 --- a/core/resource/index.ts +++ b/core/resource/index.ts @@ -6,11 +6,6 @@ export { saveResourceDescriptor } from "./save.ts" export { validateResourceMetadata } from "./validate.ts" export { convertResourceFromDescriptor } from "./convert/fromDescriptor.ts" export { convertResourceToDescriptor } from "./convert/toDescriptor.ts" +export { isRemoteResource } from "./helpers.ts" export type { Source } from "./Source.ts" export type { License } from "./License.ts" -export { loadResourceDialect } from "./dialect.ts" -export { loadResourceSchema } from "./schema.ts" -export { isRemoteResource } from "./helpers.ts" - -// TODO: Remove in v2 -export { validateResourceMetadata as validateResourceDescriptor } from "./validate.ts" diff --git a/core/schema/index.ts b/core/schema/index.ts index d78cb87d..fcca68a9 100644 --- a/core/schema/index.ts +++ b/core/schema/index.ts @@ -8,3 +8,4 @@ export { convertSchemaFromDescriptor } from "./convert/fromDescriptor.ts" export { convertSchemaToDescriptor } from "./convert/toDescriptor.ts" export { convertSchemaFromJsonSchema } from "./convert/fromJsonSchema.ts" export { convertSchemaToJsonSchema } from "./convert/toJsonSchema.ts" +export { resolveSchema } from "./resolve.ts" diff --git a/core/resource/schema.ts b/core/schema/resolve.ts similarity index 61% rename from core/resource/schema.ts rename to core/schema/resolve.ts index 499543f6..33102eb7 100644 --- a/core/resource/schema.ts +++ b/core/schema/resolve.ts @@ -1,7 +1,7 @@ import { loadSchema } from "../schema/index.ts" -import type { Resource } from "./Resource.ts" +import type { Schema } from "./Schema.ts" -export async function loadResourceSchema(schema: Resource["schema"]) { +export async function resolveSchema(schema?: Schema | string) { if (!schema) { return undefined } diff --git a/csv/table/load.spec.ts b/csv/table/load.spec.ts index c74c98c3..93031c16 100644 --- a/csv/table/load.spec.ts +++ b/csv/table/load.spec.ts @@ -94,7 +94,7 @@ describe("loadCsvTable", () => { ]) }) - it.skip("should handle custom line terminator", async () => { + it("should handle custom line terminator", async () => { const path = await writeTempFile("id,name|1,alice|2,bob") const table = await loadCsvTable({ path, diff --git a/csv/table/load.ts b/csv/table/load.ts index b451dc66..bfd0e560 100644 --- a/csv/table/load.ts +++ b/csv/table/load.ts @@ -1,5 +1,5 @@ import type { Dialect, Resource } from "@dpkit/core" -import { loadResourceDialect, loadResourceSchema } from "@dpkit/core" +import { resolveDialect, resolveSchema } from "@dpkit/core" import { prefetchFiles } from "@dpkit/file" import type { Table } from "@dpkit/table" import { inferSchemaFromTable, normalizeTable } from "@dpkit/table" @@ -25,7 +25,7 @@ export async function loadCsvTable( throw new Error("Resource path is not defined") } - let dialect = await loadResourceDialect(resource.dialect) + let dialect = await resolveDialect(resource.dialect) if (!dialect) { dialect = await inferCsvDialect({ ...resource, path: paths[0] }, options) } @@ -55,7 +55,7 @@ export async function loadCsvTable( } if (!options?.denormalized) { - let schema = await loadResourceSchema(resource.schema) + let schema = await resolveSchema(resource.schema) if (!schema) schema = await inferSchemaFromTable(table, options) table = await normalizeTable(table, schema) } @@ -84,12 +84,9 @@ function getScanOptions(resource: Partial, dialect?: Dialect) { options.skipRows = getRowsToSkip(dialect) options.hasHeader = dialect?.header !== false + options.eolChar = dialect?.lineTerminator ?? "\n" options.sep = dialect?.delimiter ?? "," - // TODO: enable after this polars issues is fixed - // https://github.com/pola-rs/nodejs-polars/issues/333 - //options.eolChar = dialect?.lineTerminator ?? "\n" - // TODO: try convincing nodejs-polars to support escapeChar // https://github.com/pola-rs/polars/issues/3074 //options.escapeChar = dialect?.escapeChar diff --git a/database/package/save.ts b/database/package/save.ts index 12a9a393..c74bc06b 100644 --- a/database/package/save.ts +++ b/database/package/save.ts @@ -1,5 +1,5 @@ import type { Package } from "@dpkit/core" -import { loadResourceSchema } from "@dpkit/core" +import { resolveSchema } from "@dpkit/core" import { isRemoteResource } from "@dpkit/core" import type { SavePackageOptions } from "@dpkit/core" import type { TablePlugin } from "@dpkit/table" @@ -24,7 +24,7 @@ export async function savePackageToDatabase( if (table) { const dialect = { table: resource.name } - const schema = await loadResourceSchema(resource.schema) + const schema = await resolveSchema(resource.schema) // TODO: support parallel saving? await saveDatabaseTable(table, { diff --git a/database/schema/infer.ts b/database/schema/infer.ts index 7105b5b2..7bf0dbd8 100644 --- a/database/schema/infer.ts +++ b/database/schema/infer.ts @@ -1,5 +1,5 @@ import type { Resource } from "@dpkit/core" -import { loadResourceDialect } from "@dpkit/core" +import { resolveDialect } from "@dpkit/core" import { createAdapter } from "../adapters/create.ts" export async function inferDatabaseSchema( @@ -10,7 +10,7 @@ export async function inferDatabaseSchema( throw new Error("Supported database format is not defined") } - const dialect = await loadResourceDialect(resource.dialect) + const dialect = await resolveDialect(resource.dialect) if (!dialect?.table) { throw new Error("Table name is not defined in dialect") } diff --git a/database/table/load.ts b/database/table/load.ts index 02121f0b..57317889 100644 --- a/database/table/load.ts +++ b/database/table/load.ts @@ -1,4 +1,4 @@ -import { loadResourceDialect, loadResourceSchema } from "@dpkit/core" +import { resolveDialect, resolveSchema } from "@dpkit/core" import type { Resource } from "@dpkit/core" import { normalizeTable } from "@dpkit/table" import type { LoadTableOptions } from "@dpkit/table" @@ -13,7 +13,7 @@ export async function loadDatabaseTable( resource: Partial & { format: "postgresql" | "mysql" | "sqlite" }, options?: LoadTableOptions, ) { - const dialect = await loadResourceDialect(resource.dialect) + const dialect = await resolveDialect(resource.dialect) if (!dialect?.table) { throw new Error("Table name is not defined in dialect") } @@ -31,7 +31,7 @@ export async function loadDatabaseTable( let table = DataFrame(records).lazy() if (!options?.denormalized) { - let schema = await loadResourceSchema(resource.schema) + let schema = await resolveSchema(resource.schema) if (!schema) schema = await inferDatabaseSchema(resource) table = await normalizeTable(table, schema) } diff --git a/inline/table/load.ts b/inline/table/load.ts index 4cb75937..7ef5e51c 100644 --- a/inline/table/load.ts +++ b/inline/table/load.ts @@ -1,6 +1,6 @@ import type { Resource } from "@dpkit/core" -import { loadResourceDialect } from "@dpkit/core" -import { loadResourceSchema } from "@dpkit/core" +import { resolveDialect } from "@dpkit/core" +import { resolveSchema } from "@dpkit/core" import { getRecordsFromRows } from "@dpkit/table" import type { LoadTableOptions } from "@dpkit/table" import { inferSchemaFromTable, normalizeTable } from "@dpkit/table" @@ -15,14 +15,14 @@ export async function loadInlineTable( throw new Error("Resource data is not defined or tabular") } - const dialect = await loadResourceDialect(resource.dialect) + const dialect = await resolveDialect(resource.dialect) const isRows = data.every(row => Array.isArray(row)) const records = isRows ? getRecordsFromRows(data, dialect) : data let table = DataFrame(records).lazy() if (!options?.denormalized) { - let schema = await loadResourceSchema(resource.schema) + let schema = await resolveSchema(resource.schema) if (!schema) schema = await inferSchemaFromTable(table, options) table = await normalizeTable(table, schema) } diff --git a/json/table/load.ts b/json/table/load.ts index 03b7a9c3..9b866c93 100644 --- a/json/table/load.ts +++ b/json/table/load.ts @@ -1,6 +1,6 @@ import type { Dialect, Resource } from "@dpkit/core" -import { loadResourceDialect } from "@dpkit/core" -import { loadResourceSchema } from "@dpkit/core" +import { resolveDialect } from "@dpkit/core" +import { resolveSchema } from "@dpkit/core" import { loadFile, prefetchFiles } from "@dpkit/file" import type { LoadTableOptions } from "@dpkit/table" import { inferSchemaFromTable, normalizeTable } from "@dpkit/table" @@ -20,7 +20,7 @@ export async function loadJsonTable( throw new Error("Resource path is not defined") } - const dialect = await loadResourceDialect(resource.dialect) + const dialect = await resolveDialect(resource.dialect) const tables: Table[] = [] for (const path of paths) { @@ -43,7 +43,7 @@ export async function loadJsonTable( let table = concat(tables) if (!options?.denormalized) { - let schema = await loadResourceSchema(resource.schema) + let schema = await resolveSchema(resource.schema) if (!schema) schema = await inferSchemaFromTable(table, options) table = await normalizeTable(table, schema) } diff --git a/lib/package/validate.spec.ts b/lib/package/validate.spec.ts index ef097e63..ade13e8a 100644 --- a/lib/package/validate.spec.ts +++ b/lib/package/validate.spec.ts @@ -182,6 +182,7 @@ describe("validatePackage", () => { rowNumber: 3, type: "cell/type", fieldName: "longitude", + fieldType: "number", cell: "bad", resource: "deployments", }, diff --git a/lib/resource/validate.spec.ts b/lib/resource/validate.spec.ts index 366796f9..9d397ec3 100644 --- a/lib/resource/validate.spec.ts +++ b/lib/resource/validate.spec.ts @@ -19,10 +19,10 @@ describe("validateResource", () => { }, } - const result = await validateResource(resource) + const report = await validateResource(resource) - expect(result.valid).toBe(false) - expect(result.errors.length).toEqual(2) + expect(report.valid).toBe(false) + expect(report.errors.length).toEqual(1) }) it("should validate correct tabular data", async () => { @@ -41,10 +41,10 @@ describe("validateResource", () => { }, } - const result = await validateResource(resource) + const report = await validateResource(resource) - expect(result.valid).toBe(true) - expect(result.errors).toEqual([]) + expect(report.valid).toBe(true) + expect(report.errors).toEqual([]) }) it("should catch multiple validation errors", async () => { @@ -69,9 +69,9 @@ describe("validateResource", () => { }, } - const result = await validateResource(resource) + const report = await validateResource(resource) - expect(result.valid).toBe(false) - expect(result.errors.length).toEqual(3) + expect(report.valid).toBe(false) + expect(report.errors.length).toEqual(3) }) }) diff --git a/lib/resource/validate.ts b/lib/resource/validate.ts index ec74b4e4..725863e7 100644 --- a/lib/resource/validate.ts +++ b/lib/resource/validate.ts @@ -1,5 +1,5 @@ import type { Descriptor, Resource } from "@dpkit/core" -import { loadResourceSchema } from "@dpkit/core" +import { resolveSchema } from "@dpkit/core" import { loadDescriptor, validateResourceMetadata } from "@dpkit/core" import { validateFile } from "@dpkit/file" import { validateTable } from "@dpkit/table" @@ -48,7 +48,7 @@ export async function validateResourceData( const table = await loadTable(resource, { denormalized: true }) if (table) { - let schema = await loadResourceSchema(resource.schema) + let schema = await resolveSchema(resource.schema) if (!schema) schema = await inferSchema(resource, options) const tableReport = await validateTable(table, { schema }) diff --git a/lib/schema/infer.spec.ts b/lib/schema/infer.spec.ts index 2435c302..b51607aa 100644 --- a/lib/schema/infer.spec.ts +++ b/lib/schema/infer.spec.ts @@ -10,11 +10,11 @@ describe("inferSchema", () => { const schema = await inferSchema(resource) expect(schema).toBeDefined() - expect(schema.fields).toBeDefined() - expect(schema.fields?.length).toBe(3) - expect(schema.fields?.[0]?.name).toBe("id") - expect(schema.fields?.[1]?.name).toBe("name") - expect(schema.fields?.[2]?.name).toBe("age") + expect(schema?.fields).toBeDefined() + expect(schema?.fields?.length).toBe(3) + expect(schema?.fields?.[0]?.name).toBe("id") + expect(schema?.fields?.[1]?.name).toBe("name") + expect(schema?.fields?.[2]?.name).toBe("age") }) it("should infer field types correctly", async () => { @@ -25,9 +25,9 @@ describe("inferSchema", () => { const schema = await inferSchema(resource) - expect(schema.fields?.[0]?.type).toBe("integer") - expect(schema.fields?.[1]?.type).toBe("string") - expect(schema.fields?.[2]?.type).toBe("number") + expect(schema?.fields?.[0]?.type).toBe("integer") + expect(schema?.fields?.[1]?.type).toBe("string") + expect(schema?.fields?.[2]?.type).toBe("number") }) it("should infer schema from inline data", async () => { @@ -42,10 +42,10 @@ describe("inferSchema", () => { const schema = await inferSchema(resource) expect(schema).toBeDefined() - expect(schema.fields).toBeDefined() - expect(schema.fields?.length).toBe(2) - expect(schema.fields?.[0]?.name).toBe("id") - expect(schema.fields?.[1]?.name).toBe("name") + expect(schema?.fields).toBeDefined() + expect(schema?.fields?.length).toBe(2) + expect(schema?.fields?.[0]?.name).toBe("id") + expect(schema?.fields?.[1]?.name).toBe("name") }) it("should infer schema with custom delimiter", async () => { @@ -59,11 +59,11 @@ describe("inferSchema", () => { const schema = await inferSchema(resource) expect(schema).toBeDefined() - expect(schema.fields).toBeDefined() - expect(schema.fields?.length).toBe(3) - expect(schema.fields?.[0]?.name).toBe("id") - expect(schema.fields?.[1]?.name).toBe("name") - expect(schema.fields?.[2]?.name).toBe("value") + expect(schema?.fields).toBeDefined() + expect(schema?.fields?.length).toBe(3) + expect(schema?.fields?.[0]?.name).toBe("id") + expect(schema?.fields?.[1]?.name).toBe("name") + expect(schema?.fields?.[2]?.name).toBe("value") }) it("should handle boolean fields", async () => { @@ -72,7 +72,7 @@ describe("inferSchema", () => { const schema = await inferSchema(resource) - expect(schema.fields?.[1]?.type).toBe("boolean") + expect(schema?.fields?.[1]?.type).toBe("boolean") }) it("should handle date fields", async () => { @@ -83,7 +83,7 @@ describe("inferSchema", () => { const schema = await inferSchema(resource) - expect(schema.fields?.[1]?.type).toBe("date") + expect(schema?.fields?.[1]?.type).toBe("date") }) it("should handle mixed numeric types", async () => { @@ -92,7 +92,7 @@ describe("inferSchema", () => { const schema = await inferSchema(resource) - expect(schema.fields?.[1]?.type).toBe("string") + expect(schema?.fields?.[1]?.type).toBe("string") }) it("should infer schema from single row", async () => { @@ -102,8 +102,8 @@ describe("inferSchema", () => { const schema = await inferSchema(resource) expect(schema).toBeDefined() - expect(schema.fields).toBeDefined() - expect(schema.fields?.length).toBe(2) + expect(schema?.fields).toBeDefined() + expect(schema?.fields?.length).toBe(2) }) it("should handle empty string values", async () => { @@ -115,8 +115,8 @@ describe("inferSchema", () => { const schema = await inferSchema(resource) expect(schema).toBeDefined() - expect(schema.fields?.length).toBe(3) - expect(schema.fields?.[2]?.name).toBe("email") + expect(schema?.fields?.length).toBe(3) + expect(schema?.fields?.[2]?.name).toBe("email") }) it("should infer schema with sampleRows option", async () => { @@ -128,8 +128,8 @@ describe("inferSchema", () => { const schema = await inferSchema(resource, { sampleRows: 2 }) expect(schema).toBeDefined() - expect(schema.fields).toBeDefined() - expect(schema.fields?.length).toBe(2) + expect(schema?.fields).toBeDefined() + expect(schema?.fields?.length).toBe(2) }) it("should handle resources with headers only", async () => { @@ -139,8 +139,8 @@ describe("inferSchema", () => { const schema = await inferSchema(resource) expect(schema).toBeDefined() - expect(schema.fields).toBeDefined() - expect(schema.fields?.length).toBe(3) + expect(schema?.fields).toBeDefined() + expect(schema?.fields?.length).toBe(3) }) it("should infer schema from complex inline data", async () => { @@ -167,11 +167,11 @@ describe("inferSchema", () => { const schema = await inferSchema(resource) expect(schema).toBeDefined() - expect(schema.fields?.length).toBe(5) - expect(schema.fields?.find(f => f.name === "id")?.type).toBe("number") - expect(schema.fields?.find(f => f.name === "name")?.type).toBe("string") - expect(schema.fields?.find(f => f.name === "score")?.type).toBe("number") - expect(schema.fields?.find(f => f.name === "active")?.type).toBe("boolean") - expect(schema.fields?.find(f => f.name === "created")?.type).toBe("date") + expect(schema?.fields?.length).toBe(5) + expect(schema?.fields?.find(f => f.name === "id")?.type).toBe("integer") + expect(schema?.fields?.find(f => f.name === "name")?.type).toBe("string") + expect(schema?.fields?.find(f => f.name === "score")?.type).toBe("number") + expect(schema?.fields?.find(f => f.name === "active")?.type).toBe("boolean") + expect(schema?.fields?.find(f => f.name === "created")?.type).toBe("date") }) }) diff --git a/lib/table/infer.ts b/lib/table/infer.ts index dc25105e..ad282128 100644 --- a/lib/table/infer.ts +++ b/lib/table/infer.ts @@ -1,11 +1,11 @@ import type { Resource } from "@dpkit/core" -import { loadResourceDialect, loadResourceSchema } from "@dpkit/core" +import { resolveDialect, resolveSchema } from "@dpkit/core" import { inferSchemaFromTable } from "@dpkit/table" import { inferDialect } from "../dialect/index.ts" import { loadTable } from "./load.ts" export async function inferTable(resource: Partial) { - let dialect = await loadResourceDialect(resource.dialect) + let dialect = await resolveDialect(resource.dialect) if (!dialect) { dialect = await inferDialect(resource) } @@ -19,7 +19,7 @@ export async function inferTable(resource: Partial) { return undefined } - let schema = await loadResourceSchema(resource.schema) + let schema = await resolveSchema(resource.schema) if (!schema) { schema = await inferSchemaFromTable(table) } diff --git a/ods/table/load.ts b/ods/table/load.ts index 38ef5a04..4d4461a3 100644 --- a/ods/table/load.ts +++ b/ods/table/load.ts @@ -1,6 +1,6 @@ -import { loadResourceDialect } from "@dpkit/core" +import { resolveDialect } from "@dpkit/core" import type { Resource } from "@dpkit/core" -import { loadResourceSchema } from "@dpkit/core" +import { resolveSchema } from "@dpkit/core" import { loadFile, prefetchFiles } from "@dpkit/file" import type { LoadTableOptions } from "@dpkit/table" import { inferSchemaFromTable, normalizeTable } from "@dpkit/table" @@ -18,7 +18,7 @@ export async function loadOdsTable( throw new Error("Resource path is not defined") } - const dialect = await loadResourceDialect(resource.dialect) + const dialect = await resolveDialect(resource.dialect) const tables: Table[] = [] for (const path of paths) { @@ -45,7 +45,7 @@ export async function loadOdsTable( let table = concat(tables) if (!options?.denormalized) { - let schema = await loadResourceSchema(resource.schema) + let schema = await resolveSchema(resource.schema) if (!schema) schema = await inferSchemaFromTable(table, options) table = await normalizeTable(table, schema) } diff --git a/ods/table/save.ts b/ods/table/save.ts index 162d0c9d..cacaea77 100644 --- a/ods/table/save.ts +++ b/ods/table/save.ts @@ -1,4 +1,4 @@ -import { loadResourceDialect } from "@dpkit/core" +import { resolveDialect } from "@dpkit/core" import { saveFile } from "@dpkit/file" import { denormalizeTable, inferSchemaFromTable } from "@dpkit/table" import type { SaveTableOptions, Table } from "@dpkit/table" @@ -19,7 +19,7 @@ export async function saveOdsTable(table: Table, options: SaveTableOptions) { }) const df = await table.collect() - const dialect = await loadResourceDialect(options.dialect) + const dialect = await resolveDialect(options.dialect) const sheetName = dialect?.sheetName ?? "Sheet1" const sheet = utils.json_to_sheet(df.toRecords()) diff --git a/parquet/table/load.ts b/parquet/table/load.ts index 3c8e0df8..bb83e7bb 100644 --- a/parquet/table/load.ts +++ b/parquet/table/load.ts @@ -1,5 +1,5 @@ import type { Resource } from "@dpkit/core" -import { loadResourceSchema } from "@dpkit/core" +import { resolveSchema } from "@dpkit/core" import { prefetchFiles } from "@dpkit/file" import type { LoadTableOptions } from "@dpkit/table" import { inferSchemaFromTable, normalizeTable } from "@dpkit/table" @@ -21,7 +21,7 @@ export async function loadParquetTable( } if (!options?.denormalized) { - let schema = await loadResourceSchema(resource.schema) + let schema = await resolveSchema(resource.schema) if (!schema) schema = await inferSchemaFromTable(table, options) table = await normalizeTable(table, schema) } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 87a7534a..a26e3475 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -859,6 +859,9 @@ importers: nodejs-polars: specifier: ^0.22.1 version: 0.22.1 + p-all: + specifier: ^5.0.1 + version: 5.0.1 test: dependencies: @@ -5042,6 +5045,10 @@ packages: openapi-types@12.1.3: resolution: {integrity: sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw==} + p-all@5.0.1: + resolution: {integrity: sha512-LMT7WX9ZSaq3J1zjloApkIVmtz0ZdMFSIqbuiEa3txGYPLjUPOvgOPOx3nFjo+f37ZYL+1aY666I2SG7GVwLOA==} + engines: {node: '>=16'} + p-each-series@3.0.0: resolution: {integrity: sha512-lastgtAdoH9YaLyDa5i5z64q+kzOcQHsQ5SsZJD3q0VEyI8mq872S3geuNbRUQLVAE9siMfgKrpj7MloKFHruw==} engines: {node: '>=12'} @@ -5066,6 +5073,10 @@ packages: resolution: {integrity: sha512-nQja7m7gSKuewoVRen45CtVfODR3crN3goVQ0DDZ9N3yHxgpkuBhZqsaiotSQRrADUrne346peY7kT3TSACykg==} engines: {node: '>=4'} + p-map@6.0.0: + resolution: {integrity: sha512-T8BatKGY+k5rU+Q/GTYgrEf2r4xRMevAN5mtXc2aPc4rS1j3s+vWTaO2Wag94neXuCAUAs8cxBL9EeB5EA6diw==} + engines: {node: '>=16'} + p-map@7.0.3: resolution: {integrity: sha512-VkndIv2fIB99swvQoA65bm+fsmt6UNdGeIB0oxBs+WhAhdh08QA04JXpI7rbB9r08/nkbysKoya9rtDERYOYMA==} engines: {node: '>=18'} @@ -11417,6 +11428,10 @@ snapshots: openapi-types@12.1.3: {} + p-all@5.0.1: + dependencies: + p-map: 6.0.0 + p-each-series@3.0.0: {} p-filter@4.1.0: @@ -11437,6 +11452,8 @@ snapshots: dependencies: p-limit: 1.3.0 + p-map@6.0.0: {} + p-map@7.0.3: {} p-queue@8.1.1: diff --git a/table/error/Cell.ts b/table/error/Cell.ts index 2eaf5407..09bc7412 100644 --- a/table/error/Cell.ts +++ b/table/error/Cell.ts @@ -1,5 +1,19 @@ +import type { FieldType } from "@dpkit/core" import type { BaseTableError } from "./Base.ts" +export type CellError = + | CellTypeError + | CellRequiredError + | CellMinimumError + | CellMaximumError + | CellExclusiveMinimumError + | CellExclusiveMaximumError + | CellMinLengthError + | CellMaxLengthError + | CellPatternError + | CellUniqueError + | CellEnumError + export interface BaseCellError extends BaseTableError { fieldName: string rowNumber: number @@ -8,6 +22,8 @@ export interface BaseCellError extends BaseTableError { export interface CellTypeError extends BaseCellError { type: "cell/type" + fieldType: FieldType + fieldFormat?: string } export interface CellRequiredError extends BaseCellError { @@ -16,30 +32,37 @@ export interface CellRequiredError extends BaseCellError { export interface CellMinimumError extends BaseCellError { type: "cell/minimum" + minimum: string } export interface CellMaximumError extends BaseCellError { type: "cell/maximum" + maximum: string } export interface CellExclusiveMinimumError extends BaseCellError { type: "cell/exclusiveMinimum" + minimum: string } export interface CellExclusiveMaximumError extends BaseCellError { type: "cell/exclusiveMaximum" + maximum: string } export interface CellMinLengthError extends BaseCellError { type: "cell/minLength" + minLength: number } export interface CellMaxLengthError extends BaseCellError { type: "cell/maxLength" + maxLength: number } export interface CellPatternError extends BaseCellError { type: "cell/pattern" + pattern: string } export interface CellUniqueError extends BaseCellError { @@ -48,4 +71,5 @@ export interface CellUniqueError extends BaseCellError { export interface CellEnumError extends BaseCellError { type: "cell/enum" + enum: string[] } diff --git a/table/error/Field.ts b/table/error/Field.ts index 936196d2..df3af39f 100644 --- a/table/error/Field.ts +++ b/table/error/Field.ts @@ -1,6 +1,8 @@ import type { FieldType } from "@dpkit/core" import type { BaseTableError } from "./Base.ts" +export type FieldError = FieldNameError | FieldTypeError + export interface BaseFieldError extends BaseTableError { fieldName: string } diff --git a/table/error/Fields.ts b/table/error/Fields.ts index 1e7da0a7..0a30ca84 100644 --- a/table/error/Fields.ts +++ b/table/error/Fields.ts @@ -1,5 +1,7 @@ import type { BaseTableError } from "./Base.ts" +export type FieldsError = FieldsMissingError | FieldsExtraError + export interface BaseFieldsError extends BaseTableError { fieldNames: string[] } diff --git a/table/error/Row.ts b/table/error/Row.ts index 0569a3e3..aa9833ec 100644 --- a/table/error/Row.ts +++ b/table/error/Row.ts @@ -1,5 +1,7 @@ import type { BaseTableError } from "./Base.ts" +export type RowError = RowUniqueError + export interface BaseRowError extends BaseTableError { rowNumber: number } diff --git a/table/error/Table.ts b/table/error/Table.ts index e6f1d362..b9be6b7f 100644 --- a/table/error/Table.ts +++ b/table/error/Table.ts @@ -1,34 +1,6 @@ -import type { - CellEnumError, - CellExclusiveMaximumError, - CellExclusiveMinimumError, - CellMaxLengthError, - CellMaximumError, - CellMinLengthError, - CellMinimumError, - CellPatternError, - CellRequiredError, - CellTypeError, - CellUniqueError, -} from "./Cell.ts" -import type { FieldNameError, FieldTypeError } from "./Field.ts" -import type { FieldsExtraError, FieldsMissingError } from "./Fields.ts" -import type { RowUniqueError } from "./Row.ts" +import type { CellError } from "./Cell.ts" +import type { FieldError } from "./Field.ts" +import type { FieldsError } from "./Fields.ts" +import type { RowError } from "./Row.ts" -export type TableError = - | FieldsMissingError - | FieldsExtraError - | FieldNameError - | FieldTypeError - | RowUniqueError - | CellTypeError - | CellRequiredError - | CellMinimumError - | CellMaximumError - | CellExclusiveMinimumError - | CellExclusiveMaximumError - | CellMinLengthError - | CellMaxLengthError - | CellPatternError - | CellUniqueError - | CellEnumError +export type TableError = FieldsError | FieldError | RowError | CellError diff --git a/table/field/Mapping.ts b/table/field/Mapping.ts new file mode 100644 index 00000000..a6899dc2 --- /dev/null +++ b/table/field/Mapping.ts @@ -0,0 +1,13 @@ +import type { Field } from "@dpkit/core" +import type { Expr } from "nodejs-polars" +import type { PolarsField } from "./Field.ts" + +export interface FieldMapping { + source: PolarsField + target: Field +} + +export interface CellMapping { + source: Expr + target: Expr +} diff --git a/table/field/checks/enum.spec.ts b/table/field/checks/enum.spec.ts index 576b6450..58b504ea 100644 --- a/table/field/checks/enum.spec.ts +++ b/table/field/checks/enum.spec.ts @@ -26,6 +26,8 @@ describe("validateTable (cell/enum)", () => { }) it("should report errors for values not in the enum", async () => { + const allowedValues = ["pending", "approved", "rejected"] + const table = DataFrame({ status: ["pending", "approved", "unknown", "cancelled", "rejected"], }).lazy() @@ -36,7 +38,7 @@ describe("validateTable (cell/enum)", () => { name: "status", type: "string", constraints: { - enum: ["pending", "approved", "rejected"], + enum: allowedValues, }, }, ], @@ -47,12 +49,14 @@ describe("validateTable (cell/enum)", () => { expect(errors).toContainEqual({ type: "cell/enum", fieldName: "status", + enum: allowedValues, rowNumber: 3, cell: "unknown", }) expect(errors).toContainEqual({ type: "cell/enum", fieldName: "status", + enum: allowedValues, rowNumber: 4, cell: "cancelled", }) @@ -80,6 +84,8 @@ describe("validateTable (cell/enum)", () => { }) it("should handle case sensitivity correctly", async () => { + const allowedValues = ["pending", "approved", "rejected"] + const table = DataFrame({ status: ["Pending", "APPROVED", "rejected"], }).lazy() @@ -90,7 +96,7 @@ describe("validateTable (cell/enum)", () => { name: "status", type: "string", constraints: { - enum: ["pending", "approved", "rejected"], + enum: allowedValues, }, }, ], @@ -101,14 +107,248 @@ describe("validateTable (cell/enum)", () => { expect(errors).toContainEqual({ type: "cell/enum", fieldName: "status", + enum: allowedValues, rowNumber: 1, cell: "Pending", }) expect(errors).toContainEqual({ type: "cell/enum", fieldName: "status", + enum: allowedValues, rowNumber: 2, cell: "APPROVED", }) }) + + it("should handle integer enum with string values", async () => { + const allowedValues = ["1", "2", "3"] + + const table = DataFrame({ + priority: ["1", "2", "5"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "priority", + type: "integer", + constraints: { + enum: allowedValues, + }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/enum", + fieldName: "priority", + enum: allowedValues, + rowNumber: 3, + cell: "5", + }, + ]) + }) + + it("should handle number enum with string values", async () => { + const allowedValues = ["1.5", "2.5", "3.5"] + + const table = DataFrame({ + rating: ["1.5", "2.5", "4.5"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "rating", + type: "number", + constraints: { + enum: allowedValues, + }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/enum", + fieldName: "rating", + enum: allowedValues, + rowNumber: 3, + cell: "4.5", + }, + ]) + }) + + it.skip("should handle date enum with string values", async () => { + const allowedValues = ["2024-01-01", "2024-02-01", "2024-03-01"] + + const table = DataFrame({ + date: ["2024-01-01", "2024-02-01", "2024-05-01"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "date", + type: "date", + constraints: { + enum: allowedValues, + }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/enum", + fieldName: "date", + enum: allowedValues, + rowNumber: 3, + cell: "2024-05-01", + }, + ]) + }) + + it.skip("should handle datetime enum with string values", async () => { + const allowedValues = [ + "2024-01-01T10:00:00", + "2024-01-01T14:00:00", + "2024-01-01T18:00:00", + ] + + const table = DataFrame({ + timestamp: [ + "2024-01-01T10:00:00", + "2024-01-01T14:00:00", + "2024-01-01T20:00:00", + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "timestamp", + type: "datetime", + constraints: { + enum: allowedValues, + }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/enum", + fieldName: "timestamp", + enum: allowedValues, + rowNumber: 3, + cell: "2024-01-01T20:00:00", + }, + ]) + }) + + it("should handle year enum with string values", async () => { + const allowedValues = ["2020", "2021", "2022"] + + const table = DataFrame({ + year: ["2020", "2021", "2023"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "year", + type: "year", + constraints: { + enum: allowedValues, + }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/enum", + fieldName: "year", + enum: allowedValues, + rowNumber: 3, + cell: "2023", + }, + ]) + }) + + it.skip("should handle time enum with string values", async () => { + const allowedValues = ["10:00:00", "14:00:00", "18:00:00"] + + const table = DataFrame({ + time: ["10:00:00", "14:00:00", "20:00:00"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "time", + type: "time", + constraints: { + enum: allowedValues, + }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/enum", + fieldName: "time", + enum: allowedValues, + rowNumber: 3, + cell: "20:00:00", + }, + ]) + }) + + it.skip("should handle yearmonth enum with string values", async () => { + const allowedValues = ["2024-01", "2024-02", "2024-03"] + + const table = DataFrame({ + yearmonth: ["2024-01", "2024-02", "2024-05"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "yearmonth", + type: "yearmonth", + constraints: { + enum: allowedValues, + }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/enum", + fieldName: "yearmonth", + enum: allowedValues, + rowNumber: 3, + cell: "2024-05", + }, + ]) + }) }) diff --git a/table/field/checks/enum.ts b/table/field/checks/enum.ts index 5ad37c52..52341ffe 100644 --- a/table/field/checks/enum.ts +++ b/table/field/checks/enum.ts @@ -1,20 +1,76 @@ import type { Field } from "@dpkit/core" -import { col } from "nodejs-polars" -import type { Table } from "../../table/index.ts" +import type { Expr } from "nodejs-polars" +import * as pl from "nodejs-polars" +import type { CellEnumError } from "../../error/index.ts" +import { evaluateExpression } from "../../helpers.ts" +import type { CellMapping } from "../Mapping.ts" +import { parseDateField } from "../types/date.ts" +import { parseDatetimeField } from "../types/datetime.ts" +import { parseIntegerField } from "../types/integer.ts" +import { parseNumberField } from "../types/number.ts" +import { parseTimeField } from "../types/time.ts" +import { parseYearField } from "../types/year.ts" +import { parseYearmonthField } from "../types/yearmonth.ts" -export function checkCellEnum(field: Field, errorTable: Table) { - if (field.type === "string") { - const rawEnum = field.constraints?.enum +export function checkCellEnum(field: Field, mapping: CellMapping) { + if ( + field.type !== "string" && + field.type !== "integer" && + field.type !== "number" && + field.type !== "date" && + field.type !== "time" && + field.type !== "datetime" && + field.type !== "year" && + field.type !== "yearmonth" + ) { + return undefined + } + + const rawEnum = field.constraints?.enum + if (!rawEnum) return undefined + + let isErrorExpr: Expr + try { + const parsedEnum = parseConstraint(field, rawEnum) + isErrorExpr = mapping.target.isIn(parsedEnum).not() + } catch (error) { + isErrorExpr = pl.lit(true) + } + + const errorTemplate: CellEnumError = { + type: "cell/enum", + fieldName: field.name, + enum: rawEnum.map(String), + rowNumber: 0, + cell: "", + } + + return { isErrorExpr, errorTemplate } +} + +function parseConstraint(field: Field, value: number[] | string[]) { + return value.map(it => parseConstraintItem(field, it)) +} - if (rawEnum) { - const target = col(`target:${field.name}`) - const errorName = `error:cell/enum:${field.name}` +function parseConstraintItem(field: Field, value: number | string) { + if (typeof value !== "string") return value - errorTable = errorTable - .withColumn(target.isIn(rawEnum).not().alias(errorName)) - .withColumn(col("error").or(col(errorName)).alias("error")) - } + let expr = pl.lit(value) + if (field.type === "integer") { + expr = parseIntegerField(field, expr) + } else if (field.type === "number") { + expr = parseNumberField(field, expr) + } else if (field.type === "date") { + expr = parseDateField(field, expr) + } else if (field.type === "time") { + expr = parseTimeField(field, expr) + } else if (field.type === "datetime") { + expr = parseDatetimeField(field, expr) + } else if (field.type === "year") { + expr = parseYearField(field, expr) + } else if (field.type === "yearmonth") { + expr = parseYearmonthField(field, expr) } - return errorTable + return evaluateExpression(expr) } diff --git a/table/field/checks/maxLength.spec.ts b/table/field/checks/maxLength.spec.ts index b8dd585f..1916d13c 100644 --- a/table/field/checks/maxLength.spec.ts +++ b/table/field/checks/maxLength.spec.ts @@ -43,6 +43,7 @@ describe("validateTable (cell/maxLength)", () => { expect(errors).toContainEqual({ type: "cell/maxLength", fieldName: "username", + maxLength: 8, rowNumber: 3, cell: "christopher", }) diff --git a/table/field/checks/maxLength.ts b/table/field/checks/maxLength.ts index d1b6ef2a..84306b21 100644 --- a/table/field/checks/maxLength.ts +++ b/table/field/checks/maxLength.ts @@ -1,20 +1,22 @@ import type { Field } from "@dpkit/core" -import { col } from "nodejs-polars" -import type { Table } from "../../table/index.ts" +import type { CellMaxLengthError } from "../../error/index.ts" +import type { CellMapping } from "../Mapping.ts" -export function checkCellMaxLength(field: Field, errorTable: Table) { - if (field.type === "string") { - const maxLength = field.constraints?.maxLength +export function checkCellMaxLength(field: Field, mapping: CellMapping) { + if (field.type !== "string") return undefined - if (maxLength !== undefined) { - const target = col(`target:${field.name}`) - const errorName = `error:cell/maxLength:${field.name}` + const maxLength = field.constraints?.maxLength + if (!maxLength) return undefined - errorTable = errorTable - .withColumn(target.str.lengths().gt(maxLength).alias(errorName)) - .withColumn(col("error").or(col(errorName)).alias("error")) - } + const isErrorExpr = mapping.target.str.lengths().gt(maxLength) + + const errorTemplate: CellMaxLengthError = { + type: "cell/maxLength", + fieldName: field.name, + maxLength: maxLength, + rowNumber: 0, + cell: "", } - return errorTable + return { isErrorExpr, errorTemplate } } diff --git a/table/field/checks/maximum.spec.ts b/table/field/checks/maximum.spec.ts index e68e47ca..3929b184 100644 --- a/table/field/checks/maximum.spec.ts +++ b/table/field/checks/maximum.spec.ts @@ -43,6 +43,7 @@ describe("validateTable (cell/maximum)", () => { expect(errors).toContainEqual({ type: "cell/maximum", fieldName: "temperature", + maximum: "40", rowNumber: 4, cell: "50.5", }) @@ -70,14 +71,439 @@ describe("validateTable (cell/maximum)", () => { expect(errors).toContainEqual({ type: "cell/exclusiveMaximum", fieldName: "temperature", + maximum: "40", rowNumber: 3, cell: "40", }) expect(errors).toContainEqual({ type: "cell/exclusiveMaximum", fieldName: "temperature", + maximum: "40", rowNumber: 4, cell: "50.5", }) }) + + it("should handle maximum as string", async () => { + const table = DataFrame({ + price: [10.5, 20.75, 55.0], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "price", + type: "number", + constraints: { maximum: "50" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/maximum", + fieldName: "price", + maximum: "50", + rowNumber: 3, + cell: "55", + }, + ]) + }) + + it("should handle exclusiveMaximum as string", async () => { + const table = DataFrame({ + temperature: [20.5, 40.0, 50.5], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "temperature", + type: "number", + constraints: { exclusiveMaximum: "40" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/exclusiveMaximum", + fieldName: "temperature", + maximum: "40", + rowNumber: 2, + cell: "40", + }, + { + type: "cell/exclusiveMaximum", + fieldName: "temperature", + maximum: "40", + rowNumber: 3, + cell: "50.5", + }, + ]) + }) + + it("should handle maximum as string with groupChar", async () => { + const table = DataFrame({ + price: ["5,000", "10,500", "15,000"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "price", + type: "integer", + groupChar: ",", + constraints: { maximum: "12,000" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/maximum", + fieldName: "price", + maximum: "12,000", + rowNumber: 3, + cell: "15,000", + }, + ]) + }) + + it("should handle maximum as string with decimalChar", async () => { + const table = DataFrame({ + price: ["5,5", "10,75", "15,3"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "price", + type: "number", + decimalChar: ",", + constraints: { maximum: "12,0" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/maximum", + fieldName: "price", + maximum: "12,0", + rowNumber: 3, + cell: "15,3", + }, + ]) + }) + + it("should handle maximum as string with groupChar and decimalChar", async () => { + const table = DataFrame({ + price: ["5.000,50", "10.500,75", "15.000,30"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "price", + type: "number", + groupChar: ".", + decimalChar: ",", + constraints: { maximum: "12.000,00" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/maximum", + fieldName: "price", + maximum: "12.000,00", + rowNumber: 3, + cell: "15.000,30", + }, + ]) + }) + + it("should handle maximum as string with bareNumber false", async () => { + const table = DataFrame({ + price: ["$5.00", "$10.50", "$15.50"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "price", + type: "number", + bareNumber: false, + constraints: { maximum: "$12.00" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/maximum", + fieldName: "price", + maximum: "$12.00", + rowNumber: 3, + cell: "$15.50", + }, + ]) + }) + + it("should handle maximum for date fields", async () => { + const table = DataFrame({ + date: ["2024-01-15", "2024-02-20", "2024-03-25"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "date", + type: "date", + constraints: { maximum: "2024-02-28" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/maximum", + fieldName: "date", + maximum: "2024-02-28", + rowNumber: 3, + cell: "2024-03-25", + }, + ]) + }) + + it.skip("should handle maximum for time fields", async () => { + const table = DataFrame({ + time: ["14:30:00", "16:45:00", "18:00:00"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "time", + type: "time", + constraints: { maximum: "17:00:00" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/maximum", + fieldName: "time", + maximum: "17:00:00", + rowNumber: 3, + cell: "18:00:00", + }, + ]) + }) + + it("should handle maximum for datetime fields", async () => { + const table = DataFrame({ + timestamp: [ + "2024-01-15T14:30:00", + "2024-02-20T08:15:00", + "2024-03-25T10:00:00", + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "timestamp", + type: "datetime", + constraints: { maximum: "2024-02-28T23:59:59" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/maximum", + fieldName: "timestamp", + maximum: "2024-02-28T23:59:59", + rowNumber: 3, + cell: "2024-03-25T10:00:00", + }, + ]) + }) + + it("should handle maximum for date fields with custom format", async () => { + const table = DataFrame({ + date: ["15/01/2024", "20/02/2024", "25/03/2024"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "date", + type: "date", + format: "%d/%m/%Y", + constraints: { maximum: "28/02/2024" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/maximum", + fieldName: "date", + maximum: "28/02/2024", + rowNumber: 3, + cell: "25/03/2024", + }, + ]) + }) + + it("should handle maximum for year fields", async () => { + const table = DataFrame({ + year: ["2020", "2021", "2023"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "year", + type: "year", + constraints: { maximum: "2022" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/maximum", + fieldName: "year", + maximum: "2022", + rowNumber: 3, + cell: "2023", + }, + ]) + }) + + it("should handle exclusiveMaximum for year fields", async () => { + const table = DataFrame({ + year: ["2020", "2021", "2022", "2023"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "year", + type: "year", + constraints: { exclusiveMaximum: "2022" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/exclusiveMaximum", + fieldName: "year", + maximum: "2022", + rowNumber: 3, + cell: "2022", + }, + { + type: "cell/exclusiveMaximum", + fieldName: "year", + maximum: "2022", + rowNumber: 4, + cell: "2023", + }, + ]) + }) + + it.skip("should handle maximum for yearmonth fields", async () => { + const table = DataFrame({ + yearmonth: ["2024-01", "2024-03", "2024-06"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "yearmonth", + type: "yearmonth", + constraints: { maximum: "2024-05" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/maximum", + fieldName: "yearmonth", + maximum: "2024-05", + rowNumber: 3, + cell: "2024-06", + }, + ]) + }) + + it.skip("should handle exclusiveMaximum for yearmonth fields", async () => { + const table = DataFrame({ + yearmonth: ["2024-01", "2024-03", "2024-05", "2024-06"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "yearmonth", + type: "yearmonth", + constraints: { exclusiveMaximum: "2024-05" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/exclusiveMaximum", + fieldName: "yearmonth", + maximum: "2024-05", + rowNumber: 3, + cell: "2024-05", + }, + { + type: "cell/exclusiveMaximum", + fieldName: "yearmonth", + maximum: "2024-05", + rowNumber: 4, + cell: "2024-06", + }, + ]) + }) }) diff --git a/table/field/checks/maximum.ts b/table/field/checks/maximum.ts index ccb7d8be..5f9f06a4 100644 --- a/table/field/checks/maximum.ts +++ b/table/field/checks/maximum.ts @@ -1,47 +1,78 @@ import type { Field } from "@dpkit/core" -import { col, lit } from "nodejs-polars" -import type { Table } from "../../table/index.ts" - -export function checkCellMaximum( - field: Field, - errorTable: Table, - options?: { - isExclusive?: boolean - }, -) { - if (field.type === "integer" || field.type === "number") { +import * as pl from "nodejs-polars" +import type { Expr } from "nodejs-polars" +import type { CellExclusiveMaximumError } from "../../error/index.ts" +import type { CellMaximumError } from "../../error/index.ts" +import { evaluateExpression } from "../../helpers.ts" +import type { CellMapping } from "../Mapping.ts" +import { parseDateField } from "../types/date.ts" +import { parseDatetimeField } from "../types/datetime.ts" +import { parseIntegerField } from "../types/integer.ts" +import { parseNumberField } from "../types/number.ts" +import { parseTimeField } from "../types/time.ts" +import { parseYearField } from "../types/year.ts" +import { parseYearmonthField } from "../types/yearmonth.ts" + +export function createCheckCellMaximum(options?: { isExclusive?: boolean }) { + return (field: Field, mapping: CellMapping) => { + if ( + field.type !== "integer" && + field.type !== "number" && + field.type !== "date" && + field.type !== "time" && + field.type !== "datetime" && + field.type !== "year" && + field.type !== "yearmonth" + ) { + return undefined + } + const maximum = options?.isExclusive ? field.constraints?.exclusiveMaximum : field.constraints?.maximum + if (maximum === undefined) return undefined + + let isErrorExpr: Expr + try { + const parsedMaximum = parseConstraint(field, maximum) + isErrorExpr = options?.isExclusive + ? mapping.target.gtEq(parsedMaximum) + : mapping.target.gt(parsedMaximum) + } catch (error) { + isErrorExpr = pl.lit(true) + } - if (maximum !== undefined) { - const target = col(`target:${field.name}`) - const errorName = options?.isExclusive - ? `error:cell/exclusiveMaximum:${field.name}` - : `error:cell/maximum:${field.name}` - - // TODO: Support numeric options (decimalChar, groupChar, etc) - const parser = - field.type === "integer" ? Number.parseInt : Number.parseFloat - - try { - const parsedMaximum = - typeof maximum === "string" ? parser(maximum) : maximum - - errorTable = errorTable - .withColumn( - options?.isExclusive - ? target.gtEq(parsedMaximum).alias(errorName) - : target.gt(parsedMaximum).alias(errorName), - ) - .withColumn(col("error").or(col(errorName)).alias("error")) - } catch (error) { - errorTable = errorTable - .withColumn(lit(true).alias(errorName)) - .withColumn(lit(true).alias("error")) - } + const errorTemplate: CellMaximumError | CellExclusiveMaximumError = { + type: options?.isExclusive ? "cell/exclusiveMaximum" : "cell/maximum", + fieldName: field.name, + maximum: String(maximum), + rowNumber: 0, + cell: "", } + + return { isErrorExpr, errorTemplate } + } +} + +function parseConstraint(field: Field, value: number | string) { + if (typeof value !== "string") return value + + let expr = pl.lit(value) + if (field.type === "integer") { + expr = parseIntegerField(field, expr) + } else if (field.type === "number") { + expr = parseNumberField(field, expr) + } else if (field.type === "date") { + expr = parseDateField(field, expr) + } else if (field.type === "time") { + expr = parseTimeField(field, expr) + } else if (field.type === "datetime") { + expr = parseDatetimeField(field, expr) + } else if (field.type === "year") { + expr = parseYearField(field, expr) + } else if (field.type === "yearmonth") { + expr = parseYearmonthField(field, expr) } - return errorTable + return evaluateExpression(expr) } diff --git a/table/field/checks/minLength.spec.ts b/table/field/checks/minLength.spec.ts index 7c34c7bf..d4ac46f3 100644 --- a/table/field/checks/minLength.spec.ts +++ b/table/field/checks/minLength.spec.ts @@ -43,12 +43,14 @@ describe("validateTable (cell/minLength)", () => { expect(errors).toContainEqual({ type: "cell/minLength", fieldName: "username", + minLength: 3, rowNumber: 2, cell: "a", }) expect(errors).toContainEqual({ type: "cell/minLength", fieldName: "username", + minLength: 3, rowNumber: 4, cell: "ab", }) diff --git a/table/field/checks/minLength.ts b/table/field/checks/minLength.ts index aa9106a3..b9bfce53 100644 --- a/table/field/checks/minLength.ts +++ b/table/field/checks/minLength.ts @@ -1,20 +1,22 @@ import type { Field } from "@dpkit/core" -import { col } from "nodejs-polars" -import type { Table } from "../../table/index.ts" +import type { CellMinLengthError } from "../../error/index.ts" +import type { CellMapping } from "../Mapping.ts" -export function checkCellMinLength(field: Field, errorTable: Table) { - if (field.type === "string") { - const minLength = field.constraints?.minLength +export function checkCellMinLength(field: Field, mapping: CellMapping) { + if (field.type !== "string") return undefined - if (minLength !== undefined) { - const target = col(`target:${field.name}`) - const errorName = `error:cell/minLength:${field.name}` + const minLength = field.constraints?.minLength + if (!minLength) return undefined - errorTable = errorTable - .withColumn(target.str.lengths().lt(minLength).alias(errorName)) - .withColumn(col("error").or(col(errorName)).alias("error")) - } + const isErrorExpr = mapping.target.str.lengths().lt(minLength) + + const errorTemplate: CellMinLengthError = { + type: "cell/minLength", + fieldName: field.name, + minLength: minLength, + rowNumber: 0, + cell: "", } - return errorTable + return { isErrorExpr, errorTemplate } } diff --git a/table/field/checks/minimum.spec.ts b/table/field/checks/minimum.spec.ts index 1e44e6c1..9ce9b35a 100644 --- a/table/field/checks/minimum.spec.ts +++ b/table/field/checks/minimum.spec.ts @@ -43,6 +43,7 @@ describe("validateTable (cell/minimum)", () => { expect(errors).toContainEqual({ type: "cell/minimum", fieldName: "temperature", + minimum: "10", rowNumber: 4, cell: "3.5", }) @@ -70,14 +71,439 @@ describe("validateTable (cell/minimum)", () => { expect(errors).toContainEqual({ type: "cell/exclusiveMinimum", fieldName: "temperature", + minimum: "10", rowNumber: 3, cell: "10", }) expect(errors).toContainEqual({ type: "cell/exclusiveMinimum", fieldName: "temperature", + minimum: "10", rowNumber: 4, cell: "5.5", }) }) + + it("should handle minimum as string", async () => { + const table = DataFrame({ + price: [10.5, 20.75, 3.0], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "price", + type: "number", + constraints: { minimum: "5" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/minimum", + fieldName: "price", + minimum: "5", + rowNumber: 3, + cell: "3", + }, + ]) + }) + + it("should handle exclusiveMinimum as string", async () => { + const table = DataFrame({ + temperature: [20.5, 10.0, 5.5], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "temperature", + type: "number", + constraints: { exclusiveMinimum: "10" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/exclusiveMinimum", + fieldName: "temperature", + minimum: "10", + rowNumber: 2, + cell: "10", + }, + { + type: "cell/exclusiveMinimum", + fieldName: "temperature", + minimum: "10", + rowNumber: 3, + cell: "5.5", + }, + ]) + }) + + it("should handle minimum as string with groupChar", async () => { + const table = DataFrame({ + price: ["5,000", "10,500", "2,500"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "price", + type: "integer", + groupChar: ",", + constraints: { minimum: "3,000" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/minimum", + fieldName: "price", + minimum: "3,000", + rowNumber: 3, + cell: "2,500", + }, + ]) + }) + + it("should handle minimum as string with decimalChar", async () => { + const table = DataFrame({ + price: ["5,5", "10,75", "2,3"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "price", + type: "number", + decimalChar: ",", + constraints: { minimum: "3,0" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/minimum", + fieldName: "price", + minimum: "3,0", + rowNumber: 3, + cell: "2,3", + }, + ]) + }) + + it("should handle minimum as string with groupChar and decimalChar", async () => { + const table = DataFrame({ + price: ["5.000,50", "10.500,75", "2.500,30"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "price", + type: "number", + groupChar: ".", + decimalChar: ",", + constraints: { minimum: "3.000,00" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/minimum", + fieldName: "price", + minimum: "3.000,00", + rowNumber: 3, + cell: "2.500,30", + }, + ]) + }) + + it("should handle minimum as string with bareNumber false", async () => { + const table = DataFrame({ + price: ["$5.00", "$10.50", "$2.50"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "price", + type: "number", + bareNumber: false, + constraints: { minimum: "$3.00" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/minimum", + fieldName: "price", + minimum: "$3.00", + rowNumber: 3, + cell: "$2.50", + }, + ]) + }) + + it("should handle minimum for date fields", async () => { + const table = DataFrame({ + date: ["2024-01-15", "2024-02-20", "2024-01-05"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "date", + type: "date", + constraints: { minimum: "2024-01-10" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/minimum", + fieldName: "date", + minimum: "2024-01-10", + rowNumber: 3, + cell: "2024-01-05", + }, + ]) + }) + + it.skip("should handle minimum for time fields", async () => { + const table = DataFrame({ + time: ["14:30:00", "16:45:00", "12:15:00"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "time", + type: "time", + constraints: { minimum: "13:00:00" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/minimum", + fieldName: "time", + minimum: "13:00:00", + rowNumber: 3, + cell: "12:15:00", + }, + ]) + }) + + it("should handle minimum for datetime fields", async () => { + const table = DataFrame({ + timestamp: [ + "2024-01-15T14:30:00", + "2024-02-20T08:15:00", + "2024-01-10T10:00:00", + ], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "timestamp", + type: "datetime", + constraints: { minimum: "2024-01-15T00:00:00" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/minimum", + fieldName: "timestamp", + minimum: "2024-01-15T00:00:00", + rowNumber: 3, + cell: "2024-01-10T10:00:00", + }, + ]) + }) + + it("should handle minimum for date fields with custom format", async () => { + const table = DataFrame({ + date: ["15/01/2024", "20/02/2024", "05/01/2024"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "date", + type: "date", + format: "%d/%m/%Y", + constraints: { minimum: "10/01/2024" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/minimum", + fieldName: "date", + minimum: "10/01/2024", + rowNumber: 3, + cell: "05/01/2024", + }, + ]) + }) + + it("should handle minimum for year fields", async () => { + const table = DataFrame({ + year: ["2020", "2021", "2018"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "year", + type: "year", + constraints: { minimum: "2019" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/minimum", + fieldName: "year", + minimum: "2019", + rowNumber: 3, + cell: "2018", + }, + ]) + }) + + it("should handle exclusiveMinimum for year fields", async () => { + const table = DataFrame({ + year: ["2020", "2021", "2019", "2018"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "year", + type: "year", + constraints: { exclusiveMinimum: "2019" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/exclusiveMinimum", + fieldName: "year", + minimum: "2019", + rowNumber: 3, + cell: "2019", + }, + { + type: "cell/exclusiveMinimum", + fieldName: "year", + minimum: "2019", + rowNumber: 4, + cell: "2018", + }, + ]) + }) + + it.skip("should handle minimum for yearmonth fields", async () => { + const table = DataFrame({ + yearmonth: ["2024-03", "2024-05", "2024-01"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "yearmonth", + type: "yearmonth", + constraints: { minimum: "2024-02" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/minimum", + fieldName: "yearmonth", + minimum: "2024-02", + rowNumber: 3, + cell: "2024-01", + }, + ]) + }) + + it.skip("should handle exclusiveMinimum for yearmonth fields", async () => { + const table = DataFrame({ + yearmonth: ["2024-03", "2024-05", "2024-02", "2024-01"], + }).lazy() + + const schema: Schema = { + fields: [ + { + name: "yearmonth", + type: "yearmonth", + constraints: { exclusiveMinimum: "2024-02" }, + }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/exclusiveMinimum", + fieldName: "yearmonth", + minimum: "2024-02", + rowNumber: 3, + cell: "2024-02", + }, + { + type: "cell/exclusiveMinimum", + fieldName: "yearmonth", + minimum: "2024-02", + rowNumber: 4, + cell: "2024-01", + }, + ]) + }) }) diff --git a/table/field/checks/minimum.ts b/table/field/checks/minimum.ts index c193df3c..1e33434d 100644 --- a/table/field/checks/minimum.ts +++ b/table/field/checks/minimum.ts @@ -1,46 +1,78 @@ import type { Field } from "@dpkit/core" -import { col, lit } from "nodejs-polars" -import type { Table } from "../../table/index.ts" - -export function checkCellMinimum( - field: Field, - errorTable: Table, - options?: { - isExclusive?: boolean - }, -) { - if (field.type === "integer" || field.type === "number") { +import * as pl from "nodejs-polars" +import type { Expr } from "nodejs-polars" +import type { CellExclusiveMinimumError } from "../../error/index.ts" +import type { CellMinimumError } from "../../error/index.ts" +import { evaluateExpression } from "../../helpers.ts" +import type { CellMapping } from "../Mapping.ts" +import { parseDateField } from "../types/date.ts" +import { parseDatetimeField } from "../types/datetime.ts" +import { parseIntegerField } from "../types/integer.ts" +import { parseNumberField } from "../types/number.ts" +import { parseTimeField } from "../types/time.ts" +import { parseYearField } from "../types/year.ts" +import { parseYearmonthField } from "../types/yearmonth.ts" + +export function createCheckCellMinimum(options?: { isExclusive?: boolean }) { + return (field: Field, mapping: CellMapping) => { + if ( + field.type !== "integer" && + field.type !== "number" && + field.type !== "date" && + field.type !== "time" && + field.type !== "datetime" && + field.type !== "year" && + field.type !== "yearmonth" + ) { + return undefined + } + const minimum = options?.isExclusive ? field.constraints?.exclusiveMinimum : field.constraints?.minimum + if (minimum === undefined) return undefined + + let isErrorExpr: Expr + try { + const parsedMinimum = parseConstraint(field, minimum) + isErrorExpr = options?.isExclusive + ? mapping.target.ltEq(parsedMinimum) + : mapping.target.lt(parsedMinimum) + } catch (error) { + isErrorExpr = pl.lit(true) + } - if (minimum !== undefined) { - const target = col(`target:${field.name}`) - const errorName = options?.isExclusive - ? `error:cell/exclusiveMinimum:${field.name}` - : `error:cell/minimum:${field.name}` - - const parser = - field.type === "integer" ? Number.parseInt : Number.parseFloat - - try { - const parsedMinimum = - typeof minimum === "string" ? parser(minimum) : minimum - - errorTable = errorTable - .withColumn( - options?.isExclusive - ? target.ltEq(parsedMinimum).alias(errorName) - : target.lt(parsedMinimum).alias(errorName), - ) - .withColumn(col("error").or(col(errorName)).alias("error")) - } catch (error) { - errorTable = errorTable - .withColumn(lit(true).alias(errorName)) - .withColumn(lit(true).alias("error")) - } + const errorTemplate: CellMinimumError | CellExclusiveMinimumError = { + type: options?.isExclusive ? "cell/exclusiveMinimum" : "cell/minimum", + fieldName: field.name, + minimum: String(minimum), + rowNumber: 0, + cell: "", } + + return { isErrorExpr, errorTemplate } + } +} + +function parseConstraint(field: Field, value: number | string) { + if (typeof value !== "string") return value + + let expr = pl.lit(value) + if (field.type === "integer") { + expr = parseIntegerField(field, expr) + } else if (field.type === "number") { + expr = parseNumberField(field, expr) + } else if (field.type === "date") { + expr = parseDateField(field, expr) + } else if (field.type === "time") { + expr = parseTimeField(field, expr) + } else if (field.type === "datetime") { + expr = parseDatetimeField(field, expr) + } else if (field.type === "year") { + expr = parseYearField(field, expr) + } else if (field.type === "yearmonth") { + expr = parseYearmonthField(field, expr) } - return errorTable + return evaluateExpression(expr) } diff --git a/table/field/checks/pattern.spec.ts b/table/field/checks/pattern.spec.ts index 39eb1ba6..afc8a255 100644 --- a/table/field/checks/pattern.spec.ts +++ b/table/field/checks/pattern.spec.ts @@ -26,6 +26,8 @@ describe("validateTable (cell/pattern)", () => { }) it("should report an error for strings that don't match the pattern", async () => { + const pattern = "^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$" + const table = DataFrame({ email: ["john@example.com", "alice@domain", "test.io", "valid@email.com"], }).lazy() @@ -36,7 +38,7 @@ describe("validateTable (cell/pattern)", () => { name: "email", type: "string", constraints: { - pattern: "^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$", + pattern, }, }, ], @@ -47,13 +49,15 @@ describe("validateTable (cell/pattern)", () => { expect(errors).toContainEqual({ type: "cell/pattern", fieldName: "email", - rowNumber: 2, // Second row (alice@domain) + pattern, + rowNumber: 2, cell: "alice@domain", }) expect(errors).toContainEqual({ type: "cell/pattern", fieldName: "email", - rowNumber: 3, // Third row (test.io) + pattern, + rowNumber: 3, cell: "test.io", }) }) diff --git a/table/field/checks/pattern.ts b/table/field/checks/pattern.ts index 82acbe3c..e4c31d0c 100644 --- a/table/field/checks/pattern.ts +++ b/table/field/checks/pattern.ts @@ -1,20 +1,22 @@ import type { Field } from "@dpkit/core" -import { col } from "nodejs-polars" -import type { Table } from "../../table/index.ts" +import type { CellPatternError } from "../../error/index.ts" +import type { CellMapping } from "../Mapping.ts" -export function checkCellPattern(field: Field, errorTable: Table) { - if (field.type === "string") { - const pattern = field.constraints?.pattern +export function checkCellPattern(field: Field, mapping: CellMapping) { + if (field.type !== "string") return undefined - if (pattern) { - const target = col(`target:${field.name}`) - const errorName = `error:cell/pattern:${field.name}` + const pattern = field.constraints?.pattern + if (!pattern) return undefined - errorTable = errorTable - .withColumn(target.str.contains(pattern).not().alias(errorName)) - .withColumn(col("error").or(col(errorName)).alias("error")) - } + const isErrorExpr = mapping.target.str.contains(pattern).not() + + const errorTemplate: CellPatternError = { + type: "cell/pattern", + fieldName: field.name, + pattern: pattern, + rowNumber: 0, + cell: "", } - return errorTable + return { isErrorExpr, errorTemplate } } diff --git a/table/field/checks/required.ts b/table/field/checks/required.ts index b7eac84d..08602f26 100644 --- a/table/field/checks/required.ts +++ b/table/field/checks/required.ts @@ -1,16 +1,19 @@ import type { Field } from "@dpkit/core" -import { col } from "nodejs-polars" -import type { Table } from "../../table/index.ts" +import type { CellRequiredError } from "../../error/index.ts" +import type { CellMapping } from "../Mapping.ts" -export function checkCellRequired(field: Field, errorTable: Table) { - if (field.constraints?.required) { - const target = col(`target:${field.name}`) - const errorName = `error:cell/required:${field.name}` +export function checkCellRequired(field: Field, mapping: CellMapping) { + const required = field.constraints?.required + if (!required) return undefined - errorTable = errorTable - .withColumn(target.isNull().alias(errorName)) - .withColumn(col("error").or(col(errorName)).alias("error")) + const isErrorExpr = mapping.target.isNull() + + const errorTemplate: CellRequiredError = { + type: "cell/required", + fieldName: field.name, + rowNumber: 0, + cell: "", } - return errorTable + return { isErrorExpr, errorTemplate } } diff --git a/table/field/checks/type.spec.ts b/table/field/checks/type.spec.ts index 641f5a89..3b0304e2 100644 --- a/table/field/checks/type.spec.ts +++ b/table/field/checks/type.spec.ts @@ -20,12 +20,14 @@ describe("validateTable", () => { type: "cell/type", cell: "bad", fieldName: "id", + fieldType: "integer", rowNumber: 2, }) expect(errors).toContainEqual({ type: "cell/type", cell: "4x", fieldName: "id", + fieldType: "integer", rowNumber: 4, }) }) @@ -46,12 +48,14 @@ describe("validateTable", () => { type: "cell/type", cell: "twenty", fieldName: "price", + fieldType: "number", rowNumber: 2, }) expect(errors).toContainEqual({ type: "cell/type", cell: "$40", fieldName: "price", + fieldType: "number", rowNumber: 4, }) }) @@ -72,6 +76,7 @@ describe("validateTable", () => { type: "cell/type", cell: "yes", fieldName: "active", + fieldType: "boolean", rowNumber: 2, }) }) @@ -92,18 +97,21 @@ describe("validateTable", () => { type: "cell/type", cell: "Jan 15, 2023", fieldName: "created", + fieldType: "date", rowNumber: 2, }) expect(errors).toContainEqual({ type: "cell/type", cell: "20230115", fieldName: "created", + fieldType: "date", rowNumber: 3, }) expect(errors).toContainEqual({ type: "cell/type", cell: "not-a-date", fieldName: "created", + fieldType: "date", rowNumber: 4, }) }) @@ -124,22 +132,49 @@ describe("validateTable", () => { type: "cell/type", cell: "2:30pm", fieldName: "time", + fieldType: "time", rowNumber: 2, }) expect(errors).toContainEqual({ type: "cell/type", cell: "invalid", fieldName: "time", + fieldType: "time", rowNumber: 3, }) expect(errors).toContainEqual({ type: "cell/type", cell: "14h30", fieldName: "time", + fieldType: "time", rowNumber: 4, }) }) + it("should validate string to time conversion errors with custom format", async () => { + const table = DataFrame({ + time: ["14:30", "invalid"], + }).lazy() + + const schema: Schema = { + fields: [{ name: "time", type: "time", format: "%H:%M" }], + } + + const { errors } = await validateTable(table, { schema }) + + console.log(errors) + + expect(errors).toHaveLength(1) + expect(errors).toContainEqual({ + type: "cell/type", + cell: "invalid", + fieldName: "time", + fieldType: "time", + fieldFormat: "%H:%M", + rowNumber: 2, + }) + }) + it("should validate string to year conversion errors", async () => { const table = DataFrame({ year: ["2023", "23", "MMXXIII", "two-thousand-twenty-three"], @@ -156,18 +191,21 @@ describe("validateTable", () => { type: "cell/type", cell: "23", fieldName: "year", + fieldType: "year", rowNumber: 2, }) expect(errors).toContainEqual({ type: "cell/type", cell: "MMXXIII", fieldName: "year", + fieldType: "year", rowNumber: 3, }) expect(errors).toContainEqual({ type: "cell/type", cell: "two-thousand-twenty-three", fieldName: "year", + fieldType: "year", rowNumber: 4, }) }) @@ -183,7 +221,7 @@ describe("validateTable", () => { }).lazy() const schema: Schema = { - fields: [{ name: "timestamp", type: "datetime" }], + fields: [{ name: "datetime", type: "datetime" }], } const { errors } = await validateTable(table, { schema }) @@ -195,14 +233,16 @@ describe("validateTable", () => { expect(errors).toContainEqual({ type: "cell/type", cell: "January 15, 2023 2:30 PM", - fieldName: "timestamp", + fieldName: "datetime", + fieldType: "datetime", rowNumber: 2, }) expect(errors).toContainEqual({ type: "cell/type", cell: "not-a-datetime", - fieldName: "timestamp", + fieldName: "datetime", + fieldType: "datetime", rowNumber: 4, }) }) diff --git a/table/field/checks/type.ts b/table/field/checks/type.ts index eb5db3e8..0e88dd62 100644 --- a/table/field/checks/type.ts +++ b/table/field/checks/type.ts @@ -1,15 +1,19 @@ import type { Field } from "@dpkit/core" -import { col } from "nodejs-polars" -import type { Table } from "../../table/index.ts" +import type { CellTypeError } from "../../error/index.ts" +import type { CellMapping } from "../Mapping.ts" -export function checkCellType(field: Field, errorTable: Table) { - const source = col(`source:${field.name}`) - const target = col(`target:${field.name}`) - const errorName = `error:cell/type:${field.name}` +export function checkCellType(field: Field, mapping: CellMapping) { + const isErrorExpr = mapping.source.isNotNull().and(mapping.target.isNull()) - errorTable = errorTable - .withColumn(source.isNotNull().and(target.isNull()).alias(errorName)) - .withColumn(col("error").or(col(errorName)).alias("error")) + const errorTemplate: CellTypeError = { + type: "cell/type", + fieldName: field.name, + fieldType: field.type ?? "any", + // @ts-ignore + fieldFormat: field.format, + rowNumber: 0, + cell: "", + } - return errorTable + return { isErrorExpr, errorTemplate } } diff --git a/table/field/checks/unique.ts b/table/field/checks/unique.ts index 5078afba..b6f3078d 100644 --- a/table/field/checks/unique.ts +++ b/table/field/checks/unique.ts @@ -1,21 +1,22 @@ import type { Field } from "@dpkit/core" -import { col } from "nodejs-polars" -import type { Table } from "../../table/index.ts" +import type { CellUniqueError } from "../../error/index.ts" +import type { CellMapping } from "../Mapping.ts" // TODO: Support schema.primaryKey and schema.uniqueKeys -export function checkCellUnique(field: Field, errorTable: Table) { +export function checkCellUnique(field: Field, mapping: CellMapping) { const unique = field.constraints?.unique + if (!unique) return undefined - if (unique) { - const target = col(`target:${field.name}`) - const errorName = `error:cell/unique:${field.name}` + const isErrorExpr = mapping.target + .isNotNull() + .and(mapping.target.isFirstDistinct().not()) - errorTable = errorTable - .withColumn( - target.isNotNull().and(target.isFirstDistinct().not()).alias(errorName), - ) - .withColumn(col("error").or(col(errorName)).alias("error")) + const errorTemplate: CellUniqueError = { + type: "cell/unique", + fieldName: field.name, + rowNumber: 0, + cell: "", } - return errorTable + return { isErrorExpr, errorTemplate } } diff --git a/table/field/denormalize.ts b/table/field/denormalize.ts index e0c3ec2e..3a45539e 100644 --- a/table/field/denormalize.ts +++ b/table/field/denormalize.ts @@ -1,23 +1,23 @@ import type { Field } from "@dpkit/core" -import { col, lit, when } from "nodejs-polars" -import type { Expr } from "nodejs-polars" +import { col } from "nodejs-polars" +import { desubstituteField } from "./desubstitute.ts" import { stringifyField } from "./stringify.ts" -const DEFAULT_MISSING_VALUE = "" - -export function denormalizeField(field: Field, expr?: Expr) { - expr = expr ?? col(field.name) - expr = stringifyField(field, expr) +export type DenormalizeFieldOptions = { + nativeTypes?: Exclude[] +} - const flattenMissingValues = field.missingValues?.map(it => - typeof it === "string" ? it : it.value, - ) +export function denormalizeField( + field: Field, + options?: DenormalizeFieldOptions, +) { + let expr = col(field.name) + const { nativeTypes } = options ?? {} - const missingValue = flattenMissingValues?.[0] ?? DEFAULT_MISSING_VALUE - expr = when(expr.isNull()) - .then(lit(missingValue)) - .otherwise(expr) - .alias(field.name) + if (!nativeTypes?.includes(field.type ?? "any")) { + expr = stringifyField(field, expr) + expr = desubstituteField(field, expr) + } return expr } diff --git a/table/field/desubstitute.ts b/table/field/desubstitute.ts new file mode 100644 index 00000000..e14396db --- /dev/null +++ b/table/field/desubstitute.ts @@ -0,0 +1,19 @@ +import type { Field } from "@dpkit/core" +import { lit, when } from "nodejs-polars" +import type { Expr } from "nodejs-polars" + +const DEFAULT_MISSING_VALUE = "" + +export function desubstituteField(field: Field, fieldExpr: Expr) { + const flattenMissingValues = field.missingValues?.map(it => + typeof it === "string" ? it : it.value, + ) + + const missingValue = flattenMissingValues?.[0] ?? DEFAULT_MISSING_VALUE + fieldExpr = when(fieldExpr.isNull()) + .then(lit(missingValue)) + .otherwise(fieldExpr) + .alias(field.name) + + return fieldExpr +} diff --git a/table/field/index.ts b/table/field/index.ts index 990d8eb1..902f73d4 100644 --- a/table/field/index.ts +++ b/table/field/index.ts @@ -1,7 +1,8 @@ export { denormalizeField } from "./denormalize.ts" export { parseField } from "./parse.ts" export { validateField } from "./validate.ts" -export { matchField } from "./match.ts" export { normalizeField } from "./normalize.ts" export { stringifyField } from "./stringify.ts" export type { PolarsField } from "./Field.ts" +export type { FieldMapping } from "./Mapping.ts" +export type { DenormalizeFieldOptions } from "./denormalize.ts" diff --git a/table/field/match.ts b/table/field/match.ts deleted file mode 100644 index ce9b33e9..00000000 --- a/table/field/match.ts +++ /dev/null @@ -1,14 +0,0 @@ -import type { Field, Schema } from "@dpkit/core" -import type { PolarsSchema } from "../schema/index.ts" - -export function matchField( - index: number, - field: Field, - schema: Schema, - polarsSchema: PolarsSchema, -) { - const fieldsMatch = schema.fieldsMatch ?? "exact" - return fieldsMatch !== "exact" - ? polarsSchema.fields.find(polarsField => polarsField.name === field.name) - : polarsSchema.fields[index] -} diff --git a/table/field/narrow.spec.ts b/table/field/narrow.spec.ts new file mode 100644 index 00000000..8a7f9628 --- /dev/null +++ b/table/field/narrow.spec.ts @@ -0,0 +1,56 @@ +import type { Schema } from "@dpkit/core" +import { DataFrame } from "nodejs-polars" +import { describe, expect, it } from "vitest" +import { normalizeTable } from "../table/normalize.ts" +import { validateTable } from "../table/validate.ts" + +describe("narrowField", () => { + it("should narrow float to integer", async () => { + const table = DataFrame({ + id: [1.0, 2.0, 3.0], + name: ["a", "b", "c"], + }).lazy() + + const schema: Schema = { + fields: [ + { name: "id", type: "integer" }, + { name: "name", type: "string" }, + ], + } + + const ldf = await normalizeTable(table, schema) + const df = await ldf.collect() + + expect(df.toRecords()).toEqual([ + { id: 1, name: "a" }, + { id: 2, name: "b" }, + { id: 3, name: "c" }, + ]) + }) + + it("should detect error when float cannot be narrowed to integer", async () => { + const table = DataFrame({ + id: [1.0, 2.0, 3.5], + name: ["a", "b", "c"], + }).lazy() + + const schema: Schema = { + fields: [ + { name: "id", type: "integer" }, + { name: "name", type: "string" }, + ], + } + + const { errors } = await validateTable(table, { schema }) + + expect(errors).toEqual([ + { + type: "cell/type", + fieldName: "id", + fieldType: "integer", + rowNumber: 3, + cell: "3.5", + }, + ]) + }) +}) diff --git a/table/field/narrow.ts b/table/field/narrow.ts new file mode 100644 index 00000000..9918e38a --- /dev/null +++ b/table/field/narrow.ts @@ -0,0 +1,18 @@ +import { DataType } from "nodejs-polars" +import { lit, when } from "nodejs-polars" +import type { Expr } from "nodejs-polars" +import type { FieldMapping } from "./Mapping.ts" + +export function narrowField(mapping: FieldMapping, fieldExpr: Expr) { + const variant = mapping.source.type.variant + + if (mapping.target.type === "integer") { + if (["Float32", "Float64"].includes(variant)) { + fieldExpr = when(fieldExpr.eq(fieldExpr.round(0))) + .then(fieldExpr.cast(DataType.Int64)) + .otherwise(lit(null)) + } + } + + return fieldExpr +} diff --git a/table/field/normalize.ts b/table/field/normalize.ts index f1329b67..838f68e4 100644 --- a/table/field/normalize.ts +++ b/table/field/normalize.ts @@ -1,31 +1,20 @@ -import type { Field } from "@dpkit/core" -import { col, lit, when } from "nodejs-polars" -import type { Expr } from "nodejs-polars" +import { col } from "nodejs-polars" +import type { FieldMapping } from "./Mapping.ts" +import { narrowField } from "./narrow.ts" import { parseField } from "./parse.ts" - -const DEFAULT_MISSING_VALUES = [""] +import { substituteField } from "./substitute.ts" export function normalizeField( - field: Field, - expr?: Expr, - options?: { dontParse?: boolean }, + mapping: FieldMapping, + options?: { keepType?: boolean }, ) { - expr = expr ?? col(field.name) - - const flattenMissingValues = - field.missingValues?.map(it => (typeof it === "string" ? it : it.value)) ?? - DEFAULT_MISSING_VALUES - - if (flattenMissingValues.length) { - expr = when(expr.isIn(flattenMissingValues)) - .then(lit(null)) - .otherwise(expr) - .alias(field.name) - } + let fieldExpr = col(mapping.source.name) + fieldExpr = substituteField(mapping, fieldExpr) - if (options?.dontParse) { - return expr + if (!options?.keepType) { + fieldExpr = parseField(mapping, fieldExpr) + fieldExpr = narrowField(mapping, fieldExpr) } - return parseField(field, expr) + return fieldExpr.alias(mapping.target.name) } diff --git a/table/field/parse.ts b/table/field/parse.ts index 64a8eb2d..896417fa 100644 --- a/table/field/parse.ts +++ b/table/field/parse.ts @@ -1,6 +1,6 @@ -import type { Field } from "@dpkit/core" -import { col } from "nodejs-polars" import type { Expr } from "nodejs-polars" +import { DataType } from "nodejs-polars" +import type { FieldMapping } from "./Mapping.ts" import { parseArrayField } from "./types/array.ts" import { parseBooleanField } from "./types/boolean.ts" import { parseDateField } from "./types/date.ts" @@ -17,41 +17,42 @@ import { parseTimeField } from "./types/time.ts" import { parseYearField } from "./types/year.ts" import { parseYearmonthField } from "./types/yearmonth.ts" -export function parseField(field: Field, expr?: Expr) { - expr = expr ?? col(field.name) +export function parseField(mapping: FieldMapping, fieldExpr: Expr) { + if (!mapping.source.type.equals(DataType.String)) return fieldExpr + const field = mapping.target switch (field.type) { case "array": - return parseArrayField(field, expr) + return parseArrayField(field, fieldExpr) case "boolean": - return parseBooleanField(field, expr) + return parseBooleanField(field, fieldExpr) case "date": - return parseDateField(field, expr) + return parseDateField(field, fieldExpr) case "datetime": - return parseDatetimeField(field, expr) + return parseDatetimeField(field, fieldExpr) case "duration": - return parseDurationField(field, expr) + return parseDurationField(field, fieldExpr) case "geojson": - return parseGeojsonField(field, expr) + return parseGeojsonField(field, fieldExpr) case "geopoint": - return parseGeopointField(field, expr) + return parseGeopointField(field, fieldExpr) case "integer": - return parseIntegerField(field, expr) + return parseIntegerField(field, fieldExpr) case "list": - return parseListField(field, expr) + return parseListField(field, fieldExpr) case "number": - return parseNumberField(field, expr) + return parseNumberField(field, fieldExpr) case "object": - return parseObjectField(field, expr) + return parseObjectField(field, fieldExpr) case "string": - return parseStringField(field, expr) + return parseStringField(field, fieldExpr) case "time": - return parseTimeField(field, expr) + return parseTimeField(field, fieldExpr) case "year": - return parseYearField(field, expr) + return parseYearField(field, fieldExpr) case "yearmonth": - return parseYearmonthField(field, expr) + return parseYearmonthField(field, fieldExpr) default: - return expr + return fieldExpr } } diff --git a/table/field/stringify.ts b/table/field/stringify.ts index f2d2ad8c..f03b4dce 100644 --- a/table/field/stringify.ts +++ b/table/field/stringify.ts @@ -1,5 +1,4 @@ import type { Field } from "@dpkit/core" -import { col } from "nodejs-polars" import type { Expr } from "nodejs-polars" import { stringifyArrayField } from "./types/array.ts" import { stringifyBooleanField } from "./types/boolean.ts" @@ -17,41 +16,39 @@ import { stringifyTimeField } from "./types/time.ts" import { stringifyYearField } from "./types/year.ts" import { stringifyYearmonthField } from "./types/yearmonth.ts" -export function stringifyField(field: Field, expr?: Expr) { - expr = expr ?? col(field.name) - +export function stringifyField(field: Field, fieldExpr: Expr) { switch (field.type) { case "array": - return stringifyArrayField(field, expr) + return stringifyArrayField(field, fieldExpr) case "boolean": - return stringifyBooleanField(field, expr) + return stringifyBooleanField(field, fieldExpr) case "date": - return stringifyDateField(field, expr) + return stringifyDateField(field, fieldExpr) case "datetime": - return stringifyDatetimeField(field, expr) + return stringifyDatetimeField(field, fieldExpr) case "duration": - return stringifyDurationField(field, expr) + return stringifyDurationField(field, fieldExpr) case "geojson": - return stringifyGeojsonField(field, expr) + return stringifyGeojsonField(field, fieldExpr) case "geopoint": - return stringifyGeopointField(field, expr) + return stringifyGeopointField(field, fieldExpr) case "integer": - return stringifyIntegerField(field, expr) + return stringifyIntegerField(field, fieldExpr) case "list": - return stringifyListField(field, expr) + return stringifyListField(field, fieldExpr) case "number": - return stringifyNumberField(field, expr) + return stringifyNumberField(field, fieldExpr) case "object": - return stringifyObjectField(field, expr) + return stringifyObjectField(field, fieldExpr) case "string": - return stringifyStringField(field, expr) + return stringifyStringField(field, fieldExpr) case "time": - return stringifyTimeField(field, expr) + return stringifyTimeField(field, fieldExpr) case "year": - return stringifyYearField(field, expr) + return stringifyYearField(field, fieldExpr) case "yearmonth": - return stringifyYearmonthField(field, expr) + return stringifyYearmonthField(field, fieldExpr) default: - return expr + return fieldExpr } } diff --git a/table/field/substitute.ts b/table/field/substitute.ts new file mode 100644 index 00000000..7234e0d4 --- /dev/null +++ b/table/field/substitute.ts @@ -0,0 +1,24 @@ +import { DataType } from "nodejs-polars" +import { lit, when } from "nodejs-polars" +import type { Expr } from "nodejs-polars" +import type { FieldMapping } from "./Mapping.ts" + +const DEFAULT_MISSING_VALUES = [""] + +export function substituteField(mapping: FieldMapping, fieldExpr: Expr) { + if (!mapping.source.type.equals(DataType.String)) return fieldExpr + + const flattenMissingValues = + mapping.target.missingValues?.map(it => + typeof it === "string" ? it : it.value, + ) ?? DEFAULT_MISSING_VALUES + + if (flattenMissingValues.length) { + fieldExpr = when(fieldExpr.isIn(flattenMissingValues)) + .then(lit(null)) + .otherwise(fieldExpr) + .alias(mapping.target.name) + } + + return fieldExpr +} diff --git a/table/field/types/array.ts b/table/field/types/array.ts index 7c21ec25..f3c7ddb3 100644 --- a/table/field/types/array.ts +++ b/table/field/types/array.ts @@ -1,19 +1,17 @@ import type { ArrayField } from "@dpkit/core" -import { col, lit, when } from "nodejs-polars" +import { lit, when } from "nodejs-polars" import type { Expr } from "nodejs-polars" // TODO: Is there a better way to do this? // Polars does not support really support free-form JSON // So we just make a basic check and return as it is -export function parseArrayField(field: ArrayField, expr?: Expr) { - expr = expr ?? col(field.name) - - return when(expr.str.contains("^\\[")).then(expr).otherwise(lit(null)) +export function parseArrayField(_field: ArrayField, fieldExpr: Expr) { + return when(fieldExpr.str.contains("^\\[")) + .then(fieldExpr) + .otherwise(lit(null)) } -export function stringifyArrayField(field: ArrayField, expr?: Expr) { - expr = expr ?? col(field.name) - +export function stringifyArrayField(_field: ArrayField, fieldExpr: Expr) { // TODO: implement - return expr + return fieldExpr } diff --git a/table/field/types/boolean.ts b/table/field/types/boolean.ts index 054cae60..0183da39 100644 --- a/table/field/types/boolean.ts +++ b/table/field/types/boolean.ts @@ -1,25 +1,23 @@ import type { BooleanField } from "@dpkit/core" import { DataType } from "nodejs-polars" -import { col, lit, when } from "nodejs-polars" +import { lit, when } from "nodejs-polars" import type { Expr } from "nodejs-polars" const DEFAULT_TRUE_VALUES = ["true", "True", "TRUE", "1"] const DEFAULT_FALSE_VALUES = ["false", "False", "FALSE", "0"] -export function parseBooleanField(field: BooleanField, expr?: Expr) { - expr = expr ?? col(field.name) - +export function parseBooleanField(field: BooleanField, fieldExpr: Expr) { const trueValues = field.trueValues ?? DEFAULT_TRUE_VALUES const falseValues = field.falseValues ?? DEFAULT_FALSE_VALUES - for (const value of trueValues) expr = expr.replace(value, "1") - for (const value of falseValues) expr = expr.replace(value, "0") + for (const value of trueValues) fieldExpr = fieldExpr.replace(value, "1") + for (const value of falseValues) fieldExpr = fieldExpr.replace(value, "0") - expr = expr.cast(DataType.Int8) + fieldExpr = fieldExpr.cast(DataType.Int8) - return when(expr.eq(1)) + return when(fieldExpr.eq(1)) .then(lit(true)) - .when(expr.eq(0)) + .when(fieldExpr.eq(0)) .then(lit(false)) .otherwise(lit(null)) .alias(field.name) @@ -28,13 +26,11 @@ export function parseBooleanField(field: BooleanField, expr?: Expr) { const DEFAULT_TRUE_VALUE = "true" const DEFAULT_FALSE_VALUE = "false" -export function stringifyBooleanField(field: BooleanField, expr?: Expr) { - expr = expr ?? col(field.name) - +export function stringifyBooleanField(field: BooleanField, fieldExpr: Expr) { const trueValue = field.trueValues?.[0] ?? DEFAULT_TRUE_VALUE const falseValue = field.falseValues?.[0] ?? DEFAULT_FALSE_VALUE - return when(expr.eq(lit(true))) + return when(fieldExpr.eq(lit(true))) .then(lit(trueValue)) .otherwise(lit(falseValue)) .alias(field.name) diff --git a/table/field/types/date.ts b/table/field/types/date.ts index 3199a413..bedf0dfd 100644 --- a/table/field/types/date.ts +++ b/table/field/types/date.ts @@ -1,25 +1,20 @@ import type { DateField } from "@dpkit/core" import { DataType } from "nodejs-polars" -import { col } from "nodejs-polars" import type { Expr } from "nodejs-polars" const DEFAULT_FORMAT = "%Y-%m-%d" -export function parseDateField(field: DateField, expr?: Expr) { - expr = expr ?? col(field.name) - +export function parseDateField(field: DateField, fieldExpr: Expr) { let format = DEFAULT_FORMAT if (field.format && field.format !== "default" && field.format !== "any") { format = field.format } - return expr.str.strptime(DataType.Date, format) + return fieldExpr.str.strptime(DataType.Date, format) } -export function stringifyDateField(field: DateField, expr?: Expr) { - expr = expr ?? col(field.name) - +export function stringifyDateField(field: DateField, fieldExpr: Expr) { const format = field.format ?? DEFAULT_FORMAT - return expr.date.strftime(format) + return fieldExpr.date.strftime(format) } diff --git a/table/field/types/datetime.ts b/table/field/types/datetime.ts index c12ab8a4..bee920ac 100644 --- a/table/field/types/datetime.ts +++ b/table/field/types/datetime.ts @@ -1,26 +1,21 @@ import type { DatetimeField } from "@dpkit/core" import { DataType } from "nodejs-polars" -import { col } from "nodejs-polars" import type { Expr } from "nodejs-polars" const DEFAULT_FORMAT = "%Y-%m-%dT%H:%M:%S" // TODO: Add support for timezone handling -export function parseDatetimeField(field: DatetimeField, expr?: Expr) { - expr = expr ?? col(field.name) - +export function parseDatetimeField(field: DatetimeField, fieldExpr: Expr) { let format = DEFAULT_FORMAT if (field.format && field.format !== "default" && field.format !== "any") { format = field.format } - return expr.str.strptime(DataType.Datetime, format) + return fieldExpr.str.strptime(DataType.Datetime, format) } -export function stringifyDatetimeField(field: DatetimeField, expr?: Expr) { - expr = expr ?? col(field.name) - +export function stringifyDatetimeField(field: DatetimeField, fieldExpr: Expr) { const format = field.format ?? DEFAULT_FORMAT - return expr.date.strftime(format) + return fieldExpr.date.strftime(format) } diff --git a/table/field/types/duration.ts b/table/field/types/duration.ts index cca2d21e..8b16d4e5 100644 --- a/table/field/types/duration.ts +++ b/table/field/types/duration.ts @@ -1,17 +1,12 @@ import type { DurationField } from "@dpkit/core" -import { col } from "nodejs-polars" import type { Expr } from "nodejs-polars" // TODO: raise an issue on nodejs-polars repo as this is not supported yet // So we do nothing on this column type for now -export function parseDurationField(field: DurationField, expr?: Expr) { - expr = expr ?? col(field.name) - - return expr +export function parseDurationField(_field: DurationField, fieldExpr: Expr) { + return fieldExpr } -export function stringifyDurationField(field: DurationField, expr?: Expr) { - expr = expr ?? col(field.name) - - return expr +export function stringifyDurationField(_field: DurationField, fieldExpr: Expr) { + return fieldExpr } diff --git a/table/field/types/geojson.ts b/table/field/types/geojson.ts index d4986074..da7a404f 100644 --- a/table/field/types/geojson.ts +++ b/table/field/types/geojson.ts @@ -1,18 +1,16 @@ import type { GeojsonField } from "@dpkit/core" -import { col, lit, when } from "nodejs-polars" +import { lit, when } from "nodejs-polars" import type { Expr } from "nodejs-polars" // TODO: Is there a better way to do this? // Polars does not support really support free-form JSON // So we just make a basic check and return as it is -export function parseGeojsonField(field: GeojsonField, expr?: Expr) { - expr = expr ?? col(field.name) - - return when(expr.str.contains("^\\{")).then(expr).otherwise(lit(null)) +export function parseGeojsonField(_field: GeojsonField, fieldExpr: Expr) { + return when(fieldExpr.str.contains("^\\{")) + .then(fieldExpr) + .otherwise(lit(null)) } -export function stringifyGeojsonField(field: GeojsonField, expr?: Expr) { - expr = expr ?? col(field.name) - - return expr +export function stringifyGeojsonField(_field: GeojsonField, fieldExpr: Expr) { + return fieldExpr } diff --git a/table/field/types/geopoint.ts b/table/field/types/geopoint.ts index 601bd5d7..7ab063cd 100644 --- a/table/field/types/geopoint.ts +++ b/table/field/types/geopoint.ts @@ -1,5 +1,5 @@ import type { GeopointField } from "@dpkit/core" -import { DataType, col, concatList, concatString, lit } from "nodejs-polars" +import { DataType, concatList, concatString, lit } from "nodejs-polars" import type { Expr } from "nodejs-polars" // TODO: @@ -8,50 +8,46 @@ import type { Expr } from "nodejs-polars" // - Check the values are within -180..180 and -90..90 // - Return null instead of list if any of the values are out of range -export function parseGeopointField(field: GeopointField, expr?: Expr) { - expr = expr ?? col(field.name) - +export function parseGeopointField(field: GeopointField, fieldExpr: Expr) { // Default format is "lon,lat" string const format = field.format ?? "default" if (format === "default") { - expr = expr.str.split(",").cast(DataType.List(DataType.Float64)) + fieldExpr = fieldExpr.str.split(",").cast(DataType.List(DataType.Float64)) } if (format === "array") { - expr = expr.str + fieldExpr = fieldExpr.str .replaceAll("[\\[\\]\\s]", "") .str.split(",") .cast(DataType.List(DataType.Float64)) } if (format === "object") { - expr = concatList([ - expr.str.jsonPathMatch("$.lon").cast(DataType.Float64), - expr.str.jsonPathMatch("$.lat").cast(DataType.Float64), + fieldExpr = concatList([ + fieldExpr.str.jsonPathMatch("$.lon").cast(DataType.Float64), + fieldExpr.str.jsonPathMatch("$.lat").cast(DataType.Float64), ]).alias(field.name) } - return expr + return fieldExpr } -export function stringifyGeopointField(field: GeopointField, expr?: Expr) { - expr = expr ?? col(field.name) - +export function stringifyGeopointField(field: GeopointField, fieldExpr: Expr) { // Default format is "lon,lat" string const format = field.format ?? "default" if (format === "default") { - return expr.cast(DataType.List(DataType.String)).lst.join(",") + return fieldExpr.cast(DataType.List(DataType.String)).lst.join(",") } if (format === "array") { return concatString( [ lit("["), - expr.lst.get(0).cast(DataType.String), + fieldExpr.lst.get(0).cast(DataType.String), lit(","), - expr.lst.get(1).cast(DataType.String), + fieldExpr.lst.get(1).cast(DataType.String), lit("]"), ], "", @@ -62,14 +58,14 @@ export function stringifyGeopointField(field: GeopointField, expr?: Expr) { return concatString( [ lit('{"lon":'), - expr.lst.get(0).cast(DataType.String), + fieldExpr.lst.get(0).cast(DataType.String), lit(',"lat":'), - expr.lst.get(1).cast(DataType.String), + fieldExpr.lst.get(1).cast(DataType.String), lit("}"), ], "", ).alias(field.name) as Expr } - return expr + return fieldExpr } diff --git a/table/field/types/integer.ts b/table/field/types/integer.ts index 0765a912..0a1d04c3 100644 --- a/table/field/types/integer.ts +++ b/table/field/types/integer.ts @@ -1,12 +1,10 @@ import type { IntegerField } from "@dpkit/core" -import { DataType, col, lit, when } from "nodejs-polars" +import { DataType, lit, when } from "nodejs-polars" import type { Expr } from "nodejs-polars" // TODO: support categories // TODO: support categoriesOrder -export function parseIntegerField(field: IntegerField, expr?: Expr) { - expr = expr ?? col(field.name) - +export function parseIntegerField(field: IntegerField, fieldExpr: Expr) { const groupChar = field.groupChar const bareNumber = field.bareNumber const flattenCategories = field.categories?.map(it => @@ -16,36 +14,34 @@ export function parseIntegerField(field: IntegerField, expr?: Expr) { // Handle non-bare numbers (with currency symbols, percent signs, etc.) if (bareNumber === false) { // Preserve the minus sign when removing leading characters - expr = expr.str.replaceAll("^[^\\d\\-]+", "") - expr = expr.str.replaceAll("[^\\d\\-]+$", "") + fieldExpr = fieldExpr.str.replaceAll("^[^\\d\\-]+", "") + fieldExpr = fieldExpr.str.replaceAll("[^\\d\\-]+$", "") } // Handle group character (thousands separator) if (groupChar) { // Escape special characters for regex const escapedGroupChar = groupChar.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") - expr = expr.str.replaceAll(escapedGroupChar, "") + fieldExpr = fieldExpr.str.replaceAll(escapedGroupChar, "") } // Cast to int64 (will handle values up to 2^63-1) - expr = expr.cast(DataType.Int64) + fieldExpr = fieldExpr.cast(DataType.Int64) // Currently, only string categories are supported if (flattenCategories) { - return when(expr.isIn(flattenCategories)) - .then(expr) + return when(fieldExpr.isIn(flattenCategories)) + .then(fieldExpr) .otherwise(lit(null)) .alias(field.name) } - return expr + return fieldExpr } -export function stringifyIntegerField(field: IntegerField, expr?: Expr) { - expr = expr ?? col(field.name) - +export function stringifyIntegerField(_field: IntegerField, fieldExpr: Expr) { // Convert to string - expr = expr.cast(DataType.String) + fieldExpr = fieldExpr.cast(DataType.String) //const groupChar = field.groupChar //const bareNumber = field.bareNumber @@ -53,5 +49,5 @@ export function stringifyIntegerField(field: IntegerField, expr?: Expr) { // TODO: Add group character formatting (thousands separator) when needed // TODO: Add non-bare number formatting (currency symbols, etc.) when needed - return expr + return fieldExpr } diff --git a/table/field/types/list.ts b/table/field/types/list.ts index 09cad5c3..926954f3 100644 --- a/table/field/types/list.ts +++ b/table/field/types/list.ts @@ -1,13 +1,11 @@ import type { ListField } from "@dpkit/core" -import { DataType, col } from "nodejs-polars" +import { DataType } from "nodejs-polars" import type { Expr } from "nodejs-polars" // TODO: // Add more validation: // - Return null instead of list if all array values are nulls? -export function parseListField(field: ListField, expr?: Expr) { - expr = expr ?? col(field.name) - +export function parseListField(field: ListField, fieldExpr: Expr) { const delimiter = field.delimiter ?? "," const itemType = field.itemType @@ -19,17 +17,15 @@ export function parseListField(field: ListField, expr?: Expr) { if (itemType === "date") dtype = DataType.Date if (itemType === "time") dtype = DataType.Time - expr = expr.str.split(delimiter).cast(DataType.List(dtype)) + fieldExpr = fieldExpr.str.split(delimiter).cast(DataType.List(dtype)) - return expr + return fieldExpr } -export function stringifyListField(field: ListField, expr?: Expr) { - expr = expr ?? col(field.name) - +export function stringifyListField(field: ListField, fieldExpr: Expr) { const delimiter = field.delimiter ?? "," - return expr + return fieldExpr .cast(DataType.List(DataType.String)) .lst.join({ separator: delimiter, ignoreNulls: true }) } diff --git a/table/field/types/number.ts b/table/field/types/number.ts index c5791117..de55a395 100644 --- a/table/field/types/number.ts +++ b/table/field/types/number.ts @@ -1,11 +1,8 @@ import type { NumberField } from "@dpkit/core" import { DataType } from "nodejs-polars" -import { col } from "nodejs-polars" import type { Expr } from "nodejs-polars" -export function parseNumberField(field: NumberField, expr?: Expr) { - expr = expr ?? col(field.name) - +export function parseNumberField(field: NumberField, fieldExpr: Expr) { // Extract the decimal and group characters const decimalChar = field.decimalChar ?? "." const groupChar = field.groupChar ?? "" @@ -16,43 +13,44 @@ export function parseNumberField(field: NumberField, expr?: Expr) { // Remove leading non-digit characters (except minus sign and allowed decimal points) const allowedDecimalChars = decimalChar === "." ? "\\." : `\\.${decimalChar}` - expr = expr.str.replaceAll(`^[^\\d\\-${allowedDecimalChars}]+`, "") + fieldExpr = fieldExpr.str.replaceAll( + `^[^\\d\\-${allowedDecimalChars}]+`, + "", + ) // Remove trailing non-digit characters - expr = expr.str.replaceAll(`[^\\d${allowedDecimalChars}]+$`, "") + fieldExpr = fieldExpr.str.replaceAll(`[^\\d${allowedDecimalChars}]+$`, "") } // Special case handling for European number format where "." is group and "," is decimal if (groupChar === "." && decimalChar === ",") { // First temporarily replace the decimal comma with a placeholder - expr = expr.str.replaceAll(",", "###DECIMAL###") + fieldExpr = fieldExpr.str.replaceAll(",", "###DECIMAL###") // Remove the group dots - expr = expr.str.replaceAll("\\.", "") + fieldExpr = fieldExpr.str.replaceAll("\\.", "") // Replace the placeholder with an actual decimal point - expr = expr.str.replaceAll("###DECIMAL###", ".") + fieldExpr = fieldExpr.str.replaceAll("###DECIMAL###", ".") } else { // Standard case: first remove group characters if (groupChar) { // Escape special characters for regex const escapedGroupChar = groupChar.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") - expr = expr.str.replaceAll(escapedGroupChar, "") + fieldExpr = fieldExpr.str.replaceAll(escapedGroupChar, "") } // Then handle decimal character if (decimalChar && decimalChar !== ".") { - expr = expr.str.replaceAll(decimalChar, ".") + fieldExpr = fieldExpr.str.replaceAll(decimalChar, ".") } } // Cast to float64 - expr = expr.cast(DataType.Float64) - return expr + fieldExpr = fieldExpr.cast(DataType.Float64) + return fieldExpr } -export function stringifyNumberField(field: NumberField, expr?: Expr) { - expr = expr ?? col(field.name) - +export function stringifyNumberField(_field: NumberField, fieldExpr: Expr) { // Convert to string - expr = expr.cast(DataType.String) + fieldExpr = fieldExpr.cast(DataType.String) //const decimalChar = field.decimalChar ?? "." //const groupChar = field.groupChar ?? "" @@ -61,5 +59,5 @@ export function stringifyNumberField(field: NumberField, expr?: Expr) { // TODO: Add group character formatting (thousands separator) when needed // TODO: Add non-bare number formatting (currency symbols, etc.) when needed - return expr + return fieldExpr } diff --git a/table/field/types/object.ts b/table/field/types/object.ts index 164d7f94..02e0c8c5 100644 --- a/table/field/types/object.ts +++ b/table/field/types/object.ts @@ -1,18 +1,16 @@ import type { ObjectField } from "@dpkit/core" -import { col, lit, when } from "nodejs-polars" +import { lit, when } from "nodejs-polars" import type { Expr } from "nodejs-polars" // TODO: Is there a better way to do this? // Polars does not support really support free-form JSON // So we just make a basic check and return as it is -export function parseObjectField(field: ObjectField, expr?: Expr) { - expr = expr ?? col(field.name) - - return when(expr.str.contains("^\\{")).then(expr).otherwise(lit(null)) +export function parseObjectField(_field: ObjectField, fieldExpr: Expr) { + return when(fieldExpr.str.contains("^\\{")) + .then(fieldExpr) + .otherwise(lit(null)) } -export function stringifyObjectField(field: ObjectField, expr?: Expr) { - expr = expr ?? col(field.name) - - return expr +export function stringifyObjectField(_field: ObjectField, fieldExpr: Expr) { + return fieldExpr } diff --git a/table/field/types/string.ts b/table/field/types/string.ts index 69bc6871..996303aa 100644 --- a/table/field/types/string.ts +++ b/table/field/types/string.ts @@ -1,5 +1,5 @@ import type { StringField } from "@dpkit/core" -import { DataType, col, lit, when } from "nodejs-polars" +import { DataType, lit, when } from "nodejs-polars" import type { Expr } from "nodejs-polars" const FORMAT_REGEX = { @@ -11,34 +11,30 @@ const FORMAT_REGEX = { } as const // TODO: support categoriesOrder? -export function parseStringField(field: StringField, expr?: Expr) { - expr = expr ?? col(field.name) - +export function parseStringField(field: StringField, fieldExpr: Expr) { const format = field.format const flattenCategories = field.categories?.map(it => typeof it === "string" ? it : it.value, ) if (flattenCategories) { - return when(expr.isIn(flattenCategories)) - .then(expr.cast(DataType.Categorical)) + return when(fieldExpr.isIn(flattenCategories)) + .then(fieldExpr.cast(DataType.Categorical)) .otherwise(lit(null)) .alias(field.name) } if (format) { const regex = FORMAT_REGEX[format] - return when(expr.str.contains(regex)) - .then(expr) + return when(fieldExpr.str.contains(regex)) + .then(fieldExpr) .otherwise(lit(null)) .alias(field.name) } - return expr + return fieldExpr } -export function stringifyStringField(field: StringField, expr?: Expr) { - expr = expr ?? col(field.name) - - return expr +export function stringifyStringField(_field: StringField, fieldExpr: Expr) { + return fieldExpr } diff --git a/table/field/types/time.ts b/table/field/types/time.ts index 13ebe1c8..23ad2370 100644 --- a/table/field/types/time.ts +++ b/table/field/types/time.ts @@ -1,28 +1,25 @@ import type { TimeField } from "@dpkit/core" import { DataType } from "nodejs-polars" -import { col, concatString, lit } from "nodejs-polars" +import * as pl from "nodejs-polars" import type { Expr } from "nodejs-polars" const DEFAULT_FORMAT = "%H:%M:%S" -export function parseTimeField(field: TimeField, expr?: Expr) { - expr = expr ?? col(field.name) - +export function parseTimeField(field: TimeField, fieldExpr: Expr) { let format = DEFAULT_FORMAT if (field.format && field.format !== "default" && field.format !== "any") { format = field.format } - return concatString([lit("1970-01-01T"), expr], "") + return pl + .concatString([pl.lit("1970-01-01T"), fieldExpr], "") .str.strptime(DataType.Datetime, `%Y-%m-%dT${format}`) .cast(DataType.Time) .alias(field.name) } -export function stringifyTimeField(field: TimeField, expr?: Expr) { - expr = expr ?? col(field.name) - +export function stringifyTimeField(field: TimeField, fieldExpr: Expr) { const format = field.format ?? DEFAULT_FORMAT - return expr.date.strftime(format) + return fieldExpr.date.strftime(format) } diff --git a/table/field/types/year.ts b/table/field/types/year.ts index 380e0631..48c85750 100644 --- a/table/field/types/year.ts +++ b/table/field/types/year.ts @@ -1,23 +1,18 @@ import type { YearField } from "@dpkit/core" import { DataType, lit, when } from "nodejs-polars" -import { col } from "nodejs-polars" import type { Expr } from "nodejs-polars" -export function parseYearField(field: YearField, expr?: Expr) { - expr = expr ?? col(field.name) - - expr = when(expr.str.lengths().eq(4)) - .then(expr) +export function parseYearField(_field: YearField, fieldExpr: Expr) { + fieldExpr = when(fieldExpr.str.lengths().eq(4)) + .then(fieldExpr) .otherwise(lit(null)) .cast(DataType.Int16) - return when(expr.gtEq(0).and(expr.ltEq(9999))) - .then(expr) + return when(fieldExpr.gtEq(0).and(fieldExpr.ltEq(9999))) + .then(fieldExpr) .otherwise(lit(null)) } -export function stringifyYearField(field: YearField, expr?: Expr) { - expr = expr ?? col(field.name) - - return expr.cast(DataType.String).str.zFill(4) +export function stringifyYearField(_field: YearField, fieldExpr: Expr) { + return fieldExpr.cast(DataType.String).str.zFill(4) } diff --git a/table/field/types/yearmonth.ts b/table/field/types/yearmonth.ts index b284b87c..d9644723 100644 --- a/table/field/types/yearmonth.ts +++ b/table/field/types/yearmonth.ts @@ -1,31 +1,21 @@ import type { YearmonthField } from "@dpkit/core" -import { DataType, col, concatString } from "nodejs-polars" +import { DataType, concatString } from "nodejs-polars" import type { Expr } from "nodejs-polars" -// TODO: -// Add more validation: -// - Check the length of the list is 2 (no list.lenghts in polars currently) -// - Check the values are year and month limits -// - Return null instead of list if any of the values are out of range -// - Rebase on Struct when lst.toStruct() is available? +export function parseYearmonthField(_field: YearmonthField, fieldExpr: Expr) { + fieldExpr = fieldExpr.str.split("-").cast(DataType.List(DataType.Int16)) -export function parseYearmonthField(field: YearmonthField, expr?: Expr) { - expr = expr ?? col(field.name) - - expr = expr.str.split("-").cast(DataType.List(DataType.Int16)) - - return expr + return fieldExpr } -export function stringifyYearmonthField(field: YearmonthField, expr?: Expr) { - expr = expr ?? col(field.name) - - // TODO: remove int casting when resolved: - // https://github.com/pola-rs/nodejs-polars/issues/362 +export function stringifyYearmonthField( + field: YearmonthField, + fieldExpr: Expr, +) { return concatString( [ - expr.lst.get(0).cast(DataType.Int16).cast(DataType.String).str.zFill(4), - expr.lst.get(1).cast(DataType.Int16).cast(DataType.String).str.zFill(2), + fieldExpr.lst.get(0).cast(DataType.String).str.zFill(4), + fieldExpr.lst.get(1).cast(DataType.String).str.zFill(2), ], "-", ).alias(field.name) as Expr diff --git a/table/field/validate.spec.ts b/table/field/validate.spec.ts index 7ea3d0f3..4c9369d4 100644 --- a/table/field/validate.spec.ts +++ b/table/field/validate.spec.ts @@ -138,12 +138,14 @@ describe("validateField", () => { type: "cell/type", cell: "bad", fieldName: "id", + fieldType: "integer", rowNumber: 2, }) expect(errors).toContainEqual({ type: "cell/type", cell: "4x", fieldName: "id", + fieldType: "integer", rowNumber: 4, }) }) @@ -169,12 +171,14 @@ describe("validateField", () => { type: "cell/type", cell: "twenty", fieldName: "price", + fieldType: "number", rowNumber: 2, }) expect(errors).toContainEqual({ type: "cell/type", cell: "$40", fieldName: "price", + fieldType: "number", rowNumber: 4, }) }) @@ -200,6 +204,7 @@ describe("validateField", () => { type: "cell/type", cell: "yes", fieldName: "active", + fieldType: "boolean", rowNumber: 2, }) }) @@ -225,18 +230,21 @@ describe("validateField", () => { type: "cell/type", cell: "Jan 15, 2023", fieldName: "created", + fieldType: "date", rowNumber: 2, }) expect(errors).toContainEqual({ type: "cell/type", cell: "20230115", fieldName: "created", + fieldType: "date", rowNumber: 3, }) expect(errors).toContainEqual({ type: "cell/type", cell: "not-a-date", fieldName: "created", + fieldType: "date", rowNumber: 4, }) }) @@ -262,18 +270,21 @@ describe("validateField", () => { type: "cell/type", cell: "2:30pm", fieldName: "time", + fieldType: "time", rowNumber: 2, }) expect(errors).toContainEqual({ type: "cell/type", cell: "invalid", fieldName: "time", + fieldType: "time", rowNumber: 3, }) expect(errors).toContainEqual({ type: "cell/type", cell: "14h30", fieldName: "time", + fieldType: "time", rowNumber: 4, }) }) @@ -299,18 +310,21 @@ describe("validateField", () => { type: "cell/type", cell: "23", fieldName: "year", + fieldType: "year", rowNumber: 2, }) expect(errors).toContainEqual({ type: "cell/type", cell: "MMXXIII", fieldName: "year", + fieldType: "year", rowNumber: 3, }) expect(errors).toContainEqual({ type: "cell/type", cell: "two-thousand-twenty-three", fieldName: "year", + fieldType: "year", rowNumber: 4, }) }) @@ -344,6 +358,7 @@ describe("validateField", () => { type: "cell/type", cell: "January 15, 2023 2:30 PM", fieldName: "timestamp", + fieldType: "datetime", rowNumber: 2, }) @@ -351,6 +366,7 @@ describe("validateField", () => { type: "cell/type", cell: "not-a-datetime", fieldName: "timestamp", + fieldType: "datetime", rowNumber: 4, }) }) diff --git a/table/field/validate.ts b/table/field/validate.ts index 7220a772..963c02b1 100644 --- a/table/field/validate.ts +++ b/table/field/validate.ts @@ -1,102 +1,155 @@ import type { Field } from "@dpkit/core" -import type { TableError } from "../error/index.ts" +import { col, lit, when } from "nodejs-polars" +import type { CellError, FieldError, TableError } from "../error/index.ts" import type { Table } from "../table/index.ts" -import type { PolarsField } from "./Field.ts" +import type { FieldMapping } from "./Mapping.ts" import { checkCellEnum } from "./checks/enum.ts" import { checkCellMaxLength } from "./checks/maxLength.ts" -import { checkCellMaximum } from "./checks/maximum.ts" +import { createCheckCellMaximum } from "./checks/maximum.ts" import { checkCellMinLength } from "./checks/minLength.ts" -import { checkCellMinimum } from "./checks/minimum.ts" +import { createCheckCellMinimum } from "./checks/minimum.ts" import { checkCellPattern } from "./checks/pattern.ts" import { checkCellRequired } from "./checks/required.ts" import { checkCellType } from "./checks/type.ts" import { checkCellUnique } from "./checks/unique.ts" +import { normalizeField } from "./normalize.ts" -export function validateField( - field: Field, +export async function validateField( + mapping: FieldMapping, + table: Table, options: { - errorTable: Table - polarsField: PolarsField + maxErrors: number }, ) { - const { polarsField } = options + const { maxErrors } = options const errors: TableError[] = [] - const nameErrors = validateName(field, polarsField) + const nameErrors = validateName(mapping) errors.push(...nameErrors) - const typeErrors = validateType(field, polarsField) + const typeErrors = validateType(mapping) errors.push(...typeErrors) - const errorTable = !typeErrors.length - ? validateCells(field, options.errorTable) - : options.errorTable + if (!typeErrors.length) { + const dataErorrs = await validateCells(mapping, table, { maxErrors }) + errors.push(...dataErorrs) + } - return { valid: !errors.length, errors, errorTable } + return { errors, valid: !errors.length } } -function validateName(field: Field, polarsField: PolarsField) { - const errors: TableError[] = [] +function validateName(mapping: FieldMapping) { + const errors: FieldError[] = [] - if (field.name !== polarsField.name) { + if (mapping.source.name !== mapping.target.name) { errors.push({ type: "field/name", - fieldName: field.name, - actualFieldName: polarsField.name, + fieldName: mapping.target.name, + actualFieldName: mapping.source.name, }) } return errors } -function validateType(field: Field, polarsField: PolarsField) { - const errors: TableError[] = [] +function validateType(mapping: FieldMapping) { + const errors: FieldError[] = [] - const mapping: Record = { - Bool: "boolean", - Date: "date", - Datetime: "datetime", - Float32: "number", - Float64: "number", - Int16: "integer", - Int32: "integer", - Int64: "integer", - Int8: "integer", - List: "list", - String: "string", - Time: "time", - UInt16: "integer", - UInt32: "integer", - UInt64: "integer", - UInt8: "integer", - Utf8: "string", + const compatMapping: Record = { + Bool: ["boolean"], + Date: ["date"], + Datetime: ["datetime"], + Float32: ["number", "integer"], + Float64: ["number", "integer"], + Int16: ["integer"], + Int32: ["integer"], + Int64: ["integer"], + Int8: ["integer"], + List: ["list"], + Time: ["time"], + UInt16: ["integer"], + UInt32: ["integer"], + UInt64: ["integer"], + UInt8: ["integer"], + Utf8: ["string"], } - const actualFieldType = mapping[polarsField.type.variant] ?? "any" + const compatTypes = compatMapping[mapping.source.type.variant] + if (!compatTypes) return errors - if (actualFieldType !== field.type && actualFieldType !== "string") { + if (!compatTypes.includes(mapping.target.type)) { errors.push({ type: "field/type", - fieldName: field.name, - fieldType: field.type ?? "any", - actualFieldType, + fieldName: mapping.target.name, + fieldType: mapping.target.type ?? "any", + actualFieldType: compatTypes[0] ?? "any", }) } return errors } -function validateCells(field: Field, errorTable: Table) { - errorTable = checkCellType(field, errorTable) - errorTable = checkCellRequired(field, errorTable) - errorTable = checkCellPattern(field, errorTable) - errorTable = checkCellEnum(field, errorTable) - errorTable = checkCellMinimum(field, errorTable) - errorTable = checkCellMaximum(field, errorTable) - errorTable = checkCellMinimum(field, errorTable, { isExclusive: true }) - errorTable = checkCellMaximum(field, errorTable, { isExclusive: true }) - errorTable = checkCellMinLength(field, errorTable) - errorTable = checkCellMaxLength(field, errorTable) - errorTable = checkCellUnique(field, errorTable) - return errorTable +async function validateCells( + mapping: FieldMapping, + table: Table, + options: { + maxErrors: number + }, +) { + const { maxErrors } = options + const errors: CellError[] = [] + + let fieldCheckTable = table + .withRowCount() + .select( + col("row_nr").add(1).alias("number"), + normalizeField(mapping).alias("target"), + normalizeField(mapping, { keepType: true }).alias("source"), + lit(null).alias("error"), + ) + + for (const checkCell of [ + checkCellType, + checkCellRequired, + checkCellPattern, + checkCellEnum, + createCheckCellMinimum(), + createCheckCellMaximum(), + createCheckCellMinimum({ isExclusive: true }), + createCheckCellMaximum({ isExclusive: true }), + checkCellMinLength, + checkCellMaxLength, + checkCellUnique, + ]) { + const cellMapping = { source: col("source"), target: col("target") } + + const check = checkCell(mapping.target, cellMapping) + if (!check) continue + + fieldCheckTable = fieldCheckTable.withColumn( + when(col("error").isNotNull()) + .then(col("error")) + .when(check.isErrorExpr) + .then(lit(JSON.stringify(check.errorTemplate))) + .otherwise(lit(null)) + .alias("error"), + ) + } + + const fieldCheckFrame = await fieldCheckTable + .filter(col("error").isNotNull()) + .drop(["target"]) + .head(maxErrors) + .collect() + + for (const row of fieldCheckFrame.toRecords() as any[]) { + const errorTemplate = JSON.parse(row.error) as CellError + errors.push({ + ...errorTemplate, + rowNumber: row.number, + cell: String(row.source ?? ""), + }) + } + + return errors } diff --git a/table/helpers.ts b/table/helpers.ts new file mode 100644 index 00000000..032b745c --- /dev/null +++ b/table/helpers.ts @@ -0,0 +1,11 @@ +import type { Expr } from "nodejs-polars" +import * as pl from "nodejs-polars" + +export function arrayDiff(a: string[], b: string[]) { + return a.filter(x => !b.includes(x)) +} + +export function evaluateExpression(expr: Expr) { + // @ts-ignore + return pl.select(expr.alias("value")).toRecords()[0].value +} diff --git a/table/package.json b/table/package.json index faebc257..d30f4eb4 100644 --- a/table/package.json +++ b/table/package.json @@ -25,6 +25,7 @@ }, "dependencies": { "@dpkit/core": "workspace:*", - "nodejs-polars": "^0.22.1" + "nodejs-polars": "^0.22.1", + "p-all": "^5.0.1" } } diff --git a/table/row/checks/unique.ts b/table/row/checks/unique.ts deleted file mode 100644 index 3ed1e21e..00000000 --- a/table/row/checks/unique.ts +++ /dev/null @@ -1,32 +0,0 @@ -import type { Schema } from "@dpkit/core" -import { col, concatList } from "nodejs-polars" -import type { Table } from "../../table/Table.ts" - -// TODO: fold is not available so we use a tricky way to eliminate list nulls -// TODO: Using comma as separator might rarely clash with comma in field names -export function checkRowUnique(schema: Schema, errorTable: Table) { - const uniqueKeys = schema.uniqueKeys ?? [] - - if (schema.primaryKey) { - uniqueKeys.push(schema.primaryKey) - } - - for (const uniqueKey of uniqueKeys) { - const targetNames = uniqueKey.map(field => `target:${field}`) - const errorName = `error:row/unique:${uniqueKey.join(",")}` - - errorTable = errorTable - .withColumn(concatList(targetNames).alias(errorName)) - .withColumn( - col(errorName) - .lst.min() - .isNull() - .not() - .and(col(errorName).isFirstDistinct().not()) - .alias(errorName), - ) - .withColumn(col("error").or(col(errorName)).alias("error")) - } - - return errorTable -} diff --git a/table/row/index.ts b/table/row/index.ts deleted file mode 100644 index 0628f872..00000000 --- a/table/row/index.ts +++ /dev/null @@ -1 +0,0 @@ -export { validateRows } from "./validate.ts" diff --git a/table/row/validate.ts b/table/row/validate.ts deleted file mode 100644 index 6e4f33cd..00000000 --- a/table/row/validate.ts +++ /dev/null @@ -1,12 +0,0 @@ -import type { Schema } from "@dpkit/core" -import type { TableError } from "../error/Table.ts" -import type { Table } from "../table/Table.ts" -import { checkRowUnique } from "./checks/unique.ts" - -export function validateRows(schema: Schema, errorTable: Table) { - const errors: TableError[] = [] - - errorTable = checkRowUnique(schema, errorTable) - - return { valid: !errors.length, errors, errorTable } -} diff --git a/table/schema/Mapping.ts b/table/schema/Mapping.ts new file mode 100644 index 00000000..baaee2cc --- /dev/null +++ b/table/schema/Mapping.ts @@ -0,0 +1,7 @@ +import type { Schema } from "@dpkit/core" +import type { PolarsSchema } from "./Schema.ts" + +export interface SchemaMapping { + source: PolarsSchema + target: Schema +} diff --git a/table/schema/index.ts b/table/schema/index.ts index d888123b..8cd4331e 100644 --- a/table/schema/index.ts +++ b/table/schema/index.ts @@ -4,3 +4,5 @@ export type { SchemaOptions } from "./Options.ts" export { inferSchemaFromTable } from "./infer.ts" export { inferSchemaFromSample } from "./infer.ts" export type { InferSchemaOptions } from "./infer.ts" +export type { SchemaMapping } from "./Mapping.ts" +export { matchSchemaField } from "./match.ts" diff --git a/table/schema/infer.spec.ts b/table/schema/infer.spec.ts index 62208283..141081ea 100644 --- a/table/schema/infer.spec.ts +++ b/table/schema/infer.spec.ts @@ -20,6 +20,22 @@ describe("inferSchemaFromTable", () => { expect(await inferSchemaFromTable(table)).toEqual(schema) }) + it("should infer integers from floats", async () => { + const table = DataFrame({ + id: [1.0, 2.0, 3.0], + count: [10.0, 20.0, 30.0], + }).lazy() + + const schema = { + fields: [ + { name: "id", type: "integer" }, + { name: "count", type: "integer" }, + ], + } + + expect(await inferSchemaFromTable(table)).toEqual(schema) + }) + it("should infer numeric", async () => { const table = DataFrame({ name1: ["1", "2", "3"], diff --git a/table/schema/infer.ts b/table/schema/infer.ts index 4c5b54ae..d5b7232c 100644 --- a/table/schema/infer.ts +++ b/table/schema/infer.ts @@ -58,21 +58,38 @@ export function inferSchemaFromSample( variant = variant.slice(0, -1) } - let type = fieldTypes?.[name] ?? typeMapping[variant] ?? "any" + const type = fieldTypes?.[name] ?? typeMapping[variant] ?? "any" + let field = { name, type } - if (type === "array" && options?.arrayType === "list") { - type = "list" - } + if (!fieldTypes?.[name]) { + if (type === "array") { + if (options?.arrayType === "list") { + field.type = "list" + } + } - let field = { name, type } - if (!keepStrings && type === "string" && !fieldTypes?.[name]) { - for (const [regex, patch] of Object.entries(regexMapping)) { + if (type === "string") { + if (!keepStrings) { + for (const [regex, patch] of Object.entries(regexMapping)) { + const failures = sample + .filter(col(name).str.contains(regex).not()) + .head(failureThreshold).height + + if (failures < failureThreshold) { + field = { ...field, ...patch } + break + } + } + } + } + + if (type === "number") { const failures = sample - .filter(col(name).str.contains(regex).not()) + .filter(col(name).eq(col(name).round(0)).not()) .head(failureThreshold).height + if (failures < failureThreshold) { - field = { ...field, ...patch } - break + field.type = "integer" } } } diff --git a/table/schema/match.ts b/table/schema/match.ts new file mode 100644 index 00000000..24583b0f --- /dev/null +++ b/table/schema/match.ts @@ -0,0 +1,17 @@ +import type { Field } from "@dpkit/core" +import type { SchemaMapping } from "./Mapping.ts" + +export function matchSchemaField( + mapping: SchemaMapping, + field: Field, + index: number, +) { + const fieldsMatch = mapping.target.fieldsMatch ?? "exact" + + const polarsField = + fieldsMatch !== "exact" + ? mapping.source.fields.find(it => it.name === field.name) + : mapping.source.fields[index] + + return polarsField ? { source: polarsField, target: field } : undefined +} diff --git a/table/row/checks/unique.spec.ts b/table/table/checks/unique.spec.ts similarity index 99% rename from table/row/checks/unique.spec.ts rename to table/table/checks/unique.spec.ts index 22f39af4..c3365a80 100644 --- a/table/row/checks/unique.spec.ts +++ b/table/table/checks/unique.spec.ts @@ -37,6 +37,7 @@ describe("validateTable (row/unique)", () => { } const { errors } = await validateTable(table, { schema }) + expect(errors.filter(e => e.type === "row/unique")).toHaveLength(1) expect(errors).toContainEqual({ type: "row/unique", @@ -178,6 +179,7 @@ describe("validateTable (row/unique)", () => { } const { errors } = await validateTable(table, { schema }) + console.log(errors) expect(errors).toHaveLength(2) expect(errors).toContainEqual({ diff --git a/table/table/checks/unique.ts b/table/table/checks/unique.ts new file mode 100644 index 00000000..77d085fd --- /dev/null +++ b/table/table/checks/unique.ts @@ -0,0 +1,29 @@ +import { concatList } from "nodejs-polars" +import type { RowUniqueError } from "../../error/index.ts" +import type { SchemaMapping } from "../../schema/index.ts" + +export function createChecksRowUnique(mapping: SchemaMapping) { + const uniqueKeys = mapping.target.uniqueKeys ?? [] + + if (mapping.target.primaryKey) { + uniqueKeys.push(mapping.target.primaryKey) + } + + return uniqueKeys.map(createCheckRowUnique) +} + +function createCheckRowUnique(uniqueKey: string[]) { + const isErrorExpr = concatList(uniqueKey) + .isFirstDistinct() + .not() + // Fold is not available so we use a tricky way to eliminate nulls + .and(concatList(uniqueKey).lst.min().isNotNull()) + + const errorTemplate: RowUniqueError = { + type: "row/unique", + fieldNames: uniqueKey, + rowNumber: 0, + } + + return { isErrorExpr, errorTemplate } +} diff --git a/table/table/denormalize.ts b/table/table/denormalize.ts index f39be815..37b7bfae 100644 --- a/table/table/denormalize.ts +++ b/table/table/denormalize.ts @@ -1,53 +1,28 @@ -import type { Field, Schema } from "@dpkit/core" -import { col, lit } from "nodejs-polars" +import type { Schema } from "@dpkit/core" import type { Expr } from "nodejs-polars" import { denormalizeField } from "../field/index.ts" -import type { PolarsSchema } from "../schema/index.ts" -import { getPolarsSchema } from "../schema/index.ts" +import type { DenormalizeFieldOptions } from "../field/index.ts" import type { Table } from "./Table.ts" -const HEAD_ROWS = 100 - -type DenormalizeTableOptions = { - nativeTypes?: Exclude[] -} - export async function denormalizeTable( table: Table, schema: Schema, - options?: DenormalizeTableOptions, + options?: DenormalizeFieldOptions, ) { - const head = await table.head(HEAD_ROWS).collect() - const polarsSchema = getPolarsSchema(head.schema) - - return table.select( - ...Object.values(denormalizeFields(schema, polarsSchema, options)), - ) + return table.select(...Object.values(denormalizeFields(schema, options))) } export function denormalizeFields( schema: Schema, - polarsSchema: PolarsSchema, - options?: DenormalizeTableOptions, + options?: DenormalizeFieldOptions, ) { - const { nativeTypes } = options ?? {} const exprs: Record = {} for (const field of schema.fields) { - const polarsField = polarsSchema.fields.find(f => f.name === field.name) - let expr = lit(null).alias(field.name) - - if (polarsField) { - expr = col(polarsField.name).alias(field.name) - - // TODO: Move this logic to denormalizeField? - if (!nativeTypes?.includes(field.type ?? "any")) { - const missingValues = field.missingValues ?? schema.missingValues - const mergedField = { ...field, missingValues } - expr = denormalizeField(mergedField, expr) - } - } + const missingValues = field.missingValues ?? schema.missingValues + const mergedField = { ...field, missingValues } + const expr = denormalizeField(mergedField, options) exprs[field.name] = expr } diff --git a/table/table/normalize.ts b/table/table/normalize.ts index 39cded33..8d23b81e 100644 --- a/table/table/normalize.ts +++ b/table/table/normalize.ts @@ -1,55 +1,35 @@ import type { Schema } from "@dpkit/core" import type { Expr } from "nodejs-polars" -import { DataType } from "nodejs-polars" -import { col, lit } from "nodejs-polars" -import { matchField } from "../field/index.ts" +import { lit } from "nodejs-polars" import { normalizeField } from "../field/index.ts" +import { matchSchemaField } from "../schema/index.ts" import { getPolarsSchema } from "../schema/index.ts" -import type { PolarsSchema } from "../schema/index.ts" +import type { SchemaMapping } from "../schema/index.ts" import type { Table } from "./Table.ts" const HEAD_ROWS = 100 -export async function normalizeTable( - table: Table, - schema: Schema, - options?: { - dontParse?: boolean - }, -) { - const { dontParse } = options ?? {} - +export async function normalizeTable(table: Table, schema: Schema) { const head = await table.head(HEAD_ROWS).collect() const polarsSchema = getPolarsSchema(head.schema) - return table.select( - ...Object.values(normalizeFields(schema, polarsSchema, { dontParse })), - ) + const mapping = { source: polarsSchema, target: schema } + return table.select(...Object.values(normalizeFields(mapping))) } -export function normalizeFields( - schema: Schema, - polarsSchema: PolarsSchema, - options?: { - dontParse?: boolean - }, -) { - const { dontParse } = options ?? {} +export function normalizeFields(mapping: SchemaMapping) { const exprs: Record = {} - for (const [index, field] of schema.fields.entries()) { - const polarsField = matchField(index, field, schema, polarsSchema) + for (const [index, field] of mapping.target.fields.entries()) { + const fieldMapping = matchSchemaField(mapping, field, index) let expr = lit(null).alias(field.name) - if (polarsField) { - expr = col(polarsField.name).alias(field.name) + if (fieldMapping) { + const missingValues = field.missingValues ?? mapping.target.missingValues + const mergedField = { ...field, missingValues } - // TODO: Move this logic to normalizeField? - if (polarsField.type.equals(DataType.String)) { - const missingValues = field.missingValues ?? schema.missingValues - const mergedField = { ...field, missingValues } - expr = normalizeField(mergedField, expr, { dontParse }) - } + const column = { source: fieldMapping.source, target: mergedField } + expr = normalizeField(column) } exprs[field.name] = expr diff --git a/table/table/validate.ts b/table/table/validate.ts index 9ee59b89..8ec28107 100644 --- a/table/table/validate.ts +++ b/table/table/validate.ts @@ -1,55 +1,55 @@ -import type { Schema } from "@dpkit/core" -import { col, lit } from "nodejs-polars" +import os from "node:os" +import type { Field, Schema } from "@dpkit/core" +import { col, lit, when } from "nodejs-polars" +import pAll from "p-all" +import type { RowError } from "../error/index.ts" import type { TableError } from "../error/index.ts" -import { matchField } from "../field/index.ts" import { validateField } from "../field/index.ts" -import { validateRows } from "../row/index.ts" +import { arrayDiff } from "../helpers.ts" +import { matchSchemaField } from "../schema/index.ts" import { getPolarsSchema } from "../schema/index.ts" -import type { PolarsSchema } from "../schema/index.ts" +import type { SchemaMapping } from "../schema/index.ts" import type { Table } from "./Table.ts" -import { normalizeFields } from "./normalize.ts" +import { createChecksRowUnique } from "./checks/unique.ts" export async function validateTable( table: Table, options?: { schema?: Schema sampleRows?: number - invalidRowsLimit?: number + maxErrors?: number }, ) { - const { schema, sampleRows = 100, invalidRowsLimit = 100 } = options ?? {} + const { schema, sampleRows = 100, maxErrors = 1000 } = options ?? {} const errors: TableError[] = [] if (schema) { const sample = await table.head(sampleRows).collect() const polarsSchema = getPolarsSchema(sample.schema) + const mapping = { source: polarsSchema, target: schema } - const matchErrors = validateFieldsMatch({ schema, polarsSchema }) + const matchErrors = validateFieldsMatch(mapping) errors.push(...matchErrors) - const fieldErrors = await validateFields( - table, - schema, - polarsSchema, - invalidRowsLimit, - ) + const fieldErrors = await validateFields(mapping, table, { maxErrors }) errors.push(...fieldErrors) + + const rowErrors = await validateRows(mapping, table, { maxErrors }) + errors.push(...rowErrors) } - return { errors, valid: !errors.length } + return { + errors: errors.slice(0, maxErrors), + valid: !errors.length, + } } -function validateFieldsMatch(props: { - schema: Schema - polarsSchema: PolarsSchema -}) { - const { schema, polarsSchema } = props - +function validateFieldsMatch(mapping: SchemaMapping) { const errors: TableError[] = [] - const fieldsMatch = schema.fieldsMatch ?? "exact" + const fieldsMatch = mapping.target.fieldsMatch ?? "exact" - const fields = schema.fields - const polarsFields = polarsSchema.fields + const fields = mapping.target.fields + const polarsFields = mapping.source.fields const names = fields.map(field => field.name) const polarsNames = polarsFields.map(field => field.name) @@ -128,95 +128,97 @@ function validateFieldsMatch(props: { } async function validateFields( + mapping: SchemaMapping, table: Table, - schema: Schema, - polarsSchema: PolarsSchema, - invalidRowsLimit: number, + options: { + maxErrors: number + }, ) { + const { maxErrors } = options const errors: TableError[] = [] - const targetNames: string[] = [] - - const sources = Object.entries( - normalizeFields(schema, polarsSchema, { dontParse: true }), - ).map(([name, expr]) => { - return expr.alias(`source:${name}`) - }) - - const targets = Object.entries( - normalizeFields(schema, polarsSchema, { dontParse: false }), - ).map(([name, expr]) => { - const targetName = `target:${name}` - targetNames.push(targetName) - return expr.alias(targetName) - }) - - let errorTable = table - .withRowCount() - .select( - col("row_nr").add(1), - lit(false).alias("error"), - ...sources, - ...targets, - ) - - for (const [index, field] of schema.fields.entries()) { - const polarsField = matchField(index, field, schema, polarsSchema) - if (polarsField) { - const fieldResult = validateField(field, { errorTable, polarsField }) - errorTable = fieldResult.errorTable - errors.push(...fieldResult.errors) + const fields = mapping.target.fields + const concurrency = os.cpus().length - 1 + const abortController = new AbortController() + const maxFieldErrors = Math.ceil(maxErrors / fields.length) + + const collectFieldErrors = async (index: number, field: Field) => { + const fieldMapping = matchSchemaField(mapping, field, index) + if (!fieldMapping) return + + const report = await validateField(fieldMapping, table, { + maxErrors: maxFieldErrors, + }) + + errors.push(...report.errors) + if (errors.length > maxErrors) { + abortController.abort() } } - const rowsResult = validateRows(schema, errorTable) - errorTable = rowsResult.errorTable - errors.push(...rowsResult.errors) - - const errorFrame = await errorTable - .filter(col("error").eq(true)) - .head(invalidRowsLimit) - .drop(targetNames) - .collect() - - for (const record of errorFrame.toRecords() as any[]) { - const typeErrorInFields: string[] = [] - for (const [key, value] of Object.entries(record)) { - const [kind, type, name] = key.split(":") - if (kind === "error" && value === true && type && name) { - const rowNumber = record.row_nr - - // Cell-level errors - if (type.startsWith("cell/")) { - if (!typeErrorInFields.includes(name)) { - errors.push({ - rowNumber, - type: type as any, - fieldName: name as any, - cell: (record[`source:${name}`] ?? "").toString(), - }) - } - - // Type error is a terminating error for a cell - if (type === "cell/type") { - typeErrorInFields.push(name) - } - } - - // Row-level errors - if (type.startsWith("row/")) { - errors.push({ - rowNumber, - type: type as any, - fieldNames: name.split(","), - }) - } - } - } + try { + await pAll( + fields.map((field, index) => () => collectFieldErrors(index, field)), + { concurrency }, + ) + } catch (error) { + const isAborted = error instanceof Error && error.name === "AbortError" + if (!isAborted) throw error } return errors } -function arrayDiff(a: string[], b: string[]) { - return a.filter(x => !b.includes(x)) +async function validateRows( + mapping: SchemaMapping, + table: Table, + options: { maxErrors: number }, +) { + const { maxErrors } = options + const errors: TableError[] = [] + const fields = mapping.target.fields + const concurrency = os.cpus().length - 1 + const abortController = new AbortController() + const maxRowErrors = Math.ceil(maxErrors / fields.length) + + const collectRowErrors = async (check: any) => { + const rowCheckTable = table + .withRowCount() + .withColumn(col("row_nr").add(1)) + .rename({ row_nr: "dpkit:number" }) + .withColumn( + when(check.isErrorExpr) + .then(lit(JSON.stringify(check.errorTemplate))) + .otherwise(lit(null)) + .alias("dpkit:error"), + ) + + const rowCheckFrame = await rowCheckTable + .filter(col("dpkit:error").isNotNull()) + .head(maxRowErrors) + .collect() + + for (const row of rowCheckFrame.toRecords() as any[]) { + const errorTemplate = JSON.parse(row["dpkit:error"]) as RowError + errors.push({ + ...errorTemplate, + rowNumber: row["dpkit:number"], + }) + } + + if (errors.length > maxErrors) { + abortController.abort() + } + } + + try { + await pAll( + [...createChecksRowUnique(mapping)].map(it => () => collectRowErrors(it)), + { concurrency }, + ) + } catch (error) { + const isAborted = error instanceof Error && error.name === "AbortError" + if (!isAborted) throw error + } + + return errors } diff --git a/xlsx/table/load.ts b/xlsx/table/load.ts index 786515c9..ce59d5f9 100644 --- a/xlsx/table/load.ts +++ b/xlsx/table/load.ts @@ -1,6 +1,6 @@ -import { loadResourceDialect } from "@dpkit/core" +import { resolveDialect } from "@dpkit/core" import type { Resource } from "@dpkit/core" -import { loadResourceSchema } from "@dpkit/core" +import { resolveSchema } from "@dpkit/core" import { loadFile, prefetchFiles } from "@dpkit/file" import type { LoadTableOptions } from "@dpkit/table" import { inferSchemaFromTable, normalizeTable } from "@dpkit/table" @@ -21,7 +21,7 @@ export async function loadXlsxTable( throw new Error("Resource path is not defined") } - const dialect = await loadResourceDialect(resource.dialect) + const dialect = await resolveDialect(resource.dialect) const tables: Table[] = [] for (const path of paths) { @@ -48,7 +48,7 @@ export async function loadXlsxTable( let table = concat(tables) if (!options?.denormalized) { - let schema = await loadResourceSchema(resource.schema) + let schema = await resolveSchema(resource.schema) if (!schema) schema = await inferSchemaFromTable(table, options) table = await normalizeTable(table, schema) } diff --git a/xlsx/table/save.ts b/xlsx/table/save.ts index 849918eb..9c5420d6 100644 --- a/xlsx/table/save.ts +++ b/xlsx/table/save.ts @@ -1,4 +1,4 @@ -import { loadResourceDialect } from "@dpkit/core" +import { resolveDialect } from "@dpkit/core" import { saveFile } from "@dpkit/file" import { denormalizeTable, inferSchemaFromTable } from "@dpkit/table" import type { SaveTableOptions, Table } from "@dpkit/table" @@ -22,7 +22,7 @@ export async function saveXlsxTable(table: Table, options: SaveTableOptions) { }) const df = await table.collect() - const dialect = await loadResourceDialect(options.dialect) + const dialect = await resolveDialect(options.dialect) const sheetName = dialect?.sheetName ?? "Sheet1" const sheet = utils.json_to_sheet(df.toRecords())