feat: Add xlsx/ods support

.changeset/grumpy-seals-exist.md

@@ -0,0 +1,12 @@
+---
+"@dpkit/inline": minor
+"dpkit": minor
+"@dpkit/table": minor
+"@dpkit/core": minor
+"@dpkit/json": minor
+"@dpkit/xlsx": minor
+"@dpkit/csv": minor
+"@dpkit/ods": minor
+---
+
+Added xlsx/ods support

core/dialect/assert.ts

@@ -6,8 +6,8 @@ import { validateDialect } from "./validate.ts"
 /**
  * Assert a Dialect descriptor (JSON Object) against its profile
  */
-export async function assertDialect(descriptor: Descriptor | Dialect) {
-  const { dialect, errors } = await validateDialect(descriptor)
+export async function assertDialect(source: Descriptor | Dialect) {
+  const { dialect, errors } = await validateDialect(source)
   if (!dialect) throw new AssertionError(errors)
   return dialect
 }

core/dialect/validate.ts

@@ -8,10 +8,8 @@ const DEFAULT_PROFILE = "https://datapackage.org/profiles/1.0/tabledialect.json"
 /**
  * Validate a Dialect descriptor (JSON Object) against its profile
  */
-export async function validateDialect(
-  descriptorOrDialect: Descriptor | Dialect,
-) {
-  const descriptor = descriptorOrDialect as Descriptor
+export async function validateDialect(source: Descriptor | Dialect) {
+  const descriptor = source as Descriptor
 
   const $schema =
     typeof descriptor.$schema === "string"

core/package/assert.ts

@@ -7,13 +7,13 @@ import { validatePackageDescriptor } from "./validate.ts"
  * Assert a Package descriptor (JSON Object) against its profile
  */
 export async function assertPackage(
-  descriptorOrPackage: Descriptor | Package,
+  source: Descriptor | Package,
   options?: {
     basepath?: string
   },
 ) {
   const { errors, dataPackage } = await validatePackageDescriptor(
-    descriptorOrPackage,
+    source,
     options,
   )
 

core/package/validate.ts

@@ -9,12 +9,12 @@ const DEFAULT_PROFILE = "https://datapackage.org/profiles/1.0/datapackage.json"
  * Validate a Package descriptor (JSON Object) against its profile
  */
 export async function validatePackageDescriptor(
-  descriptorOrPackage: Descriptor | Package,
+  source: Descriptor | Package,
   options?: {
     basepath?: string
   },
 ) {
-  const descriptor = descriptorOrPackage as Descriptor
+  const descriptor = source as Descriptor
 
   const $schema =
     typeof descriptor.$schema === "string"

core/resource/assert.ts

@@ -7,15 +7,12 @@ import { validateResourceDescriptor } from "./validate.ts"
  * Assert a Resource descriptor (JSON Object) against its profile
  */
 export async function assertResource(
-  descriptorOrResource: Descriptor | Resource,
+  source: Descriptor | Resource,
   options?: {
     basepath?: string
   },
 ) {
-  const { errors, resource } = await validateResourceDescriptor(
-    descriptorOrResource,
-    options,
-  )
+  const { errors, resource } = await validateResourceDescriptor(source, options)
 
   if (!resource) throw new AssertionError(errors)
   return resource

core/resource/dialect.ts

@@ -0,0 +1,14 @@
+import { loadDialect } from "../dialect/index.ts"
+import type { Resource } from "./Resource.ts"
+
+export async function loadResourceDialect(dialect: Resource["dialect"]) {
+  if (!dialect) {
+    return undefined
+  }
+
+  if (typeof dialect !== "string") {
+    return dialect
+  }
+
+  return await loadDialect(dialect)
+}

core/resource/index.ts

@@ -8,3 +8,5 @@ export { normalizeResource } from "./process/normalize.ts"
 export { denormalizeResource } from "./process/denormalize.ts"
 export type { Source } from "./Source.ts"
 export type { License } from "./License.ts"
+export { loadResourceDialect } from "./dialect.ts"
+export { loadResourceSchema } from "./schema.ts"

core/resource/schema.ts

@@ -0,0 +1,14 @@
+import { loadSchema } from "../schema/index.ts"
+import type { Resource } from "./Resource.ts"
+
+export async function loadResourceSchema(schema: Resource["schema"]) {
+  if (!schema) {
+    return undefined
+  }
+
+  if (typeof schema !== "string") {
+    return schema
+  }
+
+  return await loadSchema(schema)
+}

core/resource/validate.ts

@@ -12,12 +12,12 @@ const DEFAULT_PROFILE = "https://datapackage.org/profiles/1.0/dataresource.json"
  * Validate a Resource descriptor (JSON Object) against its profile
  */
 export async function validateResourceDescriptor(
-  descriptorOrResource: Descriptor | Resource,
+  source: Descriptor | Resource,
   options?: {
     basepath?: string
   },
 ) {
-  const descriptor = descriptorOrResource as Descriptor
+  const descriptor = source as Descriptor
 
   const $schema =
     typeof descriptor.$schema === "string"

core/schema/assert.ts

@@ -6,8 +6,8 @@ import { validateSchema } from "./validate.ts"
 /**
  * Assert a Schema descriptor (JSON Object) against its profile
  */
-export async function assertSchema(descriptor: Descriptor | Schema) {
-  const { schema, errors } = await validateSchema(descriptor)
+export async function assertSchema(source: Descriptor | Schema) {
+  const { schema, errors } = await validateSchema(source)
   if (!schema) throw new AssertionError(errors)
   return schema
 }

core/schema/validate.ts

@@ -8,8 +8,8 @@ const DEFAULT_PROFILE = "https://datapackage.org/profiles/1.0/tableschema.json"
 /**
  * Validate a Schema descriptor (JSON Object) against its profile
  */
-export async function validateSchema(descriptorOrSchema: Descriptor | Schema) {
-  const descriptor = descriptorOrSchema as Descriptor
+export async function validateSchema(source: Descriptor | Schema) {
+  const descriptor = source as Descriptor
 
   const $schema =
     typeof descriptor.$schema === "string"

csv/table/load.ts

@@ -1,8 +1,10 @@
 import type { Dialect, Resource } from "@dpkit/core"
-import { loadDialect } from "@dpkit/core"
+import { loadResourceDialect } from "@dpkit/core"
 import { prefetchFiles } from "@dpkit/file"
-import type { Table } from "@dpkit/table"
-import { DataFrame, col, scanCSV } from "nodejs-polars"
+import { stripInitialSpace } from "@dpkit/table"
+import { joinHeaderRows } from "@dpkit/table"
+import { skipCommentRows } from "@dpkit/table"
+import { DataFrame, scanCSV } from "nodejs-polars"
 import type { ScanCsvOptions } from "nodejs-polars"
 import { Utf8, concat } from "nodejs-polars"
 
@@ -16,11 +18,7 @@ export async function loadCsvTable(resource: Partial<Resource>) {
     return DataFrame().lazy()
   }
 
-  const dialect =
-    typeof resource.dialect === "string"
-      ? await loadDialect(resource.dialect)
-      : resource.dialect
-
+  const dialect = await loadResourceDialect(resource.dialect)
   const scanOptions = getScanOptions(resource, dialect)
   let table = scanCSV(firstPath, scanOptions)
 
@@ -42,9 +40,9 @@ export async function loadCsvTable(resource: Partial<Resource>) {
   }
 
   if (dialect) {
-    table = await joinHeaderRows(table, dialect)
-    table = skipCommentRows(table, dialect)
-    table = stripInitialSpace(table, dialect)
+    table = await joinHeaderRows(table, { dialect })
+    table = skipCommentRows(table, { dialect })
+    table = stripInitialSpace(table, { dialect })
   }
 
   return table
@@ -86,81 +84,11 @@ function getScanOptions(resource: Partial<Resource>, dialect?: Dialect) {
   return options
 }
 
-// TODO: move to the table package?
-
-async function joinHeaderRows(table: Table, dialect: Dialect) {
-  const headerOffset = getHeaderOffset(dialect)
-  const headerRows = getHeaderRows(dialect)
-  const headerJoin = dialect?.headerJoin ?? " "
-  if (headerRows.length < 2) {
-    return table
-  }
-
-  const extraLabelsFrame = await table
-    .withRowCount()
-    .withColumn(col("row_nr").add(1))
-    .filter(col("row_nr").add(headerOffset).isIn(headerRows))
-    .select(table.columns.map(name => col(name).str.concat(headerJoin)))
-    .collect()
-
-  const labels = table.columns
-  const extraLabels = extraLabelsFrame.row(0)
-
-  const mapping = Object.fromEntries(
-    labels.map((label, index) => [
-      label,
-      [label, extraLabels[index]].join(headerJoin),
-    ]),
-  )
-
-  return table
-    .withRowCount()
-    .withColumn(col("row_nr").add(1))
-    .filter(col("row_nr").add(headerOffset).isIn(headerRows).not())
-    .rename(mapping)
-    .drop("row_nr")
-}
-
-function skipCommentRows(table: Table, dialect: Dialect) {
-  const commentOffset = getCommentOffset(dialect)
-  if (!dialect?.commentRows) {
-    return table
-  }
-
-  return table
-    .withRowCount()
-    .withColumn(col("row_nr").add(1))
-    .filter(col("row_nr").add(commentOffset).isIn(dialect.commentRows).not())
-    .drop("row_nr")
-}
-
-function stripInitialSpace(table: Table, dialect: Dialect) {
-  if (!dialect?.skipInitialSpace) {
-    return table
-  }
-
-  return table.select(
-    // TODO: rebase on stripCharsStart when it's fixed in polars
-    // https://github.com/pola-rs/nodejs-polars/issues/336
-    table.columns.map(name => col(name).str.strip().as(name)),
-  )
-}
-
 function getRowsToSkip(dialect?: Dialect) {
   const headerRows = getHeaderRows(dialect)
   return headerRows[0] ? headerRows[0] - 1 : 0
 }
 
-function getHeaderOffset(dialect?: Dialect) {
-  const headerRows = getHeaderRows(dialect)
-  return headerRows.at(0) ?? 0
-}
-
-function getCommentOffset(dialect?: Dialect) {
-  const headerRows = getHeaderRows(dialect)
-  return headerRows.at(-1) ?? 0
-}
-
 function getHeaderRows(dialect?: Dialect) {
   return dialect?.header !== false ? (dialect?.headerRows ?? [1]) : []
 }

docs/astro.config.ts

@@ -17,8 +17,11 @@ const PACKAGES = {
   "@dpkit/file": "../file",
   "@dpkit/github": "../github",
   "@dpkit/inline": "../inline",
+  "@dpkit/json": "../json",
+  "@dpkit/ods": "../ods",
   "@dpkit/parquet": "../parquet",
   "@dpkit/table": "../table",
+  "@dpkit/xlsx": "../xlsx",
   "@dpkit/zenodo": "../zenodo",
   "@dpkit/zip": "../zip",
 }
@@ -109,7 +112,7 @@ function generatePackageEntrypoints() {
 }
 
 function generatePackageSidebars() {
-  return Object.entries(PACKAGES).map(([name, path]) =>
+  return Object.entries(PACKAGES).map(([name, _path]) =>
     generatePackageSidebar({ name }),
   )
 }