From a2109390baf58de92bf7a70d932eee2c7859a7ff Mon Sep 17 00:00:00 2001 From: Dmitry Dorofeev Date: Wed, 3 Jun 2026 13:58:44 +0300 Subject: [PATCH] feat: apache-arrow read_arrow() function via bundled nanoarrow extension Adds first-class Apache Arrow IPC reading to pg_duckdb, mirroring the read_parquet pattern: * third_party/pg_duckdb_extensions.cmake: link paleolimbot/duckdb-nanoarrow PR #47 (commit 42e4199), which targets DuckDB 1.5. * sql/pg_duckdb--1.1.0--1.2.0.sql: register read_arrow(text) and read_arrow(text[]) as duckdb_only_function stubs. The planner reroutes any query containing these calls to DuckDB, which executes the function natively via the statically-linked nanoarrow extension. * pg_duckdb.control: bump default_version 1.1.0 -> 1.2.0. * src/pgduckdb_hooks.cpp:ContainsDuckdbRowReturningFunction(): add read_arrow to the strstr filter so the planner-routing pre-check recognises it without requiring duckdb.force_execution = true. * src/pgduckdb_metadata_cache.cpp:BuildDuckdbOnlyFunctions(): add read_arrow to the known-function list so duckdb_only_function lookups cache its OID. * test/regression/sql/arrow.sql + expected/arrow.out: round-trip both Arrow IPC stream (.arrows) and Arrow IPC file (.arrow) formats through COPY ... TO + read_arrow(). Array-variant test included. * test/regression/schedule: register the arrow test. Note: nanoarrow rejects Arrow files with Dictionary-encoded columns. A common JS producer pitfall is apache-arrow's tableFromArrays(), which auto-builds Dictionary for string columns. Producers should emit plain Utf8 vectors via vectorFromArray() and stream-format IPC. Co-Authored-By: Claude Opus 4.7 (1M context) --- pg_duckdb.control | 2 +- sql/pg_duckdb--1.1.0--1.2.0.sql | 18 +++++++++ src/pgduckdb_hooks.cpp | 3 +- src/pgduckdb_metadata_cache.cpp | 1 + test/regression/expected/arrow.out | 51 ++++++++++++++++++++++++++ test/regression/schedule | 1 + test/regression/sql/arrow.sql | 31 ++++++++++++++++ third_party/pg_duckdb_extensions.cmake | 6 +++ 8 files changed, 111 insertions(+), 2 deletions(-) create mode 100644 sql/pg_duckdb--1.1.0--1.2.0.sql create mode 100644 test/regression/expected/arrow.out create mode 100644 test/regression/sql/arrow.sql diff --git a/pg_duckdb.control b/pg_duckdb.control index 8ca89f31..3ed847a7 100644 --- a/pg_duckdb.control +++ b/pg_duckdb.control @@ -1,4 +1,4 @@ comment = 'DuckDB Embedded in Postgres' -default_version = '1.1.0' +default_version = '1.2.0' module_pathname = '$libdir/pg_duckdb' relocatable = false diff --git a/sql/pg_duckdb--1.1.0--1.2.0.sql b/sql/pg_duckdb--1.1.0--1.2.0.sql new file mode 100644 index 00000000..e7652629 --- /dev/null +++ b/sql/pg_duckdb--1.1.0--1.2.0.sql @@ -0,0 +1,18 @@ +-- Apache Arrow support via the bundled nanoarrow DuckDB extension. +-- Mirrors the read_parquet pattern: the C body is the duckdb_only_function +-- stub; the planner reroutes any query that mentions these functions to +-- DuckDB, which executes read_arrow natively. + +-- read_arrow function for single path (Arrow IPC stream or file, auto-detected) +CREATE FUNCTION @extschema@.read_arrow(path text) +RETURNS SETOF duckdb.row +SET search_path = pg_catalog, pg_temp +AS 'MODULE_PATHNAME', 'duckdb_only_function' +LANGUAGE C; + +-- read_arrow function for array of paths +CREATE FUNCTION @extschema@.read_arrow(path text[]) +RETURNS SETOF duckdb.row +SET search_path = pg_catalog, pg_temp +AS 'MODULE_PATHNAME', 'duckdb_only_function' +LANGUAGE C; diff --git a/src/pgduckdb_hooks.cpp b/src/pgduckdb_hooks.cpp index 8157c3b1..13ad0fcb 100644 --- a/src/pgduckdb_hooks.cpp +++ b/src/pgduckdb_hooks.cpp @@ -444,7 +444,8 @@ static bool ContainsDuckdbRowReturningFunction(const char *query_string) { return strstr(query_string, "read_parquet") || strstr(query_string, "read_csv") || strstr(query_string, "read_json") || strstr(query_string, "delta_scan") || - strstr(query_string, "iceberg_scan") || strstr(query_string, "duckdb.query"); + strstr(query_string, "iceberg_scan") || strstr(query_string, "read_arrow") || + strstr(query_string, "duckdb.query"); } static void diff --git a/src/pgduckdb_metadata_cache.cpp b/src/pgduckdb_metadata_cache.cpp index b11543da..ad25f845 100644 --- a/src/pgduckdb_metadata_cache.cpp +++ b/src/pgduckdb_metadata_cache.cpp @@ -152,6 +152,7 @@ BuildDuckdbOnlyFunctions() { "iceberg_snapshots", "delta_scan", "read_json", + "read_arrow", "approx_count_distinct", "query", "view", diff --git a/test/regression/expected/arrow.out b/test/regression/expected/arrow.out new file mode 100644 index 00000000..6cc18d6f --- /dev/null +++ b/test/regression/expected/arrow.out @@ -0,0 +1,51 @@ +-- Apache Arrow support via the bundled nanoarrow extension. +-- Verifies that nanoarrow is loaded by default and that read_arrow() works +-- for both Arrow IPC stream (.arrows) and Arrow IPC file (.arrow) formats. +SET duckdb.force_execution = true; +-- Sanity: nanoarrow is loaded as a built-in (no install required). +SET duckdb.force_execution = false; +SELECT * FROM duckdb.query($$ SELECT extension_name, loaded FROM duckdb_extensions() WHERE extension_name = 'nanoarrow' $$); + extension_name | loaded +----------------+-------- + nanoarrow | t +(1 row) + +SET duckdb.force_execution = true; +-- Set up a small dataset and round-trip it through Arrow IPC stream. +\set pwd `pwd` +\set arrow_stream_path '\'' :pwd '/tmp_check/pg_duckdb_arrow.arrows' '\'' +\set arrow_file_path '\'' :pwd '/tmp_check/pg_duckdb_arrow.arrow' '\'' +CREATE TABLE t_arrow(i INT, label TEXT); +INSERT INTO t_arrow SELECT g, 'row_' || g FROM generate_series(1, 5) g; +-- Arrow IPC stream round-trip. +COPY t_arrow TO :arrow_stream_path (FORMAT ARROWS); +SELECT r['i'], r['label'] FROM read_arrow(:arrow_stream_path) r ORDER BY r['i']; + i | label +---+------- + 1 | row_1 + 2 | row_2 + 3 | row_3 + 4 | row_4 + 5 | row_5 +(5 rows) + +-- Arrow IPC file round-trip. +COPY t_arrow TO :arrow_file_path (FORMAT ARROW); +SELECT r['i'], r['label'] FROM read_arrow(:arrow_file_path) r ORDER BY r['i']; + i | label +---+------- + 1 | row_1 + 2 | row_2 + 3 | row_3 + 4 | row_4 + 5 | row_5 +(5 rows) + +-- Array variant: union the two files. +SELECT count(*) FROM read_arrow(ARRAY[:arrow_stream_path, :arrow_file_path]); + count +------- + 10 +(1 row) + +DROP TABLE t_arrow; diff --git a/test/regression/schedule b/test/regression/schedule index eb3fd68c..5b156b2a 100644 --- a/test/regression/schedule +++ b/test/regression/schedule @@ -3,6 +3,7 @@ test: altered_tables test: approx_count_distinct test: array_problems test: array_type_support +test: arrow test: basic test: case_insensitivity test: concurrency diff --git a/test/regression/sql/arrow.sql b/test/regression/sql/arrow.sql new file mode 100644 index 00000000..10bda6b9 --- /dev/null +++ b/test/regression/sql/arrow.sql @@ -0,0 +1,31 @@ +-- Apache Arrow support via the bundled nanoarrow extension. +-- Verifies that nanoarrow is loaded by default and that read_arrow() works +-- for both Arrow IPC stream (.arrows) and Arrow IPC file (.arrow) formats. + +SET duckdb.force_execution = true; + +-- Sanity: nanoarrow is loaded as a built-in (no install required). +SET duckdb.force_execution = false; +SELECT * FROM duckdb.query($$ SELECT extension_name, loaded FROM duckdb_extensions() WHERE extension_name = 'nanoarrow' $$); +SET duckdb.force_execution = true; + +-- Set up a small dataset and round-trip it through Arrow IPC stream. +\set pwd `pwd` +\set arrow_stream_path '\'' :pwd '/tmp_check/pg_duckdb_arrow.arrows' '\'' +\set arrow_file_path '\'' :pwd '/tmp_check/pg_duckdb_arrow.arrow' '\'' + +CREATE TABLE t_arrow(i INT, label TEXT); +INSERT INTO t_arrow SELECT g, 'row_' || g FROM generate_series(1, 5) g; + +-- Arrow IPC stream round-trip. +COPY t_arrow TO :arrow_stream_path (FORMAT ARROWS); +SELECT r['i'], r['label'] FROM read_arrow(:arrow_stream_path) r ORDER BY r['i']; + +-- Arrow IPC file round-trip. +COPY t_arrow TO :arrow_file_path (FORMAT ARROW); +SELECT r['i'], r['label'] FROM read_arrow(:arrow_file_path) r ORDER BY r['i']; + +-- Array variant: union the two files. +SELECT count(*) FROM read_arrow(ARRAY[:arrow_stream_path, :arrow_file_path]); + +DROP TABLE t_arrow; diff --git a/third_party/pg_duckdb_extensions.cmake b/third_party/pg_duckdb_extensions.cmake index 3ee82bad..73e8e075 100644 --- a/third_party/pg_duckdb_extensions.cmake +++ b/third_party/pg_duckdb_extensions.cmake @@ -4,3 +4,9 @@ duckdb_extension_load(httpfs GIT_URL https://github.com/duckdb/duckdb-httpfs GIT_TAG 9de3296f40ed03e8e063394887f0d6a46144e847 ) +# paleolimbot/duckdb-nanoarrow main HEAD (42e4199, PR #47) targets DuckDB +# 1.5; pinned here because pg_duckdb's vendored DuckDB is now v1.5.3. +duckdb_extension_load(nanoarrow + GIT_URL https://github.com/paleolimbot/duckdb-nanoarrow + GIT_TAG 42e4199a67c4cd0789087562a025e87e7130fdc3 +)