feat: apache-arrow read_arrow() function via bundled nanoarrow extension

pg_duckdb.control

@@ -1,4 +1,4 @@
 comment = 'DuckDB Embedded in Postgres'
-default_version = '1.1.0'
+default_version = '1.2.0'
 module_pathname = '$libdir/pg_duckdb'
 relocatable = false

sql/pg_duckdb--1.1.0--1.2.0.sql

@@ -0,0 +1,18 @@
+-- Apache Arrow support via the bundled nanoarrow DuckDB extension.
+-- Mirrors the read_parquet pattern: the C body is the duckdb_only_function
+-- stub; the planner reroutes any query that mentions these functions to
+-- DuckDB, which executes read_arrow natively.
+
+-- read_arrow function for single path (Arrow IPC stream or file, auto-detected)
+CREATE FUNCTION @extschema@.read_arrow(path text)
+RETURNS SETOF duckdb.row
+SET search_path = pg_catalog, pg_temp
+AS 'MODULE_PATHNAME', 'duckdb_only_function'
+LANGUAGE C;
+
+-- read_arrow function for array of paths
+CREATE FUNCTION @extschema@.read_arrow(path text[])
+RETURNS SETOF duckdb.row
+SET search_path = pg_catalog, pg_temp
+AS 'MODULE_PATHNAME', 'duckdb_only_function'
+LANGUAGE C;

src/pgduckdb_hooks.cpp

@@ -444,7 +444,8 @@ static bool
 ContainsDuckdbRowReturningFunction(const char *query_string) {
 	return strstr(query_string, "read_parquet") || strstr(query_string, "read_csv") ||
 	       strstr(query_string, "read_json") || strstr(query_string, "delta_scan") ||
-	       strstr(query_string, "iceberg_scan") || strstr(query_string, "duckdb.query");
+	       strstr(query_string, "iceberg_scan") || strstr(query_string, "read_arrow") ||
+	       strstr(query_string, "duckdb.query");
 }
 
 static void

src/pgduckdb_metadata_cache.cpp

@@ -152,6 +152,7 @@ BuildDuckdbOnlyFunctions() {
 	                                "iceberg_snapshots",
 	                                "delta_scan",
 	                                "read_json",
+	                                "read_arrow",
 	                                "approx_count_distinct",
 	                                "query",
 	                                "view",

test/regression/expected/arrow.out

@@ -0,0 +1,51 @@
+-- Apache Arrow support via the bundled nanoarrow extension.
+-- Verifies that nanoarrow is loaded by default and that read_arrow() works
+-- for both Arrow IPC stream (.arrows) and Arrow IPC file (.arrow) formats.
+SET duckdb.force_execution = true;
+-- Sanity: nanoarrow is loaded as a built-in (no install required).
+SET duckdb.force_execution = false;
+SELECT * FROM duckdb.query($$ SELECT extension_name, loaded FROM duckdb_extensions() WHERE extension_name = 'nanoarrow' $$);
+ extension_name | loaded 
+----------------+--------
+ nanoarrow      | t
+(1 row)
+
+SET duckdb.force_execution = true;
+-- Set up a small dataset and round-trip it through Arrow IPC stream.
+\set pwd `pwd`
+\set arrow_stream_path '\'' :pwd '/tmp_check/pg_duckdb_arrow.arrows' '\''
+\set arrow_file_path   '\'' :pwd '/tmp_check/pg_duckdb_arrow.arrow'  '\''
+CREATE TABLE t_arrow(i INT, label TEXT);
+INSERT INTO t_arrow SELECT g, 'row_' || g FROM generate_series(1, 5) g;
+-- Arrow IPC stream round-trip.
+COPY t_arrow TO :arrow_stream_path (FORMAT ARROWS);
+SELECT r['i'], r['label'] FROM read_arrow(:arrow_stream_path) r ORDER BY r['i'];
+ i | label 
+---+-------
+ 1 | row_1
+ 2 | row_2
+ 3 | row_3
+ 4 | row_4
+ 5 | row_5
+(5 rows)
+
+-- Arrow IPC file round-trip.
+COPY t_arrow TO :arrow_file_path (FORMAT ARROW);
+SELECT r['i'], r['label'] FROM read_arrow(:arrow_file_path) r ORDER BY r['i'];
+ i | label 
+---+-------
+ 1 | row_1
+ 2 | row_2
+ 3 | row_3
+ 4 | row_4
+ 5 | row_5
+(5 rows)
+
+-- Array variant: union the two files.
+SELECT count(*) FROM read_arrow(ARRAY[:arrow_stream_path, :arrow_file_path]);
+ count 
+-------
+    10
+(1 row)
+
+DROP TABLE t_arrow;

test/regression/schedule

@@ -3,6 +3,7 @@ test: altered_tables
 test: approx_count_distinct
 test: array_problems
 test: array_type_support
+test: arrow
 test: basic
 test: case_insensitivity
 test: concurrency

test/regression/sql/arrow.sql

@@ -0,0 +1,31 @@
+-- Apache Arrow support via the bundled nanoarrow extension.
+-- Verifies that nanoarrow is loaded by default and that read_arrow() works
+-- for both Arrow IPC stream (.arrows) and Arrow IPC file (.arrow) formats.
+
+SET duckdb.force_execution = true;
+
+-- Sanity: nanoarrow is loaded as a built-in (no install required).
+SET duckdb.force_execution = false;
+SELECT * FROM duckdb.query($$ SELECT extension_name, loaded FROM duckdb_extensions() WHERE extension_name = 'nanoarrow' $$);
+SET duckdb.force_execution = true;
+
+-- Set up a small dataset and round-trip it through Arrow IPC stream.
+\set pwd `pwd`
+\set arrow_stream_path '\'' :pwd '/tmp_check/pg_duckdb_arrow.arrows' '\''
+\set arrow_file_path   '\'' :pwd '/tmp_check/pg_duckdb_arrow.arrow'  '\''
+
+CREATE TABLE t_arrow(i INT, label TEXT);
+INSERT INTO t_arrow SELECT g, 'row_' || g FROM generate_series(1, 5) g;
+
+-- Arrow IPC stream round-trip.
+COPY t_arrow TO :arrow_stream_path (FORMAT ARROWS);
+SELECT r['i'], r['label'] FROM read_arrow(:arrow_stream_path) r ORDER BY r['i'];
+
+-- Arrow IPC file round-trip.
+COPY t_arrow TO :arrow_file_path (FORMAT ARROW);
+SELECT r['i'], r['label'] FROM read_arrow(:arrow_file_path) r ORDER BY r['i'];
+
+-- Array variant: union the two files.
+SELECT count(*) FROM read_arrow(ARRAY[:arrow_stream_path, :arrow_file_path]);
+
+DROP TABLE t_arrow;

third_party/pg_duckdb_extensions.cmake

@@ -4,3 +4,9 @@ duckdb_extension_load(httpfs
     GIT_URL https://github.com/duckdb/duckdb-httpfs
     GIT_TAG 9de3296f40ed03e8e063394887f0d6a46144e847
 )
+# paleolimbot/duckdb-nanoarrow main HEAD (42e4199, PR #47) targets DuckDB
+# 1.5; pinned here because pg_duckdb's vendored DuckDB is now v1.5.3.
+duckdb_extension_load(nanoarrow
+    GIT_URL https://github.com/paleolimbot/duckdb-nanoarrow
+    GIT_TAG 42e4199a67c4cd0789087562a025e87e7130fdc3
+)