diff --git a/pg_ducklake/include/pgducklake/pgducklake_metadata_manager.hpp b/pg_ducklake/include/pgducklake/pgducklake_metadata_manager.hpp index 3d1bb926..d13e817b 100644 --- a/pg_ducklake/include/pgducklake/pgducklake_metadata_manager.hpp +++ b/pg_ducklake/include/pgducklake/pgducklake_metadata_manager.hpp @@ -40,6 +40,8 @@ class PgDuckLakeMetadataManager : public duckdb::PostgresMetadataManager { ReadAllInlinedDataForFlush(duckdb::DuckLakeSnapshot snapshot, const duckdb::string &inlined_table_name, const duckdb::vector &columns_to_read) override; + duckdb::vector GetFlushPartitionSQLExpressions(const duckdb::DuckLakeTableEntry &table) override; + static bool IsInitialized(); // In-process SPI backend has no DuckDB metadata connection: the base AttachMetadata // ATTACHes on it (null here) and MetadataExists probes ducklake_metadata in a way that diff --git a/pg_ducklake/sql/pg_ducklake--1.0.0--1.1.0.sql b/pg_ducklake/sql/pg_ducklake--1.0.0--1.1.0.sql index 2cb41186..07b988af 100644 --- a/pg_ducklake/sql/pg_ducklake--1.0.0--1.1.0.sql +++ b/pg_ducklake/sql/pg_ducklake--1.0.0--1.1.0.sql @@ -178,3 +178,134 @@ CREATE OPERATOR CLASS ducklake.unresolved_type_hash_ops DEFAULT FOR TYPE ducklake.unresolved_type USING hash AS OPERATOR 1 = (ducklake.unresolved_type, ducklake.unresolved_type), FUNCTION 1 ducklake.unresolved_type_hash(ducklake.unresolved_type); + +-- DuckDB-dialect partition transform functions (GitHub issue #223). +-- +-- DuckLake generates flush_inlined_data()'s per-file "already deleted rows" +-- bookkeeping query with the table's partition expression in DuckDB dialect: +-- year(col)/month(col)/day(col)/hour(col) and, for bucket partitions, +-- (murmur3_32(col) & 2147483647) % N. pg_ducklake executes that query in +-- PostgreSQL over SPI (search_path forced to this schema), so these DuckDB +-- function names must exist here with DuckDB-identical semantics. Each +-- expression compares against the partition value the DuckDB writer baked +-- into the data file, so any semantic drift mis-tracks deleted rows. + +-- Time transforms. DuckDB's year()/month()/day()/hour() return BIGINT; the +-- timestamptz variants depend on the session TimeZone (like DuckDB's, which +-- is synced from PostgreSQL when the embedded instance is created), hence +-- STABLE rather than IMMUTABLE. +CREATE FUNCTION ducklake.year(date) RETURNS bigint + SET search_path = pg_catalog, pg_temp + LANGUAGE sql IMMUTABLE STRICT + AS $$ SELECT EXTRACT(YEAR FROM $1)::bigint $$; +CREATE FUNCTION ducklake.month(date) RETURNS bigint + SET search_path = pg_catalog, pg_temp + LANGUAGE sql IMMUTABLE STRICT + AS $$ SELECT EXTRACT(MONTH FROM $1)::bigint $$; +CREATE FUNCTION ducklake.day(date) RETURNS bigint + SET search_path = pg_catalog, pg_temp + LANGUAGE sql IMMUTABLE STRICT + AS $$ SELECT EXTRACT(DAY FROM $1)::bigint $$; +-- hour of a date is 0 (DuckDB implicitly casts date -> timestamp at midnight) +CREATE FUNCTION ducklake.hour(date) RETURNS bigint + SET search_path = pg_catalog, pg_temp + LANGUAGE sql IMMUTABLE STRICT + AS $$ SELECT EXTRACT(HOUR FROM $1::timestamp)::bigint $$; + +CREATE FUNCTION ducklake.year(timestamp) RETURNS bigint + SET search_path = pg_catalog, pg_temp + LANGUAGE sql IMMUTABLE STRICT + AS $$ SELECT EXTRACT(YEAR FROM $1)::bigint $$; +CREATE FUNCTION ducklake.month(timestamp) RETURNS bigint + SET search_path = pg_catalog, pg_temp + LANGUAGE sql IMMUTABLE STRICT + AS $$ SELECT EXTRACT(MONTH FROM $1)::bigint $$; +CREATE FUNCTION ducklake.day(timestamp) RETURNS bigint + SET search_path = pg_catalog, pg_temp + LANGUAGE sql IMMUTABLE STRICT + AS $$ SELECT EXTRACT(DAY FROM $1)::bigint $$; +CREATE FUNCTION ducklake.hour(timestamp) RETURNS bigint + SET search_path = pg_catalog, pg_temp + LANGUAGE sql IMMUTABLE STRICT + AS $$ SELECT EXTRACT(HOUR FROM $1)::bigint $$; + +-- DuckDB accepts only hour() on TIME (year/month/day fail to bind there) +CREATE FUNCTION ducklake.hour(time) RETURNS bigint + SET search_path = pg_catalog, pg_temp + LANGUAGE sql IMMUTABLE STRICT + AS $$ SELECT EXTRACT(HOUR FROM $1)::bigint $$; + +CREATE FUNCTION ducklake.year(timestamptz) RETURNS bigint + SET search_path = pg_catalog, pg_temp + LANGUAGE sql STABLE STRICT + AS $$ SELECT EXTRACT(YEAR FROM $1)::bigint $$; +CREATE FUNCTION ducklake.month(timestamptz) RETURNS bigint + SET search_path = pg_catalog, pg_temp + LANGUAGE sql STABLE STRICT + AS $$ SELECT EXTRACT(MONTH FROM $1)::bigint $$; +CREATE FUNCTION ducklake.day(timestamptz) RETURNS bigint + SET search_path = pg_catalog, pg_temp + LANGUAGE sql STABLE STRICT + AS $$ SELECT EXTRACT(DAY FROM $1)::bigint $$; +CREATE FUNCTION ducklake.hour(timestamptz) RETURNS bigint + SET search_path = pg_catalog, pg_temp + LANGUAGE sql STABLE STRICT + AS $$ SELECT EXTRACT(HOUR FROM $1)::bigint $$; + +-- Iceberg-compatible murmur3 x86 32-bit hash (seed 0), bit-exact with +-- DuckLake's murmur3_32. One overload per exact input type: the generated +-- SQL always casts the column to its DuckLake type, and exact matches keep +-- PostgreSQL's function resolution from picking a wrong-encoding overload +-- (e.g. an int must hash as a sign-extended 8-byte long, not as a double). +CREATE FUNCTION ducklake.murmur3_32(boolean) RETURNS integer + SET search_path = pg_catalog, pg_temp + LANGUAGE C IMMUTABLE STRICT AS 'MODULE_PATHNAME', 'ducklake_murmur3_bool'; +CREATE FUNCTION ducklake.murmur3_32(smallint) RETURNS integer + SET search_path = pg_catalog, pg_temp + LANGUAGE C IMMUTABLE STRICT AS 'MODULE_PATHNAME', 'ducklake_murmur3_int2'; +CREATE FUNCTION ducklake.murmur3_32(integer) RETURNS integer + SET search_path = pg_catalog, pg_temp + LANGUAGE C IMMUTABLE STRICT AS 'MODULE_PATHNAME', 'ducklake_murmur3_int4'; +CREATE FUNCTION ducklake.murmur3_32(bigint) RETURNS integer + SET search_path = pg_catalog, pg_temp + LANGUAGE C IMMUTABLE STRICT AS 'MODULE_PATHNAME', 'ducklake_murmur3_int8'; +CREATE FUNCTION ducklake.murmur3_32(real) RETURNS integer + SET search_path = pg_catalog, pg_temp + LANGUAGE C IMMUTABLE STRICT AS 'MODULE_PATHNAME', 'ducklake_murmur3_float4'; +CREATE FUNCTION ducklake.murmur3_32(double precision) RETURNS integer + SET search_path = pg_catalog, pg_temp + LANGUAGE C IMMUTABLE STRICT AS 'MODULE_PATHNAME', 'ducklake_murmur3_float8'; +CREATE FUNCTION ducklake.murmur3_32(text) RETURNS integer + SET search_path = pg_catalog, pg_temp + LANGUAGE C IMMUTABLE STRICT AS 'MODULE_PATHNAME', 'ducklake_murmur3_text'; +-- raw bytes, for BLOB columns and VARCHAR columns' BYTEA storage +CREATE FUNCTION ducklake.murmur3_32(bytea) RETURNS integer + SET search_path = pg_catalog, pg_temp + LANGUAGE C IMMUTABLE STRICT AS 'MODULE_PATHNAME', 'ducklake_murmur3_text'; +CREATE FUNCTION ducklake.murmur3_32(date) RETURNS integer + SET search_path = pg_catalog, pg_temp + LANGUAGE C IMMUTABLE STRICT AS 'MODULE_PATHNAME', 'ducklake_murmur3_date'; +CREATE FUNCTION ducklake.murmur3_32(time) RETURNS integer + SET search_path = pg_catalog, pg_temp + LANGUAGE C IMMUTABLE STRICT AS 'MODULE_PATHNAME', 'ducklake_murmur3_time'; +CREATE FUNCTION ducklake.murmur3_32(timestamp) RETURNS integer + SET search_path = pg_catalog, pg_temp + LANGUAGE C IMMUTABLE STRICT AS 'MODULE_PATHNAME', 'ducklake_murmur3_timestamp'; +CREATE FUNCTION ducklake.murmur3_32(timestamptz) RETURNS integer + SET search_path = pg_catalog, pg_temp + LANGUAGE C IMMUTABLE STRICT AS 'MODULE_PATHNAME', 'ducklake_murmur3_timestamp'; +-- DuckDB hashes a DECIMAL of width <= 18 as its unscaled value in an 8-byte +-- long (DECIMAL(18,3) 12.5 hashes as 12500). Wider decimals hash their +-- fixed-scale string form; the flush filter renders those as ::text instead +-- of calling this overload. Without an exact numeric overload, PostgreSQL +-- would resolve murmur3_32(numeric) to the double precision one (the +-- preferred numeric type) and silently mis-hash every bucket. +CREATE FUNCTION ducklake.murmur3_32(numeric) RETURNS integer + SET search_path = pg_catalog, pg_temp + LANGUAGE sql IMMUTABLE STRICT + AS $$ SELECT ducklake.murmur3_32(($1 * power(10::numeric, scale($1)))::bigint) $$; +-- DuckDB hashes a UUID's lowercase hyphenated string form +CREATE FUNCTION ducklake.murmur3_32(uuid) RETURNS integer + SET search_path = pg_catalog, pg_temp + LANGUAGE sql IMMUTABLE STRICT + AS $$ SELECT ducklake.murmur3_32($1::text) $$; diff --git a/pg_ducklake/src/murmur3.cpp b/pg_ducklake/src/murmur3.cpp new file mode 100644 index 00000000..3064ae44 --- /dev/null +++ b/pg_ducklake/src/murmur3.cpp @@ -0,0 +1,118 @@ +/* + * PG-side murmur3_32 for DuckLake's Iceberg-compatible bucket partition + * transform. flush_inlined_data()'s bookkeeping SQL embeds + * (murmur3_32(col) & 2147483647) % N and runs over SPI, so PostgreSQL needs + * the hash with byte-for-byte DuckLake semantics. The core hash is reused + * from the vendored header; each overload mirrors the input encoding of + * DuckLake's Murmur3ScalarFunction (ducklake_murmur3.cpp): + * - bool/int16/int32/int64: sign-extended to int64, hashed as 8 bytes + * - float/double: widened to double, -0.0 normalized, IEEE754 bits as int64 + * - text: raw UTF-8 bytes + * - date: days since 1970-01-01 as int64 (PG stores days since 2000-01-01) + * - time: microseconds since midnight (same encoding in PG and DuckDB) + * - timestamp/timestamptz: microseconds since 1970-01-01 UTC (PG epoch 2000) + */ + +#include + +#include + +extern "C" { +#include "postgres.h" + +#if PG_VERSION_NUM >= 160000 +#include "varatt.h" +#endif + +#include "fmgr.h" +#include "utils/date.h" +#include "utils/timestamp.h" +} + +static constexpr int64_t PG_TO_UNIX_EPOCH_DAYS = POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE; +static constexpr int64_t PG_TO_UNIX_EPOCH_USECS = PG_TO_UNIX_EPOCH_DAYS * USECS_PER_DAY; + +static int32_t +MurmurInt64(int64_t val) { + return duckdb::DuckLakeMurmur3::HashValue(val); +} + +static int32_t +MurmurDouble(double val) { + if (val == 0.0) { + val = 0.0; // normalize -0.0, as DuckLake does + } + int64_t bits; + memcpy(&bits, &val, sizeof(bits)); + return MurmurInt64(bits); +} + +extern "C" { + +PG_FUNCTION_INFO_V1(ducklake_murmur3_bool); +Datum +ducklake_murmur3_bool(PG_FUNCTION_ARGS) { + PG_RETURN_INT32(MurmurInt64(PG_GETARG_BOOL(0) ? 1 : 0)); +} + +PG_FUNCTION_INFO_V1(ducklake_murmur3_int2); +Datum +ducklake_murmur3_int2(PG_FUNCTION_ARGS) { + PG_RETURN_INT32(MurmurInt64(PG_GETARG_INT16(0))); +} + +PG_FUNCTION_INFO_V1(ducklake_murmur3_int4); +Datum +ducklake_murmur3_int4(PG_FUNCTION_ARGS) { + PG_RETURN_INT32(MurmurInt64(PG_GETARG_INT32(0))); +} + +PG_FUNCTION_INFO_V1(ducklake_murmur3_int8); +Datum +ducklake_murmur3_int8(PG_FUNCTION_ARGS) { + PG_RETURN_INT32(MurmurInt64(PG_GETARG_INT64(0))); +} + +PG_FUNCTION_INFO_V1(ducklake_murmur3_float4); +Datum +ducklake_murmur3_float4(PG_FUNCTION_ARGS) { + PG_RETURN_INT32(MurmurDouble((double)PG_GETARG_FLOAT4(0))); +} + +PG_FUNCTION_INFO_V1(ducklake_murmur3_float8); +Datum +ducklake_murmur3_float8(PG_FUNCTION_ARGS) { + PG_RETURN_INT32(MurmurDouble(PG_GETARG_FLOAT8(0))); +} + +/* Backs both the text and bytea overloads: DuckLake hashes a VARCHAR's UTF-8 + * bytes and a BLOB's raw bytes, the same raw-varlena-bytes rule. */ +PG_FUNCTION_INFO_V1(ducklake_murmur3_text); +Datum +ducklake_murmur3_text(PG_FUNCTION_ARGS) { + text *val = PG_GETARG_TEXT_PP(0); + auto data = reinterpret_cast(VARDATA_ANY(val)); + PG_RETURN_INT32(duckdb::DuckLakeMurmur3::Hash(data, VARSIZE_ANY_EXHDR(val))); +} + +PG_FUNCTION_INFO_V1(ducklake_murmur3_date); +Datum +ducklake_murmur3_date(PG_FUNCTION_ARGS) { + PG_RETURN_INT32(MurmurInt64((int64_t)PG_GETARG_DATEADT(0) + PG_TO_UNIX_EPOCH_DAYS)); +} + +PG_FUNCTION_INFO_V1(ducklake_murmur3_time); +Datum +ducklake_murmur3_time(PG_FUNCTION_ARGS) { + PG_RETURN_INT32(MurmurInt64(PG_GETARG_TIMEADT(0))); +} + +/* Timestamp and TimestampTz share the representation (microseconds since + * 2000-01-01 UTC), so this one entry point backs both SQL overloads. */ +PG_FUNCTION_INFO_V1(ducklake_murmur3_timestamp); +Datum +ducklake_murmur3_timestamp(PG_FUNCTION_ARGS) { + PG_RETURN_INT32(MurmurInt64(PG_GETARG_TIMESTAMP(0) + PG_TO_UNIX_EPOCH_USECS)); +} + +} // extern "C" diff --git a/pg_ducklake/src/pgducklake_metadata_manager.cpp b/pg_ducklake/src/pgducklake_metadata_manager.cpp index 183a2bfb..2db257d0 100644 --- a/pg_ducklake/src/pgducklake_metadata_manager.cpp +++ b/pg_ducklake/src/pgducklake_metadata_manager.cpp @@ -22,6 +22,9 @@ #include #include #include +#include +#include +#include extern "C" { #include "postgres.h" @@ -96,6 +99,9 @@ SPIExecuteInSubtransaction(const duckdb::string &query, bool &had_error, duckdb: /* Suppress NOTICEs: DuckLake re-runs CREATE TABLE IF NOT EXISTS, whose NOTICE would leak to the client. */ int save_nestlevel = NewGUCNestLevel(); ::SetConfigOption("client_min_messages", "warning", PGC_USERSET, PGC_S_SESSION); + /* DuckLake-generated SQL calls DuckDB-dialect functions (month(), murmur3_32(), ...) unqualified; + * resolve them to the ducklake-schema UDFs deterministically, independent of the caller's search_path. */ + ::SetConfigOption("search_path", "ducklake", PGC_USERSET, PGC_S_SESSION); SetAllowSubtransaction(true); BeginInternalSubTransaction(NULL); @@ -351,6 +357,53 @@ ORDER BY row_id, begin_snapshot;)", return transaction.ExecuteRaw(query); } +/* + * The flush's deleted-rows filter runs over SPI against the inlined heap table, so each partition + * column must be rendered as PG SQL that recovers the DuckLake-typed value from its storage type: + * - VARCHAR is stored as BYTEA; the upstream CAST(col AS VARCHAR) would yield PG's hex form + * ('\x6170706c65'), silently mis-hashing every bucket/identity comparison. convert_from() + * restores the original string. + * - BLOB is stored as BYTEA; pass it through raw (murmur3_32(bytea) hashes the bytes directly, + * matching DuckDB, which hashes a BLOB's raw bytes). + * - Types stored as VARCHAR (date/timestamp family) keep the upstream CAST to their DuckDB type + * name, which PG parses from the stored text form. + * - Native types cast via GetColumnTypeInternal so the type name is PG-parseable (e.g. DuckDB + * DOUBLE -> DOUBLE PRECISION). + * The transform wrapper (month(...), (murmur3_32(...) & ...) % N) then resolves to the ducklake + * schema UDFs via the search_path forced in SPIExecuteInSubtransaction. + */ +duckdb::vector +PgDuckLakeMetadataManager::GetFlushPartitionSQLExpressions(const duckdb::DuckLakeTableEntry &table) { + duckdb::vector result; + auto partition_data = table.GetPartitionData(); + if (!partition_data) { + return result; + } + for (auto &field : partition_data->fields) { + auto &col = table.GetColumnByFieldId(field.field_id); + auto col_name = duckdb::KeywordHelper::WriteOptionallyQuoted(col.GetName()); + auto type = col.GetType(); + duckdb::string rendered; + if (type.id() == duckdb::LogicalTypeId::VARCHAR) { + rendered = "convert_from(" + col_name + ", 'UTF8')"; + } else if (type.id() == duckdb::LogicalTypeId::BLOB) { + rendered = col_name; + } else if (GetColumnTypeInternal(type) == "VARCHAR") { + rendered = "CAST(" + col_name + " AS " + type.ToString() + ")"; + } else { + rendered = "CAST(" + col_name + " AS " + GetColumnTypeInternal(type) + ")"; + } + /* DuckDB hashes DECIMALs wider than 18 digits (INT128 storage) by their fixed-scale + * string form; ducklake.murmur3_32(numeric) implements the narrow unscaled-long rule. */ + if (field.transform.type == duckdb::DuckLakeTransformType::BUCKET && + type.id() == duckdb::LogicalTypeId::DECIMAL && duckdb::DecimalType::GetWidth(type) > 18) { + rendered = "CAST(" + rendered + " AS TEXT)"; + } + result.push_back(duckdb::DuckLakePartitionUtils::GetPartitionSQLExpression(field.transform, rendered)); + } + return result; +} + duckdb::unique_ptr PgDuckLakeMetadataManager::Query(duckdb::DuckLakeSnapshot snapshot, duckdb::string query) { DuckLakeMetadataManager::FillSnapshotArgs(query, snapshot); diff --git a/pg_ducklake/test/regression/expected/partition_transform_flush.out b/pg_ducklake/test/regression/expected/partition_transform_flush.out new file mode 100644 index 00000000..0520eea4 --- /dev/null +++ b/pg_ducklake/test/regression/expected/partition_transform_flush.out @@ -0,0 +1,507 @@ +-- Regression for GitHub issue #223: +-- ducklake.flush_inlined_data() on a table whose partition uses a non-identity +-- transform (month(), year(), day(), hour(), bucket()) failed with +-- IO Error: SPI execution failed: function month(timestamp with time zone) does not exist +-- +-- The flush's per-file "which inlined rows were already deleted" query embeds +-- the DuckDB-dialect partition expression (month(col), murmur3_32(col)) and +-- pg_ducklake ships it to PostgreSQL via SPI. The fix provides those functions +-- as UDFs in the ducklake schema (resolved via forced search_path) with +-- DuckDB-identical semantics, so the filter agrees with the partition values +-- the DuckDB writer baked into the data files. +-- +-- Caveat: the timestamptz transforms depend on the session TimeZone in both +-- engines; DuckDB's TimeZone is synced from PostgreSQL once, when the embedded +-- instance is created (a later SET timezone only affects the PG side). +CALL ducklake.set_option('data_inlining_row_limit', 100); +-- ------------------------------------------------------------------ +-- 1. month() transform on TIMESTAMPTZ (the exact issue case), +-- including a NULL partition value (filter uses IS NULL) +-- ------------------------------------------------------------------ +CREATE TABLE flush_month (id int, ts timestamptz, val int) USING ducklake; +CALL ducklake.set_partition('flush_month'::regclass, 'month(ts)'); +-- Small batch -> inlined (not written to Parquet yet) +INSERT INTO flush_month VALUES + (1, '2024-01-15 00:00:00+00', 10), + (2, '2024-01-20 00:00:00+00', 20), + (3, '2024-06-10 00:00:00+00', 30), + (4, '2024-06-25 00:00:00+00', 40), + (5, NULL, 50), + (6, NULL, 60); +-- Delete + update inlined rows before flush: this leaves inlined versions with +-- end_snapshot set, which is exactly what the fixed query enumerates. +DELETE FROM flush_month WHERE id = 2; +UPDATE flush_month SET val = 99 WHERE id = 3; +DELETE FROM flush_month WHERE id = 5; -- row in the NULL partition +-- Pre-fix: ERROR (function month(timestamp with time zone) does not exist) +SELECT * FROM ducklake.flush_inlined_data('flush_month'::regclass); + schema_name | table_name | rows_flushed +-------------+-------------+-------------- + public | flush_month | 7 +(1 row) + +-- Deleted rows 2 and 5 gone, updated row 3 reflects new value, live rows intact. +SELECT id, val FROM flush_month ORDER BY id; + id | val +----+----- + 1 | 10 + 3 | 99 + 4 | 40 + 6 | 60 +(4 rows) + +DROP TABLE flush_month; +-- ------------------------------------------------------------------ +-- 2. bucket(N, col) on integer -- PostgreSQL has no murmur3_32() at all +-- ------------------------------------------------------------------ +CREATE TABLE flush_bucket (id int, val int) USING ducklake; +CALL ducklake.set_partition('flush_bucket'::regclass, 'bucket(4, id)'); +INSERT INTO flush_bucket VALUES (1, 10), (2, 20), (3, 30), (4, 40), (5, 50); +DELETE FROM flush_bucket WHERE id = 4; +SELECT * FROM ducklake.flush_inlined_data('flush_bucket'::regclass); + schema_name | table_name | rows_flushed +-------------+--------------+-------------- + public | flush_bucket | 5 +(1 row) + +SELECT id, val FROM flush_bucket ORDER BY id; + id | val +----+----- + 1 | 10 + 2 | 20 + 3 | 30 + 5 | 50 +(4 rows) + +DROP TABLE flush_bucket; +-- ------------------------------------------------------------------ +-- 3. bucket(N, col) on text -- a hash-encoding mismatch here would make the +-- delete filter match nothing and silently resurrect the deleted row +-- ------------------------------------------------------------------ +CREATE TABLE flush_btext (k text, val int) USING ducklake; +CALL ducklake.set_partition('flush_btext'::regclass, 'bucket(3, k)'); +INSERT INTO flush_btext VALUES + ('apple', 1), ('banana', 2), ('cherry', 3), ('', 4), (chr(233) || 't' || chr(233), 5); +DELETE FROM flush_btext WHERE k = 'banana'; +DELETE FROM flush_btext WHERE k = chr(233) || 't' || chr(233); +SELECT * FROM ducklake.flush_inlined_data('flush_btext'::regclass); + schema_name | table_name | rows_flushed +-------------+-------------+-------------- + public | flush_btext | 5 +(1 row) + +SELECT k, val FROM flush_btext ORDER BY val; + k | val +--------+----- + apple | 1 + cherry | 3 + | 4 +(3 rows) + +DROP TABLE flush_btext; +-- ------------------------------------------------------------------ +-- 4. identity partition on text: the inlined table stores text as BYTEA, so +-- the filter must recover the string (convert_from), not its hex form +-- ------------------------------------------------------------------ +CREATE TABLE flush_idtext (k text, val int) USING ducklake; +CALL ducklake.set_partition('flush_idtext'::regclass, 'k'); +INSERT INTO flush_idtext VALUES ('red', 1), ('blue', 2), ('red', 3); +DELETE FROM flush_idtext WHERE val = 3; +SELECT * FROM ducklake.flush_inlined_data('flush_idtext'::regclass); + schema_name | table_name | rows_flushed +-------------+--------------+-------------- + public | flush_idtext | 3 +(1 row) + +SELECT k, val FROM flush_idtext ORDER BY val; + k | val +------+----- + red | 1 + blue | 2 +(2 rows) + +DROP TABLE flush_idtext; +-- ------------------------------------------------------------------ +-- 5. bucket on numeric. DECIMAL(18,3) hashes its unscaled bigint; without an +-- exact numeric overload PG would silently pick the double one. The wide +-- DECIMAL(30,10) hashes its fixed-scale string form instead. +-- ------------------------------------------------------------------ +CREATE TABLE flush_num (d numeric(18,3), val int) USING ducklake; +CALL ducklake.set_partition('flush_num'::regclass, 'bucket(4, d)'); +INSERT INTO flush_num VALUES (12.5, 1), (0.001, 2), (-7.25, 3); +DELETE FROM flush_num WHERE val = 2; +SELECT * FROM ducklake.flush_inlined_data('flush_num'::regclass); + schema_name | table_name | rows_flushed +-------------+------------+-------------- + public | flush_num | 3 +(1 row) + +SELECT d, val FROM flush_num ORDER BY val; + d | val +--------+----- + 12.500 | 1 + -7.250 | 3 +(2 rows) + +DROP TABLE flush_num; +CREATE TABLE flush_widenum (d numeric(30,10), val int) USING ducklake; +CALL ducklake.set_partition('flush_widenum'::regclass, 'bucket(4, d)'); +INSERT INTO flush_widenum VALUES (12.5, 1), (98765432109876543210.5, 2), (-0.0000000001, 3); +DELETE FROM flush_widenum WHERE val = 2; +SELECT * FROM ducklake.flush_inlined_data('flush_widenum'::regclass); + schema_name | table_name | rows_flushed +-------------+---------------+-------------- + public | flush_widenum | 3 +(1 row) + +SELECT d, val FROM flush_widenum ORDER BY val; + d | val +---------------+----- + 12.5000000000 | 1 + -0.0000000001 | 3 +(2 rows) + +DROP TABLE flush_widenum; +-- ------------------------------------------------------------------ +-- 6. bucket on uuid (hashes the lowercase hyphenated string form) +-- ------------------------------------------------------------------ +CREATE TABLE flush_uuid (u uuid, val int) USING ducklake; +CALL ducklake.set_partition('flush_uuid'::regclass, 'bucket(4, u)'); +INSERT INTO flush_uuid VALUES + ('f79c3e09-677c-4bbd-a479-3f349cb785e7', 1), + ('00000000-0000-0000-0000-000000000000', 2), + ('ffffffff-ffff-ffff-ffff-ffffffffffff', 3); +DELETE FROM flush_uuid WHERE val = 2; +SELECT * FROM ducklake.flush_inlined_data('flush_uuid'::regclass); + schema_name | table_name | rows_flushed +-------------+------------+-------------- + public | flush_uuid | 3 +(1 row) + +SELECT u, val FROM flush_uuid ORDER BY val; + u | val +--------------------------------------+----- + f79c3e09-677c-4bbd-a479-3f349cb785e7 | 1 + ffffffff-ffff-ffff-ffff-ffffffffffff | 3 +(2 rows) + +DROP TABLE flush_uuid; +-- ------------------------------------------------------------------ +-- 7. bucket on bytea (raw bytes; the column is passed through unconverted). +-- Identity partition on bytea is not covered here: DuckLake itself fails +-- to build the partition value from non-UTF8 bytes ("Invalid unicode ... +-- in value construction"), independent of the metadata backend. +-- ------------------------------------------------------------------ +CREATE TABLE flush_bbytea (b bytea, val int) USING ducklake; +CALL ducklake.set_partition('flush_bbytea'::regclass, 'bucket(4, b)'); +INSERT INTO flush_bbytea VALUES ('\x0102ff', 1), ('\x616263', 2), ('\x', 3); +DELETE FROM flush_bbytea WHERE val = 2; +SELECT * FROM ducklake.flush_inlined_data('flush_bbytea'::regclass); + schema_name | table_name | rows_flushed +-------------+--------------+-------------- + public | flush_bbytea | 3 +(1 row) + +SELECT b, val FROM flush_bbytea ORDER BY val; + b | val +----------+----- + \x0102ff | 1 + \x | 3 +(2 rows) + +DROP TABLE flush_bbytea; +-- ------------------------------------------------------------------ +-- 8. hour() on time (the only transform DuckDB accepts on TIME; +-- year/month/day fail to bind there) +-- ------------------------------------------------------------------ +CREATE TABLE flush_time (t time, val int) USING ducklake; +CALL ducklake.set_partition('flush_time'::regclass, 'hour(t)'); +INSERT INTO flush_time VALUES ('22:31:08', 1), ('05:00:00', 2), ('22:59:59', 3); +DELETE FROM flush_time WHERE val = 2; +SELECT * FROM ducklake.flush_inlined_data('flush_time'::regclass); + schema_name | table_name | rows_flushed +-------------+------------+-------------- + public | flush_time | 3 +(1 row) + +SELECT t, val FROM flush_time ORDER BY val; + t | val +----------+----- + 22:31:08 | 1 + 22:59:59 | 3 +(2 rows) + +DROP TABLE flush_time; +CALL ducklake.set_option('data_inlining_row_limit', 0); +-- ------------------------------------------------------------------ +-- 9. Equivalence: the PG UDFs must return exactly what DuckDB returns for the +-- same expression. The DuckDB side is evaluated via ducklake.query() and +-- captured with \gset; a false anywhere below is a dialect divergence. +-- ------------------------------------------------------------------ +-- 9a. year/month/day/hour on TIMESTAMP (signature = year.month.day.hour) +SELECT * FROM ducklake.query($$ SELECT + concat(year(t), '.', month(t), '.', day(t), '.', hour(t)) AS ts1 + FROM (SELECT TIMESTAMP '2024-01-01 00:00:00' AS t) $$) \gset duck_ +SELECT * FROM ducklake.query($$ SELECT + concat(year(t), '.', month(t), '.', day(t), '.', hour(t)) AS ts2 + FROM (SELECT TIMESTAMP '2023-12-31 23:59:59.999999' AS t) $$) \gset duck_ +SELECT * FROM ducklake.query($$ SELECT + concat(year(t), '.', month(t), '.', day(t), '.', hour(t)) AS ts3 + FROM (SELECT TIMESTAMP '2024-02-29 12:34:56' AS t) $$) \gset duck_ +SELECT * FROM ducklake.query($$ SELECT + concat(year(t), '.', month(t), '.', day(t), '.', hour(t)) AS ts4 + FROM (SELECT TIMESTAMP '1970-01-01 00:00:00' AS t) $$) \gset duck_ +SELECT * FROM ducklake.query($$ SELECT + concat(year(t), '.', month(t), '.', day(t), '.', hour(t)) AS ts5 + FROM (SELECT TIMESTAMP '1969-12-31 23:59:59.999999' AS t) $$) \gset duck_ +SELECT * FROM ducklake.query($$ SELECT + concat(year(t), '.', month(t), '.', day(t), '.', hour(t)) AS ts6 + FROM (SELECT TIMESTAMP '0001-01-01 00:00:00' AS t) $$) \gset duck_ +SELECT * FROM ducklake.query($$ SELECT + concat(year(t), '.', month(t), '.', day(t), '.', hour(t)) AS ts7 + FROM (SELECT TIMESTAMP '9999-12-31 23:59:59' AS t) $$) \gset duck_ +WITH v(name, duck, t) AS (VALUES + ('ts1', :'duck_ts1', TIMESTAMP '2024-01-01 00:00:00'), + ('ts2', :'duck_ts2', TIMESTAMP '2023-12-31 23:59:59.999999'), + ('ts3', :'duck_ts3', TIMESTAMP '2024-02-29 12:34:56'), + ('ts4', :'duck_ts4', TIMESTAMP '1970-01-01 00:00:00'), + ('ts5', :'duck_ts5', TIMESTAMP '1969-12-31 23:59:59.999999'), + ('ts6', :'duck_ts6', TIMESTAMP '0001-01-01 00:00:00'), + ('ts7', :'duck_ts7', TIMESTAMP '9999-12-31 23:59:59')) +SELECT name, duck, + duck = concat(ducklake.year(t), '.', ducklake.month(t), '.', ducklake.day(t), '.', ducklake.hour(t)) AS match +FROM v ORDER BY name; + name | duck | match +------+---------------+------- + ts1 | 2024.1.1.0 | t + ts2 | 2023.12.31.23 | t + ts3 | 2024.2.29.12 | t + ts4 | 1970.1.1.0 | t + ts5 | 1969.12.31.23 | t + ts6 | 1.1.1.0 | t + ts7 | 9999.12.31.23 | t +(7 rows) + +-- 9b. year/month/day/hour on TIMESTAMPTZ (session TimeZone dependent; both +-- engines run with the same synced TimeZone here) +SELECT * FROM ducklake.query($$ SELECT + concat(year(t), '.', month(t), '.', day(t), '.', hour(t)) AS tz1 + FROM (SELECT TIMESTAMPTZ '2024-06-30 23:30:00+00' AS t) $$) \gset duck_ +SELECT * FROM ducklake.query($$ SELECT + concat(year(t), '.', month(t), '.', day(t), '.', hour(t)) AS tz2 + FROM (SELECT TIMESTAMPTZ '2024-07-01 06:59:59+00' AS t) $$) \gset duck_ +SELECT * FROM ducklake.query($$ SELECT + concat(year(t), '.', month(t), '.', day(t), '.', hour(t)) AS tz3 + FROM (SELECT TIMESTAMPTZ '2024-07-01 07:00:00+00' AS t) $$) \gset duck_ +SELECT * FROM ducklake.query($$ SELECT + concat(year(t), '.', month(t), '.', day(t), '.', hour(t)) AS tz4 + FROM (SELECT TIMESTAMPTZ '2024-01-01 07:59:59+00' AS t) $$) \gset duck_ +SELECT * FROM ducklake.query($$ SELECT + concat(year(t), '.', month(t), '.', day(t), '.', hour(t)) AS tz5 + FROM (SELECT TIMESTAMPTZ '1970-01-01 00:00:00+00' AS t) $$) \gset duck_ +WITH v(name, duck, t) AS (VALUES + ('tz1', :'duck_tz1', TIMESTAMPTZ '2024-06-30 23:30:00+00'), + ('tz2', :'duck_tz2', TIMESTAMPTZ '2024-07-01 06:59:59+00'), + ('tz3', :'duck_tz3', TIMESTAMPTZ '2024-07-01 07:00:00+00'), + ('tz4', :'duck_tz4', TIMESTAMPTZ '2024-01-01 07:59:59+00'), + ('tz5', :'duck_tz5', TIMESTAMPTZ '1970-01-01 00:00:00+00')) +SELECT name, duck, + duck = concat(ducklake.year(t), '.', ducklake.month(t), '.', ducklake.day(t), '.', ducklake.hour(t)) AS match +FROM v ORDER BY name; + name | duck | match +------+---------------+------- + tz1 | 2024.6.30.16 | t + tz2 | 2024.6.30.23 | t + tz3 | 2024.7.1.0 | t + tz4 | 2023.12.31.23 | t + tz5 | 1969.12.31.16 | t +(5 rows) + +-- 9c. year/month/day/hour on DATE +SELECT * FROM ducklake.query($$ SELECT + concat(year(t), '.', month(t), '.', day(t), '.', hour(t)) AS d1 + FROM (SELECT DATE '2024-01-01' AS t) $$) \gset duck_ +SELECT * FROM ducklake.query($$ SELECT + concat(year(t), '.', month(t), '.', day(t), '.', hour(t)) AS d2 + FROM (SELECT DATE '2024-02-29' AS t) $$) \gset duck_ +SELECT * FROM ducklake.query($$ SELECT + concat(year(t), '.', month(t), '.', day(t), '.', hour(t)) AS d3 + FROM (SELECT DATE '1969-12-31' AS t) $$) \gset duck_ +SELECT * FROM ducklake.query($$ SELECT + concat(year(t), '.', month(t), '.', day(t), '.', hour(t)) AS d4 + FROM (SELECT DATE '9999-12-31' AS t) $$) \gset duck_ +WITH v(name, duck, t) AS (VALUES + ('d1', :'duck_d1', DATE '2024-01-01'), + ('d2', :'duck_d2', DATE '2024-02-29'), + ('d3', :'duck_d3', DATE '1969-12-31'), + ('d4', :'duck_d4', DATE '9999-12-31')) +SELECT name, duck, + duck = concat(ducklake.year(t), '.', ducklake.month(t), '.', ducklake.day(t), '.', ducklake.hour(t)) AS match +FROM v ORDER BY name; + name | duck | match +------+--------------+------- + d1 | 2024.1.1.0 | t + d2 | 2024.2.29.0 | t + d3 | 1969.12.31.0 | t + d4 | 9999.12.31.0 | t +(4 rows) + +-- 9d. murmur3_32 integer family. 34 hashes as a sign-extended 8-byte long for +-- every integer width (2017239379 is the Iceberg spec reference vector). +SELECT * FROM ducklake.query($$ SELECT + murmur3_32(CAST(34 AS BIGINT)) AS h_i8_34, + murmur3_32(CAST(0 AS BIGINT)) AS h_i8_0, + murmur3_32(CAST(-1 AS BIGINT)) AS h_i8_neg1, + murmur3_32(CAST(-34 AS BIGINT)) AS h_i8_neg34, + murmur3_32(CAST(9223372036854775807 AS BIGINT)) AS h_i8_max, + murmur3_32(CAST(-9223372036854775808 AS BIGINT)) AS h_i8_min, + murmur3_32(CAST(34 AS INTEGER)) AS h_i4_34, + murmur3_32(CAST(-2147483648 AS INTEGER)) AS h_i4_min, + murmur3_32(CAST(2147483647 AS INTEGER)) AS h_i4_max, + murmur3_32(CAST(34 AS SMALLINT)) AS h_i2_34, + murmur3_32(CAST(-1 AS SMALLINT)) AS h_i2_neg1, + murmur3_32(true) AS h_bool_t, + murmur3_32(false) AS h_bool_f $$) \gset duck_ +SELECT :'duck_h_i8_34'::int = ducklake.murmur3_32(34::bigint) AS i8_34, + :'duck_h_i8_34'::int = 2017239379 AS i8_34_iceberg_vector, + :'duck_h_i8_0'::int = ducklake.murmur3_32(0::bigint) AS i8_0, + :'duck_h_i8_neg1'::int = ducklake.murmur3_32((-1)::bigint) AS i8_neg1, + :'duck_h_i8_neg34'::int = ducklake.murmur3_32((-34)::bigint) AS i8_neg34, + :'duck_h_i8_max'::int = ducklake.murmur3_32(9223372036854775807::bigint) AS i8_max, + :'duck_h_i8_min'::int = ducklake.murmur3_32((-9223372036854775808)::bigint) AS i8_min, + :'duck_h_i4_34'::int = ducklake.murmur3_32(34::integer) AS i4_34, + :'duck_h_i4_34'::int = ducklake.murmur3_32(34::bigint) AS i4_equals_i8, + :'duck_h_i4_min'::int = ducklake.murmur3_32((-2147483648)::integer) AS i4_min, + :'duck_h_i4_max'::int = ducklake.murmur3_32(2147483647::integer) AS i4_max, + :'duck_h_i2_34'::int = ducklake.murmur3_32(34::smallint) AS i2_34, + :'duck_h_i2_neg1'::int = ducklake.murmur3_32((-1)::smallint) AS i2_neg1, + :'duck_h_bool_t'::int = ducklake.murmur3_32(true) AS bool_t, + :'duck_h_bool_f'::int = ducklake.murmur3_32(false) AS bool_f; + i8_34 | i8_34_iceberg_vector | i8_0 | i8_neg1 | i8_neg34 | i8_max | i8_min | i4_34 | i4_equals_i8 | i4_min | i4_max | i2_34 | i2_neg1 | bool_t | bool_f +-------+----------------------+------+---------+----------+--------+--------+-------+--------------+--------+--------+-------+---------+--------+-------- + t | t | t | t | t | t | t | t | t | t | t | t | t | t | t +(1 row) + +-- 9e. murmur3_32 text ('iceberg' = 1210000089 in the Iceberg spec; empty +-- string, a 4-byte block, block+tail, and non-ASCII UTF-8) +SELECT * FROM ducklake.query($$ SELECT + murmur3_32('iceberg') AS h_iceberg, + murmur3_32('') AS h_empty, + murmur3_32('abcd') AS h_abcd, + murmur3_32('abcde') AS h_abcde, + murmur3_32(chr(233) || 't' || chr(233)) AS h_unicode $$) \gset duck_ +SELECT :'duck_h_iceberg'::int = ducklake.murmur3_32('iceberg') AS str_iceberg, + :'duck_h_iceberg'::int = 1210000089 AS str_iceberg_vector, + :'duck_h_empty'::int = ducklake.murmur3_32('') AS str_empty, + :'duck_h_abcd'::int = ducklake.murmur3_32('abcd') AS str_abcd, + :'duck_h_abcde'::int = ducklake.murmur3_32('abcde') AS str_abcde, + :'duck_h_unicode'::int = ducklake.murmur3_32(chr(233) || 't' || chr(233)) AS str_unicode; + str_iceberg | str_iceberg_vector | str_empty | str_abcd | str_abcde | str_unicode +-------------+--------------------+-----------+----------+-----------+------------- + t | t | t | t | t | t +(1 row) + +-- 9f. murmur3_32 float/double (widened to double, -0.0 normalized to +0.0) +SELECT * FROM ducklake.query($$ SELECT + murmur3_32(CAST(1.0 AS FLOAT)) AS h_f4_1, + murmur3_32(CAST(0.0 AS FLOAT)) AS h_f4_0, + murmur3_32(CAST(-0.0 AS FLOAT)) AS h_f4_neg0, + murmur3_32(CAST(1.0 AS DOUBLE)) AS h_f8_1, + murmur3_32(CAST(-1.5 AS DOUBLE)) AS h_f8_neg15 $$) \gset duck_ +SELECT :'duck_h_f4_1'::int = ducklake.murmur3_32(1.0::real) AS f4_1, + :'duck_h_f4_0'::int = ducklake.murmur3_32(0.0::real) AS f4_0, + :'duck_h_f4_neg0'::int = ducklake.murmur3_32('-0.0'::real) AS f4_neg0, + :'duck_h_f4_neg0'::int = ducklake.murmur3_32(0.0::real) AS f4_neg0_normalized, + :'duck_h_f4_1'::int = ducklake.murmur3_32(1.0::double precision) AS f4_equals_f8, + :'duck_h_f8_1'::int = ducklake.murmur3_32(1.0::double precision) AS f8_1, + :'duck_h_f8_neg15'::int = ducklake.murmur3_32((-1.5)::double precision) AS f8_neg15; + f4_1 | f4_0 | f4_neg0 | f4_neg0_normalized | f4_equals_f8 | f8_1 | f8_neg15 +------+------+---------+--------------------+--------------+------+---------- + t | t | t | t | t | t | t +(1 row) + +-- 9g. murmur3_32 date/time/timestamp/timestamptz (Iceberg vectors: +-- date 2017-11-16 = -653330422, time 22:31:08 = -662762989, +-- timestamp 2017-11-16T22:31:08 = -2047944441) +SELECT * FROM ducklake.query($$ SELECT + murmur3_32(DATE '2017-11-16') AS h_d_iceberg, + murmur3_32(DATE '1970-01-01') AS h_d_epoch, + murmur3_32(DATE '1969-12-31') AS h_d_preepoch, + murmur3_32(TIME '22:31:08') AS h_t_iceberg, + murmur3_32(TIMESTAMP '2017-11-16 22:31:08') AS h_ts_iceberg, + murmur3_32(TIMESTAMP '1970-01-01 00:00:00') AS h_ts_epoch, + murmur3_32(TIMESTAMPTZ '2017-11-16 22:31:08+00') AS h_tstz_utc, + murmur3_32(TIMESTAMPTZ '2017-11-16 14:31:08-08') AS h_tstz_offset $$) \gset duck_ +SELECT :'duck_h_d_iceberg'::int = ducklake.murmur3_32(DATE '2017-11-16') AS d_iceberg, + :'duck_h_d_iceberg'::int = -653330422 AS d_iceberg_vector, + :'duck_h_d_epoch'::int = ducklake.murmur3_32(DATE '1970-01-01') AS d_epoch, + :'duck_h_d_preepoch'::int = ducklake.murmur3_32(DATE '1969-12-31') AS d_preepoch, + :'duck_h_t_iceberg'::int = ducklake.murmur3_32(TIME '22:31:08') AS t_iceberg, + :'duck_h_t_iceberg'::int = -662762989 AS t_iceberg_vector, + :'duck_h_ts_iceberg'::int = ducklake.murmur3_32(TIMESTAMP '2017-11-16 22:31:08') AS ts_iceberg, + :'duck_h_ts_iceberg'::int = -2047944441 AS ts_iceberg_vector, + :'duck_h_ts_epoch'::int = ducklake.murmur3_32(TIMESTAMP '1970-01-01 00:00:00') AS ts_epoch, + :'duck_h_tstz_utc'::int = ducklake.murmur3_32(TIMESTAMPTZ '2017-11-16 22:31:08+00') AS tstz_utc, + :'duck_h_tstz_offset'::int = ducklake.murmur3_32(TIMESTAMPTZ '2017-11-16 22:31:08+00') AS tstz_same_instant; + d_iceberg | d_iceberg_vector | d_epoch | d_preepoch | t_iceberg | t_iceberg_vector | ts_iceberg | ts_iceberg_vector | ts_epoch | tstz_utc | tstz_same_instant +-----------+------------------+---------+------------+-----------+------------------+------------+-------------------+----------+----------+------------------- + t | t | t | t | t | t | t | t | t | t | t +(1 row) + +-- 9h. murmur3_32 numeric/uuid/bytea. DECIMAL(18,3) 12.5 must hash like its +-- unscaled long 12500; DECIMAL(30,10) hashes its fixed-scale string form; +-- UUID hashes its lowercase hyphenated string; BYTEA hashes raw bytes. +-- hour(TIME) is the one temporal transform DuckDB accepts on TIME. +SELECT * FROM ducklake.query($$ SELECT + murmur3_32(CAST(12.5 AS DECIMAL(18,3))) AS h_dec, + murmur3_32(CAST(-7.25 AS DECIMAL(18,3))) AS h_dec_neg, + murmur3_32(CAST(0.001 AS DECIMAL(18,3))) AS h_dec_small, + murmur3_32(CAST(0 AS DECIMAL(10,0))) AS h_dec_zero, + murmur3_32(CAST(12.5 AS DECIMAL(30,10))) AS h_widedec, + murmur3_32(UUID 'f79c3e09-677c-4bbd-a479-3f349cb785e7') AS h_uuid, + murmur3_32(UUID '00000000-0000-0000-0000-000000000000') AS h_uuid_zero, + murmur3_32(BLOB '\x01\x02\xFF') AS h_blob, + murmur3_32(BLOB '') AS h_blob_empty, + hour(TIME '22:31:08') AS h_hour_time $$) \gset duck_ +SELECT :'duck_h_dec'::int = ducklake.murmur3_32(12.5::numeric(18,3)) AS dec, + :'duck_h_dec'::int = ducklake.murmur3_32(12500::bigint) AS dec_is_unscaled_long, + :'duck_h_dec_neg'::int = ducklake.murmur3_32((-7.25)::numeric(18,3)) AS dec_neg, + :'duck_h_dec_small'::int = ducklake.murmur3_32(0.001::numeric(18,3)) AS dec_small, + :'duck_h_dec_zero'::int = ducklake.murmur3_32(0::numeric(10,0)) AS dec_zero, + :'duck_h_widedec'::int = ducklake.murmur3_32((12.5::numeric(30,10))::text) AS widedec_is_string, + :'duck_h_uuid'::int = ducklake.murmur3_32('f79c3e09-677c-4bbd-a479-3f349cb785e7'::uuid) AS uuid, + :'duck_h_uuid_zero'::int = ducklake.murmur3_32('00000000-0000-0000-0000-000000000000'::uuid) AS uuid_zero, + :'duck_h_blob'::int = ducklake.murmur3_32('\x0102ff'::bytea) AS blob, + :'duck_h_blob_empty'::int = ducklake.murmur3_32(''::bytea) AS blob_empty, + :'duck_h_hour_time'::bigint = ducklake.hour('22:31:08'::time) AS hour_time; + dec | dec_is_unscaled_long | dec_neg | dec_small | dec_zero | widedec_is_string | uuid | uuid_zero | blob | blob_empty | hour_time +-----+----------------------+---------+-----------+----------+-------------------+------+-----------+------+------------+----------- + t | t | t | t | t | t | t | t | t | t | t +(1 row) + +-- 9i. NULL handling: both engines yield NULL (the flush filter then uses +-- "expr IS NULL" for NULL partition values, which matches in both) +SELECT * FROM ducklake.query($$ SELECT + month(NULL::TIMESTAMPTZ) IS NULL AS duck_month_null, + murmur3_32(NULL::INTEGER) IS NULL AS duck_murmur_null $$); + duck_month_null | duck_murmur_null +-----------------+------------------ + t | t +(1 row) + +SELECT ducklake.month(NULL::timestamptz) IS NULL AS pg_month_null, + ducklake.murmur3_32(NULL::integer) IS NULL AS pg_murmur_null; + pg_month_null | pg_murmur_null +---------------+---------------- + t | t +(1 row) + +-- Full bucket expression parity, exactly as generated by DuckLake +SELECT * FROM ducklake.query($$ SELECT + (murmur3_32(CAST(42 AS INTEGER)) & 2147483647) % 4 AS bucket42 $$) \gset duck_ +SELECT :'duck_bucket42'::int = (ducklake.murmur3_32(CAST(42 AS INTEGER)) & 2147483647) % 4 AS bucket_expr_match; + bucket_expr_match +------------------- + t +(1 row) + diff --git a/pg_ducklake/test/regression/schedule b/pg_ducklake/test/regression/schedule index eab6deb4..37c04c5f 100644 --- a/pg_ducklake/test/regression/schedule +++ b/pg_ducklake/test/regression/schedule @@ -38,6 +38,7 @@ test: vacuum # test: maintenance test: commit_message test: partition +test: partition_transform_flush test: sorted_table test: hybrid_scan test: order_by_pushdown diff --git a/pg_ducklake/test/regression/sql/partition_transform_flush.sql b/pg_ducklake/test/regression/sql/partition_transform_flush.sql new file mode 100644 index 00000000..e6ab9b02 --- /dev/null +++ b/pg_ducklake/test/regression/sql/partition_transform_flush.sql @@ -0,0 +1,379 @@ +-- Regression for GitHub issue #223: +-- ducklake.flush_inlined_data() on a table whose partition uses a non-identity +-- transform (month(), year(), day(), hour(), bucket()) failed with +-- IO Error: SPI execution failed: function month(timestamp with time zone) does not exist +-- +-- The flush's per-file "which inlined rows were already deleted" query embeds +-- the DuckDB-dialect partition expression (month(col), murmur3_32(col)) and +-- pg_ducklake ships it to PostgreSQL via SPI. The fix provides those functions +-- as UDFs in the ducklake schema (resolved via forced search_path) with +-- DuckDB-identical semantics, so the filter agrees with the partition values +-- the DuckDB writer baked into the data files. +-- +-- Caveat: the timestamptz transforms depend on the session TimeZone in both +-- engines; DuckDB's TimeZone is synced from PostgreSQL once, when the embedded +-- instance is created (a later SET timezone only affects the PG side). + +CALL ducklake.set_option('data_inlining_row_limit', 100); + +-- ------------------------------------------------------------------ +-- 1. month() transform on TIMESTAMPTZ (the exact issue case), +-- including a NULL partition value (filter uses IS NULL) +-- ------------------------------------------------------------------ +CREATE TABLE flush_month (id int, ts timestamptz, val int) USING ducklake; +CALL ducklake.set_partition('flush_month'::regclass, 'month(ts)'); + +-- Small batch -> inlined (not written to Parquet yet) +INSERT INTO flush_month VALUES + (1, '2024-01-15 00:00:00+00', 10), + (2, '2024-01-20 00:00:00+00', 20), + (3, '2024-06-10 00:00:00+00', 30), + (4, '2024-06-25 00:00:00+00', 40), + (5, NULL, 50), + (6, NULL, 60); + +-- Delete + update inlined rows before flush: this leaves inlined versions with +-- end_snapshot set, which is exactly what the fixed query enumerates. +DELETE FROM flush_month WHERE id = 2; +UPDATE flush_month SET val = 99 WHERE id = 3; +DELETE FROM flush_month WHERE id = 5; -- row in the NULL partition + +-- Pre-fix: ERROR (function month(timestamp with time zone) does not exist) +SELECT * FROM ducklake.flush_inlined_data('flush_month'::regclass); + +-- Deleted rows 2 and 5 gone, updated row 3 reflects new value, live rows intact. +SELECT id, val FROM flush_month ORDER BY id; + +DROP TABLE flush_month; + +-- ------------------------------------------------------------------ +-- 2. bucket(N, col) on integer -- PostgreSQL has no murmur3_32() at all +-- ------------------------------------------------------------------ +CREATE TABLE flush_bucket (id int, val int) USING ducklake; +CALL ducklake.set_partition('flush_bucket'::regclass, 'bucket(4, id)'); + +INSERT INTO flush_bucket VALUES (1, 10), (2, 20), (3, 30), (4, 40), (5, 50); +DELETE FROM flush_bucket WHERE id = 4; + +SELECT * FROM ducklake.flush_inlined_data('flush_bucket'::regclass); +SELECT id, val FROM flush_bucket ORDER BY id; + +DROP TABLE flush_bucket; + +-- ------------------------------------------------------------------ +-- 3. bucket(N, col) on text -- a hash-encoding mismatch here would make the +-- delete filter match nothing and silently resurrect the deleted row +-- ------------------------------------------------------------------ +CREATE TABLE flush_btext (k text, val int) USING ducklake; +CALL ducklake.set_partition('flush_btext'::regclass, 'bucket(3, k)'); + +INSERT INTO flush_btext VALUES + ('apple', 1), ('banana', 2), ('cherry', 3), ('', 4), (chr(233) || 't' || chr(233), 5); +DELETE FROM flush_btext WHERE k = 'banana'; +DELETE FROM flush_btext WHERE k = chr(233) || 't' || chr(233); + +SELECT * FROM ducklake.flush_inlined_data('flush_btext'::regclass); +SELECT k, val FROM flush_btext ORDER BY val; + +DROP TABLE flush_btext; + +-- ------------------------------------------------------------------ +-- 4. identity partition on text: the inlined table stores text as BYTEA, so +-- the filter must recover the string (convert_from), not its hex form +-- ------------------------------------------------------------------ +CREATE TABLE flush_idtext (k text, val int) USING ducklake; +CALL ducklake.set_partition('flush_idtext'::regclass, 'k'); + +INSERT INTO flush_idtext VALUES ('red', 1), ('blue', 2), ('red', 3); +DELETE FROM flush_idtext WHERE val = 3; + +SELECT * FROM ducklake.flush_inlined_data('flush_idtext'::regclass); +SELECT k, val FROM flush_idtext ORDER BY val; + +DROP TABLE flush_idtext; + +-- ------------------------------------------------------------------ +-- 5. bucket on numeric. DECIMAL(18,3) hashes its unscaled bigint; without an +-- exact numeric overload PG would silently pick the double one. The wide +-- DECIMAL(30,10) hashes its fixed-scale string form instead. +-- ------------------------------------------------------------------ +CREATE TABLE flush_num (d numeric(18,3), val int) USING ducklake; +CALL ducklake.set_partition('flush_num'::regclass, 'bucket(4, d)'); +INSERT INTO flush_num VALUES (12.5, 1), (0.001, 2), (-7.25, 3); +DELETE FROM flush_num WHERE val = 2; +SELECT * FROM ducklake.flush_inlined_data('flush_num'::regclass); +SELECT d, val FROM flush_num ORDER BY val; +DROP TABLE flush_num; + +CREATE TABLE flush_widenum (d numeric(30,10), val int) USING ducklake; +CALL ducklake.set_partition('flush_widenum'::regclass, 'bucket(4, d)'); +INSERT INTO flush_widenum VALUES (12.5, 1), (98765432109876543210.5, 2), (-0.0000000001, 3); +DELETE FROM flush_widenum WHERE val = 2; +SELECT * FROM ducklake.flush_inlined_data('flush_widenum'::regclass); +SELECT d, val FROM flush_widenum ORDER BY val; +DROP TABLE flush_widenum; + +-- ------------------------------------------------------------------ +-- 6. bucket on uuid (hashes the lowercase hyphenated string form) +-- ------------------------------------------------------------------ +CREATE TABLE flush_uuid (u uuid, val int) USING ducklake; +CALL ducklake.set_partition('flush_uuid'::regclass, 'bucket(4, u)'); +INSERT INTO flush_uuid VALUES + ('f79c3e09-677c-4bbd-a479-3f349cb785e7', 1), + ('00000000-0000-0000-0000-000000000000', 2), + ('ffffffff-ffff-ffff-ffff-ffffffffffff', 3); +DELETE FROM flush_uuid WHERE val = 2; +SELECT * FROM ducklake.flush_inlined_data('flush_uuid'::regclass); +SELECT u, val FROM flush_uuid ORDER BY val; +DROP TABLE flush_uuid; + +-- ------------------------------------------------------------------ +-- 7. bucket on bytea (raw bytes; the column is passed through unconverted). +-- Identity partition on bytea is not covered here: DuckLake itself fails +-- to build the partition value from non-UTF8 bytes ("Invalid unicode ... +-- in value construction"), independent of the metadata backend. +-- ------------------------------------------------------------------ +CREATE TABLE flush_bbytea (b bytea, val int) USING ducklake; +CALL ducklake.set_partition('flush_bbytea'::regclass, 'bucket(4, b)'); +INSERT INTO flush_bbytea VALUES ('\x0102ff', 1), ('\x616263', 2), ('\x', 3); +DELETE FROM flush_bbytea WHERE val = 2; +SELECT * FROM ducklake.flush_inlined_data('flush_bbytea'::regclass); +SELECT b, val FROM flush_bbytea ORDER BY val; +DROP TABLE flush_bbytea; + +-- ------------------------------------------------------------------ +-- 8. hour() on time (the only transform DuckDB accepts on TIME; +-- year/month/day fail to bind there) +-- ------------------------------------------------------------------ +CREATE TABLE flush_time (t time, val int) USING ducklake; +CALL ducklake.set_partition('flush_time'::regclass, 'hour(t)'); +INSERT INTO flush_time VALUES ('22:31:08', 1), ('05:00:00', 2), ('22:59:59', 3); +DELETE FROM flush_time WHERE val = 2; +SELECT * FROM ducklake.flush_inlined_data('flush_time'::regclass); +SELECT t, val FROM flush_time ORDER BY val; +DROP TABLE flush_time; + +CALL ducklake.set_option('data_inlining_row_limit', 0); + +-- ------------------------------------------------------------------ +-- 9. Equivalence: the PG UDFs must return exactly what DuckDB returns for the +-- same expression. The DuckDB side is evaluated via ducklake.query() and +-- captured with \gset; a false anywhere below is a dialect divergence. +-- ------------------------------------------------------------------ + +-- 9a. year/month/day/hour on TIMESTAMP (signature = year.month.day.hour) +SELECT * FROM ducklake.query($$ SELECT + concat(year(t), '.', month(t), '.', day(t), '.', hour(t)) AS ts1 + FROM (SELECT TIMESTAMP '2024-01-01 00:00:00' AS t) $$) \gset duck_ +SELECT * FROM ducklake.query($$ SELECT + concat(year(t), '.', month(t), '.', day(t), '.', hour(t)) AS ts2 + FROM (SELECT TIMESTAMP '2023-12-31 23:59:59.999999' AS t) $$) \gset duck_ +SELECT * FROM ducklake.query($$ SELECT + concat(year(t), '.', month(t), '.', day(t), '.', hour(t)) AS ts3 + FROM (SELECT TIMESTAMP '2024-02-29 12:34:56' AS t) $$) \gset duck_ +SELECT * FROM ducklake.query($$ SELECT + concat(year(t), '.', month(t), '.', day(t), '.', hour(t)) AS ts4 + FROM (SELECT TIMESTAMP '1970-01-01 00:00:00' AS t) $$) \gset duck_ +SELECT * FROM ducklake.query($$ SELECT + concat(year(t), '.', month(t), '.', day(t), '.', hour(t)) AS ts5 + FROM (SELECT TIMESTAMP '1969-12-31 23:59:59.999999' AS t) $$) \gset duck_ +SELECT * FROM ducklake.query($$ SELECT + concat(year(t), '.', month(t), '.', day(t), '.', hour(t)) AS ts6 + FROM (SELECT TIMESTAMP '0001-01-01 00:00:00' AS t) $$) \gset duck_ +SELECT * FROM ducklake.query($$ SELECT + concat(year(t), '.', month(t), '.', day(t), '.', hour(t)) AS ts7 + FROM (SELECT TIMESTAMP '9999-12-31 23:59:59' AS t) $$) \gset duck_ + +WITH v(name, duck, t) AS (VALUES + ('ts1', :'duck_ts1', TIMESTAMP '2024-01-01 00:00:00'), + ('ts2', :'duck_ts2', TIMESTAMP '2023-12-31 23:59:59.999999'), + ('ts3', :'duck_ts3', TIMESTAMP '2024-02-29 12:34:56'), + ('ts4', :'duck_ts4', TIMESTAMP '1970-01-01 00:00:00'), + ('ts5', :'duck_ts5', TIMESTAMP '1969-12-31 23:59:59.999999'), + ('ts6', :'duck_ts6', TIMESTAMP '0001-01-01 00:00:00'), + ('ts7', :'duck_ts7', TIMESTAMP '9999-12-31 23:59:59')) +SELECT name, duck, + duck = concat(ducklake.year(t), '.', ducklake.month(t), '.', ducklake.day(t), '.', ducklake.hour(t)) AS match +FROM v ORDER BY name; + +-- 9b. year/month/day/hour on TIMESTAMPTZ (session TimeZone dependent; both +-- engines run with the same synced TimeZone here) +SELECT * FROM ducklake.query($$ SELECT + concat(year(t), '.', month(t), '.', day(t), '.', hour(t)) AS tz1 + FROM (SELECT TIMESTAMPTZ '2024-06-30 23:30:00+00' AS t) $$) \gset duck_ +SELECT * FROM ducklake.query($$ SELECT + concat(year(t), '.', month(t), '.', day(t), '.', hour(t)) AS tz2 + FROM (SELECT TIMESTAMPTZ '2024-07-01 06:59:59+00' AS t) $$) \gset duck_ +SELECT * FROM ducklake.query($$ SELECT + concat(year(t), '.', month(t), '.', day(t), '.', hour(t)) AS tz3 + FROM (SELECT TIMESTAMPTZ '2024-07-01 07:00:00+00' AS t) $$) \gset duck_ +SELECT * FROM ducklake.query($$ SELECT + concat(year(t), '.', month(t), '.', day(t), '.', hour(t)) AS tz4 + FROM (SELECT TIMESTAMPTZ '2024-01-01 07:59:59+00' AS t) $$) \gset duck_ +SELECT * FROM ducklake.query($$ SELECT + concat(year(t), '.', month(t), '.', day(t), '.', hour(t)) AS tz5 + FROM (SELECT TIMESTAMPTZ '1970-01-01 00:00:00+00' AS t) $$) \gset duck_ + +WITH v(name, duck, t) AS (VALUES + ('tz1', :'duck_tz1', TIMESTAMPTZ '2024-06-30 23:30:00+00'), + ('tz2', :'duck_tz2', TIMESTAMPTZ '2024-07-01 06:59:59+00'), + ('tz3', :'duck_tz3', TIMESTAMPTZ '2024-07-01 07:00:00+00'), + ('tz4', :'duck_tz4', TIMESTAMPTZ '2024-01-01 07:59:59+00'), + ('tz5', :'duck_tz5', TIMESTAMPTZ '1970-01-01 00:00:00+00')) +SELECT name, duck, + duck = concat(ducklake.year(t), '.', ducklake.month(t), '.', ducklake.day(t), '.', ducklake.hour(t)) AS match +FROM v ORDER BY name; + +-- 9c. year/month/day/hour on DATE +SELECT * FROM ducklake.query($$ SELECT + concat(year(t), '.', month(t), '.', day(t), '.', hour(t)) AS d1 + FROM (SELECT DATE '2024-01-01' AS t) $$) \gset duck_ +SELECT * FROM ducklake.query($$ SELECT + concat(year(t), '.', month(t), '.', day(t), '.', hour(t)) AS d2 + FROM (SELECT DATE '2024-02-29' AS t) $$) \gset duck_ +SELECT * FROM ducklake.query($$ SELECT + concat(year(t), '.', month(t), '.', day(t), '.', hour(t)) AS d3 + FROM (SELECT DATE '1969-12-31' AS t) $$) \gset duck_ +SELECT * FROM ducklake.query($$ SELECT + concat(year(t), '.', month(t), '.', day(t), '.', hour(t)) AS d4 + FROM (SELECT DATE '9999-12-31' AS t) $$) \gset duck_ + +WITH v(name, duck, t) AS (VALUES + ('d1', :'duck_d1', DATE '2024-01-01'), + ('d2', :'duck_d2', DATE '2024-02-29'), + ('d3', :'duck_d3', DATE '1969-12-31'), + ('d4', :'duck_d4', DATE '9999-12-31')) +SELECT name, duck, + duck = concat(ducklake.year(t), '.', ducklake.month(t), '.', ducklake.day(t), '.', ducklake.hour(t)) AS match +FROM v ORDER BY name; + +-- 9d. murmur3_32 integer family. 34 hashes as a sign-extended 8-byte long for +-- every integer width (2017239379 is the Iceberg spec reference vector). +SELECT * FROM ducklake.query($$ SELECT + murmur3_32(CAST(34 AS BIGINT)) AS h_i8_34, + murmur3_32(CAST(0 AS BIGINT)) AS h_i8_0, + murmur3_32(CAST(-1 AS BIGINT)) AS h_i8_neg1, + murmur3_32(CAST(-34 AS BIGINT)) AS h_i8_neg34, + murmur3_32(CAST(9223372036854775807 AS BIGINT)) AS h_i8_max, + murmur3_32(CAST(-9223372036854775808 AS BIGINT)) AS h_i8_min, + murmur3_32(CAST(34 AS INTEGER)) AS h_i4_34, + murmur3_32(CAST(-2147483648 AS INTEGER)) AS h_i4_min, + murmur3_32(CAST(2147483647 AS INTEGER)) AS h_i4_max, + murmur3_32(CAST(34 AS SMALLINT)) AS h_i2_34, + murmur3_32(CAST(-1 AS SMALLINT)) AS h_i2_neg1, + murmur3_32(true) AS h_bool_t, + murmur3_32(false) AS h_bool_f $$) \gset duck_ + +SELECT :'duck_h_i8_34'::int = ducklake.murmur3_32(34::bigint) AS i8_34, + :'duck_h_i8_34'::int = 2017239379 AS i8_34_iceberg_vector, + :'duck_h_i8_0'::int = ducklake.murmur3_32(0::bigint) AS i8_0, + :'duck_h_i8_neg1'::int = ducklake.murmur3_32((-1)::bigint) AS i8_neg1, + :'duck_h_i8_neg34'::int = ducklake.murmur3_32((-34)::bigint) AS i8_neg34, + :'duck_h_i8_max'::int = ducklake.murmur3_32(9223372036854775807::bigint) AS i8_max, + :'duck_h_i8_min'::int = ducklake.murmur3_32((-9223372036854775808)::bigint) AS i8_min, + :'duck_h_i4_34'::int = ducklake.murmur3_32(34::integer) AS i4_34, + :'duck_h_i4_34'::int = ducklake.murmur3_32(34::bigint) AS i4_equals_i8, + :'duck_h_i4_min'::int = ducklake.murmur3_32((-2147483648)::integer) AS i4_min, + :'duck_h_i4_max'::int = ducklake.murmur3_32(2147483647::integer) AS i4_max, + :'duck_h_i2_34'::int = ducklake.murmur3_32(34::smallint) AS i2_34, + :'duck_h_i2_neg1'::int = ducklake.murmur3_32((-1)::smallint) AS i2_neg1, + :'duck_h_bool_t'::int = ducklake.murmur3_32(true) AS bool_t, + :'duck_h_bool_f'::int = ducklake.murmur3_32(false) AS bool_f; + +-- 9e. murmur3_32 text ('iceberg' = 1210000089 in the Iceberg spec; empty +-- string, a 4-byte block, block+tail, and non-ASCII UTF-8) +SELECT * FROM ducklake.query($$ SELECT + murmur3_32('iceberg') AS h_iceberg, + murmur3_32('') AS h_empty, + murmur3_32('abcd') AS h_abcd, + murmur3_32('abcde') AS h_abcde, + murmur3_32(chr(233) || 't' || chr(233)) AS h_unicode $$) \gset duck_ + +SELECT :'duck_h_iceberg'::int = ducklake.murmur3_32('iceberg') AS str_iceberg, + :'duck_h_iceberg'::int = 1210000089 AS str_iceberg_vector, + :'duck_h_empty'::int = ducklake.murmur3_32('') AS str_empty, + :'duck_h_abcd'::int = ducklake.murmur3_32('abcd') AS str_abcd, + :'duck_h_abcde'::int = ducklake.murmur3_32('abcde') AS str_abcde, + :'duck_h_unicode'::int = ducklake.murmur3_32(chr(233) || 't' || chr(233)) AS str_unicode; + +-- 9f. murmur3_32 float/double (widened to double, -0.0 normalized to +0.0) +SELECT * FROM ducklake.query($$ SELECT + murmur3_32(CAST(1.0 AS FLOAT)) AS h_f4_1, + murmur3_32(CAST(0.0 AS FLOAT)) AS h_f4_0, + murmur3_32(CAST(-0.0 AS FLOAT)) AS h_f4_neg0, + murmur3_32(CAST(1.0 AS DOUBLE)) AS h_f8_1, + murmur3_32(CAST(-1.5 AS DOUBLE)) AS h_f8_neg15 $$) \gset duck_ + +SELECT :'duck_h_f4_1'::int = ducklake.murmur3_32(1.0::real) AS f4_1, + :'duck_h_f4_0'::int = ducklake.murmur3_32(0.0::real) AS f4_0, + :'duck_h_f4_neg0'::int = ducklake.murmur3_32('-0.0'::real) AS f4_neg0, + :'duck_h_f4_neg0'::int = ducklake.murmur3_32(0.0::real) AS f4_neg0_normalized, + :'duck_h_f4_1'::int = ducklake.murmur3_32(1.0::double precision) AS f4_equals_f8, + :'duck_h_f8_1'::int = ducklake.murmur3_32(1.0::double precision) AS f8_1, + :'duck_h_f8_neg15'::int = ducklake.murmur3_32((-1.5)::double precision) AS f8_neg15; + +-- 9g. murmur3_32 date/time/timestamp/timestamptz (Iceberg vectors: +-- date 2017-11-16 = -653330422, time 22:31:08 = -662762989, +-- timestamp 2017-11-16T22:31:08 = -2047944441) +SELECT * FROM ducklake.query($$ SELECT + murmur3_32(DATE '2017-11-16') AS h_d_iceberg, + murmur3_32(DATE '1970-01-01') AS h_d_epoch, + murmur3_32(DATE '1969-12-31') AS h_d_preepoch, + murmur3_32(TIME '22:31:08') AS h_t_iceberg, + murmur3_32(TIMESTAMP '2017-11-16 22:31:08') AS h_ts_iceberg, + murmur3_32(TIMESTAMP '1970-01-01 00:00:00') AS h_ts_epoch, + murmur3_32(TIMESTAMPTZ '2017-11-16 22:31:08+00') AS h_tstz_utc, + murmur3_32(TIMESTAMPTZ '2017-11-16 14:31:08-08') AS h_tstz_offset $$) \gset duck_ + +SELECT :'duck_h_d_iceberg'::int = ducklake.murmur3_32(DATE '2017-11-16') AS d_iceberg, + :'duck_h_d_iceberg'::int = -653330422 AS d_iceberg_vector, + :'duck_h_d_epoch'::int = ducklake.murmur3_32(DATE '1970-01-01') AS d_epoch, + :'duck_h_d_preepoch'::int = ducklake.murmur3_32(DATE '1969-12-31') AS d_preepoch, + :'duck_h_t_iceberg'::int = ducklake.murmur3_32(TIME '22:31:08') AS t_iceberg, + :'duck_h_t_iceberg'::int = -662762989 AS t_iceberg_vector, + :'duck_h_ts_iceberg'::int = ducklake.murmur3_32(TIMESTAMP '2017-11-16 22:31:08') AS ts_iceberg, + :'duck_h_ts_iceberg'::int = -2047944441 AS ts_iceberg_vector, + :'duck_h_ts_epoch'::int = ducklake.murmur3_32(TIMESTAMP '1970-01-01 00:00:00') AS ts_epoch, + :'duck_h_tstz_utc'::int = ducklake.murmur3_32(TIMESTAMPTZ '2017-11-16 22:31:08+00') AS tstz_utc, + :'duck_h_tstz_offset'::int = ducklake.murmur3_32(TIMESTAMPTZ '2017-11-16 22:31:08+00') AS tstz_same_instant; + +-- 9h. murmur3_32 numeric/uuid/bytea. DECIMAL(18,3) 12.5 must hash like its +-- unscaled long 12500; DECIMAL(30,10) hashes its fixed-scale string form; +-- UUID hashes its lowercase hyphenated string; BYTEA hashes raw bytes. +-- hour(TIME) is the one temporal transform DuckDB accepts on TIME. +SELECT * FROM ducklake.query($$ SELECT + murmur3_32(CAST(12.5 AS DECIMAL(18,3))) AS h_dec, + murmur3_32(CAST(-7.25 AS DECIMAL(18,3))) AS h_dec_neg, + murmur3_32(CAST(0.001 AS DECIMAL(18,3))) AS h_dec_small, + murmur3_32(CAST(0 AS DECIMAL(10,0))) AS h_dec_zero, + murmur3_32(CAST(12.5 AS DECIMAL(30,10))) AS h_widedec, + murmur3_32(UUID 'f79c3e09-677c-4bbd-a479-3f349cb785e7') AS h_uuid, + murmur3_32(UUID '00000000-0000-0000-0000-000000000000') AS h_uuid_zero, + murmur3_32(BLOB '\x01\x02\xFF') AS h_blob, + murmur3_32(BLOB '') AS h_blob_empty, + hour(TIME '22:31:08') AS h_hour_time $$) \gset duck_ + +SELECT :'duck_h_dec'::int = ducklake.murmur3_32(12.5::numeric(18,3)) AS dec, + :'duck_h_dec'::int = ducklake.murmur3_32(12500::bigint) AS dec_is_unscaled_long, + :'duck_h_dec_neg'::int = ducklake.murmur3_32((-7.25)::numeric(18,3)) AS dec_neg, + :'duck_h_dec_small'::int = ducklake.murmur3_32(0.001::numeric(18,3)) AS dec_small, + :'duck_h_dec_zero'::int = ducklake.murmur3_32(0::numeric(10,0)) AS dec_zero, + :'duck_h_widedec'::int = ducklake.murmur3_32((12.5::numeric(30,10))::text) AS widedec_is_string, + :'duck_h_uuid'::int = ducklake.murmur3_32('f79c3e09-677c-4bbd-a479-3f349cb785e7'::uuid) AS uuid, + :'duck_h_uuid_zero'::int = ducklake.murmur3_32('00000000-0000-0000-0000-000000000000'::uuid) AS uuid_zero, + :'duck_h_blob'::int = ducklake.murmur3_32('\x0102ff'::bytea) AS blob, + :'duck_h_blob_empty'::int = ducklake.murmur3_32(''::bytea) AS blob_empty, + :'duck_h_hour_time'::bigint = ducklake.hour('22:31:08'::time) AS hour_time; + +-- 9i. NULL handling: both engines yield NULL (the flush filter then uses +-- "expr IS NULL" for NULL partition values, which matches in both) +SELECT * FROM ducklake.query($$ SELECT + month(NULL::TIMESTAMPTZ) IS NULL AS duck_month_null, + murmur3_32(NULL::INTEGER) IS NULL AS duck_murmur_null $$); +SELECT ducklake.month(NULL::timestamptz) IS NULL AS pg_month_null, + ducklake.murmur3_32(NULL::integer) IS NULL AS pg_murmur_null; + +-- Full bucket expression parity, exactly as generated by DuckLake +SELECT * FROM ducklake.query($$ SELECT + (murmur3_32(CAST(42 AS INTEGER)) & 2147483647) % 4 AS bucket42 $$) \gset duck_ +SELECT :'duck_bucket42'::int = (ducklake.murmur3_32(CAST(42 AS INTEGER)) & 2147483647) % 4 AS bucket_expr_match; diff --git a/pg_ducklake/third_party/ducklake-010-flush-partition-filter-dialect.patch b/pg_ducklake/third_party/ducklake-010-flush-partition-filter-dialect.patch new file mode 100644 index 00000000..e956499b --- /dev/null +++ b/pg_ducklake/third_party/ducklake-010-flush-partition-filter-dialect.patch @@ -0,0 +1,40 @@ +diff --git a/src/include/storage/ducklake_metadata_manager.hpp b/src/include/storage/ducklake_metadata_manager.hpp +--- a/src/include/storage/ducklake_metadata_manager.hpp ++++ b/src/include/storage/ducklake_metadata_manager.hpp +@@ -347,6 +347,10 @@ + virtual unique_ptr ReadAllInlinedDataForFlush(DuckLakeSnapshot snapshot, + const string &inlined_table_name, + const vector &columns_to_read); ++ //! Partition-transform SQL expressions for the flush's deleted-rows filter, which runs against ++ //! the inlined data table. Overridable so backends can render each column according to how the ++ //! inlined table actually stores it (e.g. Postgres stores VARCHAR columns as BYTEA). ++ virtual vector GetFlushPartitionSQLExpressions(const DuckLakeTableEntry &table); + virtual shared_ptr TransformInlinedData(QueryResult &result, + const vector &expected_types); + +diff --git a/src/storage/ducklake_metadata_manager.cpp b/src/storage/ducklake_metadata_manager.cpp +--- a/src/storage/ducklake_metadata_manager.cpp ++++ b/src/storage/ducklake_metadata_manager.cpp +@@ -3092,6 +3092,10 @@ + ORDER BY row_id, begin_snapshot;)", + projection, inlined_table_name)); + return result; ++} ++ ++vector DuckLakeMetadataManager::GetFlushPartitionSQLExpressions(const DuckLakeTableEntry &table) { ++ return table.GetPartitionSQLExpressions(); + } + + string DuckLakeMetadataManager::GetPathForSchema(SchemaIndex schema_id, +diff --git a/src/functions/ducklake_flush_inlined_data.cpp b/src/functions/ducklake_flush_inlined_data.cpp +--- a/src/functions/ducklake_flush_inlined_data.cpp ++++ b/src/functions/ducklake_flush_inlined_data.cpp +@@ -118,7 +118,7 @@ + + if (!global_state.written_files.empty()) { + DeletesPerFile deletes_per_file; +- auto partition_sql_exprs = table.GetPartitionSQLExpressions(); ++ auto partition_sql_exprs = transaction.GetMetadataManager().GetFlushPartitionSQLExpressions(table); + + // Track cumulative row offset per partition so each file knows its range + unordered_map partition_row_offsets;