From bfb432485e397273c6af759acf0bfc8732804308 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 10 Mar 2026 15:12:03 +0000 Subject: [PATCH 01/59] fix: rewrite_data_files should not skip large files with delete files The file size check (>= target_file_size) was intended for MERGE_ADJACENT_TABLES to avoid merging already-large files, but it incorrectly also gated REWRITE_DELETES. This caused data files larger than target_file_size (default 512 MB) with associated delete files to be silently skipped by ducklake_rewrite_data_files, leaving delete files that can never be eliminated. Fixes #821 --- src/functions/ducklake_compaction_functions.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/functions/ducklake_compaction_functions.cpp b/src/functions/ducklake_compaction_functions.cpp index 6c2ba53be61..571671a55dc 100644 --- a/src/functions/ducklake_compaction_functions.cpp +++ b/src/functions/ducklake_compaction_functions.cpp @@ -222,8 +222,9 @@ void DuckLakeCompactor::GenerateCompactions(DuckLakeTableEntry &table, compaction_map_t candidates; for (idx_t file_idx = 0; file_idx < files.size(); file_idx++) { auto &candidate = files[file_idx]; - if (candidate.file.data.file_size_bytes >= target_file_size) { + if (candidate.file.data.file_size_bytes >= target_file_size && type != CompactionType::REWRITE_DELETES) { // this file by itself exceeds the threshold - skip merging + // (does not apply to REWRITE_DELETES - delete files must be rewritten regardless of data file size) continue; } if ((!candidate.delete_files.empty() && type == CompactionType::MERGE_ADJACENT_TABLES) || From 9eea9e1455a0004a8428efbf862b60e0fc65fabd Mon Sep 17 00:00:00 2001 From: Kumo Date: Wed, 11 Mar 2026 16:11:16 +0000 Subject: [PATCH 02/59] add test: rewrite_data_files should rewrite large files with delete files Co-Authored-By: Claude Opus 4.6 --- .../test_rewrite_large_file_with_deletes.test | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 test/sql/rewrite_data_files/test_rewrite_large_file_with_deletes.test diff --git a/test/sql/rewrite_data_files/test_rewrite_large_file_with_deletes.test b/test/sql/rewrite_data_files/test_rewrite_large_file_with_deletes.test new file mode 100644 index 00000000000..38698b65037 --- /dev/null +++ b/test/sql/rewrite_data_files/test_rewrite_large_file_with_deletes.test @@ -0,0 +1,89 @@ +# name: test/sql/rewrite_data_files/test_rewrite_large_file_with_deletes.test +# description: Test that rewrite_data_files does not skip large files with delete files +# group: [rewrite_data_files] + +require ducklake + +require parquet + +test-env DUCKLAKE_CONNECTION __TEST_DIR__/{UUID}.db + +test-env DATA_PATH __TEST_DIR__ + + +statement ok +ATTACH 'ducklake:${DUCKLAKE_CONNECTION}' AS ducklake (DATA_PATH '${DATA_PATH}/rewrite_large_deletes', METADATA_CATALOG 'ducklake_metadata') + +statement ok +USE ducklake + +# Set a very small target_file_size so that the data file exceeds it +statement ok +CALL ducklake.set_option('target_file_size', '1KB'); + +statement ok +CREATE TABLE test (key INTEGER, value VARCHAR); + +statement ok +INSERT INTO test SELECT i, concat('thisisastring_', i) FROM range(1000) t(i) + +statement ok +CALL ducklake_flush_inlined_data('ducklake') + +# Verify we have 1 data file, no delete files +query I +SELECT count(*) FROM ducklake_metadata.ducklake_data_file WHERE end_snapshot IS NULL +---- +1 + +query I +SELECT count(*) FROM ducklake_metadata.ducklake_delete_file WHERE end_snapshot IS NULL +---- +0 + +# Delete some rows to create a delete file +statement ok +DELETE FROM test WHERE key < 500 + +statement ok +CALL ducklake_flush_inlined_data('ducklake') + +# Verify we now have a delete file +query I +SELECT count(*) FROM ducklake_metadata.ducklake_delete_file WHERE end_snapshot IS NULL +---- +1 + +# The data file should be larger than target_file_size (1KB) +# Without the fix, rewrite_data_files would skip this file +query III +SELECT table_name, files_processed, files_created FROM ducklake_rewrite_data_files('ducklake', delete_threshold => 0) ORDER BY ALL +---- +test 1 1 + +# After rewrite: delete files should be gone, data file should be rewritten +query I +SELECT count(*) FROM ducklake_metadata.ducklake_delete_file WHERE end_snapshot IS NULL +---- +0 + +query I +SELECT count(*) FROM ducklake_metadata.ducklake_data_file WHERE end_snapshot IS NULL +---- +1 + +# Verify data correctness +query I +SELECT count(*) FROM test +---- +500 + +query I +SELECT min(key) FROM test +---- +500 + +query I +SELECT max(key) FROM test +---- +999 From aaca98e3d04bf957f1d8e2139133bddec0b4db24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Rafael?= Date: Wed, 11 Mar 2026 14:42:58 +0000 Subject: [PATCH 03/59] Fix race in transaction local change reads and add metadata query stress tests --- src/include/storage/ducklake_transaction.hpp | 2 +- src/storage/ducklake_transaction.cpp | 41 +++++- .../metadata/ducklake_ui_catalog_query.test | 49 +++++++ ...lake_ui_catalog_query_parallel_stress.test | 135 ++++++++++++++++++ 4 files changed, 220 insertions(+), 7 deletions(-) create mode 100644 test/sql/metadata/ducklake_ui_catalog_query.test create mode 100644 test/sql/metadata/ducklake_ui_catalog_query_parallel_stress.test diff --git a/src/include/storage/ducklake_transaction.hpp b/src/include/storage/ducklake_transaction.hpp index 7e61f753a42..84db958e640 100644 --- a/src/include/storage/ducklake_transaction.hpp +++ b/src/include/storage/ducklake_transaction.hpp @@ -261,7 +261,7 @@ class DuckLakeTransaction : public Transaction, public enable_shared_from_this new_schemas; map> dropped_schemas; //! Local changes made to tables - mutex table_data_changes_lock; + mutable mutex table_data_changes_lock; map table_data_changes; //! Snapshot cache for the AT (...) conditions that are referenced in the transaction value_map_t snapshot_cache; diff --git a/src/storage/ducklake_transaction.cpp b/src/storage/ducklake_transaction.cpp index b58d70bcb0d..1037fe9d5db 100644 --- a/src/storage/ducklake_transaction.cpp +++ b/src/storage/ducklake_transaction.cpp @@ -68,6 +68,7 @@ void DuckLakeTransaction::Commit() { } connection.reset(); + lock_guard guard(table_data_changes_lock); table_data_changes.clear(); } @@ -79,6 +80,7 @@ void DuckLakeTransaction::Rollback() { } CleanupFiles(); + lock_guard guard(table_data_changes_lock); table_data_changes.clear(); } @@ -111,6 +113,7 @@ bool DuckLakeTransaction::SchemaChangesMade() { } bool DuckLakeTransaction::ChangesMade() { + lock_guard guard(table_data_changes_lock); return SchemaChangesMade() || !table_data_changes.empty() || !dropped_files.empty() || !new_name_maps.name_maps.empty(); } @@ -280,6 +283,7 @@ TransactionChangeInformation DuckLakeTransaction::GetTransactionChanges() { } } changes.tables_deleted_from = tables_deleted_from; + lock_guard guard(table_data_changes_lock); for (auto &entry : table_data_changes) { auto table_id = entry.first; if (table_id.IsTransactionLocal()) { @@ -395,6 +399,7 @@ string DuckLakeTransaction::WriteSnapshotChanges(DuckLakeCommitState &commit_sta // re-add all inserted tables - transaction-local table identifiers should have been converted at this stage changes.tables_deleted_from = tables_deleted_from; + lock_guard guard(table_data_changes_lock); for (auto &entry : table_data_changes) { auto table_id = commit_state.GetTableId(entry.first); auto &table_changes = entry.second; @@ -489,6 +494,7 @@ void DuckLakeTransaction::CleanupFiles() { // remove any files that were written auto context_ref = context.lock(); auto &fs = FileSystem::GetFileSystem(db); + lock_guard guard(table_data_changes_lock); for (auto &entry : table_data_changes) { auto &table_changes = entry.second; for (auto &file : table_changes.new_data_files) { @@ -657,6 +663,7 @@ void DuckLakeTransaction::CheckForConflicts(const TransactionChangeInformation & if (check_for_matches) { // If we have deletes on the tables, check for files being deleted const auto deleted_files = metadata_manager->GetFilesDeletedOrDroppedAfterSnapshot(transaction_snapshot); + lock_guard guard(table_data_changes_lock); for (auto &entry : table_data_changes) { auto &table_changes = entry.second; for (auto &file_entry : table_changes.new_delete_files) { @@ -781,6 +788,7 @@ DuckLakePartitionInfo DuckLakeTransaction::GetNewPartitionKey(DuckLakeCommitStat } // if we wrote any data with this partition id - rewrite it to the latest partition id + lock_guard guard(table_data_changes_lock); for (auto &entry : table_data_changes) { auto &table_changes = entry.second; for (auto &file : table_changes.new_data_files) { @@ -1356,6 +1364,7 @@ NewDataInfo DuckLakeTransaction::GetNewDataFiles(string &batch_query, DuckLakeCo auto &schema = ducklake_catalog.GetSchemaForSnapshot(*this, GetSnapshot()); dl_stats = ducklake_catalog.ConstructStatsMap(*stats, schema); } + lock_guard guard(table_data_changes_lock); for (auto &entry : table_data_changes) { auto table_id = commit_state.GetTableId(entry.first); if (table_id.IsTransactionLocal()) { @@ -1449,6 +1458,7 @@ vector DuckLakeTransaction::GetNewDeleteFiles(const DuckLakeCommitState &commit_state, vector &overwritten_delete_files) const { vector result; + lock_guard guard(table_data_changes_lock); for (auto &entry : table_data_changes) { auto table_id = commit_state.GetTableId(entry.first); auto &table_changes = entry.second; @@ -1525,6 +1535,7 @@ NewNameMapInfo DuckLakeTransaction::GetNewNameMaps(DuckLakeCommitState &commit_s remap_mapping_index[local_map_id] = new_map_id; } // iterate over the data files to point them towards any new mapping ids + lock_guard guard(table_data_changes_lock); for (auto &entry : table_data_changes) { auto &table_changes = entry.second; for (auto &data_file : table_changes.new_data_files) { @@ -1542,6 +1553,7 @@ NewNameMapInfo DuckLakeTransaction::GetNewNameMaps(DuckLakeCommitState &commit_s vector DuckLakeTransaction::GetNewInlinedDeletes(DuckLakeCommitState &commit_state) { vector result; + lock_guard guard(table_data_changes_lock); for (auto &entry : table_data_changes) { auto table_id = commit_state.GetTableId(entry.first); auto &table_changes = entry.second; @@ -1639,7 +1651,12 @@ string DuckLakeTransaction::CommitChanges(DuckLakeCommitState &commit_state, } // write new data / data files - if (!table_data_changes.empty()) { + bool has_table_data_changes = false; + { + lock_guard guard(table_data_changes_lock); + has_table_data_changes = !table_data_changes.empty(); + } + if (has_table_data_changes) { auto result = GetNewDataFiles(batch_queries, commit_state, stats); batch_queries += metadata_manager->WriteNewDataFiles(commit_snapshot, result.new_files, new_tables_result, new_schemas_result); batch_queries += metadata_manager->WriteNewInlinedData(commit_snapshot, result.new_inlined_data, @@ -1655,7 +1672,7 @@ string DuckLakeTransaction::CommitChanges(DuckLakeCommitState &commit_state, batch_queries += metadata_manager->DropDataFiles(dropped_indexes); } - if (!table_data_changes.empty()) { + if (has_table_data_changes) { // write new delete files vector overwritten_delete_files; auto file_list = GetNewDeleteFiles(commit_state, overwritten_delete_files); @@ -1707,6 +1724,7 @@ string DuckLakeTransaction::CommitChanges(DuckLakeCommitState &commit_state, CompactionInformation DuckLakeTransaction::GetCompactionChanges(DuckLakeSnapshot &commit_snapshot, CompactionType type) { CompactionInformation result; + lock_guard guard(table_data_changes_lock); for (auto &entry : table_data_changes) { auto table_id = entry.first; auto &table_changes = entry.second; @@ -1992,6 +2010,7 @@ idx_t DuckLakeTransaction::GetLocalCatalogId() { } bool DuckLakeTransaction::HasTransactionLocalInserts(TableIndex table_id) const { + lock_guard guard(table_data_changes_lock); auto entry = table_data_changes.find(table_id); if (entry == table_data_changes.end()) { return false; @@ -2001,6 +2020,7 @@ bool DuckLakeTransaction::HasTransactionLocalInserts(TableIndex table_id) const } bool DuckLakeTransaction::HasTransactionInlinedData(TableIndex table_id) const { + lock_guard guard(table_data_changes_lock); auto entry = table_data_changes.find(table_id); if (entry == table_data_changes.end()) { return false; @@ -2010,6 +2030,7 @@ bool DuckLakeTransaction::HasTransactionInlinedData(TableIndex table_id) const { } vector DuckLakeTransaction::GetTransactionLocalFiles(TableIndex table_id) { + lock_guard guard(table_data_changes_lock); auto entry = table_data_changes.find(table_id); if (entry == table_data_changes.end()) { return vector(); @@ -2018,6 +2039,7 @@ vector DuckLakeTransaction::GetTransactionLocalFiles(TableInde } shared_ptr DuckLakeTransaction::GetTransactionLocalInlinedData(TableIndex table_id) { + lock_guard guard(table_data_changes_lock); auto entry = table_data_changes.find(table_id); if (entry == table_data_changes.end()) { return nullptr; @@ -2037,6 +2059,7 @@ shared_ptr DuckLakeTransaction::GetTransactionLocalInlinedD } void DuckLakeTransaction::DropTransactionLocalFile(TableIndex table_id, const string &path) { + lock_guard guard(table_data_changes_lock); auto entry = table_data_changes.find(table_id); if (entry == table_data_changes.end()) { throw InternalException( @@ -2355,6 +2378,7 @@ void DuckLakeTransaction::AddNewInlinedFileDeletes(TableIndex table_id, idx_t fi vector DuckLakeTransaction::GetNewInlinedFileDeletes(DuckLakeCommitState &commit_state) { vector result; + lock_guard guard(table_data_changes_lock); for (auto &entry : table_data_changes) { auto table_id = commit_state.GetTableId(entry.first); auto &table_changes = entry.second; @@ -2412,6 +2436,7 @@ void DuckLakeTransaction::AddCompaction(TableIndex table_id, DuckLakeCompactionE } bool DuckLakeTransaction::HasLocalDeletes(TableIndex table_id) { + lock_guard guard(table_data_changes_lock); auto entry = table_data_changes.find(table_id); if (entry == table_data_changes.end()) { return false; @@ -2420,6 +2445,7 @@ bool DuckLakeTransaction::HasLocalDeletes(TableIndex table_id) { } bool DuckLakeTransaction::HasAnyLocalChanges(TableIndex table_id) { + lock_guard guard(table_data_changes_lock); auto entry = table_data_changes.find(table_id); if (entry != table_data_changes.end() && !entry->second.IsEmpty()) { return true; @@ -2428,6 +2454,7 @@ bool DuckLakeTransaction::HasAnyLocalChanges(TableIndex table_id) { } void DuckLakeTransaction::GetLocalDeleteForFile(TableIndex table_id, const string &path, DuckLakeFileData &result) { + lock_guard guard(table_data_changes_lock); auto entry = table_data_changes.find(table_id); if (entry == table_data_changes.end()) { return; @@ -2479,6 +2506,7 @@ void DuckLakeTransaction::GetLocalInlinedFileDeletesForFile(TableIndex table_id, void DuckLakeTransaction::TransactionLocalDelete(TableIndex table_id, const string &data_file_path, DuckLakeDeleteFile delete_file) { + lock_guard guard(table_data_changes_lock); auto entry = table_data_changes.find(table_id); if (entry == table_data_changes.end()) { throw InternalException( @@ -2540,10 +2568,11 @@ void DuckLakeTransaction::DropTable(DuckLakeTableEntry &table) { if (schema_entry == new_tables.end()) { throw InternalException("Dropping a transaction local table that does not exist?"); } - auto table_id = table.GetTableId(); - schema_entry->second->DropEntry(table.name); - // if we have written any files for this table - clean them up - auto table_entry = table_data_changes.find(table_id); + auto table_id = table.GetTableId(); + schema_entry->second->DropEntry(table.name); + // if we have written any files for this table - clean them up + lock_guard guard(table_data_changes_lock); + auto table_entry = table_data_changes.find(table_id); if (table_entry != table_data_changes.end()) { auto &table_changes = table_entry->second; auto context_ref = context.lock(); diff --git a/test/sql/metadata/ducklake_ui_catalog_query.test b/test/sql/metadata/ducklake_ui_catalog_query.test new file mode 100644 index 00000000000..ff8be82e78e --- /dev/null +++ b/test/sql/metadata/ducklake_ui_catalog_query.test @@ -0,0 +1,49 @@ +# name: test/sql/metadata/ducklake_ui_catalog_query.test +# description: Reproducer for UI metadata query against ducklake catalog +# group: [metadata] + +require ducklake + +require parquet + +test-env DUCKLAKE_CONNECTION __TEST_DIR__/{UUID}.db + +test-env DATA_PATH __TEST_DIR__ + +statement ok +ATTACH 'ducklake:${DUCKLAKE_CONNECTION}' AS lake (DATA_PATH '${DATA_PATH}/ducklake_ui_catalog_query_files') + +statement ok +CREATE TABLE lake.sample(i INTEGER, note VARCHAR); + +query I +SELECT COUNT(*)::INTEGER +FROM ( + SELECT + schemata.schema_name AS schema_name, + tables.table_name AS table_name, + tables.table_type AS table_type, + tables.table_comment AS table_comment, + table_metadata.estimated_size AS table_estimated_size, + columns.ordinal_position AS column_position, + columns.column_name AS column_name, + columns.data_type AS column_data_type, + columns.column_comment AS column_comment + FROM system.information_schema.schemata AS schemata + LEFT JOIN system.information_schema.tables AS tables + ON tables.table_catalog = schemata.catalog_name + AND tables.table_schema = schemata.schema_name + LEFT JOIN duckdb_tables() AS table_metadata + ON table_metadata.table_name = tables.table_name + AND table_metadata.schema_name = tables.table_schema + AND table_metadata.database_name = tables.table_catalog + LEFT JOIN system.information_schema.columns AS columns + ON columns.table_catalog = tables.table_catalog + AND columns.table_schema = tables.table_schema + AND columns.table_name = tables.table_name + WHERE schemata.catalog_name = 'lake' + AND schemata.schema_name NOT IN ('information_schema', 'pg_catalog') + ORDER BY schemata.schema_name, tables.table_name, columns.ordinal_position +) q; +---- +2 diff --git a/test/sql/metadata/ducklake_ui_catalog_query_parallel_stress.test b/test/sql/metadata/ducklake_ui_catalog_query_parallel_stress.test new file mode 100644 index 00000000000..f216faa01a8 --- /dev/null +++ b/test/sql/metadata/ducklake_ui_catalog_query_parallel_stress.test @@ -0,0 +1,135 @@ +# name: test/sql/metadata/ducklake_ui_catalog_query_parallel_stress.test +# description: Stress test for ducklake metadata query with parallel execution +# group: [metadata] + +require ducklake + +require parquet + +test-env DUCKLAKE_CONNECTION __TEST_DIR__/{UUID}.db + +test-env DATA_PATH __TEST_DIR__ + +statement ok +SET threads=8 + +statement ok +ATTACH 'ducklake:${DUCKLAKE_CONNECTION}' AS lake (DATA_PATH '${DATA_PATH}/ducklake_ui_catalog_query_parallel_stress_files') + +statement ok +CREATE TABLE lake.sample(i INTEGER, note VARCHAR) + +query I +SELECT COUNT(*)::INTEGER +FROM ( + SELECT schemata.schema_name + FROM system.information_schema.schemata AS schemata + LEFT JOIN system.information_schema.tables AS tables + ON tables.table_catalog = schemata.catalog_name + AND tables.table_schema = schemata.schema_name + LEFT JOIN duckdb_tables() AS table_metadata + ON table_metadata.table_name = tables.table_name + AND table_metadata.schema_name = tables.table_schema + AND table_metadata.database_name = tables.table_catalog + LEFT JOIN system.information_schema.columns AS columns + ON columns.table_catalog = tables.table_catalog + AND columns.table_schema = tables.table_schema + AND columns.table_name = tables.table_name + WHERE schemata.catalog_name = 'lake' + AND schemata.schema_name NOT IN ('information_schema', 'pg_catalog') + ORDER BY schemata.schema_name, tables.table_name, columns.ordinal_position +) q; +---- +2 + +query I +SELECT COUNT(*)::INTEGER +FROM ( + SELECT schemata.schema_name + FROM system.information_schema.schemata AS schemata + LEFT JOIN system.information_schema.tables AS tables + ON tables.table_catalog = schemata.catalog_name + AND tables.table_schema = schemata.schema_name + LEFT JOIN duckdb_tables() AS table_metadata + ON table_metadata.table_name = tables.table_name + AND table_metadata.schema_name = tables.table_schema + AND table_metadata.database_name = tables.table_catalog + LEFT JOIN system.information_schema.columns AS columns + ON columns.table_catalog = tables.table_catalog + AND columns.table_schema = tables.table_schema + AND columns.table_name = tables.table_name + WHERE schemata.catalog_name = 'lake' + AND schemata.schema_name NOT IN ('information_schema', 'pg_catalog') + ORDER BY schemata.schema_name, tables.table_name, columns.ordinal_position +) q; +---- +2 + +query I +SELECT COUNT(*)::INTEGER +FROM ( + SELECT schemata.schema_name + FROM system.information_schema.schemata AS schemata + LEFT JOIN system.information_schema.tables AS tables + ON tables.table_catalog = schemata.catalog_name + AND tables.table_schema = schemata.schema_name + LEFT JOIN duckdb_tables() AS table_metadata + ON table_metadata.table_name = tables.table_name + AND table_metadata.schema_name = tables.table_schema + AND table_metadata.database_name = tables.table_catalog + LEFT JOIN system.information_schema.columns AS columns + ON columns.table_catalog = tables.table_catalog + AND columns.table_schema = tables.table_schema + AND columns.table_name = tables.table_name + WHERE schemata.catalog_name = 'lake' + AND schemata.schema_name NOT IN ('information_schema', 'pg_catalog') + ORDER BY schemata.schema_name, tables.table_name, columns.ordinal_position +) q; +---- +2 + +query I +SELECT COUNT(*)::INTEGER +FROM ( + SELECT schemata.schema_name + FROM system.information_schema.schemata AS schemata + LEFT JOIN system.information_schema.tables AS tables + ON tables.table_catalog = schemata.catalog_name + AND tables.table_schema = schemata.schema_name + LEFT JOIN duckdb_tables() AS table_metadata + ON table_metadata.table_name = tables.table_name + AND table_metadata.schema_name = tables.table_schema + AND table_metadata.database_name = tables.table_catalog + LEFT JOIN system.information_schema.columns AS columns + ON columns.table_catalog = tables.table_catalog + AND columns.table_schema = tables.table_schema + AND columns.table_name = tables.table_name + WHERE schemata.catalog_name = 'lake' + AND schemata.schema_name NOT IN ('information_schema', 'pg_catalog') + ORDER BY schemata.schema_name, tables.table_name, columns.ordinal_position +) q; +---- +2 + +query I +SELECT COUNT(*)::INTEGER +FROM ( + SELECT schemata.schema_name + FROM system.information_schema.schemata AS schemata + LEFT JOIN system.information_schema.tables AS tables + ON tables.table_catalog = schemata.catalog_name + AND tables.table_schema = schemata.schema_name + LEFT JOIN duckdb_tables() AS table_metadata + ON table_metadata.table_name = tables.table_name + AND table_metadata.schema_name = tables.table_schema + AND table_metadata.database_name = tables.table_catalog + LEFT JOIN system.information_schema.columns AS columns + ON columns.table_catalog = tables.table_catalog + AND columns.table_schema = tables.table_schema + AND columns.table_name = tables.table_name + WHERE schemata.catalog_name = 'lake' + AND schemata.schema_name NOT IN ('information_schema', 'pg_catalog') + ORDER BY schemata.schema_name, tables.table_name, columns.ordinal_position +) q; +---- +2 From 7b57030b8ea315d7f9a0fc3e10b08f4e6b0848ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Rafael?= Date: Thu, 12 Mar 2026 14:40:14 +0000 Subject: [PATCH 04/59] Cleanup --- src/include/storage/ducklake_transaction.hpp | 2 +- .../metadata/ducklake_ui_catalog_query.test | 3 + ...lake_ui_catalog_query_parallel_stress.test | 135 ------------------ 3 files changed, 4 insertions(+), 136 deletions(-) delete mode 100644 test/sql/metadata/ducklake_ui_catalog_query_parallel_stress.test diff --git a/src/include/storage/ducklake_transaction.hpp b/src/include/storage/ducklake_transaction.hpp index 84db958e640..7e61f753a42 100644 --- a/src/include/storage/ducklake_transaction.hpp +++ b/src/include/storage/ducklake_transaction.hpp @@ -261,7 +261,7 @@ class DuckLakeTransaction : public Transaction, public enable_shared_from_this new_schemas; map> dropped_schemas; //! Local changes made to tables - mutable mutex table_data_changes_lock; + mutex table_data_changes_lock; map table_data_changes; //! Snapshot cache for the AT (...) conditions that are referenced in the transaction value_map_t snapshot_cache; diff --git a/test/sql/metadata/ducklake_ui_catalog_query.test b/test/sql/metadata/ducklake_ui_catalog_query.test index ff8be82e78e..25646f0e231 100644 --- a/test/sql/metadata/ducklake_ui_catalog_query.test +++ b/test/sql/metadata/ducklake_ui_catalog_query.test @@ -10,6 +10,9 @@ test-env DUCKLAKE_CONNECTION __TEST_DIR__/{UUID}.db test-env DATA_PATH __TEST_DIR__ +statement ok +SET threads=8 + statement ok ATTACH 'ducklake:${DUCKLAKE_CONNECTION}' AS lake (DATA_PATH '${DATA_PATH}/ducklake_ui_catalog_query_files') diff --git a/test/sql/metadata/ducklake_ui_catalog_query_parallel_stress.test b/test/sql/metadata/ducklake_ui_catalog_query_parallel_stress.test deleted file mode 100644 index f216faa01a8..00000000000 --- a/test/sql/metadata/ducklake_ui_catalog_query_parallel_stress.test +++ /dev/null @@ -1,135 +0,0 @@ -# name: test/sql/metadata/ducklake_ui_catalog_query_parallel_stress.test -# description: Stress test for ducklake metadata query with parallel execution -# group: [metadata] - -require ducklake - -require parquet - -test-env DUCKLAKE_CONNECTION __TEST_DIR__/{UUID}.db - -test-env DATA_PATH __TEST_DIR__ - -statement ok -SET threads=8 - -statement ok -ATTACH 'ducklake:${DUCKLAKE_CONNECTION}' AS lake (DATA_PATH '${DATA_PATH}/ducklake_ui_catalog_query_parallel_stress_files') - -statement ok -CREATE TABLE lake.sample(i INTEGER, note VARCHAR) - -query I -SELECT COUNT(*)::INTEGER -FROM ( - SELECT schemata.schema_name - FROM system.information_schema.schemata AS schemata - LEFT JOIN system.information_schema.tables AS tables - ON tables.table_catalog = schemata.catalog_name - AND tables.table_schema = schemata.schema_name - LEFT JOIN duckdb_tables() AS table_metadata - ON table_metadata.table_name = tables.table_name - AND table_metadata.schema_name = tables.table_schema - AND table_metadata.database_name = tables.table_catalog - LEFT JOIN system.information_schema.columns AS columns - ON columns.table_catalog = tables.table_catalog - AND columns.table_schema = tables.table_schema - AND columns.table_name = tables.table_name - WHERE schemata.catalog_name = 'lake' - AND schemata.schema_name NOT IN ('information_schema', 'pg_catalog') - ORDER BY schemata.schema_name, tables.table_name, columns.ordinal_position -) q; ----- -2 - -query I -SELECT COUNT(*)::INTEGER -FROM ( - SELECT schemata.schema_name - FROM system.information_schema.schemata AS schemata - LEFT JOIN system.information_schema.tables AS tables - ON tables.table_catalog = schemata.catalog_name - AND tables.table_schema = schemata.schema_name - LEFT JOIN duckdb_tables() AS table_metadata - ON table_metadata.table_name = tables.table_name - AND table_metadata.schema_name = tables.table_schema - AND table_metadata.database_name = tables.table_catalog - LEFT JOIN system.information_schema.columns AS columns - ON columns.table_catalog = tables.table_catalog - AND columns.table_schema = tables.table_schema - AND columns.table_name = tables.table_name - WHERE schemata.catalog_name = 'lake' - AND schemata.schema_name NOT IN ('information_schema', 'pg_catalog') - ORDER BY schemata.schema_name, tables.table_name, columns.ordinal_position -) q; ----- -2 - -query I -SELECT COUNT(*)::INTEGER -FROM ( - SELECT schemata.schema_name - FROM system.information_schema.schemata AS schemata - LEFT JOIN system.information_schema.tables AS tables - ON tables.table_catalog = schemata.catalog_name - AND tables.table_schema = schemata.schema_name - LEFT JOIN duckdb_tables() AS table_metadata - ON table_metadata.table_name = tables.table_name - AND table_metadata.schema_name = tables.table_schema - AND table_metadata.database_name = tables.table_catalog - LEFT JOIN system.information_schema.columns AS columns - ON columns.table_catalog = tables.table_catalog - AND columns.table_schema = tables.table_schema - AND columns.table_name = tables.table_name - WHERE schemata.catalog_name = 'lake' - AND schemata.schema_name NOT IN ('information_schema', 'pg_catalog') - ORDER BY schemata.schema_name, tables.table_name, columns.ordinal_position -) q; ----- -2 - -query I -SELECT COUNT(*)::INTEGER -FROM ( - SELECT schemata.schema_name - FROM system.information_schema.schemata AS schemata - LEFT JOIN system.information_schema.tables AS tables - ON tables.table_catalog = schemata.catalog_name - AND tables.table_schema = schemata.schema_name - LEFT JOIN duckdb_tables() AS table_metadata - ON table_metadata.table_name = tables.table_name - AND table_metadata.schema_name = tables.table_schema - AND table_metadata.database_name = tables.table_catalog - LEFT JOIN system.information_schema.columns AS columns - ON columns.table_catalog = tables.table_catalog - AND columns.table_schema = tables.table_schema - AND columns.table_name = tables.table_name - WHERE schemata.catalog_name = 'lake' - AND schemata.schema_name NOT IN ('information_schema', 'pg_catalog') - ORDER BY schemata.schema_name, tables.table_name, columns.ordinal_position -) q; ----- -2 - -query I -SELECT COUNT(*)::INTEGER -FROM ( - SELECT schemata.schema_name - FROM system.information_schema.schemata AS schemata - LEFT JOIN system.information_schema.tables AS tables - ON tables.table_catalog = schemata.catalog_name - AND tables.table_schema = schemata.schema_name - LEFT JOIN duckdb_tables() AS table_metadata - ON table_metadata.table_name = tables.table_name - AND table_metadata.schema_name = tables.table_schema - AND table_metadata.database_name = tables.table_catalog - LEFT JOIN system.information_schema.columns AS columns - ON columns.table_catalog = tables.table_catalog - AND columns.table_schema = tables.table_schema - AND columns.table_name = tables.table_name - WHERE schemata.catalog_name = 'lake' - AND schemata.schema_name NOT IN ('information_schema', 'pg_catalog') - ORDER BY schemata.schema_name, tables.table_name, columns.ordinal_position -) q; ----- -2 From bd6a3c28a6eacdf664185363ee11aa8d367b6ca4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Rafael?= Date: Thu, 12 Mar 2026 22:02:41 +0000 Subject: [PATCH 05/59] Remove guards from const methods --- src/storage/ducklake_transaction.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/storage/ducklake_transaction.cpp b/src/storage/ducklake_transaction.cpp index 1037fe9d5db..aae7f5d2624 100644 --- a/src/storage/ducklake_transaction.cpp +++ b/src/storage/ducklake_transaction.cpp @@ -1458,7 +1458,6 @@ vector DuckLakeTransaction::GetNewDeleteFiles(const DuckLakeCommitState &commit_state, vector &overwritten_delete_files) const { vector result; - lock_guard guard(table_data_changes_lock); for (auto &entry : table_data_changes) { auto table_id = commit_state.GetTableId(entry.first); auto &table_changes = entry.second; @@ -2010,7 +2009,6 @@ idx_t DuckLakeTransaction::GetLocalCatalogId() { } bool DuckLakeTransaction::HasTransactionLocalInserts(TableIndex table_id) const { - lock_guard guard(table_data_changes_lock); auto entry = table_data_changes.find(table_id); if (entry == table_data_changes.end()) { return false; @@ -2020,7 +2018,6 @@ bool DuckLakeTransaction::HasTransactionLocalInserts(TableIndex table_id) const } bool DuckLakeTransaction::HasTransactionInlinedData(TableIndex table_id) const { - lock_guard guard(table_data_changes_lock); auto entry = table_data_changes.find(table_id); if (entry == table_data_changes.end()) { return false; From 0f55a31c25a1a7d991caf5ed9e7620c931e1a541 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Rafael?= Date: Thu, 12 Mar 2026 22:30:38 +0000 Subject: [PATCH 06/59] Make ducklake transaction read helpers const and lock-free --- src/include/storage/ducklake_transaction.hpp | 30 ++++++------- src/storage/ducklake_transaction.cpp | 45 ++++++++------------ 2 files changed, 32 insertions(+), 43 deletions(-) diff --git a/src/include/storage/ducklake_transaction.hpp b/src/include/storage/ducklake_transaction.hpp index 7e61f753a42..7ed24d25263 100644 --- a/src/include/storage/ducklake_transaction.hpp +++ b/src/include/storage/ducklake_transaction.hpp @@ -97,8 +97,8 @@ class DuckLakeTransaction : public Transaction, public enable_shared_from_this GetTransactionLocalEntries(CatalogType type, const string &schema_name); optional_ptr GetTransactionLocalEntry(CatalogType catalog_type, const string &schema_name, const string &entry_name); - vector GetTransactionLocalFiles(TableIndex table_id); - shared_ptr GetTransactionLocalInlinedData(TableIndex table_id); + vector GetTransactionLocalFiles(TableIndex table_id) const; + shared_ptr GetTransactionLocalInlinedData(TableIndex table_id) const; void DropTransactionLocalFile(TableIndex table_id, const string &path); bool HasTransactionLocalInserts(TableIndex table_id) const; bool HasTransactionInlinedData(TableIndex table_id) const; @@ -117,8 +117,8 @@ class DuckLakeTransaction : public Transaction, public enable_shared_from_this GetInlinedDeletes(TableIndex table_id, const string &table_name); - vector GetNewInlinedDeletes(DuckLakeCommitState &commit_state); + optional_ptr GetInlinedDeletes(TableIndex table_id, const string &table_name) const; + vector GetNewInlinedDeletes(DuckLakeCommitState &commit_state) const; //! Add inlined file deletions (deletions from parquet files stored in metadata) void AddNewInlinedFileDeletes(TableIndex table_id, idx_t file_id, set new_deletes); @@ -135,8 +135,8 @@ class DuckLakeTransaction : public Transaction, public enable_shared_from_this &snapshots); void DeleteInlinedData(const DuckLakeInlinedTableInfo &inlined_table); - bool SchemaChangesMade(); - bool ChangesMade(); + bool SchemaChangesMade() const; + bool ChangesMade() const; idx_t GetLocalCatalogId(); static bool IsTransactionLocal(idx_t id) { return id >= DuckLakeConstants::TRANSACTION_LOCAL_ID_START; @@ -147,17 +147,17 @@ class DuckLakeTransaction : public Transaction, public enable_shared_from_this &result); + bool HasLocalInlinedFileDeletes(TableIndex table_id) const; + void GetLocalInlinedFileDeletesForFile(TableIndex table_id, idx_t file_id, set &result) const; bool HasDroppedFiles() const; bool FileIsDropped(const string &path) const; //! Check if there are any uncommitted changes for this table (inserts, deletes, or dropped files) - bool HasAnyLocalChanges(TableIndex table_id); + bool HasAnyLocalChanges(TableIndex table_id) const; string GenerateUUID() const; static string GenerateUUIDv7(); @@ -215,10 +215,10 @@ class DuckLakeTransaction : public Transaction, public enable_shared_from_this table_entry, NewTableInfo &result, TransactionChangeInformation &transaction_changes); @@ -231,7 +231,7 @@ class DuckLakeTransaction : public Transaction, public enable_shared_from_this new_entry); void AlterEntryInternal(DuckLakeViewEntry &old_entry, unique_ptr new_entry); void AddTableChanges(TableIndex table_id, const LocalTableDataChanges &table_changes, - TransactionChangeInformation &changes); + TransactionChangeInformation &changes) const; private: DuckLakeCatalog &ducklake_catalog; diff --git a/src/storage/ducklake_transaction.cpp b/src/storage/ducklake_transaction.cpp index aae7f5d2624..f8614d101ee 100644 --- a/src/storage/ducklake_transaction.cpp +++ b/src/storage/ducklake_transaction.cpp @@ -106,14 +106,13 @@ Connection &DuckLakeTransaction::GetConnection() { return *connection; } -bool DuckLakeTransaction::SchemaChangesMade() { +bool DuckLakeTransaction::SchemaChangesMade() const { return !new_tables.empty() || !dropped_tables.empty() || new_schemas || !dropped_schemas.empty() || !dropped_views.empty() || !new_macros.empty() || !dropped_scalar_macros.empty() || !dropped_table_macros.empty(); } -bool DuckLakeTransaction::ChangesMade() { - lock_guard guard(table_data_changes_lock); +bool DuckLakeTransaction::ChangesMade() const { return SchemaChangesMade() || !table_data_changes.empty() || !dropped_files.empty() || !new_name_maps.name_maps.empty(); } @@ -225,7 +224,7 @@ void GetTransactionViewChanges(reference view_entry, TransactionCh } } -TransactionChangeInformation DuckLakeTransaction::GetTransactionChanges() { +TransactionChangeInformation DuckLakeTransaction::GetTransactionChanges() const { TransactionChangeInformation changes; for (auto &dropped_table_idx : dropped_tables) { changes.dropped_tables.insert(dropped_table_idx); @@ -283,7 +282,6 @@ TransactionChangeInformation DuckLakeTransaction::GetTransactionChanges() { } } changes.tables_deleted_from = tables_deleted_from; - lock_guard guard(table_data_changes_lock); for (auto &entry : table_data_changes) { auto table_id = entry.first; if (table_id.IsTransactionLocal()) { @@ -339,7 +337,7 @@ struct DuckLakeCommitState { }; void DuckLakeTransaction::AddTableChanges(TableIndex table_id, const LocalTableDataChanges &table_changes, - TransactionChangeInformation &changes) { + TransactionChangeInformation &changes) const { bool inserted_data = false; bool flushed_inline_data = false; for (auto &file : table_changes.new_data_files) { @@ -394,12 +392,11 @@ void AddChangeInfo(DuckLakeCommitState &commit_state, SnapshotChangeInfo &change } string DuckLakeTransaction::WriteSnapshotChanges(DuckLakeCommitState &commit_state, - TransactionChangeInformation &changes) { + TransactionChangeInformation &changes) const { SnapshotChangeInfo change_info; // re-add all inserted tables - transaction-local table identifiers should have been converted at this stage changes.tables_deleted_from = tables_deleted_from; - lock_guard guard(table_data_changes_lock); for (auto &entry : table_data_changes) { auto table_id = commit_state.GetTableId(entry.first); auto &table_changes = entry.second; @@ -579,7 +576,7 @@ void ConflictCheck(const case_insensitive_map_t> & void DuckLakeTransaction::CheckForConflicts(const TransactionChangeInformation &changes, const SnapshotChangeInformation &other_changes, - DuckLakeSnapshot transaction_snapshot) { + DuckLakeSnapshot transaction_snapshot) const { // check if we are dropping the same table as another transaction for (auto &dropped_idx : changes.dropped_tables) { ConflictCheck(dropped_idx, other_changes.dropped_tables, "drop table", "dropped it already"); @@ -663,7 +660,6 @@ void DuckLakeTransaction::CheckForConflicts(const TransactionChangeInformation & if (check_for_matches) { // If we have deletes on the tables, check for files being deleted const auto deleted_files = metadata_manager->GetFilesDeletedOrDroppedAfterSnapshot(transaction_snapshot); - lock_guard guard(table_data_changes_lock); for (auto &entry : table_data_changes) { auto &table_changes = entry.second; for (auto &file_entry : table_changes.new_delete_files) { @@ -1550,9 +1546,8 @@ NewNameMapInfo DuckLakeTransaction::GetNewNameMaps(DuckLakeCommitState &commit_s return result; } -vector DuckLakeTransaction::GetNewInlinedDeletes(DuckLakeCommitState &commit_state) { +vector DuckLakeTransaction::GetNewInlinedDeletes(DuckLakeCommitState &commit_state) const { vector result; - lock_guard guard(table_data_changes_lock); for (auto &entry : table_data_changes) { auto table_id = commit_state.GetTableId(entry.first); auto &table_changes = entry.second; @@ -2026,8 +2021,7 @@ bool DuckLakeTransaction::HasTransactionInlinedData(TableIndex table_id) const { return table_changes.new_inlined_data != nullptr; } -vector DuckLakeTransaction::GetTransactionLocalFiles(TableIndex table_id) { - lock_guard guard(table_data_changes_lock); +vector DuckLakeTransaction::GetTransactionLocalFiles(TableIndex table_id) const { auto entry = table_data_changes.find(table_id); if (entry == table_data_changes.end()) { return vector(); @@ -2035,8 +2029,7 @@ vector DuckLakeTransaction::GetTransactionLocalFiles(TableInde return entry->second.new_data_files; } -shared_ptr DuckLakeTransaction::GetTransactionLocalInlinedData(TableIndex table_id) { - lock_guard guard(table_data_changes_lock); +shared_ptr DuckLakeTransaction::GetTransactionLocalInlinedData(TableIndex table_id) const { auto entry = table_data_changes.find(table_id); if (entry == table_data_changes.end()) { return nullptr; @@ -2343,8 +2336,7 @@ void DuckLakeTransaction::RemoveColumnFromLocalInlinedData(TableIndex table_id, } optional_ptr DuckLakeTransaction::GetInlinedDeletes(TableIndex table_id, - const string &table_name) { - lock_guard guard(table_data_changes_lock); + const string &table_name) const { auto entry = table_data_changes.find(table_id); if (entry == table_data_changes.end()) { return nullptr; @@ -2432,8 +2424,7 @@ void DuckLakeTransaction::AddCompaction(TableIndex table_id, DuckLakeCompactionE table_changes.compactions.push_back(std::move(entry)); } -bool DuckLakeTransaction::HasLocalDeletes(TableIndex table_id) { - lock_guard guard(table_data_changes_lock); +bool DuckLakeTransaction::HasLocalDeletes(TableIndex table_id) const { auto entry = table_data_changes.find(table_id); if (entry == table_data_changes.end()) { return false; @@ -2441,8 +2432,7 @@ bool DuckLakeTransaction::HasLocalDeletes(TableIndex table_id) { return !entry->second.new_delete_files.empty(); } -bool DuckLakeTransaction::HasAnyLocalChanges(TableIndex table_id) { - lock_guard guard(table_data_changes_lock); +bool DuckLakeTransaction::HasAnyLocalChanges(TableIndex table_id) const { auto entry = table_data_changes.find(table_id); if (entry != table_data_changes.end() && !entry->second.IsEmpty()) { return true; @@ -2450,8 +2440,8 @@ bool DuckLakeTransaction::HasAnyLocalChanges(TableIndex table_id) { return tables_deleted_from.find(table_id) != tables_deleted_from.end(); } -void DuckLakeTransaction::GetLocalDeleteForFile(TableIndex table_id, const string &path, DuckLakeFileData &result) { - lock_guard guard(table_data_changes_lock); +void DuckLakeTransaction::GetLocalDeleteForFile(TableIndex table_id, const string &path, + DuckLakeFileData &result) const { auto entry = table_data_changes.find(table_id); if (entry == table_data_changes.end()) { return; @@ -2468,8 +2458,7 @@ void DuckLakeTransaction::GetLocalDeleteForFile(TableIndex table_id, const strin result.encryption_key = delete_file.encryption_key; } -bool DuckLakeTransaction::HasLocalInlinedFileDeletes(TableIndex table_id) { - lock_guard guard(table_data_changes_lock); +bool DuckLakeTransaction::HasLocalInlinedFileDeletes(TableIndex table_id) const { auto entry = table_data_changes.find(table_id); if (entry == table_data_changes.end()) { return false; @@ -2481,8 +2470,8 @@ bool DuckLakeTransaction::HasLocalInlinedFileDeletes(TableIndex table_id) { return !table_changes.new_inlined_file_deletes->file_deletes.empty(); } -void DuckLakeTransaction::GetLocalInlinedFileDeletesForFile(TableIndex table_id, idx_t file_id, set &result) { - lock_guard guard(table_data_changes_lock); +void DuckLakeTransaction::GetLocalInlinedFileDeletesForFile(TableIndex table_id, idx_t file_id, + set &result) const { auto entry = table_data_changes.find(table_id); if (entry == table_data_changes.end()) { return; From e25f7a8ea62ab3fd964a165b0b0d4a8058e75452 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Rafael?= Date: Thu, 12 Mar 2026 23:00:41 +0000 Subject: [PATCH 07/59] Avoid AddDeletes self-deadlock during commit --- src/include/storage/ducklake_transaction.hpp | 2 ++ src/storage/ducklake_transaction.cpp | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/include/storage/ducklake_transaction.hpp b/src/include/storage/ducklake_transaction.hpp index 7ed24d25263..b859d70ffc4 100644 --- a/src/include/storage/ducklake_transaction.hpp +++ b/src/include/storage/ducklake_transaction.hpp @@ -230,6 +230,8 @@ class DuckLakeTransaction : public Transaction, public enable_shared_from_this new_entry); void AlterEntryInternal(DuckLakeViewEntry &old_entry, unique_ptr new_entry); + //! Mutates table_data_changes; caller must hold table_data_changes_lock. + void AddDeletesLocked(TableIndex table_id, vector files); void AddTableChanges(TableIndex table_id, const LocalTableDataChanges &table_changes, TransactionChangeInformation &changes) const; diff --git a/src/storage/ducklake_transaction.cpp b/src/storage/ducklake_transaction.cpp index f8614d101ee..051108ea4d3 100644 --- a/src/storage/ducklake_transaction.cpp +++ b/src/storage/ducklake_transaction.cpp @@ -1415,7 +1415,7 @@ NewDataInfo DuckLakeTransaction::GetNewDataFiles(string &batch_query, DuckLakeCo result.new_files.push_back(std::move(data_file)); } // write any deletes that were made on top of these transaction-local files - AddDeletes(table_id, std::move(delete_files)); + AddDeletesLocked(table_id, std::move(delete_files)); if (table_changes.new_inlined_data) { auto &inlined_data = *table_changes.new_inlined_data; @@ -2390,6 +2390,10 @@ void DuckLakeTransaction::AddDeletes(TableIndex table_id, vector guard(table_data_changes_lock); + AddDeletesLocked(table_id, std::move(files)); +} + +void DuckLakeTransaction::AddDeletesLocked(TableIndex table_id, vector files) { auto &table_changes = table_data_changes[table_id]; auto &table_delete_map = table_changes.new_delete_files; for (auto &file : files) { From 57a8621b167da6f87c2c76829ad8b7cd6e9d01c6 Mon Sep 17 00:00:00 2001 From: Christiaan Herrewijn Date: Fri, 13 Mar 2026 16:51:46 +0100 Subject: [PATCH 08/59] only delete .parquet files via ducklake_delete_orphaned_files() --- src/storage/ducklake_metadata_manager.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/storage/ducklake_metadata_manager.cpp b/src/storage/ducklake_metadata_manager.cpp index 22cbc44d0bb..071fe877949 100644 --- a/src/storage/ducklake_metadata_manager.cpp +++ b/src/storage/ducklake_metadata_manager.cpp @@ -3872,7 +3872,8 @@ vector DuckLakeMetadataManager::GetOrphanFilesForCleanup const string &separator) { auto query = R"(SELECT filename FROM read_blob({DATA_PATH} || '**') -WHERE filename NOT IN ( +WHERE suffix(filename, '.parquet') +AND filename NOT IN ( SELECT REPLACE( CASE WHEN NOT file_relative THEN file_path From e3f5789b0e98ae67042a0ab9e4ce8ce5eb816db2 Mon Sep 17 00:00:00 2001 From: Christiaan Herrewijn Date: Mon, 16 Mar 2026 14:50:08 +0100 Subject: [PATCH 09/59] add test --- .../remove_orphans/metadata_in_data_path.test | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 test/sql/remove_orphans/metadata_in_data_path.test diff --git a/test/sql/remove_orphans/metadata_in_data_path.test b/test/sql/remove_orphans/metadata_in_data_path.test new file mode 100644 index 00000000000..28633265ec2 --- /dev/null +++ b/test/sql/remove_orphans/metadata_in_data_path.test @@ -0,0 +1,53 @@ +# name: test/sql/remove_orphans/metadata_in_data_path.test +# description: ducklake_delete_orphaned_files should not delete the metadata database when it resides in the data path +# group: [remove_orphans] + +require ducklake + +require parquet + +# Place the metadata DB directly inside the data path directory +test-env DUCKLAKE_CONNECTION __TEST_DIR__/{UUID}.db + +test-env DATA_PATH __TEST_DIR__ + +statement ok +ATTACH 'ducklake:${DUCKLAKE_CONNECTION}' AS ducklake (DATA_PATH '${DATA_PATH}', METADATA_CATALOG 'ducklake_metadata') + +statement ok +USE ducklake + +statement ok +CREATE TABLE test (a integer) + +statement ok +INSERT INTO test VALUES (1); + +statement ok +CALL ducklake_flush_inlined_data('ducklake') + +# The metadata DB and the parquet data file must exist before the call +query I +SELECT count(*) FROM GLOB('${DATA_PATH}/*.db') +---- +1 + +query I +SELECT count(*) FROM GLOB('${DATA_PATH}/**/*.parquet') +---- +1 + +statement ok +CALL ducklake_delete_orphaned_files('ducklake', cleanup_all => true); + +# Verify the metadata DB file was NOT deleted +query I +SELECT count(*) FROM GLOB('${DATA_PATH}/*.db') +---- +1 + +# Verify the parquet data file was NOT deleted +query I +SELECT count(*) FROM GLOB('${DATA_PATH}/**/*.parquet') +---- +1 From b7c4fe80a56103b304bb970e9884f34682ff615d Mon Sep 17 00:00:00 2001 From: Christiaan Herrewijn Date: Mon, 16 Mar 2026 17:56:30 +0100 Subject: [PATCH 10/59] exclude tests from non applicable test configs --- test/configs/minio.json | 5 +++-- test/configs/postgres.json | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/test/configs/minio.json b/test/configs/minio.json index 11fb6eac24d..4fcd0ca3713 100644 --- a/test/configs/minio.json +++ b/test/configs/minio.json @@ -29,8 +29,9 @@ "test/sql/migration/v01_partitioned.test", "test/sql/delete/delete_ignore_extra_columns.test", "test/sql/add_files/add_file_with_three_level_list.test", - "test/sql/migration/test_compaction.test" + "test/sql/migration/test_compaction.test", + "test/sql/remove_orphans/metadata_in_data_path.test" ] } ] -} \ No newline at end of file +} diff --git a/test/configs/postgres.json b/test/configs/postgres.json index 8eb787eb051..e778e0b3220 100644 --- a/test/configs/postgres.json +++ b/test/configs/postgres.json @@ -24,7 +24,8 @@ "test/sql/general/default_path.test", "test/sql/autoloading/autoload_data_path.test", "test/sql/general/metadata_parameters.test", - "test/sql/metadata/ducklake_settings.test" + "test/sql/metadata/ducklake_settings.test", + "test/sql/remove_orphans/metadata_in_data_path.test" ] }, { @@ -60,4 +61,4 @@ ] } ] -} \ No newline at end of file +} From a9d4d39c7d61438561d46cbd1379b3db8d83ca32 Mon Sep 17 00:00:00 2001 From: Mytherin Date: Tue, 17 Mar 2026 09:32:22 +0100 Subject: [PATCH 11/59] Move towards isolating table data changes --- duckdb | 2 +- src/include/storage/ducklake_transaction.hpp | 13 +- src/storage/ducklake_transaction.cpp | 162 ++++++++++--------- 3 files changed, 96 insertions(+), 81 deletions(-) diff --git a/duckdb b/duckdb index 3a3967aa819..7f02bc91654 160000 --- a/duckdb +++ b/duckdb @@ -1 +1 @@ -Subproject commit 3a3967aa8190d0a2d1931d4ca4f5d920760030b4 +Subproject commit 7f02bc91654eae29997fe0df140628421ba912ad diff --git a/src/include/storage/ducklake_transaction.hpp b/src/include/storage/ducklake_transaction.hpp index b859d70ffc4..b10de60eee9 100644 --- a/src/include/storage/ducklake_transaction.hpp +++ b/src/include/storage/ducklake_transaction.hpp @@ -49,6 +49,16 @@ struct LocalTableDataChanges { bool IsEmpty() const; }; +struct LocalTableChanges { +public: + void Clear(); + bool HasChanges() const; + +// private: + mutable mutex lock; + map changes; +}; + struct SnapshotAndStats { vector stats; DuckLakeSnapshot snapshot; @@ -263,8 +273,7 @@ class DuckLakeTransaction : public Transaction, public enable_shared_from_this new_schemas; map> dropped_schemas; //! Local changes made to tables - mutex table_data_changes_lock; - map table_data_changes; + LocalTableChanges local_changes; //! Snapshot cache for the AT (...) conditions that are referenced in the transaction value_map_t snapshot_cache; //! New set of transaction-local name maps diff --git a/src/storage/ducklake_transaction.cpp b/src/storage/ducklake_transaction.cpp index 051108ea4d3..aee33b4275b 100644 --- a/src/storage/ducklake_transaction.cpp +++ b/src/storage/ducklake_transaction.cpp @@ -47,6 +47,16 @@ bool LocalTableDataChanges::IsEmpty() const { return true; } +void LocalTableChanges::Clear() { + lock_guard guard(lock); + changes.clear(); +} + +bool LocalTableChanges::HasChanges() const{ + lock_guard guard(lock); + return !changes.empty(); +} + DuckLakeTransaction::DuckLakeTransaction(DuckLakeCatalog &ducklake_catalog, TransactionManager &manager, ClientContext &context) : Transaction(manager, context), ducklake_catalog(ducklake_catalog), db(*context.db), @@ -67,9 +77,7 @@ void DuckLakeTransaction::Commit() { connection->Commit(); } connection.reset(); - - lock_guard guard(table_data_changes_lock); - table_data_changes.clear(); + local_changes.Clear(); } void DuckLakeTransaction::Rollback() { @@ -79,9 +87,7 @@ void DuckLakeTransaction::Rollback() { connection.reset(); } CleanupFiles(); - - lock_guard guard(table_data_changes_lock); - table_data_changes.clear(); + local_changes.Clear(); } Connection &DuckLakeTransaction::GetConnection() { @@ -113,7 +119,7 @@ bool DuckLakeTransaction::SchemaChangesMade() const { } bool DuckLakeTransaction::ChangesMade() const { - return SchemaChangesMade() || !table_data_changes.empty() || !dropped_files.empty() || + return SchemaChangesMade() || local_changes.HasChanges() || !dropped_files.empty() || !new_name_maps.name_maps.empty(); } @@ -282,7 +288,7 @@ TransactionChangeInformation DuckLakeTransaction::GetTransactionChanges() const } } changes.tables_deleted_from = tables_deleted_from; - for (auto &entry : table_data_changes) { + for (auto &entry : local_changes.changes) { auto table_id = entry.first; if (table_id.IsTransactionLocal()) { // don't report transaction-local tables yet - these will get added later on @@ -397,7 +403,7 @@ string DuckLakeTransaction::WriteSnapshotChanges(DuckLakeCommitState &commit_sta // re-add all inserted tables - transaction-local table identifiers should have been converted at this stage changes.tables_deleted_from = tables_deleted_from; - for (auto &entry : table_data_changes) { + for (auto &entry : local_changes.changes) { auto table_id = commit_state.GetTableId(entry.first); auto &table_changes = entry.second; AddTableChanges(table_id, table_changes, changes); @@ -491,8 +497,8 @@ void DuckLakeTransaction::CleanupFiles() { // remove any files that were written auto context_ref = context.lock(); auto &fs = FileSystem::GetFileSystem(db); - lock_guard guard(table_data_changes_lock); - for (auto &entry : table_data_changes) { + lock_guard guard(local_changes.lock); + for (auto &entry : local_changes.changes) { auto &table_changes = entry.second; for (auto &file : table_changes.new_data_files) { if (file.created_by_ducklake) { @@ -660,7 +666,7 @@ void DuckLakeTransaction::CheckForConflicts(const TransactionChangeInformation & if (check_for_matches) { // If we have deletes on the tables, check for files being deleted const auto deleted_files = metadata_manager->GetFilesDeletedOrDroppedAfterSnapshot(transaction_snapshot); - for (auto &entry : table_data_changes) { + for (auto &entry : local_changes.changes) { auto &table_changes = entry.second; for (auto &file_entry : table_changes.new_delete_files) { for (auto &file : file_entry.second) { @@ -784,8 +790,8 @@ DuckLakePartitionInfo DuckLakeTransaction::GetNewPartitionKey(DuckLakeCommitStat } // if we wrote any data with this partition id - rewrite it to the latest partition id - lock_guard guard(table_data_changes_lock); - for (auto &entry : table_data_changes) { + lock_guard guard(local_changes.lock); + for (auto &entry : local_changes.changes) { auto &table_changes = entry.second; for (auto &file : table_changes.new_data_files) { if (file.partition_id.IsValid() && file.partition_id.GetIndex() == local_partition_id) { @@ -1360,8 +1366,8 @@ NewDataInfo DuckLakeTransaction::GetNewDataFiles(string &batch_query, DuckLakeCo auto &schema = ducklake_catalog.GetSchemaForSnapshot(*this, GetSnapshot()); dl_stats = ducklake_catalog.ConstructStatsMap(*stats, schema); } - lock_guard guard(table_data_changes_lock); - for (auto &entry : table_data_changes) { + lock_guard guard(local_changes.lock); + for (auto &entry : local_changes.changes) { auto table_id = commit_state.GetTableId(entry.first); if (table_id.IsTransactionLocal()) { throw InternalException("Cannot commit transaction local files - these should have been cleaned up before"); @@ -1454,7 +1460,7 @@ vector DuckLakeTransaction::GetNewDeleteFiles(const DuckLakeCommitState &commit_state, vector &overwritten_delete_files) const { vector result; - for (auto &entry : table_data_changes) { + for (auto &entry : local_changes.changes) { auto table_id = commit_state.GetTableId(entry.first); auto &table_changes = entry.second; for (auto &file_entry : table_changes.new_delete_files) { @@ -1530,8 +1536,8 @@ NewNameMapInfo DuckLakeTransaction::GetNewNameMaps(DuckLakeCommitState &commit_s remap_mapping_index[local_map_id] = new_map_id; } // iterate over the data files to point them towards any new mapping ids - lock_guard guard(table_data_changes_lock); - for (auto &entry : table_data_changes) { + lock_guard guard(local_changes.lock); + for (auto &entry : local_changes.changes) { auto &table_changes = entry.second; for (auto &data_file : table_changes.new_data_files) { if (!data_file.mapping_id.IsValid()) { @@ -1548,7 +1554,7 @@ NewNameMapInfo DuckLakeTransaction::GetNewNameMaps(DuckLakeCommitState &commit_s vector DuckLakeTransaction::GetNewInlinedDeletes(DuckLakeCommitState &commit_state) const { vector result; - for (auto &entry : table_data_changes) { + for (auto &entry : local_changes.changes) { auto table_id = commit_state.GetTableId(entry.first); auto &table_changes = entry.second; for (auto &delete_entry : table_changes.new_inlined_data_deletes) { @@ -1647,8 +1653,8 @@ string DuckLakeTransaction::CommitChanges(DuckLakeCommitState &commit_state, // write new data / data files bool has_table_data_changes = false; { - lock_guard guard(table_data_changes_lock); - has_table_data_changes = !table_data_changes.empty(); + lock_guard guard(local_changes.lock); + has_table_data_changes = !local_changes.changes.empty(); } if (has_table_data_changes) { auto result = GetNewDataFiles(batch_queries, commit_state, stats); @@ -1718,8 +1724,8 @@ string DuckLakeTransaction::CommitChanges(DuckLakeCommitState &commit_state, CompactionInformation DuckLakeTransaction::GetCompactionChanges(DuckLakeSnapshot &commit_snapshot, CompactionType type) { CompactionInformation result; - lock_guard guard(table_data_changes_lock); - for (auto &entry : table_data_changes) { + lock_guard guard(local_changes.lock); + for (auto &entry : local_changes.changes) { auto table_id = entry.first; auto &table_changes = entry.second; for (auto &compaction : table_changes.compactions) { @@ -2004,8 +2010,8 @@ idx_t DuckLakeTransaction::GetLocalCatalogId() { } bool DuckLakeTransaction::HasTransactionLocalInserts(TableIndex table_id) const { - auto entry = table_data_changes.find(table_id); - if (entry == table_data_changes.end()) { + auto entry = local_changes.changes.find(table_id); + if (entry == local_changes.changes.end()) { return false; } auto &table_changes = entry->second; @@ -2013,8 +2019,8 @@ bool DuckLakeTransaction::HasTransactionLocalInserts(TableIndex table_id) const } bool DuckLakeTransaction::HasTransactionInlinedData(TableIndex table_id) const { - auto entry = table_data_changes.find(table_id); - if (entry == table_data_changes.end()) { + auto entry = local_changes.changes.find(table_id); + if (entry == local_changes.changes.end()) { return false; } auto &table_changes = entry->second; @@ -2022,16 +2028,16 @@ bool DuckLakeTransaction::HasTransactionInlinedData(TableIndex table_id) const { } vector DuckLakeTransaction::GetTransactionLocalFiles(TableIndex table_id) const { - auto entry = table_data_changes.find(table_id); - if (entry == table_data_changes.end()) { + auto entry = local_changes.changes.find(table_id); + if (entry == local_changes.changes.end()) { return vector(); } return entry->second.new_data_files; } shared_ptr DuckLakeTransaction::GetTransactionLocalInlinedData(TableIndex table_id) const { - auto entry = table_data_changes.find(table_id); - if (entry == table_data_changes.end()) { + auto entry = local_changes.changes.find(table_id); + if (entry == local_changes.changes.end()) { return nullptr; } auto &table_changes = entry->second; @@ -2049,9 +2055,9 @@ shared_ptr DuckLakeTransaction::GetTransactionLocalInlinedD } void DuckLakeTransaction::DropTransactionLocalFile(TableIndex table_id, const string &path) { - lock_guard guard(table_data_changes_lock); - auto entry = table_data_changes.find(table_id); - if (entry == table_data_changes.end()) { + lock_guard guard(local_changes.lock); + auto entry = local_changes.changes.find(table_id); + if (entry == local_changes.changes.end()) { throw InternalException( "DropTransactionLocalFile called for a table for which no transaction-local files exist"); } @@ -2071,7 +2077,7 @@ void DuckLakeTransaction::DropTransactionLocalFile(TableIndex table_id, const st fs.RemoveFile(path); if (table_changes.IsEmpty()) { // no more files remaining - table_data_changes.erase(entry); + local_changes.changes.erase(entry); } return; } @@ -2083,8 +2089,8 @@ void DuckLakeTransaction::AppendFiles(TableIndex table_id, vector guard(table_data_changes_lock); - auto &table_changes = table_data_changes[table_id]; + lock_guard guard(local_changes.lock); + auto &table_changes = local_changes.changes[table_id]; if (table_changes.new_data_files.empty()) { // If empty, just move the entire vector table_changes.new_data_files = std::move(files); @@ -2100,8 +2106,8 @@ void DuckLakeTransaction::AppendFiles(TableIndex table_id, vector new_data) { - lock_guard guard(table_data_changes_lock); - auto &table_changes = table_data_changes[table_id]; + lock_guard guard(local_changes.lock); + auto &table_changes = local_changes.changes[table_id]; if (table_changes.new_inlined_data) { // already exists - append auto &existing_data = *table_changes.new_inlined_data; @@ -2152,8 +2158,8 @@ void DuckLakeTransaction::AddNewInlinedDeletes(TableIndex table_id, const string if (new_deletes.empty()) { return; } - lock_guard guard(table_data_changes_lock); - auto &table_changes = table_data_changes[table_id]; + lock_guard guard(local_changes.lock); + auto &table_changes = local_changes.changes[table_id]; auto &table_deletes = table_changes.new_inlined_data_deletes; auto entry = table_deletes.find(table_name); if (entry != table_deletes.end()) { @@ -2170,9 +2176,9 @@ void DuckLakeTransaction::AddNewInlinedDeletes(TableIndex table_id, const string } void DuckLakeTransaction::DeleteFromLocalInlinedData(TableIndex table_id, set new_deletes) { - lock_guard guard(table_data_changes_lock); - auto entry = table_data_changes.find(table_id); - if (entry == table_data_changes.end()) { + lock_guard guard(local_changes.lock); + auto entry = local_changes.changes.find(table_id); + if (entry == local_changes.changes.end()) { throw InternalException("DeleteFromLocalInlinedData called but no transaction-local data exists for table"); } auto &table_changes = entry->second; @@ -2211,9 +2217,9 @@ void DuckLakeTransaction::DeleteFromLocalInlinedData(TableIndex table_id, set guard(table_data_changes_lock); - auto entry = table_data_changes.find(table_id); - if (entry == table_data_changes.end()) { + lock_guard guard(local_changes.lock); + auto entry = local_changes.changes.find(table_id); + if (entry == local_changes.changes.end()) { throw InternalException("AddColumnToLocalInlinedData called but no transaction-local data exists"); } auto &table_changes = entry->second; @@ -2285,9 +2291,9 @@ static void RemoveFieldStats(map &column_stats, void DuckLakeTransaction::RemoveColumnFromLocalInlinedData(TableIndex table_id, LogicalIndex removed_column_index, const DuckLakeFieldId &field_id) { - lock_guard guard(table_data_changes_lock); - auto entry = table_data_changes.find(table_id); - if (entry == table_data_changes.end()) { + lock_guard guard(local_changes.lock); + auto entry = local_changes.changes.find(table_id); + if (entry == local_changes.changes.end()) { throw InternalException("RemoveColumnFromLocalInlinedData called but no transaction-local data exists"); } auto &table_changes = entry->second; @@ -2337,8 +2343,8 @@ void DuckLakeTransaction::RemoveColumnFromLocalInlinedData(TableIndex table_id, optional_ptr DuckLakeTransaction::GetInlinedDeletes(TableIndex table_id, const string &table_name) const { - auto entry = table_data_changes.find(table_id); - if (entry == table_data_changes.end()) { + auto entry = local_changes.changes.find(table_id); + if (entry == local_changes.changes.end()) { return nullptr; } auto &table_changes = entry->second; @@ -2353,8 +2359,8 @@ void DuckLakeTransaction::AddNewInlinedFileDeletes(TableIndex table_id, idx_t fi if (new_deletes.empty()) { return; } - lock_guard guard(table_data_changes_lock); - auto &table_changes = table_data_changes[table_id]; + lock_guard guard(local_changes.lock); + auto &table_changes = local_changes.changes[table_id]; if (!table_changes.new_inlined_file_deletes) { table_changes.new_inlined_file_deletes = make_uniq(); } @@ -2367,8 +2373,8 @@ void DuckLakeTransaction::AddNewInlinedFileDeletes(TableIndex table_id, idx_t fi vector DuckLakeTransaction::GetNewInlinedFileDeletes(DuckLakeCommitState &commit_state) { vector result; - lock_guard guard(table_data_changes_lock); - for (auto &entry : table_data_changes) { + lock_guard guard(local_changes.lock); + for (auto &entry : local_changes.changes) { auto table_id = commit_state.GetTableId(entry.first); auto &table_changes = entry.second; if (!table_changes.new_inlined_file_deletes) { @@ -2389,12 +2395,12 @@ void DuckLakeTransaction::AddDeletes(TableIndex table_id, vector guard(table_data_changes_lock); + lock_guard guard(local_changes.lock); AddDeletesLocked(table_id, std::move(files)); } void DuckLakeTransaction::AddDeletesLocked(TableIndex table_id, vector files) { - auto &table_changes = table_data_changes[table_id]; + auto &table_changes = local_changes.changes[table_id]; auto &table_delete_map = table_changes.new_delete_files; for (auto &file : files) { auto &data_file_path = file.data_file_path; @@ -2423,22 +2429,22 @@ void DuckLakeTransaction::AddDeletesLocked(TableIndex table_id, vector guard(table_data_changes_lock); - auto &table_changes = table_data_changes[table_id]; + lock_guard guard(local_changes.lock); + auto &table_changes = local_changes.changes[table_id]; table_changes.compactions.push_back(std::move(entry)); } bool DuckLakeTransaction::HasLocalDeletes(TableIndex table_id) const { - auto entry = table_data_changes.find(table_id); - if (entry == table_data_changes.end()) { + auto entry = local_changes.changes.find(table_id); + if (entry == local_changes.changes.end()) { return false; } return !entry->second.new_delete_files.empty(); } bool DuckLakeTransaction::HasAnyLocalChanges(TableIndex table_id) const { - auto entry = table_data_changes.find(table_id); - if (entry != table_data_changes.end() && !entry->second.IsEmpty()) { + auto entry = local_changes.changes.find(table_id); + if (entry != local_changes.changes.end() && !entry->second.IsEmpty()) { return true; } return tables_deleted_from.find(table_id) != tables_deleted_from.end(); @@ -2446,8 +2452,8 @@ bool DuckLakeTransaction::HasAnyLocalChanges(TableIndex table_id) const { void DuckLakeTransaction::GetLocalDeleteForFile(TableIndex table_id, const string &path, DuckLakeFileData &result) const { - auto entry = table_data_changes.find(table_id); - if (entry == table_data_changes.end()) { + auto entry = local_changes.changes.find(table_id); + if (entry == local_changes.changes.end()) { return; } auto &table_changes = entry->second; @@ -2463,8 +2469,8 @@ void DuckLakeTransaction::GetLocalDeleteForFile(TableIndex table_id, const strin } bool DuckLakeTransaction::HasLocalInlinedFileDeletes(TableIndex table_id) const { - auto entry = table_data_changes.find(table_id); - if (entry == table_data_changes.end()) { + auto entry = local_changes.changes.find(table_id); + if (entry == local_changes.changes.end()) { return false; } auto &table_changes = entry->second; @@ -2476,8 +2482,8 @@ bool DuckLakeTransaction::HasLocalInlinedFileDeletes(TableIndex table_id) const void DuckLakeTransaction::GetLocalInlinedFileDeletesForFile(TableIndex table_id, idx_t file_id, set &result) const { - auto entry = table_data_changes.find(table_id); - if (entry == table_data_changes.end()) { + auto entry = local_changes.changes.find(table_id); + if (entry == local_changes.changes.end()) { return; } auto &table_changes = entry->second; @@ -2496,9 +2502,9 @@ void DuckLakeTransaction::GetLocalInlinedFileDeletesForFile(TableIndex table_id, void DuckLakeTransaction::TransactionLocalDelete(TableIndex table_id, const string &data_file_path, DuckLakeDeleteFile delete_file) { - lock_guard guard(table_data_changes_lock); - auto entry = table_data_changes.find(table_id); - if (entry == table_data_changes.end()) { + lock_guard guard(local_changes.lock); + auto entry = local_changes.changes.find(table_id); + if (entry == local_changes.changes.end()) { throw InternalException( "Transaction local delete called for table which does not have transaction local insertions"); } @@ -2561,16 +2567,16 @@ void DuckLakeTransaction::DropTable(DuckLakeTableEntry &table) { auto table_id = table.GetTableId(); schema_entry->second->DropEntry(table.name); // if we have written any files for this table - clean them up - lock_guard guard(table_data_changes_lock); - auto table_entry = table_data_changes.find(table_id); - if (table_entry != table_data_changes.end()) { + lock_guard guard(local_changes.lock); + auto table_entry = local_changes.changes.find(table_id); + if (table_entry != local_changes.changes.end()) { auto &table_changes = table_entry->second; auto context_ref = context.lock(); auto &fs = FileSystem::GetFileSystem(*context_ref); for (auto &file : table_changes.new_data_files) { fs.RemoveFile(file.file_name); } - table_data_changes.erase(table_entry); + local_changes.changes.erase(table_entry); } new_tables.erase(schema_entry); } else { From 5606042709a9ac10499492864b4e8be2fd290d11 Mon Sep 17 00:00:00 2001 From: Mytherin Date: Tue, 17 Mar 2026 09:44:40 +0100 Subject: [PATCH 12/59] Add thread-safe iterator over local changes --- src/include/storage/ducklake_transaction.hpp | 42 ++++++++++++ src/storage/ducklake_transaction.cpp | 70 +++++++++++++++----- 2 files changed, 97 insertions(+), 15 deletions(-) diff --git a/src/include/storage/ducklake_transaction.hpp b/src/include/storage/ducklake_transaction.hpp index b10de60eee9..cd88ddce0c4 100644 --- a/src/include/storage/ducklake_transaction.hpp +++ b/src/include/storage/ducklake_transaction.hpp @@ -38,6 +38,7 @@ struct CompactionInformation; struct DuckLakePath; struct DuckLakeCommitState; class DuckLakeFieldId; +class LocalTableChangeIterationHelper; struct LocalTableDataChanges { vector new_data_files; @@ -53,12 +54,53 @@ struct LocalTableChanges { public: void Clear(); bool HasChanges() const; + LocalTableChangeIterationHelper Changes() const; // private: mutable mutex lock; map changes; }; +class LocalTableChangeIterationHelper { +public: + LocalTableChangeIterationHelper(mutex &local_changes_lock, const map &changes); +private: + unique_lock lock; + const map &changes; + +private: + struct LocalTableChangeIteratorEntry { + friend class LocalTableChangeIterationHelper; + public: + LocalTableChangeIteratorEntry(); + TableIndex GetTableIndex() const; + const LocalTableDataChanges &GetTableChanges() const; + + private: + TableIndex table_id; + optional_ptr changes; + }; + class LocalTableChangeIterator { + public: + explicit LocalTableChangeIterator(map::const_iterator it, map::const_iterator end_it); + map::const_iterator it; + map::const_iterator end_it; + LocalTableChangeIteratorEntry entry; + + public: + LocalTableChangeIterator &operator++(); + bool operator!=(const LocalTableChangeIterator &other) const; + const LocalTableChangeIteratorEntry &operator*() const; + }; +public: + LocalTableChangeIterator begin() { // NOLINT: match stl API + return LocalTableChangeIterator(changes.begin(), changes.end()); + } + LocalTableChangeIterator end() { // NOLINT: match stl API + return LocalTableChangeIterator(changes.end(), changes.end()); + } +}; + struct SnapshotAndStats { vector stats; DuckLakeSnapshot snapshot; diff --git a/src/storage/ducklake_transaction.cpp b/src/storage/ducklake_transaction.cpp index aee33b4275b..2ac32f301d8 100644 --- a/src/storage/ducklake_transaction.cpp +++ b/src/storage/ducklake_transaction.cpp @@ -57,6 +57,47 @@ bool LocalTableChanges::HasChanges() const{ return !changes.empty(); } +LocalTableChangeIterationHelper::LocalTableChangeIterationHelper(mutex &local_changes_lock, const map &changes_p) : lock(local_changes_lock), changes(changes_p) {} + + +LocalTableChangeIterationHelper::LocalTableChangeIteratorEntry::LocalTableChangeIteratorEntry() {} + +TableIndex LocalTableChangeIterationHelper::LocalTableChangeIteratorEntry::GetTableIndex() const { + return table_id; +} + +const LocalTableDataChanges &LocalTableChangeIterationHelper::LocalTableChangeIteratorEntry::GetTableChanges() const { + return *changes; +} + +LocalTableChangeIterationHelper::LocalTableChangeIterator::LocalTableChangeIterator(map::const_iterator it_p, map::const_iterator end_it_p) : it(std::move(it_p)), end_it(std::move(end_it_p)) { + if (it != end_it) { + entry.table_id = it->first; + entry.changes = it->second; + } +} + +LocalTableChangeIterationHelper::LocalTableChangeIterator &LocalTableChangeIterationHelper::LocalTableChangeIterator::operator++() { + it++; + if (it != end_it) { + entry.table_id = it->first; + entry.changes = it->second; + } + return *this; +} + +bool LocalTableChangeIterationHelper::LocalTableChangeIterator::operator!=(const LocalTableChangeIterator &other) const { + return it != other.it; +} + +const LocalTableChangeIterationHelper::LocalTableChangeIteratorEntry &LocalTableChangeIterationHelper::LocalTableChangeIterator::operator*() const { + return entry; +} + +LocalTableChangeIterationHelper LocalTableChanges::Changes() const { + return LocalTableChangeIterationHelper(lock, changes); +} + DuckLakeTransaction::DuckLakeTransaction(DuckLakeCatalog &ducklake_catalog, TransactionManager &manager, ClientContext &context) : Transaction(manager, context), ducklake_catalog(ducklake_catalog), db(*context.db), @@ -288,13 +329,13 @@ TransactionChangeInformation DuckLakeTransaction::GetTransactionChanges() const } } changes.tables_deleted_from = tables_deleted_from; - for (auto &entry : local_changes.changes) { - auto table_id = entry.first; + for (auto &entry : local_changes.Changes()) { + auto table_id = entry.GetTableIndex(); if (table_id.IsTransactionLocal()) { // don't report transaction-local tables yet - these will get added later on continue; } - auto &table_changes = entry.second; + auto &table_changes = entry.GetTableChanges(); AddTableChanges(table_id, table_changes, changes); } return changes; @@ -403,9 +444,9 @@ string DuckLakeTransaction::WriteSnapshotChanges(DuckLakeCommitState &commit_sta // re-add all inserted tables - transaction-local table identifiers should have been converted at this stage changes.tables_deleted_from = tables_deleted_from; - for (auto &entry : local_changes.changes) { - auto table_id = commit_state.GetTableId(entry.first); - auto &table_changes = entry.second; + for (auto &entry : local_changes.Changes()) { + auto table_id = commit_state.GetTableId(entry.GetTableIndex()); + auto &table_changes = entry.GetTableChanges(); AddTableChanges(table_id, table_changes, changes); } for (auto &entry : changes.dropped_schemas) { @@ -666,8 +707,8 @@ void DuckLakeTransaction::CheckForConflicts(const TransactionChangeInformation & if (check_for_matches) { // If we have deletes on the tables, check for files being deleted const auto deleted_files = metadata_manager->GetFilesDeletedOrDroppedAfterSnapshot(transaction_snapshot); - for (auto &entry : local_changes.changes) { - auto &table_changes = entry.second; + for (auto &entry : local_changes.Changes()) { + auto &table_changes = entry.GetTableChanges(); for (auto &file_entry : table_changes.new_delete_files) { for (auto &file : file_entry.second) { ConflictCheck(file.data_file_id, deleted_files.deleted_from_files, "delete from file", @@ -1460,9 +1501,9 @@ vector DuckLakeTransaction::GetNewDeleteFiles(const DuckLakeCommitState &commit_state, vector &overwritten_delete_files) const { vector result; - for (auto &entry : local_changes.changes) { - auto table_id = commit_state.GetTableId(entry.first); - auto &table_changes = entry.second; + for (auto &entry : local_changes.Changes()) { + auto table_id = commit_state.GetTableId(entry.GetTableIndex()); + auto &table_changes = entry.GetTableChanges(); for (auto &file_entry : table_changes.new_delete_files) { for (auto &file : file_entry.second) { if (file.overwritten_delete_file.delete_file_id.IsValid()) { @@ -2373,10 +2414,9 @@ void DuckLakeTransaction::AddNewInlinedFileDeletes(TableIndex table_id, idx_t fi vector DuckLakeTransaction::GetNewInlinedFileDeletes(DuckLakeCommitState &commit_state) { vector result; - lock_guard guard(local_changes.lock); - for (auto &entry : local_changes.changes) { - auto table_id = commit_state.GetTableId(entry.first); - auto &table_changes = entry.second; + for (auto &entry : local_changes.Changes()) { + auto table_id = commit_state.GetTableId(entry.GetTableIndex()); + auto &table_changes = entry.GetTableChanges(); if (!table_changes.new_inlined_file_deletes) { continue; } From 723836c39fed3fd7ea95ba78f8f7e05125f906f5 Mon Sep 17 00:00:00 2001 From: Mytherin Date: Tue, 17 Mar 2026 09:47:31 +0100 Subject: [PATCH 13/59] Move cleanup files --- src/include/storage/ducklake_transaction.hpp | 1 + src/storage/ducklake_transaction.cpp | 45 +++++++++++--------- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/src/include/storage/ducklake_transaction.hpp b/src/include/storage/ducklake_transaction.hpp index cd88ddce0c4..f577584fd11 100644 --- a/src/include/storage/ducklake_transaction.hpp +++ b/src/include/storage/ducklake_transaction.hpp @@ -55,6 +55,7 @@ struct LocalTableChanges { void Clear(); bool HasChanges() const; LocalTableChangeIterationHelper Changes() const; + void CleanupFiles(DatabaseInstance &db); // private: mutable mutex lock; diff --git a/src/storage/ducklake_transaction.cpp b/src/storage/ducklake_transaction.cpp index 2ac32f301d8..506f4e6ffce 100644 --- a/src/storage/ducklake_transaction.cpp +++ b/src/storage/ducklake_transaction.cpp @@ -57,6 +57,29 @@ bool LocalTableChanges::HasChanges() const{ return !changes.empty(); } +void LocalTableChanges::CleanupFiles(DatabaseInstance &db) { + auto &fs = FileSystem::GetFileSystem(db); + lock_guard guard(lock); + for (auto &entry : changes) { + auto &table_changes = entry.second; + for (auto &file : table_changes.new_data_files) { + if (file.created_by_ducklake) { + fs.TryRemoveFile(file.file_name); + } + for (auto &del_file : file.delete_files) { + fs.TryRemoveFile(del_file.file_name); + } + } + for (auto &file : table_changes.new_delete_files) { + for (auto &delete_files : file.second) { + fs.TryRemoveFile(delete_files.file_name); + } + } + table_changes.new_data_files.clear(); + table_changes.new_delete_files.clear(); + } +} + LocalTableChangeIterationHelper::LocalTableChangeIterationHelper(mutex &local_changes_lock, const map &changes_p) : lock(local_changes_lock), changes(changes_p) {} @@ -536,27 +559,7 @@ string DuckLakeTransaction::WriteSnapshotChanges(DuckLakeCommitState &commit_sta void DuckLakeTransaction::CleanupFiles() { // remove any files that were written - auto context_ref = context.lock(); - auto &fs = FileSystem::GetFileSystem(db); - lock_guard guard(local_changes.lock); - for (auto &entry : local_changes.changes) { - auto &table_changes = entry.second; - for (auto &file : table_changes.new_data_files) { - if (file.created_by_ducklake) { - fs.TryRemoveFile(file.file_name); - } - for (auto &del_file : file.delete_files) { - fs.TryRemoveFile(del_file.file_name); - } - } - for (auto &file : table_changes.new_delete_files) { - for (auto &delete_files : file.second) { - fs.TryRemoveFile(delete_files.file_name); - } - } - table_changes.new_data_files.clear(); - table_changes.new_delete_files.clear(); - } + local_changes.CleanupFiles(db); } template From 84ad69842049104f3c65bd6c54561735a7262e3d Mon Sep 17 00:00:00 2001 From: Mytherin Date: Tue, 17 Mar 2026 09:59:29 +0100 Subject: [PATCH 14/59] Move methods to LocalTableChanges --- src/include/storage/ducklake_transaction.hpp | 11 ++ src/storage/ducklake_transaction.cpp | 162 ++++++++++++------- 2 files changed, 114 insertions(+), 59 deletions(-) diff --git a/src/include/storage/ducklake_transaction.hpp b/src/include/storage/ducklake_transaction.hpp index f577584fd11..a9b7de86071 100644 --- a/src/include/storage/ducklake_transaction.hpp +++ b/src/include/storage/ducklake_transaction.hpp @@ -56,6 +56,17 @@ struct LocalTableChanges { bool HasChanges() const; LocalTableChangeIterationHelper Changes() const; void CleanupFiles(DatabaseInstance &db); + bool HasTransactionLocalInserts(TableIndex table_id) const; + bool HasTransactionInlinedData(TableIndex table_id) const; + vector GetTransactionLocalFiles(TableIndex table_id) const; + shared_ptr GetTransactionLocalInlinedData(ClientContext &context, TableIndex table_id) const; + void DropTransactionLocalFile(ClientContext &context, TableIndex table_id, const string &path); + void AppendFiles(TableIndex table_id, vector files); + void AppendInlinedData(ClientContext &context, TableIndex table_id, unique_ptr new_data); + void AddNewInlinedDeletes(TableIndex table_id, const string &table_name, set new_deletes); + void DeleteFromLocalInlinedData(ClientContext &context, TableIndex table_id, set new_deletes); + void AddColumnToLocalInlinedData(ClientContext &context, TableIndex table_id, const LogicalType &new_column_type, + FieldIndex new_field_index, const Value &default_value); // private: mutable mutex lock; diff --git a/src/storage/ducklake_transaction.cpp b/src/storage/ducklake_transaction.cpp index 506f4e6ffce..65f6690ad87 100644 --- a/src/storage/ducklake_transaction.cpp +++ b/src/storage/ducklake_transaction.cpp @@ -2053,35 +2053,38 @@ idx_t DuckLakeTransaction::GetLocalCatalogId() { return local_catalog_id++; } -bool DuckLakeTransaction::HasTransactionLocalInserts(TableIndex table_id) const { - auto entry = local_changes.changes.find(table_id); - if (entry == local_changes.changes.end()) { +bool LocalTableChanges::HasTransactionLocalInserts(TableIndex table_id) const { + lock_guard guard(lock); + auto entry = changes.find(table_id); + if (entry == changes.end()) { return false; } auto &table_changes = entry->second; return !table_changes.new_data_files.empty() || table_changes.new_inlined_data; } -bool DuckLakeTransaction::HasTransactionInlinedData(TableIndex table_id) const { - auto entry = local_changes.changes.find(table_id); - if (entry == local_changes.changes.end()) { +bool LocalTableChanges::HasTransactionInlinedData(TableIndex table_id) const { + lock_guard guard(lock); + auto entry = changes.find(table_id); + if (entry == changes.end()) { return false; } auto &table_changes = entry->second; return table_changes.new_inlined_data != nullptr; } -vector DuckLakeTransaction::GetTransactionLocalFiles(TableIndex table_id) const { - auto entry = local_changes.changes.find(table_id); - if (entry == local_changes.changes.end()) { +vector LocalTableChanges::GetTransactionLocalFiles(TableIndex table_id) const { + lock_guard guard(lock); + auto entry = changes.find(table_id); + if (entry == changes.end()) { return vector(); } return entry->second.new_data_files; } -shared_ptr DuckLakeTransaction::GetTransactionLocalInlinedData(TableIndex table_id) const { - auto entry = local_changes.changes.find(table_id); - if (entry == local_changes.changes.end()) { +shared_ptr LocalTableChanges::GetTransactionLocalInlinedData(ClientContext &context, TableIndex table_id) const { + auto entry = changes.find(table_id); + if (entry == changes.end()) { return nullptr; } auto &table_changes = entry->second; @@ -2089,26 +2092,24 @@ shared_ptr DuckLakeTransaction::GetTransactionLocalInlinedD return nullptr; } auto &local_changes = *table_changes.new_inlined_data; - auto context_ref = context.lock(); auto result = make_shared_ptr(); - result->data = make_uniq(*context_ref, local_changes.data->Types()); + result->data = make_uniq(context, local_changes.data->Types()); for (auto &chunk : local_changes.data->Chunks()) { result->data->Append(chunk); } return result; } -void DuckLakeTransaction::DropTransactionLocalFile(TableIndex table_id, const string &path) { - lock_guard guard(local_changes.lock); - auto entry = local_changes.changes.find(table_id); - if (entry == local_changes.changes.end()) { +void LocalTableChanges::DropTransactionLocalFile(ClientContext &context, TableIndex table_id, const string &path) { + lock_guard guard(lock); + auto entry = changes.find(table_id); + if (entry == changes.end()) { throw InternalException( - "DropTransactionLocalFile called for a table for which no transaction-local files exist"); + "DropTransactionLocalFile called for a table for which no transaction-local files exist"); } auto &table_changes = entry->second; auto &table_files = table_changes.new_data_files; - auto context_ref = context.lock(); - auto &fs = FileSystem::GetFileSystem(*context_ref); + auto &fs = FileSystem::GetFileSystem(context); for (idx_t i = 0; i < table_files.size(); i++) { auto &file = table_files[i]; if (file.file_name == path) { @@ -2121,7 +2122,7 @@ void DuckLakeTransaction::DropTransactionLocalFile(TableIndex table_id, const st fs.RemoveFile(path); if (table_changes.IsEmpty()) { // no more files remaining - local_changes.changes.erase(entry); + changes.erase(entry); } return; } @@ -2129,12 +2130,9 @@ void DuckLakeTransaction::DropTransactionLocalFile(TableIndex table_id, const st throw InternalException("Failed to find matching transaction-local file for DropTransactionLocalFile"); } -void DuckLakeTransaction::AppendFiles(TableIndex table_id, vector files) { - if (files.empty()) { - return; - } - lock_guard guard(local_changes.lock); - auto &table_changes = local_changes.changes[table_id]; +void LocalTableChanges::AppendFiles(TableIndex table_id, vector files) { + lock_guard guard(lock); + auto &table_changes = changes[table_id]; if (table_changes.new_data_files.empty()) { // If empty, just move the entire vector table_changes.new_data_files = std::move(files); @@ -2143,15 +2141,15 @@ void DuckLakeTransaction::AppendFiles(TableIndex table_id, vector new_data) { - lock_guard guard(local_changes.lock); - auto &table_changes = local_changes.changes[table_id]; +void LocalTableChanges::AppendInlinedData(ClientContext &context, TableIndex table_id, unique_ptr new_data) { + lock_guard guard(lock); + auto &table_changes = changes[table_id]; if (table_changes.new_inlined_data) { // already exists - append auto &existing_data = *table_changes.new_inlined_data; @@ -2160,17 +2158,16 @@ void DuckLakeTransaction::AppendInlinedData(TableIndex table_id, unique_ptr(*context_ref, new_types); + auto casted_data = make_uniq(context, new_types); ColumnDataAppendState append_state; casted_data->InitializeAppend(append_state); for (auto &chunk : existing_data.data->Chunks()) { DataChunk casted_chunk; - casted_chunk.Initialize(*context_ref, new_types); + casted_chunk.Initialize(context, new_types); for (idx_t col_idx = 0; col_idx < chunk.ColumnCount(); col_idx++) { if (existing_types[col_idx] != new_types[col_idx]) { - VectorOperations::Cast(*context_ref, chunk.data[col_idx], casted_chunk.data[col_idx], - chunk.size()); + VectorOperations::Cast(context, chunk.data[col_idx], casted_chunk.data[col_idx], + chunk.size()); } else { casted_chunk.data[col_idx].Reference(chunk.data[col_idx]); } @@ -2198,12 +2195,9 @@ void DuckLakeTransaction::AppendInlinedData(TableIndex table_id, unique_ptr new_deletes) { - if (new_deletes.empty()) { - return; - } - lock_guard guard(local_changes.lock); - auto &table_changes = local_changes.changes[table_id]; +void LocalTableChanges::AddNewInlinedDeletes(TableIndex table_id, const string &table_name, set new_deletes) { + lock_guard guard(lock); + auto &table_changes = changes[table_id]; auto &table_deletes = table_changes.new_inlined_data_deletes; auto entry = table_deletes.find(table_name); if (entry != table_deletes.end()) { @@ -2219,17 +2213,16 @@ void DuckLakeTransaction::AddNewInlinedDeletes(TableIndex table_id, const string } } -void DuckLakeTransaction::DeleteFromLocalInlinedData(TableIndex table_id, set new_deletes) { - lock_guard guard(local_changes.lock); - auto entry = local_changes.changes.find(table_id); - if (entry == local_changes.changes.end()) { +void LocalTableChanges::DeleteFromLocalInlinedData(ClientContext &context, TableIndex table_id, set new_deletes) { + lock_guard guard(lock); + auto entry = changes.find(table_id); + if (entry == changes.end()) { throw InternalException("DeleteFromLocalInlinedData called but no transaction-local data exists for table"); } auto &table_changes = entry->second; auto &existing = *table_changes.new_inlined_data->data; // construct a new collection from the existing data minus the deletes - auto context_ref = context.lock(); - auto new_data = make_uniq(*context_ref, existing.Types()); + auto new_data = make_uniq(context, existing.Types()); idx_t base_row_id = 0; ColumnDataAppendState append_state; @@ -2259,11 +2252,11 @@ void DuckLakeTransaction::DeleteFromLocalInlinedData(TableIndex table_id, setdata = std::move(new_data); } -void DuckLakeTransaction::AddColumnToLocalInlinedData(TableIndex table_id, const LogicalType &new_column_type, - FieldIndex new_field_index, const Value &default_value) { - lock_guard guard(local_changes.lock); - auto entry = local_changes.changes.find(table_id); - if (entry == local_changes.changes.end()) { +void LocalTableChanges::AddColumnToLocalInlinedData(ClientContext &context, TableIndex table_id, const LogicalType &new_column_type, + FieldIndex new_field_index, const Value &default_value) { + lock_guard guard(lock); + auto entry = changes.find(table_id); + if (entry == changes.end()) { throw InternalException("AddColumnToLocalInlinedData called but no transaction-local data exists"); } auto &table_changes = entry->second; @@ -2277,8 +2270,7 @@ void DuckLakeTransaction::AddColumnToLocalInlinedData(TableIndex table_id, const auto new_types = existing.Types(); new_types.push_back(new_column_type); - auto context_ref = context.lock(); - auto new_data = make_uniq(*context_ref, new_types); + auto new_data = make_uniq(context, new_types); ColumnDataAppendState append_state; new_data->InitializeAppend(append_state); @@ -2287,7 +2279,7 @@ void DuckLakeTransaction::AddColumnToLocalInlinedData(TableIndex table_id, const for (auto &chunk : existing.Chunks()) { DataChunk new_chunk; - new_chunk.Initialize(*context_ref, new_types); + new_chunk.Initialize(context, new_types); // Copy existing columns for (idx_t col_idx = 0; col_idx < chunk.ColumnCount(); col_idx++) { @@ -2326,6 +2318,58 @@ void DuckLakeTransaction::AddColumnToLocalInlinedData(TableIndex table_id, const table_changes.new_inlined_data->data = std::move(new_data); } +bool DuckLakeTransaction::HasTransactionLocalInserts(TableIndex table_id) const { + return local_changes.HasTransactionLocalInserts(table_id); +} + +bool DuckLakeTransaction::HasTransactionInlinedData(TableIndex table_id) const { + return local_changes.HasTransactionInlinedData(table_id); +} + +vector DuckLakeTransaction::GetTransactionLocalFiles(TableIndex table_id) const { + return local_changes.GetTransactionLocalFiles(table_id); +} + +shared_ptr DuckLakeTransaction::GetTransactionLocalInlinedData(TableIndex table_id) const { + auto context_ref = context.lock(); + return local_changes.GetTransactionLocalInlinedData(*context_ref, table_id); +} + +void DuckLakeTransaction::DropTransactionLocalFile(TableIndex table_id, const string &path) { + auto context_ref = context.lock(); + local_changes.DropTransactionLocalFile(*context_ref, table_id, path); +} + +void DuckLakeTransaction::AppendFiles(TableIndex table_id, vector files) { + if (files.empty()) { + return; + } + local_changes.AppendFiles(table_id, std::move(files)); +} + +void DuckLakeTransaction::AppendInlinedData(TableIndex table_id, unique_ptr new_data) { + auto context_ref = context.lock(); + local_changes.AppendInlinedData(*context_ref, table_id, std::move(new_data)); +} + +void DuckLakeTransaction::AddNewInlinedDeletes(TableIndex table_id, const string &table_name, set new_deletes) { + if (new_deletes.empty()) { + return; + } + local_changes.AddNewInlinedDeletes(table_id, table_name, std::move(new_deletes)); +} + +void DuckLakeTransaction::DeleteFromLocalInlinedData(TableIndex table_id, set new_deletes) { + auto context_ref = context.lock(); + local_changes.DeleteFromLocalInlinedData(*context_ref, table_id, std::move(new_deletes)); +} + +void DuckLakeTransaction::AddColumnToLocalInlinedData(TableIndex table_id, const LogicalType &new_column_type, + FieldIndex new_field_index, const Value &default_value) { + auto context_ref = context.lock(); + local_changes.AddColumnToLocalInlinedData(*context_ref, table_id, new_column_type, new_field_index, default_value); +} + static void RemoveFieldStats(map &column_stats, const DuckLakeFieldId &field_id) { column_stats.erase(field_id.GetFieldIndex()); for (auto &child_id : field_id.Children()) { From 39d27ac986f7d9bc3880bbce99d14525e7bf5c8a Mon Sep 17 00:00:00 2001 From: Mytherin Date: Tue, 17 Mar 2026 10:17:44 +0100 Subject: [PATCH 15/59] Move more code into LocalTableChanges --- src/functions/ducklake_add_data_files.cpp | 8 +- src/include/storage/ducklake_transaction.hpp | 30 +- src/storage/ducklake_metadata_manager.cpp | 41 +- src/storage/ducklake_multi_file_reader.cpp | 6 +- src/storage/ducklake_transaction.cpp | 1494 +++++++++--------- 5 files changed, 836 insertions(+), 743 deletions(-) diff --git a/src/functions/ducklake_add_data_files.cpp b/src/functions/ducklake_add_data_files.cpp index 89c911bfd53..809a198c6c1 100644 --- a/src/functions/ducklake_add_data_files.cpp +++ b/src/functions/ducklake_add_data_files.cpp @@ -130,9 +130,8 @@ struct DuckLakeFileProcessor { public: DuckLakeFileProcessor(DuckLakeTransaction &transaction, ClientContext &context, const DuckLakeAddDataFilesData &bind_data) - : transaction(transaction), context(context), table(bind_data.table), - allow_missing(bind_data.allow_missing), ignore_extra_columns(bind_data.ignore_extra_columns), - hive_partitioning(bind_data.hive_partitioning) { + : transaction(transaction), context(context), table(bind_data.table), allow_missing(bind_data.allow_missing), + ignore_extra_columns(bind_data.ignore_extra_columns), hive_partitioning(bind_data.hive_partitioning) { } vector AddFiles(const vector &globs); @@ -1167,7 +1166,8 @@ void DuckLakeFileProcessor::MapPartitionColumns(ParquetFileMetadata &file) { // For YEAR/MONTH/DAY/HOUR transforms, the type is BIGINT, not the source column type auto partition_key_type = DuckLakePartitionUtils::GetPartitionKeyType(partition_field.transform.type, field_id->Type()); - auto hive_value = HivePartitioning::GetValue(context, partition_key_name, hive_entry->second, partition_key_type); + auto hive_value = + HivePartitioning::GetValue(context, partition_key_name, hive_entry->second, partition_key_type); file.hive_partition_values.emplace_back( HivePartition {partition_field.field_id, partition_key_type, hive_value}); } diff --git a/src/include/storage/ducklake_transaction.hpp b/src/include/storage/ducklake_transaction.hpp index a9b7de86071..20eb4e53bb6 100644 --- a/src/include/storage/ducklake_transaction.hpp +++ b/src/include/storage/ducklake_transaction.hpp @@ -56,6 +56,7 @@ struct LocalTableChanges { bool HasChanges() const; LocalTableChangeIterationHelper Changes() const; void CleanupFiles(DatabaseInstance &db); + void CleanupFiles(ClientContext &context, TableIndex table_id); bool HasTransactionLocalInserts(TableIndex table_id) const; bool HasTransactionInlinedData(TableIndex table_id) const; vector GetTransactionLocalFiles(TableIndex table_id) const; @@ -66,9 +67,24 @@ struct LocalTableChanges { void AddNewInlinedDeletes(TableIndex table_id, const string &table_name, set new_deletes); void DeleteFromLocalInlinedData(ClientContext &context, TableIndex table_id, set new_deletes); void AddColumnToLocalInlinedData(ClientContext &context, TableIndex table_id, const LogicalType &new_column_type, - FieldIndex new_field_index, const Value &default_value); + FieldIndex new_field_index, const Value &default_value); + void RemoveColumnFromLocalInlinedData(ClientContext &context, TableIndex table_id, + LogicalIndex removed_column_index, const DuckLakeFieldId &field_id); + optional_ptr GetInlinedDeletes(TableIndex table_id, const string &table_name) const; + void AddNewInlinedFileDeletes(TableIndex table_id, idx_t file_id, set new_deletes); + void AddCompaction(TableIndex table_id, DuckLakeCompactionEntry entry); + bool HasLocalDeletes(TableIndex table_id) const; + bool HasAnyLocalChanges(TableIndex table_id) const; + + void GetLocalDeleteForFile(TableIndex table_id, const string &path, DuckLakeFileData &result) const; + bool HasLocalInlinedFileDeletes(TableIndex table_id) const; -// private: + void GetLocalInlinedFileDeletesForFile(TableIndex table_id, idx_t file_id, set &result) const; + + void TransactionLocalDelete(ClientContext &context, TableIndex table_id, const string &data_file_path, + DuckLakeDeleteFile delete_file); + + // private: mutable mutex lock; map changes; }; @@ -76,6 +92,7 @@ struct LocalTableChanges { class LocalTableChangeIterationHelper { public: LocalTableChangeIterationHelper(mutex &local_changes_lock, const map &changes); + private: unique_lock lock; const map &changes; @@ -83,6 +100,7 @@ class LocalTableChangeIterationHelper { private: struct LocalTableChangeIteratorEntry { friend class LocalTableChangeIterationHelper; + public: LocalTableChangeIteratorEntry(); TableIndex GetTableIndex() const; @@ -94,7 +112,8 @@ class LocalTableChangeIterationHelper { }; class LocalTableChangeIterator { public: - explicit LocalTableChangeIterator(map::const_iterator it, map::const_iterator end_it); + explicit LocalTableChangeIterator(map::const_iterator it, + map::const_iterator end_it); map::const_iterator it; map::const_iterator end_it; LocalTableChangeIteratorEntry entry; @@ -104,6 +123,7 @@ class LocalTableChangeIterationHelper { bool operator!=(const LocalTableChangeIterator &other) const; const LocalTableChangeIteratorEntry &operator*() const; }; + public: LocalTableChangeIterator begin() { // NOLINT: match stl API return LocalTableChangeIterator(changes.begin(), changes.end()); @@ -268,8 +288,8 @@ class DuckLakeTransaction : public Transaction, public enable_shared_from_this> stats); vector diff --git a/src/storage/ducklake_metadata_manager.cpp b/src/storage/ducklake_metadata_manager.cpp index 22cbc44d0bb..5b32b45e85b 100644 --- a/src/storage/ducklake_metadata_manager.cpp +++ b/src/storage/ducklake_metadata_manager.cpp @@ -2818,7 +2818,6 @@ string DuckLakeMetadataManager::FromRelativePath(TableIndex table_id, const Duck return FromRelativePath(path, GetPath(table_id, {}, {})); } - // Optimized version using DuckDB Appender API for much faster inserts string DuckLakeMetadataManager::WriteNewDataFilesWithAppender(DuckLakeSnapshot &commit_snapshot, const vector &new_files, @@ -2842,8 +2841,8 @@ string DuckLakeMetadataManager::WriteNewDataFilesWithAppender(DuckLakeSnapshot & auto data_file_index = static_cast(file.id.index); auto table_id = static_cast(file.table_id.index); int64_t begin_snapshot_val = file.begin_snapshot.IsValid() - ? static_cast(file.begin_snapshot.GetIndex()) - : static_cast(commit_snapshot.snapshot_id); + ? static_cast(file.begin_snapshot.GetIndex()) + : static_cast(commit_snapshot.snapshot_id); auto path = GetRelativePath(file.table_id, file.file_name, new_tables, new_schemas_result); // ducklake_data_file columns: @@ -2851,15 +2850,15 @@ string DuckLakeMetadataManager::WriteNewDataFilesWithAppender(DuckLakeSnapshot & // file_format, record_count, file_size_bytes, footer_size, row_id_start, partition_id, // encryption_key, mapping_id, partial_max data_file_appender.BeginRow(); - data_file_appender.Append(data_file_index); // data_file_id - data_file_appender.Append(table_id); // table_id - data_file_appender.Append(begin_snapshot_val); // begin_snapshot - data_file_appender.Append(Value()); // end_snapshot (NULL) - data_file_appender.Append(Value()); // file_order (NULL) - data_file_appender.Append(string_t(path.path)); // path - data_file_appender.Append(path.path_is_relative); // path_is_relative - data_file_appender.Append(string_t("parquet")); // file_format - data_file_appender.Append(static_cast(file.row_count)); // record_count + data_file_appender.Append(data_file_index); // data_file_id + data_file_appender.Append(table_id); // table_id + data_file_appender.Append(begin_snapshot_val); // begin_snapshot + data_file_appender.Append(Value()); // end_snapshot (NULL) + data_file_appender.Append(Value()); // file_order (NULL) + data_file_appender.Append(string_t(path.path)); // path + data_file_appender.Append(path.path_is_relative); // path_is_relative + data_file_appender.Append(string_t("parquet")); // file_format + data_file_appender.Append(static_cast(file.row_count)); // record_count data_file_appender.Append(static_cast(file.file_size_bytes)); // file_size_bytes if (file.footer_size.IsValid()) { data_file_appender.Append(static_cast(file.footer_size.GetIndex())); // footer_size @@ -2877,7 +2876,8 @@ string DuckLakeMetadataManager::WriteNewDataFilesWithAppender(DuckLakeSnapshot & data_file_appender.Append(Value()); } if (!file.encryption_key.empty()) { - data_file_appender.Append(string_t(Blob::ToBase64(string_t(file.encryption_key)))); // encryption_key + data_file_appender.Append( + string_t(Blob::ToBase64(string_t(file.encryption_key)))); // encryption_key } else { data_file_appender.Append(Value()); } @@ -2887,7 +2887,8 @@ string DuckLakeMetadataManager::WriteNewDataFilesWithAppender(DuckLakeSnapshot & data_file_appender.Append(Value()); } if (file.max_partial_file_snapshot.IsValid()) { - data_file_appender.Append(static_cast(file.max_partial_file_snapshot.GetIndex())); // partial_max + data_file_appender.Append( + static_cast(file.max_partial_file_snapshot.GetIndex())); // partial_max } else { data_file_appender.Append(Value()); } @@ -2962,11 +2963,14 @@ string DuckLakeMetadataManager::WriteNewDataFilesWithAppender(DuckLakeSnapshot & variant_stats_appender.Append(table_id); variant_stats_appender.Append(column_id); variant_stats_appender.Append(string_t(variant_entry.first)); - variant_stats_appender.Append(string_t(DuckLakeTypes::ToString(variant_entry.second.shredded_type))); + variant_stats_appender.Append( + string_t(DuckLakeTypes::ToString(variant_entry.second.shredded_type))); variant_stats_appender.Append(static_cast(field_stats.column_size_bytes)); - if (field_stats.has_null_count && field_stats.has_num_values && field_stats.null_count <= field_stats.num_values) { - variant_stats_appender.Append(static_cast(field_stats.num_values - field_stats.null_count)); + if (field_stats.has_null_count && field_stats.has_num_values && + field_stats.null_count <= field_stats.num_values) { + variant_stats_appender.Append( + static_cast(field_stats.num_values - field_stats.null_count)); variant_stats_appender.Append(static_cast(field_stats.null_count)); } else { variant_stats_appender.Append(Value()); @@ -2993,7 +2997,8 @@ string DuckLakeMetadataManager::WriteNewDataFilesWithAppender(DuckLakeSnapshot & string field_extra_stats_str; if (field_stats.extra_stats && field_stats.extra_stats->TrySerialize(field_extra_stats_str)) { // TrySerialize wraps the JSON in single quotes for SQL - strip them for Appender - if (field_extra_stats_str.size() >= 2 && field_extra_stats_str.front() == '\'' && field_extra_stats_str.back() == '\'') { + if (field_extra_stats_str.size() >= 2 && field_extra_stats_str.front() == '\'' && + field_extra_stats_str.back() == '\'') { field_extra_stats_str = field_extra_stats_str.substr(1, field_extra_stats_str.size() - 2); } variant_stats_appender.Append(string_t(field_extra_stats_str)); diff --git a/src/storage/ducklake_multi_file_reader.cpp b/src/storage/ducklake_multi_file_reader.cpp index 245c7f7b0ad..21627dd769a 100644 --- a/src/storage/ducklake_multi_file_reader.cpp +++ b/src/storage/ducklake_multi_file_reader.cpp @@ -443,7 +443,8 @@ vector MapColumns(ClientContext &context, MultiFileRe column_map->source_name, reader_data.reader->file.path); } // Use GetValue to handle NULL values (__HIVE_DEFAULT_PARTITION__) and type casting - Value partition_val = HivePartitioning::GetValue(context, column_map->source_name, entry->second, result_col.type); + Value partition_val = + HivePartitioning::GetValue(context, column_map->source_name, entry->second, result_col.type); result_col.default_expression = make_uniq(std::move(partition_val)); continue; } @@ -457,7 +458,8 @@ vector MapColumns(ClientContext &context, MultiFileRe // recursively process any child nodes if (!column_map->child_entries.empty()) { bool is_list = result_col.type.id() == LogicalTypeId::LIST; - result_col.children = MapColumns(context, reader_data, result_col.children, column_map->child_entries, is_list); + result_col.children = + MapColumns(context, reader_data, result_col.children, column_map->child_entries, is_list); } } return result; diff --git a/src/storage/ducklake_transaction.cpp b/src/storage/ducklake_transaction.cpp index 65f6690ad87..5c26ab1afd7 100644 --- a/src/storage/ducklake_transaction.cpp +++ b/src/storage/ducklake_transaction.cpp @@ -52,7 +52,7 @@ void LocalTableChanges::Clear() { changes.clear(); } -bool LocalTableChanges::HasChanges() const{ +bool LocalTableChanges::HasChanges() const { lock_guard guard(lock); return !changes.empty(); } @@ -80,115 +80,597 @@ void LocalTableChanges::CleanupFiles(DatabaseInstance &db) { } } -LocalTableChangeIterationHelper::LocalTableChangeIterationHelper(mutex &local_changes_lock, const map &changes_p) : lock(local_changes_lock), changes(changes_p) {} - - -LocalTableChangeIterationHelper::LocalTableChangeIteratorEntry::LocalTableChangeIteratorEntry() {} - -TableIndex LocalTableChangeIterationHelper::LocalTableChangeIteratorEntry::GetTableIndex() const { - return table_id; +bool LocalTableChanges::HasTransactionLocalInserts(TableIndex table_id) const { + lock_guard guard(lock); + auto entry = changes.find(table_id); + if (entry == changes.end()) { + return false; + } + auto &table_changes = entry->second; + return !table_changes.new_data_files.empty() || table_changes.new_inlined_data; } -const LocalTableDataChanges &LocalTableChangeIterationHelper::LocalTableChangeIteratorEntry::GetTableChanges() const { - return *changes; +bool LocalTableChanges::HasTransactionInlinedData(TableIndex table_id) const { + lock_guard guard(lock); + auto entry = changes.find(table_id); + if (entry == changes.end()) { + return false; + } + auto &table_changes = entry->second; + return table_changes.new_inlined_data != nullptr; } -LocalTableChangeIterationHelper::LocalTableChangeIterator::LocalTableChangeIterator(map::const_iterator it_p, map::const_iterator end_it_p) : it(std::move(it_p)), end_it(std::move(end_it_p)) { - if (it != end_it) { - entry.table_id = it->first; - entry.changes = it->second; +vector LocalTableChanges::GetTransactionLocalFiles(TableIndex table_id) const { + lock_guard guard(lock); + auto entry = changes.find(table_id); + if (entry == changes.end()) { + return vector(); } + return entry->second.new_data_files; } -LocalTableChangeIterationHelper::LocalTableChangeIterator &LocalTableChangeIterationHelper::LocalTableChangeIterator::operator++() { - it++; - if (it != end_it) { - entry.table_id = it->first; - entry.changes = it->second; +shared_ptr LocalTableChanges::GetTransactionLocalInlinedData(ClientContext &context, + TableIndex table_id) const { + lock_guard guard(lock); + auto entry = changes.find(table_id); + if (entry == changes.end()) { + return nullptr; } - return *this; + auto &table_changes = entry->second; + if (!table_changes.new_inlined_data) { + return nullptr; + } + auto &local_changes = *table_changes.new_inlined_data; + auto result = make_shared_ptr(); + result->data = make_uniq(context, local_changes.data->Types()); + for (auto &chunk : local_changes.data->Chunks()) { + result->data->Append(chunk); + } + return result; } -bool LocalTableChangeIterationHelper::LocalTableChangeIterator::operator!=(const LocalTableChangeIterator &other) const { - return it != other.it; +void LocalTableChanges::DropTransactionLocalFile(ClientContext &context, TableIndex table_id, const string &path) { + lock_guard guard(lock); + auto entry = changes.find(table_id); + if (entry == changes.end()) { + throw InternalException( + "DropTransactionLocalFile called for a table for which no transaction-local files exist"); + } + auto &table_changes = entry->second; + auto &table_files = table_changes.new_data_files; + auto &fs = FileSystem::GetFileSystem(context); + for (idx_t i = 0; i < table_files.size(); i++) { + auto &file = table_files[i]; + if (file.file_name == path) { + for (auto &del_file : file.delete_files) { + fs.RemoveFile(del_file.file_name); + } + file.delete_files.clear(); + // found the file - delete it from the table list and from disk + table_files.erase_at(i); + fs.RemoveFile(path); + if (table_changes.IsEmpty()) { + // no more files remaining + changes.erase(entry); + } + return; + } + } + throw InternalException("Failed to find matching transaction-local file for DropTransactionLocalFile"); } -const LocalTableChangeIterationHelper::LocalTableChangeIteratorEntry &LocalTableChangeIterationHelper::LocalTableChangeIterator::operator*() const { - return entry; +void LocalTableChanges::AppendFiles(TableIndex table_id, vector files) { + lock_guard guard(lock); + auto &table_changes = changes[table_id]; + if (table_changes.new_data_files.empty()) { + // If empty, just move the entire vector + table_changes.new_data_files = std::move(files); + } else { + // Reserve to avoid reallocations during insertion + table_changes.new_data_files.reserve(table_changes.new_data_files.size() + files.size()); + // Use move_iterator for efficient batch move + table_changes.new_data_files.insert(table_changes.new_data_files.end(), std::make_move_iterator(files.begin()), + std::make_move_iterator(files.end())); + } } -LocalTableChangeIterationHelper LocalTableChanges::Changes() const { - return LocalTableChangeIterationHelper(lock, changes); +void LocalTableChanges::AppendInlinedData(ClientContext &context, TableIndex table_id, + unique_ptr new_data) { + lock_guard guard(lock); + auto &table_changes = changes[table_id]; + if (table_changes.new_inlined_data) { + // already exists - append + auto &existing_data = *table_changes.new_inlined_data; + auto &existing_types = existing_data.data->Types(); + auto &new_types = new_data->data->Types(); + // check if types changed (e.g. due to ALTER COLUMN TYPE) + if (existing_types != new_types) { + // if types differ we gotta add a cast. + auto casted_data = make_uniq(context, new_types); + ColumnDataAppendState append_state; + casted_data->InitializeAppend(append_state); + for (auto &chunk : existing_data.data->Chunks()) { + DataChunk casted_chunk; + casted_chunk.Initialize(context, new_types); + for (idx_t col_idx = 0; col_idx < chunk.ColumnCount(); col_idx++) { + if (existing_types[col_idx] != new_types[col_idx]) { + VectorOperations::Cast(context, chunk.data[col_idx], casted_chunk.data[col_idx], chunk.size()); + } else { + casted_chunk.data[col_idx].Reference(chunk.data[col_idx]); + } + } + casted_chunk.SetCardinality(chunk.size()); + casted_data->Append(append_state, casted_chunk); + } + existing_data.data = std::move(casted_data); + } + ColumnDataAppendState append_state; + existing_data.data->InitializeAppend(append_state); + for (auto &chunk : new_data->data->Chunks()) { + existing_data.data->Append(chunk); + } + for (auto &entry : new_data->column_stats) { + auto stats_entry = existing_data.column_stats.find(entry.first); + if (stats_entry == existing_data.column_stats.end()) { + throw InternalException("Missing stats when merging inlined data"); + } + stats_entry->second.MergeStats(entry.second); + } + } else { + // does not exist yet - set it + table_changes.new_inlined_data = std::move(new_data); + } } -DuckLakeTransaction::DuckLakeTransaction(DuckLakeCatalog &ducklake_catalog, TransactionManager &manager, - ClientContext &context) - : Transaction(manager, context), ducklake_catalog(ducklake_catalog), db(*context.db), - local_catalog_id(DuckLakeConstants::TRANSACTION_LOCAL_ID_START), catalog_version(0) { - metadata_manager = DuckLakeMetadataManager::Create(*this); +void LocalTableChanges::AddNewInlinedDeletes(TableIndex table_id, const string &table_name, set new_deletes) { + lock_guard guard(lock); + auto &table_changes = changes[table_id]; + auto &table_deletes = table_changes.new_inlined_data_deletes; + auto entry = table_deletes.find(table_name); + if (entry != table_deletes.end()) { + // merge deletes + auto &existing_rows = entry->second->rows; + for (auto &row_idx : new_deletes) { + existing_rows.insert(row_idx); + } + } else { + auto new_data = make_uniq(); + new_data->rows = std::move(new_deletes); + table_deletes.emplace(table_name, std::move(new_data)); + } } -DuckLakeTransaction::~DuckLakeTransaction() { -} +void LocalTableChanges::DeleteFromLocalInlinedData(ClientContext &context, TableIndex table_id, + set new_deletes) { + lock_guard guard(lock); + auto entry = changes.find(table_id); + if (entry == changes.end()) { + throw InternalException("DeleteFromLocalInlinedData called but no transaction-local data exists for table"); + } + auto &table_changes = entry->second; + auto &existing = *table_changes.new_inlined_data->data; + // construct a new collection from the existing data minus the deletes + auto new_data = make_uniq(context, existing.Types()); -void DuckLakeTransaction::Start() { -} + idx_t base_row_id = 0; + ColumnDataAppendState append_state; + new_data->InitializeAppend(append_state); + for (auto &chunk : existing.Chunks()) { + // slice out non-deleted rows + SelectionVector sel(chunk.size()); + idx_t selected_rows = 0; -void DuckLakeTransaction::Commit() { - if (ChangesMade()) { - FlushChanges(); - } else if (connection) { - connection->Commit(); + for (idx_t r = 0; r < chunk.size(); r++) { + auto row_id = base_row_id + r; + if (new_deletes.find(row_id) != new_deletes.end()) { + // deleted - skip + continue; + } + sel.set_index(selected_rows++, r); + } + base_row_id += chunk.size(); + if (selected_rows == 0) { + continue; + } + chunk.Slice(sel, selected_rows); + new_data->Append(append_state, chunk); } - connection.reset(); - local_changes.Clear(); + + // override the existing collection + table_changes.new_inlined_data->data = std::move(new_data); } -void DuckLakeTransaction::Rollback() { - if (connection) { - // rollback any changes made to the metadata catalog - connection->Rollback(); - connection.reset(); +static void RemoveFieldStats(map &column_stats, const DuckLakeFieldId &field_id) { + column_stats.erase(field_id.GetFieldIndex()); + for (auto &child_id : field_id.Children()) { + RemoveFieldStats(column_stats, *child_id); } - CleanupFiles(); - local_changes.Clear(); } -Connection &DuckLakeTransaction::GetConnection() { - lock_guard lock(connection_lock); - if (!connection) { - connection = make_uniq(db); - // set the search path to the metadata catalog - auto &client_data = ClientData::Get(*connection->context); - // ensure we are only looking in the ducklake catalog schema during querying - CatalogSearchEntry metadata_entry(ducklake_catalog.MetadataDatabaseName(), - ducklake_catalog.MetadataSchemaName()); - if (metadata_entry.schema.empty()) { - metadata_entry.schema = "main"; - } - client_data.catalog_search_path->Set(metadata_entry, CatalogSetPathType::SET_DIRECTLY); - - // set max error reporting to 0 so that during error reporting we don't traverse other schemas / catalogs - auto &client_config = ClientConfig::GetConfig(*connection->context); - client_config.user_settings.SetUserSetting(CatalogErrorMaxSchemasSetting::SettingIndex, Value::UBIGINT(0)); - connection->BeginTransaction(); +void LocalTableChanges::AddColumnToLocalInlinedData(ClientContext &context, TableIndex table_id, + const LogicalType &new_column_type, FieldIndex new_field_index, + const Value &default_value) { + lock_guard guard(lock); + auto entry = changes.find(table_id); + if (entry == changes.end()) { + throw InternalException("AddColumnToLocalInlinedData called but no transaction-local data exists"); + } + auto &table_changes = entry->second; + if (!table_changes.new_inlined_data) { + throw InternalException("AddColumnToLocalInlinedData called but no inlined data exists"); } - return *connection; -} -bool DuckLakeTransaction::SchemaChangesMade() const { - return !new_tables.empty() || !dropped_tables.empty() || new_schemas || !dropped_schemas.empty() || - !dropped_views.empty() || !new_macros.empty() || !dropped_scalar_macros.empty() || - !dropped_table_macros.empty(); -} + auto &existing = *table_changes.new_inlined_data->data; -bool DuckLakeTransaction::ChangesMade() const { - return SchemaChangesMade() || local_changes.HasChanges() || !dropped_files.empty() || - !new_name_maps.name_maps.empty(); -} + // New types: existing + new column + auto new_types = existing.Types(); + new_types.push_back(new_column_type); -struct TransactionChangeInformation { - case_insensitive_set_t created_schemas; + auto new_data = make_uniq(context, new_types); + + ColumnDataAppendState append_state; + new_data->InitializeAppend(append_state); + + bool has_default = !default_value.IsNull(); + + for (auto &chunk : existing.Chunks()) { + DataChunk new_chunk; + new_chunk.Initialize(context, new_types); + + // Copy existing columns + for (idx_t col_idx = 0; col_idx < chunk.ColumnCount(); col_idx++) { + new_chunk.data[col_idx].Reference(chunk.data[col_idx]); + } + + // New column: use default value or NULL + auto &new_col_vector = new_chunk.data[chunk.ColumnCount()]; + if (has_default) { + new_col_vector.Reference(default_value); + } else { + new_col_vector.SetVectorType(VectorType::CONSTANT_VECTOR); + ConstantVector::SetNull(new_col_vector, true); + } + + new_chunk.SetCardinality(chunk.size()); + new_data->Append(append_state, new_chunk); + } + + // Add stats for new column + idx_t total_rows = existing.Count(); + DuckLakeColumnStats new_col_stats(new_column_type); + new_col_stats.num_values = total_rows; + new_col_stats.has_num_values = true; + if (has_default) { + new_col_stats.null_count = 0; + new_col_stats.has_null_count = true; + new_col_stats.any_valid = true; + } else { + new_col_stats.null_count = total_rows; + new_col_stats.has_null_count = true; + new_col_stats.any_valid = false; + } + + table_changes.new_inlined_data->column_stats.emplace(new_field_index, std::move(new_col_stats)); + table_changes.new_inlined_data->data = std::move(new_data); +} + +void LocalTableChanges::RemoveColumnFromLocalInlinedData(ClientContext &context, TableIndex table_id, + LogicalIndex removed_column_index, + const DuckLakeFieldId &field_id) { + lock_guard guard(lock); + auto entry = changes.find(table_id); + if (entry == changes.end()) { + throw InternalException("RemoveColumnFromLocalInlinedData called but no transaction-local data exists"); + } + auto &table_changes = entry->second; + if (!table_changes.new_inlined_data) { + throw InternalException("RemoveColumnFromLocalInlinedData called but no inlined data exists"); + } + + auto &existing = *table_changes.new_inlined_data->data; + + // New types: existing minus the removed column + vector new_types; + for (idx_t col_idx = 0; col_idx < existing.Types().size(); col_idx++) { + if (col_idx == removed_column_index.index) { + continue; + } + new_types.push_back(existing.Types()[col_idx]); + } + + auto new_data = make_uniq(context, new_types); + + ColumnDataAppendState append_state; + new_data->InitializeAppend(append_state); + + for (auto &chunk : existing.Chunks()) { + DataChunk new_chunk; + new_chunk.Initialize(context, new_types); + + idx_t new_col_idx = 0; + for (idx_t col_idx = 0; col_idx < chunk.ColumnCount(); col_idx++) { + if (col_idx == removed_column_index.index) { + continue; + } + new_chunk.data[new_col_idx].Reference(chunk.data[col_idx]); + new_col_idx++; + } + + new_chunk.SetCardinality(chunk.size()); + new_data->Append(append_state, new_chunk); + } + + // Remove stats for the dropped field and all its children + RemoveFieldStats(table_changes.new_inlined_data->column_stats, field_id); + + table_changes.new_inlined_data->data = std::move(new_data); +} + +optional_ptr LocalTableChanges::GetInlinedDeletes(TableIndex table_id, + const string &table_name) const { + lock_guard guard(lock); + auto entry = changes.find(table_id); + if (entry == changes.end()) { + return nullptr; + } + auto &table_changes = entry->second; + auto delete_entry = table_changes.new_inlined_data_deletes.find(table_name); + if (delete_entry == table_changes.new_inlined_data_deletes.end()) { + return nullptr; + } + return delete_entry->second.get(); +} + +void LocalTableChanges::AddNewInlinedFileDeletes(TableIndex table_id, idx_t file_id, set new_deletes) { + if (new_deletes.empty()) { + return; + } + lock_guard guard(lock); + auto &table_changes = changes[table_id]; + if (!table_changes.new_inlined_file_deletes) { + table_changes.new_inlined_file_deletes = make_uniq(); + } + auto &file_deletes = table_changes.new_inlined_file_deletes->file_deletes[file_id]; + for (auto &row_id : new_deletes) { + file_deletes.insert(row_id); + } +} + +void LocalTableChanges::AddCompaction(TableIndex table_id, DuckLakeCompactionEntry entry) { + lock_guard guard(lock); + auto &table_changes = changes[table_id]; + table_changes.compactions.push_back(std::move(entry)); +} + +bool LocalTableChanges::HasLocalDeletes(TableIndex table_id) const { + lock_guard guard(lock); + auto entry = changes.find(table_id); + if (entry == changes.end()) { + return false; + } + return !entry->second.new_delete_files.empty(); +} + +bool LocalTableChanges::HasAnyLocalChanges(TableIndex table_id) const { + lock_guard guard(lock); + auto entry = changes.find(table_id); + if (entry != changes.end() && !entry->second.IsEmpty()) { + return true; + } + return false; +} + +void LocalTableChanges::GetLocalDeleteForFile(TableIndex table_id, const string &path, DuckLakeFileData &result) const { + lock_guard guard(lock); + auto entry = changes.find(table_id); + if (entry == changes.end()) { + return; + } + auto &table_changes = entry->second; + auto file_entry = table_changes.new_delete_files.find(path); + if (file_entry == table_changes.new_delete_files.end() || file_entry->second.empty()) { + return; + } + auto &delete_file = file_entry->second.back(); + result.path = delete_file.file_name; + result.file_size_bytes = delete_file.file_size_bytes; + result.footer_size = delete_file.footer_size; + result.encryption_key = delete_file.encryption_key; +} + +bool LocalTableChanges::HasLocalInlinedFileDeletes(TableIndex table_id) const { + lock_guard guard(lock); + auto entry = changes.find(table_id); + if (entry == changes.end()) { + return false; + } + auto &table_changes = entry->second; + if (!table_changes.new_inlined_file_deletes) { + return false; + } + return !table_changes.new_inlined_file_deletes->file_deletes.empty(); +} + +void LocalTableChanges::GetLocalInlinedFileDeletesForFile(TableIndex table_id, idx_t file_id, + set &result) const { + lock_guard guard(lock); + auto entry = changes.find(table_id); + if (entry == changes.end()) { + return; + } + auto &table_changes = entry->second; + if (!table_changes.new_inlined_file_deletes) { + return; + } + auto file_entry = table_changes.new_inlined_file_deletes->file_deletes.find(file_id); + if (file_entry == table_changes.new_inlined_file_deletes->file_deletes.end()) { + return; + } + // Merge the inlined deletes into the result set + for (auto &row_id : file_entry->second) { + result.insert(row_id); + } +} + +void LocalTableChanges::TransactionLocalDelete(ClientContext &context, TableIndex table_id, + const string &data_file_path, DuckLakeDeleteFile delete_file) { + lock_guard guard(lock); + auto entry = changes.find(table_id); + if (entry == changes.end()) { + throw InternalException( + "Transaction local delete called for table which does not have transaction local insertions"); + } + auto &table_changes = entry->second; + for (auto &file : table_changes.new_data_files) { + if (file.file_name == data_file_path) { + if (!file.delete_files.empty()) { + auto &fs = FileSystem::GetFileSystem(context); + vector files_to_delete; + files_to_delete.reserve(file.delete_files.size()); + for (auto &old_file : file.delete_files) { + files_to_delete.push_back(old_file.file_name); + } + fs.RemoveFiles(files_to_delete); + file.delete_files.clear(); + } + file.delete_files.push_back(std::move(delete_file)); + return; + } + } + throw InternalException("Failed to find matching transaction-local file for written delete file"); +} + +void LocalTableChanges::CleanupFiles(ClientContext &context, TableIndex table_id) { + lock_guard guard(lock); + auto table_entry = changes.find(table_id); + if (table_entry != changes.end()) { + auto &table_changes = table_entry->second; + auto &fs = FileSystem::GetFileSystem(context); + for (auto &file : table_changes.new_data_files) { + fs.RemoveFile(file.file_name); + } + changes.erase(table_entry); + } +} + +LocalTableChangeIterationHelper::LocalTableChangeIterationHelper( + mutex &local_changes_lock, const map &changes_p) + : lock(local_changes_lock), changes(changes_p) { +} + +LocalTableChangeIterationHelper::LocalTableChangeIteratorEntry::LocalTableChangeIteratorEntry() { +} + +TableIndex LocalTableChangeIterationHelper::LocalTableChangeIteratorEntry::GetTableIndex() const { + return table_id; +} + +const LocalTableDataChanges &LocalTableChangeIterationHelper::LocalTableChangeIteratorEntry::GetTableChanges() const { + return *changes; +} + +LocalTableChangeIterationHelper::LocalTableChangeIterator::LocalTableChangeIterator( + map::const_iterator it_p, + map::const_iterator end_it_p) + : it(std::move(it_p)), end_it(std::move(end_it_p)) { + if (it != end_it) { + entry.table_id = it->first; + entry.changes = it->second; + } +} + +LocalTableChangeIterationHelper::LocalTableChangeIterator & +LocalTableChangeIterationHelper::LocalTableChangeIterator::operator++() { + it++; + if (it != end_it) { + entry.table_id = it->first; + entry.changes = it->second; + } + return *this; +} + +bool LocalTableChangeIterationHelper::LocalTableChangeIterator::operator!=( + const LocalTableChangeIterator &other) const { + return it != other.it; +} + +const LocalTableChangeIterationHelper::LocalTableChangeIteratorEntry & +LocalTableChangeIterationHelper::LocalTableChangeIterator::operator*() const { + return entry; +} + +LocalTableChangeIterationHelper LocalTableChanges::Changes() const { + return LocalTableChangeIterationHelper(lock, changes); +} + +DuckLakeTransaction::DuckLakeTransaction(DuckLakeCatalog &ducklake_catalog, TransactionManager &manager, + ClientContext &context) + : Transaction(manager, context), ducklake_catalog(ducklake_catalog), db(*context.db), + local_catalog_id(DuckLakeConstants::TRANSACTION_LOCAL_ID_START), catalog_version(0) { + metadata_manager = DuckLakeMetadataManager::Create(*this); +} + +DuckLakeTransaction::~DuckLakeTransaction() { +} + +void DuckLakeTransaction::Start() { +} + +void DuckLakeTransaction::Commit() { + if (ChangesMade()) { + FlushChanges(); + } else if (connection) { + connection->Commit(); + } + connection.reset(); + local_changes.Clear(); +} + +void DuckLakeTransaction::Rollback() { + if (connection) { + // rollback any changes made to the metadata catalog + connection->Rollback(); + connection.reset(); + } + CleanupFiles(); + local_changes.Clear(); +} + +Connection &DuckLakeTransaction::GetConnection() { + lock_guard lock(connection_lock); + if (!connection) { + connection = make_uniq(db); + // set the search path to the metadata catalog + auto &client_data = ClientData::Get(*connection->context); + // ensure we are only looking in the ducklake catalog schema during querying + CatalogSearchEntry metadata_entry(ducklake_catalog.MetadataDatabaseName(), + ducklake_catalog.MetadataSchemaName()); + if (metadata_entry.schema.empty()) { + metadata_entry.schema = "main"; + } + client_data.catalog_search_path->Set(metadata_entry, CatalogSetPathType::SET_DIRECTLY); + + // set max error reporting to 0 so that during error reporting we don't traverse other schemas / catalogs + auto &client_config = ClientConfig::GetConfig(*connection->context); + client_config.user_settings.SetUserSetting(CatalogErrorMaxSchemasSetting::SettingIndex, Value::UBIGINT(0)); + connection->BeginTransaction(); + } + return *connection; +} + +bool DuckLakeTransaction::SchemaChangesMade() const { + return !new_tables.empty() || !dropped_tables.empty() || new_schemas || !dropped_schemas.empty() || + !dropped_views.empty() || !new_macros.empty() || !dropped_scalar_macros.empty() || + !dropped_table_macros.empty(); +} + +bool DuckLakeTransaction::ChangesMade() const { + return SchemaChangesMade() || local_changes.HasChanges() || !dropped_files.empty() || + !new_name_maps.name_maps.empty(); +} + +struct TransactionChangeInformation { + case_insensitive_set_t created_schemas; map> dropped_schemas; case_insensitive_map_t> created_tables; case_insensitive_map_t> created_scalar_macros; @@ -1370,7 +1852,7 @@ DuckLakeColumnStatsInfo DuckLakeColumnStatsInfo::FromColumnStats(FieldIndex fiel return column_stats; } -DuckLakeFileInfo DuckLakeTransaction::GetNewDataFile(DuckLakeDataFile &file, DuckLakeSnapshot &commit_snapshot, +DuckLakeFileInfo DuckLakeTransaction::GetNewDataFile(const DuckLakeDataFile &file, DuckLakeSnapshot &commit_snapshot, TableIndex table_id, optional_idx row_id_start) { DuckLakeFileInfo data_file; data_file.id = DataFileIndex(commit_snapshot.next_file_id++); @@ -1596,11 +2078,12 @@ NewNameMapInfo DuckLakeTransaction::GetNewNameMaps(DuckLakeCommitState &commit_s return result; } -vector DuckLakeTransaction::GetNewInlinedDeletes(DuckLakeCommitState &commit_state) const { +vector +DuckLakeTransaction::GetNewInlinedDeletes(DuckLakeCommitState &commit_state) const { vector result; - for (auto &entry : local_changes.changes) { - auto table_id = commit_state.GetTableId(entry.first); - auto &table_changes = entry.second; + for (auto &entry : local_changes.Changes()) { + auto table_id = commit_state.GetTableId(entry.GetTableIndex()); + auto &table_changes = entry.GetTableChanges(); for (auto &delete_entry : table_changes.new_inlined_data_deletes) { DuckLakeDeletedInlinedDataInfo info; info.table_id = table_id; @@ -1695,14 +2178,11 @@ string DuckLakeTransaction::CommitChanges(DuckLakeCommitState &commit_state, } // write new data / data files - bool has_table_data_changes = false; - { - lock_guard guard(local_changes.lock); - has_table_data_changes = !local_changes.changes.empty(); - } + bool has_table_data_changes = local_changes.HasChanges(); if (has_table_data_changes) { auto result = GetNewDataFiles(batch_queries, commit_state, stats); - batch_queries += metadata_manager->WriteNewDataFiles(commit_snapshot, result.new_files, new_tables_result, new_schemas_result); + batch_queries += metadata_manager->WriteNewDataFiles(commit_snapshot, result.new_files, new_tables_result, + new_schemas_result); batch_queries += metadata_manager->WriteNewInlinedData(commit_snapshot, result.new_inlined_data, new_tables_result, new_inlined_data_tables_result); } @@ -1736,12 +2216,12 @@ string DuckLakeTransaction::CommitChanges(DuckLakeCommitState &commit_state, GetCompactionChanges(commit_snapshot, CompactionType::MERGE_ADJACENT_TABLES); batch_queries += metadata_manager->WriteCompactions(compaction_merge_adjacent_changes.compacted_files, CompactionType::MERGE_ADJACENT_TABLES); - batch_queries += metadata_manager->WriteNewDataFiles(commit_snapshot, compaction_merge_adjacent_changes.new_files, - new_tables_result, new_schemas_result); + batch_queries += metadata_manager->WriteNewDataFiles( + commit_snapshot, compaction_merge_adjacent_changes.new_files, new_tables_result, new_schemas_result); auto compaction_rewrite_delete_changes = GetCompactionChanges(commit_snapshot, CompactionType::REWRITE_DELETES); - batch_queries += metadata_manager->WriteNewDataFiles(commit_snapshot, compaction_rewrite_delete_changes.new_files, - new_tables_result, new_schemas_result); + batch_queries += metadata_manager->WriteNewDataFiles( + commit_snapshot, compaction_rewrite_delete_changes.new_files, new_tables_result, new_schemas_result); batch_queries += metadata_manager->WriteCompactions(compaction_rewrite_delete_changes.compacted_files, CompactionType::REWRITE_DELETES); } @@ -1768,10 +2248,9 @@ string DuckLakeTransaction::CommitChanges(DuckLakeCommitState &commit_state, CompactionInformation DuckLakeTransaction::GetCompactionChanges(DuckLakeSnapshot &commit_snapshot, CompactionType type) { CompactionInformation result; - lock_guard guard(local_changes.lock); - for (auto &entry : local_changes.changes) { - auto table_id = entry.first; - auto &table_changes = entry.second; + for (auto &entry : local_changes.Changes()) { + auto table_id = entry.GetTableIndex(); + auto &table_changes = entry.GetTableChanges(); for (auto &compaction : table_changes.compactions) { if (type != compaction.type) { continue; @@ -1820,7 +2299,7 @@ CompactionInformation DuckLakeTransaction::GetCompactionChanges(DuckLakeSnapshot file_info.delete_file_path = compacted_file.delete_files.back().data.path; file_info.delete_file_id = compacted_file.delete_files.back().delete_file_id; file_info.start_snapshot = compacted_file.file.begin_snapshot; - file_info.table_index = entry.first; + file_info.table_index = entry.GetTableIndex(); file_info.delete_file_start_snapshot = commit_snapshot.snapshot_id; file_info.delete_file_end_snapshot = compacted_file.delete_files.back().end_snapshot; } @@ -1830,492 +2309,227 @@ CompactionInformation DuckLakeTransaction::GetCompactionChanges(DuckLakeSnapshot result.compacted_files.push_back(std::move(file_info)); } result.new_files.push_back(std::move(new_file)); - } - } - return result; -} - -bool RetryOnError(const string &original_message) { - auto message = StringUtil::Lower(original_message); - // retry on primary key errors - if (StringUtil::Contains(message, "primary key") || StringUtil::Contains(message, "unique")) { - return true; - } - // retry on conflicts - if (StringUtil::Contains(message, "conflict")) { - return true; - } - // retry on concurrent access - if (StringUtil::Contains(message, "concurrent")) { - return true; - } - return false; -} - -void DuckLakeTransaction::FlushChanges() { - if (!ChangesMade()) { - // read-only transactions don't need to do anything - return; - } - idx_t max_retry_count = 10; - idx_t retry_wait_ms = 100; - double retry_backoff = 1.5; - Value setting_val; - auto context_ref = context.lock(); - if (context_ref->TryGetCurrentSetting("ducklake_max_retry_count", setting_val)) { - max_retry_count = setting_val.GetValue(); - } - if (context_ref->TryGetCurrentSetting("ducklake_retry_wait_ms", setting_val)) { - retry_wait_ms = setting_val.GetValue(); - } - if (context_ref->TryGetCurrentSetting("ducklake_retry_backoff", setting_val)) { - retry_backoff = setting_val.GetValue(); - } - - auto transaction_snapshot = GetSnapshot(); - auto transaction_changes = GetTransactionChanges(); - SnapshotAndStats commit_stats_snapshot; - auto &commit_snapshot = commit_stats_snapshot.snapshot; - optional_ptr> stats; - for (idx_t i = 0; i < max_retry_count + 1; i++) { - bool can_retry; - try { - can_retry = false; - if (i > 0) { - // we failed our first commit due to another transaction committing - // retry - but first check for conflicts - commit_stats_snapshot = CheckForConflicts(transaction_snapshot, transaction_changes); - stats = &commit_stats_snapshot.stats; - } else { - commit_stats_snapshot.snapshot = GetSnapshot(); - } - commit_snapshot.snapshot_id++; - if (SchemaChangesMade()) { - // we changed the schema - need to get a new schema version - commit_snapshot.schema_version++; - } - can_retry = true; - DuckLakeCommitState commit_state(commit_snapshot); - // write the new snapshot - string batch_queries = metadata_manager->InsertSnapshot(); - batch_queries += CommitChanges(commit_state, transaction_changes, stats); - - batch_queries += WriteSnapshotChanges(commit_state, transaction_changes); - auto res = metadata_manager->Execute(commit_snapshot, batch_queries); - if (res->HasError()) { - res->GetErrorObject().Throw("Failed to flush changes into DuckLake: "); - } - connection->Commit(); - catalog_version = commit_snapshot.schema_version; - - // finished writing - break; - } catch (std::exception &ex) { - ErrorData error(ex); - // rollback if there is an active transaction - auto has_active_transaction = connection->context->transaction.HasActiveTransaction(); - if (has_active_transaction) { - connection->Rollback(); - } - bool retry_on_error = RetryOnError(error.Message()); - bool finished_retrying = i + 1 >= max_retry_count; - if (!can_retry || !retry_on_error || finished_retrying) { - // we abort after the max retry count - CleanupFiles(); - // Add additional information on the number of retries and suggest to increase it - std::ostringstream error_message; - error_message << "Failed to commit DuckLake transaction." << '\n'; - if (finished_retrying) { - error_message << "Exceeded the maximum retry count of " << max_retry_count - << " set by the ducklake_max_retry_count setting." << '\n' - << ". Consider increasing the value with: e.g., \"SET ducklake_max_retry_count = " - << max_retry_count * 10 << ";\"" << '\n'; - } - error.Throw(error_message.str()); - } - -#ifndef DUCKDB_NO_THREADS - RandomEngine random; - // random multiplier between 0.5 - 1.0 - double random_multiplier = (random.NextRandom() + 1.0) / 2.0; - uint64_t sleep_amount = - (uint64_t)((double)retry_wait_ms * random_multiplier * pow(retry_backoff, static_cast(i))); - std::this_thread::sleep_for(std::chrono::milliseconds(sleep_amount)); -#endif - - // retry the transaction (with a new snapshot id) - connection->BeginTransaction(); - snapshot.reset(); - } - } - // If we got here, this snapshot was successful - ducklake_catalog.SetCommittedSnapshotId(commit_snapshot.snapshot_id); -} - -void DuckLakeTransaction::SetConfigOption(const DuckLakeConfigOption &option) { - // write the config option to the metadata - metadata_manager->SetConfigOption(option); - // set the option in the catalog - ducklake_catalog.SetConfigOption(option); -} - -void DuckLakeTransaction::SetCommitMessage(const DuckLakeSnapshotCommit &option) { - commit_info = option; -} - -void DuckLakeTransaction::DeleteSnapshots(const vector &snapshots) { - auto &metadata_manager = GetMetadataManager(); - metadata_manager.DeleteSnapshots(snapshots); -} - -void DuckLakeTransaction::DeleteInlinedData(const DuckLakeInlinedTableInfo &inlined_table) { - auto &metadata_manager = GetMetadataManager(); - metadata_manager.DeleteInlinedData(inlined_table); -} - -unique_ptr DuckLakeTransaction::Query(string query) { - auto &connection = GetConnection(); - auto catalog_identifier = DuckLakeUtil::SQLIdentifierToString(ducklake_catalog.MetadataDatabaseName()); - auto catalog_literal = DuckLakeUtil::SQLLiteralToString(ducklake_catalog.MetadataDatabaseName()); - auto schema_identifier = DuckLakeUtil::SQLIdentifierToString(ducklake_catalog.MetadataSchemaName()); - auto schema_identifier_escaped = StringUtil::Replace(schema_identifier, "'", "''"); - auto schema_literal = DuckLakeUtil::SQLLiteralToString(ducklake_catalog.MetadataSchemaName()); - auto metadata_path = DuckLakeUtil::SQLLiteralToString(ducklake_catalog.MetadataPath()); - auto data_path = DuckLakeUtil::SQLLiteralToString(ducklake_catalog.DataPath()); - - query = StringUtil::Replace(query, "{METADATA_CATALOG_NAME_LITERAL}", catalog_literal); - query = StringUtil::Replace(query, "{METADATA_CATALOG_NAME_IDENTIFIER}", catalog_identifier); - query = StringUtil::Replace(query, "{METADATA_SCHEMA_NAME_LITERAL}", schema_literal); - query = StringUtil::Replace(query, "{METADATA_CATALOG}", catalog_identifier + "." + schema_identifier); - query = StringUtil::Replace(query, "{METADATA_SCHEMA_ESCAPED}", schema_identifier_escaped); - query = StringUtil::Replace(query, "{METADATA_PATH}", metadata_path); - query = StringUtil::Replace(query, "{DATA_PATH}", data_path); - return connection.Query(query); -} - -unique_ptr DuckLakeTransaction::Query(DuckLakeSnapshot snapshot, string query) { - query = StringUtil::Replace(query, "{SNAPSHOT_ID}", to_string(snapshot.snapshot_id)); - query = StringUtil::Replace(query, "{SCHEMA_VERSION}", to_string(snapshot.schema_version)); - query = StringUtil::Replace(query, "{NEXT_CATALOG_ID}", to_string(snapshot.next_catalog_id)); - query = StringUtil::Replace(query, "{NEXT_FILE_ID}", to_string(snapshot.next_file_id)); - query = StringUtil::Replace(query, "{AUTHOR}", commit_info.author.ToSQLString()); - query = StringUtil::Replace(query, "{COMMIT_MESSAGE}", commit_info.commit_message.ToSQLString()); - query = StringUtil::Replace(query, "{COMMIT_EXTRA_INFO}", commit_info.commit_extra_info.ToSQLString()); - - return Query(std::move(query)); -} - -string DuckLakeTransaction::GetDefaultSchemaName() { - auto &metadata_context = *connection->context; - auto &db_manager = DatabaseManager::Get(metadata_context); - auto metadb = db_manager.GetDatabase(metadata_context, ducklake_catalog.MetadataDatabaseName()); - return metadb->GetCatalog().GetDefaultSchema(); -} - -DuckLakeSnapshot DuckLakeTransaction::GetSnapshot() { - auto catalog_snapshot = ducklake_catalog.CatalogSnapshot(); - if (catalog_snapshot) { - // the catalog was opened at a specific snapshot - load that snapshot - return GetSnapshot(catalog_snapshot); - } - lock_guard guard(snapshot_lock); - if (!snapshot) { - // no snapshot loaded yet for this transaction - load it - snapshot = metadata_manager->GetSnapshot(); - } - return *snapshot; -} - -DuckLakeSnapshot DuckLakeTransaction::GetSnapshot(optional_ptr at_clause, SnapshotBound bound) { - if (!at_clause) { - // no AT-clause - get the latest snapshot - return GetSnapshot(); - } - // construct a struct value from the AT clause in the form of {"unit": value} (e.g. {"version": 2} - // this is used as a caching key for the snapshot - child_list_t values; - values.push_back(make_pair(at_clause->Unit(), at_clause->GetValue())); - auto snapshot_value = Value::STRUCT(std::move(values)); - - lock_guard guard(snapshot_lock); - auto entry = snapshot_cache.find(snapshot_value); - if (entry != snapshot_cache.end()) { - // we already found this snapshot - return it - return entry->second; - } - // find the snapshot and cache it - auto result_snapshot = *metadata_manager->GetSnapshot(*at_clause, bound); - snapshot_cache.insert(make_pair(std::move(snapshot_value), result_snapshot)); - return result_snapshot; -} - -idx_t DuckLakeTransaction::GetLocalCatalogId() { - return local_catalog_id++; -} - -bool LocalTableChanges::HasTransactionLocalInserts(TableIndex table_id) const { - lock_guard guard(lock); - auto entry = changes.find(table_id); - if (entry == changes.end()) { - return false; - } - auto &table_changes = entry->second; - return !table_changes.new_data_files.empty() || table_changes.new_inlined_data; -} - -bool LocalTableChanges::HasTransactionInlinedData(TableIndex table_id) const { - lock_guard guard(lock); - auto entry = changes.find(table_id); - if (entry == changes.end()) { - return false; - } - auto &table_changes = entry->second; - return table_changes.new_inlined_data != nullptr; -} - -vector LocalTableChanges::GetTransactionLocalFiles(TableIndex table_id) const { - lock_guard guard(lock); - auto entry = changes.find(table_id); - if (entry == changes.end()) { - return vector(); + } } - return entry->second.new_data_files; + return result; } -shared_ptr LocalTableChanges::GetTransactionLocalInlinedData(ClientContext &context, TableIndex table_id) const { - auto entry = changes.find(table_id); - if (entry == changes.end()) { - return nullptr; +bool RetryOnError(const string &original_message) { + auto message = StringUtil::Lower(original_message); + // retry on primary key errors + if (StringUtil::Contains(message, "primary key") || StringUtil::Contains(message, "unique")) { + return true; } - auto &table_changes = entry->second; - if (!table_changes.new_inlined_data) { - return nullptr; + // retry on conflicts + if (StringUtil::Contains(message, "conflict")) { + return true; } - auto &local_changes = *table_changes.new_inlined_data; - auto result = make_shared_ptr(); - result->data = make_uniq(context, local_changes.data->Types()); - for (auto &chunk : local_changes.data->Chunks()) { - result->data->Append(chunk); + // retry on concurrent access + if (StringUtil::Contains(message, "concurrent")) { + return true; } - return result; + return false; } -void LocalTableChanges::DropTransactionLocalFile(ClientContext &context, TableIndex table_id, const string &path) { - lock_guard guard(lock); - auto entry = changes.find(table_id); - if (entry == changes.end()) { - throw InternalException( - "DropTransactionLocalFile called for a table for which no transaction-local files exist"); +void DuckLakeTransaction::FlushChanges() { + if (!ChangesMade()) { + // read-only transactions don't need to do anything + return; } - auto &table_changes = entry->second; - auto &table_files = table_changes.new_data_files; - auto &fs = FileSystem::GetFileSystem(context); - for (idx_t i = 0; i < table_files.size(); i++) { - auto &file = table_files[i]; - if (file.file_name == path) { - for (auto &del_file : file.delete_files) { - fs.RemoveFile(del_file.file_name); - } - file.delete_files.clear(); - // found the file - delete it from the table list and from disk - table_files.erase_at(i); - fs.RemoveFile(path); - if (table_changes.IsEmpty()) { - // no more files remaining - changes.erase(entry); - } - return; - } + idx_t max_retry_count = 10; + idx_t retry_wait_ms = 100; + double retry_backoff = 1.5; + Value setting_val; + auto context_ref = context.lock(); + if (context_ref->TryGetCurrentSetting("ducklake_max_retry_count", setting_val)) { + max_retry_count = setting_val.GetValue(); } - throw InternalException("Failed to find matching transaction-local file for DropTransactionLocalFile"); -} - -void LocalTableChanges::AppendFiles(TableIndex table_id, vector files) { - lock_guard guard(lock); - auto &table_changes = changes[table_id]; - if (table_changes.new_data_files.empty()) { - // If empty, just move the entire vector - table_changes.new_data_files = std::move(files); - } else { - // Reserve to avoid reallocations during insertion - table_changes.new_data_files.reserve(table_changes.new_data_files.size() + files.size()); - // Use move_iterator for efficient batch move - table_changes.new_data_files.insert( - table_changes.new_data_files.end(), - std::make_move_iterator(files.begin()), - std::make_move_iterator(files.end())); + if (context_ref->TryGetCurrentSetting("ducklake_retry_wait_ms", setting_val)) { + retry_wait_ms = setting_val.GetValue(); + } + if (context_ref->TryGetCurrentSetting("ducklake_retry_backoff", setting_val)) { + retry_backoff = setting_val.GetValue(); } -} -void LocalTableChanges::AppendInlinedData(ClientContext &context, TableIndex table_id, unique_ptr new_data) { - lock_guard guard(lock); - auto &table_changes = changes[table_id]; - if (table_changes.new_inlined_data) { - // already exists - append - auto &existing_data = *table_changes.new_inlined_data; - auto &existing_types = existing_data.data->Types(); - auto &new_types = new_data->data->Types(); - // check if types changed (e.g. due to ALTER COLUMN TYPE) - if (existing_types != new_types) { - // if types differ we gotta add a cast. - auto casted_data = make_uniq(context, new_types); - ColumnDataAppendState append_state; - casted_data->InitializeAppend(append_state); - for (auto &chunk : existing_data.data->Chunks()) { - DataChunk casted_chunk; - casted_chunk.Initialize(context, new_types); - for (idx_t col_idx = 0; col_idx < chunk.ColumnCount(); col_idx++) { - if (existing_types[col_idx] != new_types[col_idx]) { - VectorOperations::Cast(context, chunk.data[col_idx], casted_chunk.data[col_idx], - chunk.size()); - } else { - casted_chunk.data[col_idx].Reference(chunk.data[col_idx]); - } - } - casted_chunk.SetCardinality(chunk.size()); - casted_data->Append(append_state, casted_chunk); + auto transaction_snapshot = GetSnapshot(); + auto transaction_changes = GetTransactionChanges(); + SnapshotAndStats commit_stats_snapshot; + auto &commit_snapshot = commit_stats_snapshot.snapshot; + optional_ptr> stats; + for (idx_t i = 0; i < max_retry_count + 1; i++) { + bool can_retry; + try { + can_retry = false; + if (i > 0) { + // we failed our first commit due to another transaction committing + // retry - but first check for conflicts + commit_stats_snapshot = CheckForConflicts(transaction_snapshot, transaction_changes); + stats = &commit_stats_snapshot.stats; + } else { + commit_stats_snapshot.snapshot = GetSnapshot(); } - existing_data.data = std::move(casted_data); - } - ColumnDataAppendState append_state; - existing_data.data->InitializeAppend(append_state); - for (auto &chunk : new_data->data->Chunks()) { - existing_data.data->Append(chunk); - } - for (auto &entry : new_data->column_stats) { - auto stats_entry = existing_data.column_stats.find(entry.first); - if (stats_entry == existing_data.column_stats.end()) { - throw InternalException("Missing stats when merging inlined data"); + commit_snapshot.snapshot_id++; + if (SchemaChangesMade()) { + // we changed the schema - need to get a new schema version + commit_snapshot.schema_version++; } - stats_entry->second.MergeStats(entry.second); - } - } else { - // does not exist yet - set it - table_changes.new_inlined_data = std::move(new_data); - } -} + can_retry = true; + DuckLakeCommitState commit_state(commit_snapshot); + // write the new snapshot + string batch_queries = metadata_manager->InsertSnapshot(); + batch_queries += CommitChanges(commit_state, transaction_changes, stats); -void LocalTableChanges::AddNewInlinedDeletes(TableIndex table_id, const string &table_name, set new_deletes) { - lock_guard guard(lock); - auto &table_changes = changes[table_id]; - auto &table_deletes = table_changes.new_inlined_data_deletes; - auto entry = table_deletes.find(table_name); - if (entry != table_deletes.end()) { - // merge deletes - auto &existing_rows = entry->second->rows; - for (auto &row_idx : new_deletes) { - existing_rows.insert(row_idx); - } - } else { - auto new_data = make_uniq(); - new_data->rows = std::move(new_deletes); - table_deletes.emplace(table_name, std::move(new_data)); - } -} + batch_queries += WriteSnapshotChanges(commit_state, transaction_changes); + auto res = metadata_manager->Execute(commit_snapshot, batch_queries); + if (res->HasError()) { + res->GetErrorObject().Throw("Failed to flush changes into DuckLake: "); + } + connection->Commit(); + catalog_version = commit_snapshot.schema_version; -void LocalTableChanges::DeleteFromLocalInlinedData(ClientContext &context, TableIndex table_id, set new_deletes) { - lock_guard guard(lock); - auto entry = changes.find(table_id); - if (entry == changes.end()) { - throw InternalException("DeleteFromLocalInlinedData called but no transaction-local data exists for table"); - } - auto &table_changes = entry->second; - auto &existing = *table_changes.new_inlined_data->data; - // construct a new collection from the existing data minus the deletes - auto new_data = make_uniq(context, existing.Types()); + // finished writing + break; + } catch (std::exception &ex) { + ErrorData error(ex); + // rollback if there is an active transaction + auto has_active_transaction = connection->context->transaction.HasActiveTransaction(); + if (has_active_transaction) { + connection->Rollback(); + } + bool retry_on_error = RetryOnError(error.Message()); + bool finished_retrying = i + 1 >= max_retry_count; + if (!can_retry || !retry_on_error || finished_retrying) { + // we abort after the max retry count + CleanupFiles(); + // Add additional information on the number of retries and suggest to increase it + std::ostringstream error_message; + error_message << "Failed to commit DuckLake transaction." << '\n'; + if (finished_retrying) { + error_message << "Exceeded the maximum retry count of " << max_retry_count + << " set by the ducklake_max_retry_count setting." << '\n' + << ". Consider increasing the value with: e.g., \"SET ducklake_max_retry_count = " + << max_retry_count * 10 << ";\"" << '\n'; + } + error.Throw(error_message.str()); + } - idx_t base_row_id = 0; - ColumnDataAppendState append_state; - new_data->InitializeAppend(append_state); - for (auto &chunk : existing.Chunks()) { - // slice out non-deleted rows - SelectionVector sel(chunk.size()); - idx_t selected_rows = 0; +#ifndef DUCKDB_NO_THREADS + RandomEngine random; + // random multiplier between 0.5 - 1.0 + double random_multiplier = (random.NextRandom() + 1.0) / 2.0; + uint64_t sleep_amount = + (uint64_t)((double)retry_wait_ms * random_multiplier * pow(retry_backoff, static_cast(i))); + std::this_thread::sleep_for(std::chrono::milliseconds(sleep_amount)); +#endif - for (idx_t r = 0; r < chunk.size(); r++) { - auto row_id = base_row_id + r; - if (new_deletes.find(row_id) != new_deletes.end()) { - // deleted - skip - continue; - } - sel.set_index(selected_rows++, r); - } - base_row_id += chunk.size(); - if (selected_rows == 0) { - continue; + // retry the transaction (with a new snapshot id) + connection->BeginTransaction(); + snapshot.reset(); } - chunk.Slice(sel, selected_rows); - new_data->Append(append_state, chunk); } + // If we got here, this snapshot was successful + ducklake_catalog.SetCommittedSnapshotId(commit_snapshot.snapshot_id); +} - // override the existing collection - table_changes.new_inlined_data->data = std::move(new_data); +void DuckLakeTransaction::SetConfigOption(const DuckLakeConfigOption &option) { + // write the config option to the metadata + metadata_manager->SetConfigOption(option); + // set the option in the catalog + ducklake_catalog.SetConfigOption(option); } -void LocalTableChanges::AddColumnToLocalInlinedData(ClientContext &context, TableIndex table_id, const LogicalType &new_column_type, - FieldIndex new_field_index, const Value &default_value) { - lock_guard guard(lock); - auto entry = changes.find(table_id); - if (entry == changes.end()) { - throw InternalException("AddColumnToLocalInlinedData called but no transaction-local data exists"); - } - auto &table_changes = entry->second; - if (!table_changes.new_inlined_data) { - throw InternalException("AddColumnToLocalInlinedData called but no inlined data exists"); - } +void DuckLakeTransaction::SetCommitMessage(const DuckLakeSnapshotCommit &option) { + commit_info = option; +} - auto &existing = *table_changes.new_inlined_data->data; +void DuckLakeTransaction::DeleteSnapshots(const vector &snapshots) { + auto &metadata_manager = GetMetadataManager(); + metadata_manager.DeleteSnapshots(snapshots); +} - // New types: existing + new column - auto new_types = existing.Types(); - new_types.push_back(new_column_type); +void DuckLakeTransaction::DeleteInlinedData(const DuckLakeInlinedTableInfo &inlined_table) { + auto &metadata_manager = GetMetadataManager(); + metadata_manager.DeleteInlinedData(inlined_table); +} - auto new_data = make_uniq(context, new_types); +unique_ptr DuckLakeTransaction::Query(string query) { + auto &connection = GetConnection(); + auto catalog_identifier = DuckLakeUtil::SQLIdentifierToString(ducklake_catalog.MetadataDatabaseName()); + auto catalog_literal = DuckLakeUtil::SQLLiteralToString(ducklake_catalog.MetadataDatabaseName()); + auto schema_identifier = DuckLakeUtil::SQLIdentifierToString(ducklake_catalog.MetadataSchemaName()); + auto schema_identifier_escaped = StringUtil::Replace(schema_identifier, "'", "''"); + auto schema_literal = DuckLakeUtil::SQLLiteralToString(ducklake_catalog.MetadataSchemaName()); + auto metadata_path = DuckLakeUtil::SQLLiteralToString(ducklake_catalog.MetadataPath()); + auto data_path = DuckLakeUtil::SQLLiteralToString(ducklake_catalog.DataPath()); - ColumnDataAppendState append_state; - new_data->InitializeAppend(append_state); + query = StringUtil::Replace(query, "{METADATA_CATALOG_NAME_LITERAL}", catalog_literal); + query = StringUtil::Replace(query, "{METADATA_CATALOG_NAME_IDENTIFIER}", catalog_identifier); + query = StringUtil::Replace(query, "{METADATA_SCHEMA_NAME_LITERAL}", schema_literal); + query = StringUtil::Replace(query, "{METADATA_CATALOG}", catalog_identifier + "." + schema_identifier); + query = StringUtil::Replace(query, "{METADATA_SCHEMA_ESCAPED}", schema_identifier_escaped); + query = StringUtil::Replace(query, "{METADATA_PATH}", metadata_path); + query = StringUtil::Replace(query, "{DATA_PATH}", data_path); + return connection.Query(query); +} - bool has_default = !default_value.IsNull(); +unique_ptr DuckLakeTransaction::Query(DuckLakeSnapshot snapshot, string query) { + query = StringUtil::Replace(query, "{SNAPSHOT_ID}", to_string(snapshot.snapshot_id)); + query = StringUtil::Replace(query, "{SCHEMA_VERSION}", to_string(snapshot.schema_version)); + query = StringUtil::Replace(query, "{NEXT_CATALOG_ID}", to_string(snapshot.next_catalog_id)); + query = StringUtil::Replace(query, "{NEXT_FILE_ID}", to_string(snapshot.next_file_id)); + query = StringUtil::Replace(query, "{AUTHOR}", commit_info.author.ToSQLString()); + query = StringUtil::Replace(query, "{COMMIT_MESSAGE}", commit_info.commit_message.ToSQLString()); + query = StringUtil::Replace(query, "{COMMIT_EXTRA_INFO}", commit_info.commit_extra_info.ToSQLString()); - for (auto &chunk : existing.Chunks()) { - DataChunk new_chunk; - new_chunk.Initialize(context, new_types); + return Query(std::move(query)); +} - // Copy existing columns - for (idx_t col_idx = 0; col_idx < chunk.ColumnCount(); col_idx++) { - new_chunk.data[col_idx].Reference(chunk.data[col_idx]); - } +string DuckLakeTransaction::GetDefaultSchemaName() { + auto &metadata_context = *connection->context; + auto &db_manager = DatabaseManager::Get(metadata_context); + auto metadb = db_manager.GetDatabase(metadata_context, ducklake_catalog.MetadataDatabaseName()); + return metadb->GetCatalog().GetDefaultSchema(); +} - // New column: use default value or NULL - auto &new_col_vector = new_chunk.data[chunk.ColumnCount()]; - if (has_default) { - new_col_vector.Reference(default_value); - } else { - new_col_vector.SetVectorType(VectorType::CONSTANT_VECTOR); - ConstantVector::SetNull(new_col_vector, true); - } +DuckLakeSnapshot DuckLakeTransaction::GetSnapshot() { + auto catalog_snapshot = ducklake_catalog.CatalogSnapshot(); + if (catalog_snapshot) { + // the catalog was opened at a specific snapshot - load that snapshot + return GetSnapshot(catalog_snapshot); + } + lock_guard guard(snapshot_lock); + if (!snapshot) { + // no snapshot loaded yet for this transaction - load it + snapshot = metadata_manager->GetSnapshot(); + } + return *snapshot; +} - new_chunk.SetCardinality(chunk.size()); - new_data->Append(append_state, new_chunk); +DuckLakeSnapshot DuckLakeTransaction::GetSnapshot(optional_ptr at_clause, SnapshotBound bound) { + if (!at_clause) { + // no AT-clause - get the latest snapshot + return GetSnapshot(); } + // construct a struct value from the AT clause in the form of {"unit": value} (e.g. {"version": 2} + // this is used as a caching key for the snapshot + child_list_t values; + values.push_back(make_pair(at_clause->Unit(), at_clause->GetValue())); + auto snapshot_value = Value::STRUCT(std::move(values)); - // Add stats for new column - idx_t total_rows = existing.Count(); - DuckLakeColumnStats new_col_stats(new_column_type); - new_col_stats.num_values = total_rows; - new_col_stats.has_num_values = true; - if (has_default) { - new_col_stats.null_count = 0; - new_col_stats.has_null_count = true; - new_col_stats.any_valid = true; - } else { - new_col_stats.null_count = total_rows; - new_col_stats.has_null_count = true; - new_col_stats.any_valid = false; + lock_guard guard(snapshot_lock); + auto entry = snapshot_cache.find(snapshot_value); + if (entry != snapshot_cache.end()) { + // we already found this snapshot - return it + return entry->second; } + // find the snapshot and cache it + auto result_snapshot = *metadata_manager->GetSnapshot(*at_clause, bound); + snapshot_cache.insert(make_pair(std::move(snapshot_value), result_snapshot)); + return result_snapshot; +} - table_changes.new_inlined_data->column_stats.emplace(new_field_index, std::move(new_col_stats)); - table_changes.new_inlined_data->data = std::move(new_data); +idx_t DuckLakeTransaction::GetLocalCatalogId() { + return local_catalog_id++; } bool DuckLakeTransaction::HasTransactionLocalInserts(TableIndex table_id) const { @@ -2370,92 +2584,19 @@ void DuckLakeTransaction::AddColumnToLocalInlinedData(TableIndex table_id, const local_changes.AddColumnToLocalInlinedData(*context_ref, table_id, new_column_type, new_field_index, default_value); } -static void RemoveFieldStats(map &column_stats, const DuckLakeFieldId &field_id) { - column_stats.erase(field_id.GetFieldIndex()); - for (auto &child_id : field_id.Children()) { - RemoveFieldStats(column_stats, *child_id); - } -} - void DuckLakeTransaction::RemoveColumnFromLocalInlinedData(TableIndex table_id, LogicalIndex removed_column_index, const DuckLakeFieldId &field_id) { - lock_guard guard(local_changes.lock); - auto entry = local_changes.changes.find(table_id); - if (entry == local_changes.changes.end()) { - throw InternalException("RemoveColumnFromLocalInlinedData called but no transaction-local data exists"); - } - auto &table_changes = entry->second; - if (!table_changes.new_inlined_data) { - throw InternalException("RemoveColumnFromLocalInlinedData called but no inlined data exists"); - } - - auto &existing = *table_changes.new_inlined_data->data; - - // New types: existing minus the removed column - vector new_types; - for (idx_t col_idx = 0; col_idx < existing.Types().size(); col_idx++) { - if (col_idx == removed_column_index.index) { - continue; - } - new_types.push_back(existing.Types()[col_idx]); - } - auto context_ref = context.lock(); - auto new_data = make_uniq(*context_ref, new_types); - - ColumnDataAppendState append_state; - new_data->InitializeAppend(append_state); - - for (auto &chunk : existing.Chunks()) { - DataChunk new_chunk; - new_chunk.Initialize(*context_ref, new_types); - - idx_t new_col_idx = 0; - for (idx_t col_idx = 0; col_idx < chunk.ColumnCount(); col_idx++) { - if (col_idx == removed_column_index.index) { - continue; - } - new_chunk.data[new_col_idx].Reference(chunk.data[col_idx]); - new_col_idx++; - } - - new_chunk.SetCardinality(chunk.size()); - new_data->Append(append_state, new_chunk); - } - - // Remove stats for the dropped field and all its children - RemoveFieldStats(table_changes.new_inlined_data->column_stats, field_id); - - table_changes.new_inlined_data->data = std::move(new_data); + local_changes.RemoveColumnFromLocalInlinedData(*context_ref, table_id, removed_column_index, field_id); } optional_ptr DuckLakeTransaction::GetInlinedDeletes(TableIndex table_id, const string &table_name) const { - auto entry = local_changes.changes.find(table_id); - if (entry == local_changes.changes.end()) { - return nullptr; - } - auto &table_changes = entry->second; - auto delete_entry = table_changes.new_inlined_data_deletes.find(table_name); - if (delete_entry == table_changes.new_inlined_data_deletes.end()) { - return nullptr; - } - return delete_entry->second.get(); + return local_changes.GetInlinedDeletes(table_id, table_name); } void DuckLakeTransaction::AddNewInlinedFileDeletes(TableIndex table_id, idx_t file_id, set new_deletes) { - if (new_deletes.empty()) { - return; - } - lock_guard guard(local_changes.lock); - auto &table_changes = local_changes.changes[table_id]; - if (!table_changes.new_inlined_file_deletes) { - table_changes.new_inlined_file_deletes = make_uniq(); - } - auto &file_deletes = table_changes.new_inlined_file_deletes->file_deletes[file_id]; - for (auto &row_id : new_deletes) { - file_deletes.insert(row_id); - } + local_changes.AddNewInlinedFileDeletes(table_id, file_id, std::move(new_deletes)); } vector @@ -2516,22 +2657,15 @@ void DuckLakeTransaction::AddDeletesLocked(TableIndex table_id, vector guard(local_changes.lock); - auto &table_changes = local_changes.changes[table_id]; - table_changes.compactions.push_back(std::move(entry)); + local_changes.AddCompaction(table_id, std::move(entry)); } bool DuckLakeTransaction::HasLocalDeletes(TableIndex table_id) const { - auto entry = local_changes.changes.find(table_id); - if (entry == local_changes.changes.end()) { - return false; - } - return !entry->second.new_delete_files.empty(); + return local_changes.HasLocalDeletes(table_id); } bool DuckLakeTransaction::HasAnyLocalChanges(TableIndex table_id) const { - auto entry = local_changes.changes.find(table_id); - if (entry != local_changes.changes.end() && !entry->second.IsEmpty()) { + if (local_changes.HasAnyLocalChanges(table_id)) { return true; } return tables_deleted_from.find(table_id) != tables_deleted_from.end(); @@ -2539,81 +2673,22 @@ bool DuckLakeTransaction::HasAnyLocalChanges(TableIndex table_id) const { void DuckLakeTransaction::GetLocalDeleteForFile(TableIndex table_id, const string &path, DuckLakeFileData &result) const { - auto entry = local_changes.changes.find(table_id); - if (entry == local_changes.changes.end()) { - return; - } - auto &table_changes = entry->second; - auto file_entry = table_changes.new_delete_files.find(path); - if (file_entry == table_changes.new_delete_files.end() || file_entry->second.empty()) { - return; - } - auto &delete_file = file_entry->second.back(); - result.path = delete_file.file_name; - result.file_size_bytes = delete_file.file_size_bytes; - result.footer_size = delete_file.footer_size; - result.encryption_key = delete_file.encryption_key; + local_changes.GetLocalDeleteForFile(table_id, path, result); } bool DuckLakeTransaction::HasLocalInlinedFileDeletes(TableIndex table_id) const { - auto entry = local_changes.changes.find(table_id); - if (entry == local_changes.changes.end()) { - return false; - } - auto &table_changes = entry->second; - if (!table_changes.new_inlined_file_deletes) { - return false; - } - return !table_changes.new_inlined_file_deletes->file_deletes.empty(); + return local_changes.HasLocalInlinedFileDeletes(table_id); } void DuckLakeTransaction::GetLocalInlinedFileDeletesForFile(TableIndex table_id, idx_t file_id, set &result) const { - auto entry = local_changes.changes.find(table_id); - if (entry == local_changes.changes.end()) { - return; - } - auto &table_changes = entry->second; - if (!table_changes.new_inlined_file_deletes) { - return; - } - auto file_entry = table_changes.new_inlined_file_deletes->file_deletes.find(file_id); - if (file_entry == table_changes.new_inlined_file_deletes->file_deletes.end()) { - return; - } - // Merge the inlined deletes into the result set - for (auto &row_id : file_entry->second) { - result.insert(row_id); - } + local_changes.GetLocalInlinedFileDeletesForFile(table_id, file_id, result); } void DuckLakeTransaction::TransactionLocalDelete(TableIndex table_id, const string &data_file_path, DuckLakeDeleteFile delete_file) { - lock_guard guard(local_changes.lock); - auto entry = local_changes.changes.find(table_id); - if (entry == local_changes.changes.end()) { - throw InternalException( - "Transaction local delete called for table which does not have transaction local insertions"); - } - auto &table_changes = entry->second; - for (auto &file : table_changes.new_data_files) { - if (file.file_name == data_file_path) { - if (!file.delete_files.empty()) { - auto context_ref = context.lock(); - auto &fs = FileSystem::GetFileSystem(*context_ref); - vector files_to_delete; - files_to_delete.reserve(file.delete_files.size()); - for (auto &old_file : file.delete_files) { - files_to_delete.push_back(old_file.file_name); - } - fs.RemoveFiles(files_to_delete); - file.delete_files.clear(); - } - file.delete_files.push_back(std::move(delete_file)); - return; - } - } - throw InternalException("Failed to find matching transaction-local file for written delete file"); + auto context_ref = context.lock(); + local_changes.TransactionLocalDelete(*context_ref, table_id, data_file_path, std::move(delete_file)); } DuckLakeTransaction &DuckLakeTransaction::Get(ClientContext &context, Catalog &catalog) { @@ -2651,20 +2726,11 @@ void DuckLakeTransaction::DropTable(DuckLakeTableEntry &table) { if (schema_entry == new_tables.end()) { throw InternalException("Dropping a transaction local table that does not exist?"); } - auto table_id = table.GetTableId(); - schema_entry->second->DropEntry(table.name); - // if we have written any files for this table - clean them up - lock_guard guard(local_changes.lock); - auto table_entry = local_changes.changes.find(table_id); - if (table_entry != local_changes.changes.end()) { - auto &table_changes = table_entry->second; - auto context_ref = context.lock(); - auto &fs = FileSystem::GetFileSystem(*context_ref); - for (auto &file : table_changes.new_data_files) { - fs.RemoveFile(file.file_name); - } - local_changes.changes.erase(table_entry); - } + auto table_id = table.GetTableId(); + schema_entry->second->DropEntry(table.name); + // if we have written any files for this table - clean them up + auto context_ref = context.lock(); + local_changes.CleanupFiles(*context_ref, table_id); new_tables.erase(schema_entry); } else { auto table_id = table.GetTableId(); From de262078995ef2d57048a77359c33ecab4b47671 Mon Sep 17 00:00:00 2001 From: Mytherin Date: Tue, 17 Mar 2026 10:25:20 +0100 Subject: [PATCH 16/59] Store partition id remaps in commit state instead of adjusting local changes directly --- src/include/storage/ducklake_transaction.hpp | 4 +-- src/storage/ducklake_transaction.cpp | 37 +++++++++++--------- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/src/include/storage/ducklake_transaction.hpp b/src/include/storage/ducklake_transaction.hpp index 20eb4e53bb6..06f0b4a1535 100644 --- a/src/include/storage/ducklake_transaction.hpp +++ b/src/include/storage/ducklake_transaction.hpp @@ -288,7 +288,7 @@ class DuckLakeTransaction : public Transaction, public enable_shared_from_this> stats); @@ -310,7 +310,7 @@ class DuckLakeTransaction : public Transaction, public enable_shared_from_this table_entry, NewTableInfo &result, TransactionChangeInformation &transaction_changes); - CompactionInformation GetCompactionChanges(DuckLakeSnapshot &commit_snapshot, CompactionType type); + CompactionInformation GetCompactionChanges(DuckLakeCommitState &commit_state, CompactionType type); void AlterEntryInternal(DuckLakeTableEntry &old_entry, unique_ptr new_entry); void AlterEntryInternal(DuckLakeViewEntry &old_entry, unique_ptr new_entry); diff --git a/src/storage/ducklake_transaction.cpp b/src/storage/ducklake_transaction.cpp index 5c26ab1afd7..3dd296fe1de 100644 --- a/src/storage/ducklake_transaction.cpp +++ b/src/storage/ducklake_transaction.cpp @@ -853,6 +853,7 @@ struct DuckLakeCommitState { DuckLakeSnapshot &commit_snapshot; map committed_schemas; map committed_tables; + map committed_partition_ids; void RemapIdentifier(SchemaIndex &schema_id) const { auto entry = committed_schemas.find(schema_id); @@ -866,6 +867,15 @@ struct DuckLakeCommitState { table_id = entry->second; } } + void RemapPartitionId(optional_idx &partition_id) const { + if (!partition_id.IsValid()) { + return; + } + auto entry = committed_partition_ids.find(partition_id.GetIndex()); + if (entry != committed_partition_ids.end()) { + partition_id = entry->second; + } + } SchemaIndex GetSchemaId(DuckLakeSchemaEntry &schema) const { auto schema_id = schema.GetSchemaId(); @@ -1314,17 +1324,7 @@ DuckLakePartitionInfo DuckLakeTransaction::GetNewPartitionKey(DuckLakeCommitStat } partition_key.fields.push_back(std::move(partition_field)); } - - // if we wrote any data with this partition id - rewrite it to the latest partition id - lock_guard guard(local_changes.lock); - for (auto &entry : local_changes.changes) { - auto &table_changes = entry.second; - for (auto &file : table_changes.new_data_files) { - if (file.partition_id.IsValid() && file.partition_id.GetIndex() == local_partition_id) { - file.partition_id = partition_id; - } - } - } + commit_state.committed_partition_ids[local_partition_id] = partition_id; return partition_key; } @@ -1852,8 +1852,9 @@ DuckLakeColumnStatsInfo DuckLakeColumnStatsInfo::FromColumnStats(FieldIndex fiel return column_stats; } -DuckLakeFileInfo DuckLakeTransaction::GetNewDataFile(const DuckLakeDataFile &file, DuckLakeSnapshot &commit_snapshot, +DuckLakeFileInfo DuckLakeTransaction::GetNewDataFile(const DuckLakeDataFile &file, DuckLakeCommitState &commit_state, TableIndex table_id, optional_idx row_id_start) { + auto &commit_snapshot = commit_state.commit_snapshot; DuckLakeFileInfo data_file; data_file.id = DataFileIndex(commit_snapshot.next_file_id++); data_file.table_id = table_id; @@ -1862,6 +1863,7 @@ DuckLakeFileInfo DuckLakeTransaction::GetNewDataFile(const DuckLakeDataFile &fil data_file.file_size_bytes = file.file_size_bytes; data_file.footer_size = file.footer_size; data_file.partition_id = file.partition_id; + commit_state.RemapPartitionId(data_file.partition_id); data_file.encryption_key = file.encryption_key; data_file.row_id_start = row_id_start; data_file.mapping_id = file.mapping_id; @@ -1926,7 +1928,7 @@ NewDataInfo DuckLakeTransaction::GetNewDataFiles(string &batch_query, DuckLakeCo // row_id_start auto row_id_start = file.flush_row_id_start.IsValid() ? file.flush_row_id_start.GetIndex() : new_stats.next_row_id; - auto data_file = GetNewDataFile(file, commit_state.commit_snapshot, table_id, row_id_start); + auto data_file = GetNewDataFile(file, commit_state, table_id, row_id_start); for (auto &del_file : file.delete_files) { // this transaction-local file already has deletes - write them out DuckLakeDeleteFile delete_file = del_file; @@ -2213,13 +2215,13 @@ string DuckLakeTransaction::CommitChanges(DuckLakeCommitState &commit_state, // write compactions auto compaction_merge_adjacent_changes = - GetCompactionChanges(commit_snapshot, CompactionType::MERGE_ADJACENT_TABLES); + GetCompactionChanges(commit_state, CompactionType::MERGE_ADJACENT_TABLES); batch_queries += metadata_manager->WriteCompactions(compaction_merge_adjacent_changes.compacted_files, CompactionType::MERGE_ADJACENT_TABLES); batch_queries += metadata_manager->WriteNewDataFiles( commit_snapshot, compaction_merge_adjacent_changes.new_files, new_tables_result, new_schemas_result); - auto compaction_rewrite_delete_changes = GetCompactionChanges(commit_snapshot, CompactionType::REWRITE_DELETES); + auto compaction_rewrite_delete_changes = GetCompactionChanges(commit_state, CompactionType::REWRITE_DELETES); batch_queries += metadata_manager->WriteNewDataFiles( commit_snapshot, compaction_rewrite_delete_changes.new_files, new_tables_result, new_schemas_result); batch_queries += metadata_manager->WriteCompactions(compaction_rewrite_delete_changes.compacted_files, @@ -2245,8 +2247,9 @@ string DuckLakeTransaction::CommitChanges(DuckLakeCommitState &commit_state, return batch_queries; } -CompactionInformation DuckLakeTransaction::GetCompactionChanges(DuckLakeSnapshot &commit_snapshot, +CompactionInformation DuckLakeTransaction::GetCompactionChanges(DuckLakeCommitState &commit_state, CompactionType type) { + auto &commit_snapshot = commit_state.commit_snapshot; CompactionInformation result; for (auto &entry : local_changes.Changes()) { auto table_id = entry.GetTableIndex(); @@ -2255,7 +2258,7 @@ CompactionInformation DuckLakeTransaction::GetCompactionChanges(DuckLakeSnapshot if (type != compaction.type) { continue; } - auto new_file = GetNewDataFile(compaction.written_file, commit_snapshot, table_id, compaction.row_id_start); + auto new_file = GetNewDataFile(compaction.written_file, commit_state, table_id, compaction.row_id_start); switch (type) { case CompactionType::REWRITE_DELETES: new_file.begin_snapshot = commit_snapshot.snapshot_id; From 1f1d1b9731d3388e1731f6aa2c7697fce748e53e Mon Sep 17 00:00:00 2001 From: Mytherin Date: Tue, 17 Mar 2026 10:29:22 +0100 Subject: [PATCH 17/59] Store mapping id remap in commit state instead of adjusting local changes --- src/storage/ducklake_transaction.cpp | 30 +++++++++++----------------- 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/src/storage/ducklake_transaction.cpp b/src/storage/ducklake_transaction.cpp index 3dd296fe1de..f2b4b82515c 100644 --- a/src/storage/ducklake_transaction.cpp +++ b/src/storage/ducklake_transaction.cpp @@ -854,6 +854,7 @@ struct DuckLakeCommitState { map committed_schemas; map committed_tables; map committed_partition_ids; + map committed_mapping_indexes; void RemapIdentifier(SchemaIndex &schema_id) const { auto entry = committed_schemas.find(schema_id); @@ -876,6 +877,12 @@ struct DuckLakeCommitState { partition_id = entry->second; } } + void RemapMappingIndex(MappingIndex &table_id) const { + auto entry = committed_mapping_indexes.find(table_id); + if (entry != committed_mapping_indexes.end()) { + table_id = entry->second; + } + } SchemaIndex GetSchemaId(DuckLakeSchemaEntry &schema) const { auto schema_id = schema.GetSchemaId(); @@ -1863,13 +1870,14 @@ DuckLakeFileInfo DuckLakeTransaction::GetNewDataFile(const DuckLakeDataFile &fil data_file.file_size_bytes = file.file_size_bytes; data_file.footer_size = file.footer_size; data_file.partition_id = file.partition_id; - commit_state.RemapPartitionId(data_file.partition_id); data_file.encryption_key = file.encryption_key; data_file.row_id_start = row_id_start; data_file.mapping_id = file.mapping_id; data_file.begin_snapshot = file.begin_snapshot; data_file.max_partial_file_snapshot = file.max_partial_file_snapshot; data_file.column_stats = file.column_stats; + commit_state.RemapPartitionId(data_file.partition_id); + commit_state.RemapMappingIndex(data_file.mapping_id); for (auto &partition_entry : file.partition_values) { DuckLakeFilePartitionInfo partition_info; partition_info.partition_column_idx = partition_entry.partition_column_idx; @@ -2039,7 +2047,7 @@ void ConvertNameMapColumn(const DuckLakeNameMapEntry &name_map_entry, MappingInd NewNameMapInfo DuckLakeTransaction::GetNewNameMaps(DuckLakeCommitState &commit_state) { NewNameMapInfo result; - map remap_mapping_index; + auto &committed_mapping_indexes = commit_state.committed_mapping_indexes; for (auto &entry : new_name_maps.name_maps) { // generate a new mapping id auto local_map_id = entry.first; @@ -2061,21 +2069,7 @@ NewNameMapInfo DuckLakeTransaction::GetNewNameMaps(DuckLakeCommitState &commit_s } result.new_column_mappings.push_back(std::move(map_info)); - remap_mapping_index[local_map_id] = new_map_id; - } - // iterate over the data files to point them towards any new mapping ids - lock_guard guard(local_changes.lock); - for (auto &entry : local_changes.changes) { - auto &table_changes = entry.second; - for (auto &data_file : table_changes.new_data_files) { - if (!data_file.mapping_id.IsValid()) { - continue; - } - auto entry = remap_mapping_index.find(data_file.mapping_id); - if (entry != remap_mapping_index.end()) { - data_file.mapping_id = entry->second; - } - } + committed_mapping_indexes[local_map_id] = new_map_id; } return result; } @@ -2249,7 +2243,7 @@ string DuckLakeTransaction::CommitChanges(DuckLakeCommitState &commit_state, CompactionInformation DuckLakeTransaction::GetCompactionChanges(DuckLakeCommitState &commit_state, CompactionType type) { - auto &commit_snapshot = commit_state.commit_snapshot; + auto &commit_snapshot = commit_state.commit_snapshot; CompactionInformation result; for (auto &entry : local_changes.Changes()) { auto table_id = entry.GetTableIndex(); From f005b1134d1ea53a8dfdd879da4f15487982d641 Mon Sep 17 00:00:00 2001 From: Mytherin Date: Tue, 17 Mar 2026 10:48:29 +0100 Subject: [PATCH 18/59] Fix up deletes, and make local changes members private --- src/include/storage/ducklake_transaction.hpp | 9 +- src/storage/ducklake_transaction.cpp | 128 +++++++++++-------- 2 files changed, 83 insertions(+), 54 deletions(-) diff --git a/src/include/storage/ducklake_transaction.hpp b/src/include/storage/ducklake_transaction.hpp index 06f0b4a1535..fcb21e45793 100644 --- a/src/include/storage/ducklake_transaction.hpp +++ b/src/include/storage/ducklake_transaction.hpp @@ -83,8 +83,11 @@ struct LocalTableChanges { void TransactionLocalDelete(ClientContext &context, TableIndex table_id, const string &data_file_path, DuckLakeDeleteFile delete_file); + void AddDeletes(ClientContext &context, TableIndex table_id, vector files); + static void AddDeletesToMap(ClientContext &context, vector new_deletes, + unordered_map> &delete_file_map); - // private: +private: mutable mutex lock; map changes; }; @@ -295,6 +298,8 @@ class DuckLakeTransaction : public Transaction, public enable_shared_from_this GetNewDeleteFiles(const DuckLakeCommitState &commit_state, vector &overwritten_delete_files) const; + DuckLakeDeleteFileInfo GetNewDeleteFile(TableIndex table_id, const DuckLakeCommitState &commit_state, + const DuckLakeDeleteFile &file) const; string UpdateGlobalTableStats(TableIndex table_id, const DuckLakeNewGlobalStats &new_stats); SnapshotAndStats CheckForConflicts(DuckLakeSnapshot transaction_snapshot, const TransactionChangeInformation &changes); @@ -314,8 +319,6 @@ class DuckLakeTransaction : public Transaction, public enable_shared_from_this new_entry); void AlterEntryInternal(DuckLakeViewEntry &old_entry, unique_ptr new_entry); - //! Mutates table_data_changes; caller must hold table_data_changes_lock. - void AddDeletesLocked(TableIndex table_id, vector files); void AddTableChanges(TableIndex table_id, const LocalTableDataChanges &table_changes, TransactionChangeInformation &changes) const; diff --git a/src/storage/ducklake_transaction.cpp b/src/storage/ducklake_transaction.cpp index f2b4b82515c..f65e5a5930b 100644 --- a/src/storage/ducklake_transaction.cpp +++ b/src/storage/ducklake_transaction.cpp @@ -553,6 +553,43 @@ void LocalTableChanges::CleanupFiles(ClientContext &context, TableIndex table_id } } +void LocalTableChanges::AddDeletesToMap(ClientContext &context, vector new_deletes, + unordered_map> &table_delete_map) { + for (auto &file : new_deletes) { + auto &data_file_path = file.data_file_path; + if (data_file_path.empty()) { + throw InternalException("Data file path needs to be set in delete"); + } + if (file.source == DeleteFileSource::FLUSH) { + // If we have a snapshot, this is a flushed delete file, we add it to the rooster + table_delete_map[data_file_path].push_back(std::move(file)); + } else { + // If not is a regular deletion + auto existing_entry = table_delete_map.find(data_file_path); + if (existing_entry != table_delete_map.end() && !existing_entry->second.empty()) { + // If a file already exists we remove it + auto &fs = FileSystem::GetFileSystem(context); + for (auto &old_file : existing_entry->second) { + fs.RemoveFile(old_file.file_name); + } + existing_entry->second.clear(); + } + // We add the new file in + table_delete_map[data_file_path].push_back(std::move(file)); + } + } +} + +void LocalTableChanges::AddDeletes(ClientContext &context, TableIndex table_id, vector files) { + if (files.empty()) { + return; + } + lock_guard guard(lock); + auto &table_changes = changes[table_id]; + auto &table_delete_map = table_changes.new_delete_files; + LocalTableChanges::AddDeletesToMap(context, std::move(files), table_delete_map); +} + LocalTableChangeIterationHelper::LocalTableChangeIterationHelper( mutex &local_changes_lock, const map &changes_p) : lock(local_changes_lock), changes(changes_p) { @@ -855,6 +892,7 @@ struct DuckLakeCommitState { map committed_tables; map committed_partition_ids; map committed_mapping_indexes; + map> local_delete_files; void RemapIdentifier(SchemaIndex &schema_id) const { auto entry = committed_schemas.find(schema_id); @@ -1902,13 +1940,12 @@ NewDataInfo DuckLakeTransaction::GetNewDataFiles(string &batch_query, DuckLakeCo auto &schema = ducklake_catalog.GetSchemaForSnapshot(*this, GetSnapshot()); dl_stats = ducklake_catalog.ConstructStatsMap(*stats, schema); } - lock_guard guard(local_changes.lock); - for (auto &entry : local_changes.changes) { - auto table_id = commit_state.GetTableId(entry.first); + for (auto &entry : local_changes.Changes()) { + auto table_id = commit_state.GetTableId(entry.GetTableIndex()); if (table_id.IsTransactionLocal()) { throw InternalException("Cannot commit transaction local files - these should have been cleaned up before"); } - auto &table_changes = entry.second; + auto &table_changes = entry.GetTableChanges(); if (table_changes.new_data_files.empty() && !table_changes.new_inlined_data) { // no new data - skip this entry continue; @@ -1956,8 +1993,8 @@ NewDataInfo DuckLakeTransaction::GetNewDataFiles(string &batch_query, DuckLakeCo } result.new_files.push_back(std::move(data_file)); } - // write any deletes that were made on top of these transaction-local files - AddDeletesLocked(table_id, std::move(delete_files)); + // add any delete files that were made on top of these transaction-local files + commit_state.local_delete_files[table_id] = std::move(delete_files); if (table_changes.new_inlined_data) { auto &inlined_data = *table_changes.new_inlined_data; @@ -1992,10 +2029,28 @@ NewDataInfo DuckLakeTransaction::GetNewDataFiles(string &batch_query, DuckLakeCo return result; } +DuckLakeDeleteFileInfo DuckLakeTransaction::GetNewDeleteFile(TableIndex table_id, + const DuckLakeCommitState &commit_state, + const DuckLakeDeleteFile &file) const { + DuckLakeDeleteFileInfo delete_file; + delete_file.id = DataFileIndex(commit_state.commit_snapshot.next_file_id++); + delete_file.table_id = table_id; + delete_file.data_file_id = file.data_file_id; + delete_file.path = file.file_name; + delete_file.delete_count = file.delete_count; + delete_file.file_size_bytes = file.file_size_bytes; + delete_file.footer_size = file.footer_size; + delete_file.encryption_key = file.encryption_key; + delete_file.begin_snapshot = file.begin_snapshot; + delete_file.max_snapshot = file.max_snapshot; + return delete_file; +} + vector DuckLakeTransaction::GetNewDeleteFiles(const DuckLakeCommitState &commit_state, vector &overwritten_delete_files) const { vector result; + // handle delete files made to existing files for (auto &entry : local_changes.Changes()) { auto table_id = commit_state.GetTableId(entry.GetTableIndex()); auto &table_changes = entry.GetTableChanges(); @@ -2005,21 +2060,24 @@ DuckLakeTransaction::GetNewDeleteFiles(const DuckLakeCommitState &commit_state, // track the old delete file for deletion from metadata and disk overwritten_delete_files.push_back(file.overwritten_delete_file); } - DuckLakeDeleteFileInfo delete_file; - delete_file.id = DataFileIndex(commit_state.commit_snapshot.next_file_id++); - delete_file.table_id = table_id; - delete_file.data_file_id = file.data_file_id; - delete_file.path = file.file_name; - delete_file.delete_count = file.delete_count; - delete_file.file_size_bytes = file.file_size_bytes; - delete_file.footer_size = file.footer_size; - delete_file.encryption_key = file.encryption_key; - delete_file.begin_snapshot = file.begin_snapshot; - delete_file.max_snapshot = file.max_snapshot; + auto delete_file = GetNewDeleteFile(table_id, commit_state, file); result.push_back(std::move(delete_file)); } } } + // handle any delete files that were added to data files that are ALSO added in this transaction + for (auto &entry : commit_state.local_delete_files) { + auto table_id = commit_state.GetTableId(entry.first); + for (auto &file : entry.second) { + if (file.overwritten_delete_file.delete_file_id.IsValid()) { + throw InternalException("Local delete files should not overwrite files"); + // track the old delete file for deletion from metadata and disk + overwritten_delete_files.push_back(file.overwritten_delete_file); + } + auto delete_file = GetNewDeleteFile(table_id, commit_state, file); + result.push_back(std::move(delete_file)); + } + } return result; } @@ -2617,40 +2675,8 @@ DuckLakeTransaction::GetNewInlinedFileDeletes(DuckLakeCommitState &commit_state) } void DuckLakeTransaction::AddDeletes(TableIndex table_id, vector files) { - if (files.empty()) { - return; - } - lock_guard guard(local_changes.lock); - AddDeletesLocked(table_id, std::move(files)); -} - -void DuckLakeTransaction::AddDeletesLocked(TableIndex table_id, vector files) { - auto &table_changes = local_changes.changes[table_id]; - auto &table_delete_map = table_changes.new_delete_files; - for (auto &file : files) { - auto &data_file_path = file.data_file_path; - if (data_file_path.empty()) { - throw InternalException("Data file path needs to be set in delete"); - } - if (file.source == DeleteFileSource::FLUSH) { - // If we have a snapshot, this is a flushed delete file, we add it to the rooster - table_delete_map[data_file_path].push_back(std::move(file)); - } else { - // If not is a regular deletion - auto existing_entry = table_delete_map.find(data_file_path); - if (existing_entry != table_delete_map.end() && !existing_entry->second.empty()) { - // If a file already exists we remove it - auto context_ref = context.lock(); - auto &fs = FileSystem::GetFileSystem(*context_ref); - for (auto &old_file : existing_entry->second) { - fs.RemoveFile(old_file.file_name); - } - existing_entry->second.clear(); - } - // We add the new file in - table_delete_map[data_file_path].push_back(std::move(file)); - } - } + auto context_ref = context.lock(); + local_changes.AddDeletes(*context_ref, table_id, std::move(files)); } void DuckLakeTransaction::AddCompaction(TableIndex table_id, DuckLakeCompactionEntry entry) { From 9e8795e16aa06c4a3f52b3ae4661391dbda20ef9 Mon Sep 17 00:00:00 2001 From: Mytherin Date: Tue, 17 Mar 2026 11:41:08 +0100 Subject: [PATCH 19/59] Add file to ignore list --- test/configs/sqlite.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/configs/sqlite.json b/test/configs/sqlite.json index e22a35aa9ee..af0d2acc4ad 100644 --- a/test/configs/sqlite.json +++ b/test/configs/sqlite.json @@ -109,6 +109,12 @@ "paths": [ "test/sql/deletion_inlining/test_deletion_inlining_concurrent.test" ] + }, + { + "reason": "FIXME: old sqlite extension has issues with this query, can be removed after v1.5.1", + "paths": [ + "test/sql/metadata/ducklake_ui_catalog_query.test" + ] } ] } \ No newline at end of file From d6a463092f394a255349ffdf1d12e0a234f0c3c0 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Tue, 17 Mar 2026 20:01:15 -0300 Subject: [PATCH 20/59] Fix issue when parsing table macros --- src/storage/ducklake_transaction_changes.cpp | 2 +- .../macros/test_snapshots_after_macro.test | 36 +++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 test/sql/macros/test_snapshots_after_macro.test diff --git a/src/storage/ducklake_transaction_changes.cpp b/src/storage/ducklake_transaction_changes.cpp index 5a0a61c524d..5efc1f62500 100644 --- a/src/storage/ducklake_transaction_changes.cpp +++ b/src/storage/ducklake_transaction_changes.cpp @@ -147,7 +147,7 @@ SnapshotChangeInformation SnapshotChangeInformation::ParseChangesMade(const stri } case ChangeType::CREATED_TABLE_MACRO: { auto catalog_value = DuckLakeUtil::ParseCatalogEntry(entry.change_value); - result.created_scalar_macros[catalog_value.schema].insert( + result.created_table_macros[catalog_value.schema].insert( make_pair(std::move(catalog_value.name), "table_macro")); break; } diff --git a/test/sql/macros/test_snapshots_after_macro.test b/test/sql/macros/test_snapshots_after_macro.test new file mode 100644 index 00000000000..d087b9da0ca --- /dev/null +++ b/test/sql/macros/test_snapshots_after_macro.test @@ -0,0 +1,36 @@ +# name: test/sql/macros/test_snapshots_after_macro.test +# description: test that snapshots() works after creating a macro +# group: [macro] + +require ducklake + +require parquet + +test-env DUCKLAKE_CONNECTION __TEST_DIR__/{UUID}.db + +test-env DATA_PATH __TEST_DIR__ + +statement ok +ATTACH 'ducklake:${DUCKLAKE_CONNECTION}' AS mylake (DATA_PATH '${DATA_PATH}/test_snapshots_after_macro') + +statement ok +use mylake + +statement ok +CREATE TABLE tbl (id INT, val VARCHAR) + +statement ok +INSERT INTO tbl VALUES (1, 'a'), (2, 'b') + +query I +SELECT count(*) FROM mylake.snapshots() +---- +3 + +statement ok +CREATE MACRO testmacro() AS TABLE (SELECT * FROM tbl) + +query I +SELECT count(*) FROM mylake.snapshots() +---- +4 From 7addd33f5f5cf253a8f808e8b3105cfd04b475f8 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Wed, 18 Mar 2026 14:16:49 +0100 Subject: [PATCH 21/59] Fix BeginSanpshot for tables with different schema versions --- src/functions/ducklake_flush_inlined_data.cpp | 5 +- src/include/storage/ducklake_catalog.hpp | 1 + .../storage/ducklake_metadata_manager.hpp | 1 + src/storage/ducklake_catalog.cpp | 6 ++ src/storage/ducklake_metadata_manager.cpp | 15 +++++ src/storage/ducklake_multi_file_reader.cpp | 3 +- ...nsert_select_alter_column_persistence.test | 56 +++++++++++++++++++ 7 files changed, 84 insertions(+), 3 deletions(-) create mode 100644 test/sql/alter/test_insert_select_alter_column_persistence.test diff --git a/src/functions/ducklake_flush_inlined_data.cpp b/src/functions/ducklake_flush_inlined_data.cpp index 703ac1b4c62..50e8023f692 100644 --- a/src/functions/ducklake_flush_inlined_data.cpp +++ b/src/functions/ducklake_flush_inlined_data.cpp @@ -277,8 +277,9 @@ DuckLakeDataFlusher::DuckLakeDataFlusher(ClientContext &context, DuckLakeCatalog unique_ptr DuckLakeDataFlusher::GenerateFlushCommand() { // get the table entry at the specified snapshot - DuckLakeSnapshot snapshot(catalog.GetBeginSnapshotForTable(table_id, transaction), inlined_table.schema_version, 0, - 0); + DuckLakeSnapshot snapshot( + catalog.GetBeginSnapshotForSchemaVersion(table_id, inlined_table.schema_version, transaction), + inlined_table.schema_version, 0, 0); auto entry = catalog.GetEntryById(transaction, snapshot, table_id); if (!entry) { diff --git a/src/include/storage/ducklake_catalog.hpp b/src/include/storage/ducklake_catalog.hpp index b95374d68e4..8edcaa5f5a6 100644 --- a/src/include/storage/ducklake_catalog.hpp +++ b/src/include/storage/ducklake_catalog.hpp @@ -164,6 +164,7 @@ class DuckLakeCatalog : public Catalog { optional_ptr TryGetMappingById(DuckLakeTransaction &transaction, MappingIndex mapping_id); MappingIndex TryGetCompatibleNameMap(DuckLakeTransaction &transaction, const DuckLakeNameMap &name_map); idx_t GetBeginSnapshotForTable(TableIndex table_id, DuckLakeTransaction &transaction); + idx_t GetBeginSnapshotForSchemaVersion(TableIndex table_id, idx_t schema_version, DuckLakeTransaction &transaction); static unique_ptr ConstructStatsMap(vector &global_stats, DuckLakeCatalogSet &schema); diff --git a/src/include/storage/ducklake_metadata_manager.hpp b/src/include/storage/ducklake_metadata_manager.hpp index 03ca7e1b0e9..d955f5de346 100644 --- a/src/include/storage/ducklake_metadata_manager.hpp +++ b/src/include/storage/ducklake_metadata_manager.hpp @@ -145,6 +145,7 @@ class DuckLakeMetadataManager { DuckLakeSnapshot snapshot, DuckLakeFileSizeOptions options); virtual idx_t GetBeginSnapshotForTable(TableIndex table_id); + virtual idx_t GetBeginSnapshotForSchemaVersion(TableIndex table_id, idx_t schema_version); virtual idx_t GetNetDataFileRowCount(TableIndex table_id, DuckLakeSnapshot snapshot); virtual idx_t GetNetInlinedRowCount(const string &inlined_table_name, DuckLakeSnapshot snapshot); virtual vector GetOldFilesForCleanup(const string &filter); diff --git a/src/storage/ducklake_catalog.cpp b/src/storage/ducklake_catalog.cpp index 8f76567fc2d..175385f6db9 100644 --- a/src/storage/ducklake_catalog.cpp +++ b/src/storage/ducklake_catalog.cpp @@ -185,6 +185,12 @@ idx_t DuckLakeCatalog::GetBeginSnapshotForTable(TableIndex table_id, DuckLakeTra return metadata_manager.GetBeginSnapshotForTable(table_id); } +idx_t DuckLakeCatalog::GetBeginSnapshotForSchemaVersion(TableIndex table_id, idx_t schema_version, + DuckLakeTransaction &transaction) { + auto &metadata_manager = transaction.GetMetadataManager(); + return metadata_manager.GetBeginSnapshotForSchemaVersion(table_id, schema_version); +} + DuckLakeCatalogSet &DuckLakeCatalog::GetSchemaForSnapshot(DuckLakeTransaction &transaction, DuckLakeSnapshot snapshot) { lock_guard guard(schemas_lock); auto entry = schemas.find(snapshot.schema_version); diff --git a/src/storage/ducklake_metadata_manager.cpp b/src/storage/ducklake_metadata_manager.cpp index 8c7e8ed1a4c..d355afbc1dd 100644 --- a/src/storage/ducklake_metadata_manager.cpp +++ b/src/storage/ducklake_metadata_manager.cpp @@ -403,6 +403,21 @@ WHERE table_id = {TABLE_ID})"; throw InternalException("Table %llu does not exist", table_id.index); } +idx_t DuckLakeMetadataManager::GetBeginSnapshotForSchemaVersion(TableIndex table_id, idx_t schema_version) { + string query = R"( +SELECT begin_snapshot +FROM {METADATA_CATALOG}.ducklake_schema_versions +WHERE table_id = {TABLE_ID} AND schema_version = {SCHEMA_VERSION})"; + query = StringUtil::Replace(query, "{TABLE_ID}", to_string(table_id.index)); + query = StringUtil::Replace(query, "{SCHEMA_VERSION}", to_string(schema_version)); + auto result = transaction.Query(query); + for (auto &row : *result) { + return row.GetValue(0); + } + // We need to fallback to GetBeginSnapshotForTable if this table doesnt have an alter yet + return GetBeginSnapshotForTable(table_id); +} + idx_t DuckLakeMetadataManager::GetNetDataFileRowCount(TableIndex table_id, DuckLakeSnapshot snapshot) { // Compute sum(record_count) - sum(delete_count) - inlined_deletions in a single query // Delete files are only counted if their corresponding data file is still visible diff --git a/src/storage/ducklake_multi_file_reader.cpp b/src/storage/ducklake_multi_file_reader.cpp index 21627dd769a..710110ac742 100644 --- a/src/storage/ducklake_multi_file_reader.cpp +++ b/src/storage/ducklake_multi_file_reader.cpp @@ -362,7 +362,8 @@ shared_ptr DuckLakeMultiFileReader::TryCreateInlinedDataReader(c // read the table at the specified version auto transaction = read_info.GetTransaction(); auto &catalog = transaction->GetCatalog(); - DuckLakeSnapshot snapshot(catalog.GetBeginSnapshotForTable(read_info.table.GetTableId(), *transaction), + DuckLakeSnapshot snapshot(catalog.GetBeginSnapshotForSchemaVersion(read_info.table.GetTableId(), + schema_version.GetIndex(), *transaction), schema_version.GetIndex(), 0, 0); auto entry = catalog.GetEntryById(*transaction, snapshot, read_info.table.GetTableId()); if (!entry) { diff --git a/test/sql/alter/test_insert_select_alter_column_persistence.test b/test/sql/alter/test_insert_select_alter_column_persistence.test new file mode 100644 index 00000000000..a69f7840516 --- /dev/null +++ b/test/sql/alter/test_insert_select_alter_column_persistence.test @@ -0,0 +1,56 @@ +# name: test/sql/alter/test_insert_select_alter_column_persistence.test +# description: test that INSERT ... SELECT on ALTER-added column is persisted across DETACH/ATTACH +# group: [alter] + +require ducklake + +require parquet + +statement ok +ATTACH 'ducklake:__TEST_DIR__/alter_insert_select.db' AS dl (DATA_PATH '__TEST_DIR__/alter_insert_select') + +statement ok +USE dl + +statement ok +CREATE TABLE t (id INT, name VARCHAR) + +statement ok +INSERT INTO t VALUES (1, 'a') + +statement ok +ALTER TABLE t ADD COLUMN ts TIMESTAMPTZ + +statement ok +CREATE TABLE src (id INT, name VARCHAR) + +statement ok +INSERT INTO src VALUES (2, 'b') + +statement ok +INSERT INTO t (id, name, ts) SELECT id, name, '2026-01-01'::TIMESTAMPTZ AS ts FROM src + +query IIT +SELECT * FROM t ORDER BY id +---- +1 a NULL +2 b 2026-01-01 00:00:00+00 + +# Detach and reattach +statement ok +USE memory + +statement ok +DETACH dl + +statement ok +ATTACH 'ducklake:__TEST_DIR__/alter_insert_select.db' AS dl (DATA_PATH '__TEST_DIR__/alter_insert_select') + +statement ok +USE dl + +query IIT +SELECT * FROM t ORDER BY id +---- +1 a NULL +2 b 2026-01-01 00:00:00+00 From bd9f5970acd2a3af847d9bb434b94bb87f67d19e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Rafael?= Date: Mon, 16 Mar 2026 16:03:56 +0000 Subject: [PATCH 22/59] Avoid flushing fully deleted inlined files --- src/functions/ducklake_flush_inlined_data.cpp | 18 ++++++ ..._deletes_full_file_delete_after_flush.test | 62 +++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 test/sql/compaction/rewrite_deletes_full_file_delete_after_flush.test diff --git a/src/functions/ducklake_flush_inlined_data.cpp b/src/functions/ducklake_flush_inlined_data.cpp index 703ac1b4c62..be589daea1c 100644 --- a/src/functions/ducklake_flush_inlined_data.cpp +++ b/src/functions/ducklake_flush_inlined_data.cpp @@ -108,6 +108,23 @@ SinkResultType DuckLakeFlushData::Sink(ExecutionContext &context, DataChunk &chu //===--------------------------------------------------------------------===// using DeletesPerFile = unordered_map>; +static void RemoveFullyDeletedWrittenFiles(ClientContext &context, vector &written_files, + DeletesPerFile &deletes_per_file) { + auto &fs = FileSystem::GetFileSystem(context); + vector remaining_files; + remaining_files.reserve(written_files.size()); + for (auto &written_file : written_files) { + auto delete_entry = deletes_per_file.find(written_file.file_name); + if (delete_entry != deletes_per_file.end() && delete_entry->second.size() >= written_file.row_count) { + fs.RemoveFile(written_file.file_name); + deletes_per_file.erase(delete_entry); + continue; + } + remaining_files.push_back(std::move(written_file)); + } + written_files = std::move(remaining_files); +} + static DeletesPerFile GroupDeletesByFile(QueryResult &deleted_rows_result, vector &written_files, vector &file_start_row_ids) { D_ASSERT(std::is_sorted(file_start_row_ids.begin(), file_start_row_ids.end())); @@ -167,6 +184,7 @@ SinkFinalizeType DuckLakeFlushData::Finalize(Pipeline &pipeline, Event &event, C auto deletes_per_file = GroupDeletesByFile(*deleted_rows_result, global_state.written_files, file_start_row_ids); + RemoveFullyDeletedWrittenFiles(context, global_state.written_files, deletes_per_file); if (!deletes_per_file.empty()) { auto &fs = FileSystem::GetFileSystem(context); diff --git a/test/sql/compaction/rewrite_deletes_full_file_delete_after_flush.test b/test/sql/compaction/rewrite_deletes_full_file_delete_after_flush.test new file mode 100644 index 00000000000..d8d6d216d62 --- /dev/null +++ b/test/sql/compaction/rewrite_deletes_full_file_delete_after_flush.test @@ -0,0 +1,62 @@ +# name: test/sql/compaction/rewrite_deletes_full_file_delete_after_flush.test +# description: rewrite_deletes should handle a fully deleted file created by ducklake_flush_inlined_data +# group: [compaction] + +require ducklake + +require parquet + +test-env DUCKLAKE_CONNECTION __TEST_DIR__/{UUID}.db + +test-env DATA_PATH __TEST_DIR__ + + +statement ok +ATTACH 'ducklake:${DUCKLAKE_CONNECTION}' AS ducklake (DATA_PATH '${DATA_PATH}/rewrite_deletes_full_file_delete_after_flush', DATA_INLINING_ROW_LIMIT 100, METADATA_CATALOG 'ducklake_meta') + +statement ok +USE ducklake; + +statement ok +CREATE TABLE t (id INTEGER); + +statement ok +INSERT INTO t SELECT i FROM range(20) tbl(i); + +query I +SELECT COUNT(*) FROM ducklake_meta.ducklake_data_file WHERE end_snapshot IS NULL; +---- +0 + +statement ok +DELETE FROM t WHERE id < 20; + +query I +SELECT COUNT(*) FROM t; +---- +0 + +query I +SELECT COUNT(*) FROM ducklake_flush_inlined_data('ducklake'); +---- +0 + +query II +SELECT COUNT(*), COALESCE(SUM(record_count), 0) FROM ducklake_meta.ducklake_data_file WHERE end_snapshot IS NULL; +---- +0 0 + +query II +SELECT COUNT(*), COALESCE(SUM(delete_count), 0) FROM ducklake_meta.ducklake_delete_file WHERE end_snapshot IS NULL; +---- +0 0 + +query I +SELECT COUNT(*) FROM ducklake_rewrite_data_files('ducklake', 't', delete_threshold => 0); +---- +0 + +query I +SELECT COUNT(*) FROM t; +---- +0 From e44b6e8693bb1e66c39ef11fb01efca2d8e6883d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Rafael?= Date: Wed, 18 Mar 2026 09:41:56 +0000 Subject: [PATCH 23/59] Validate timetravel --- ..._deletes_full_file_delete_after_flush.test | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/test/sql/compaction/rewrite_deletes_full_file_delete_after_flush.test b/test/sql/compaction/rewrite_deletes_full_file_delete_after_flush.test index d8d6d216d62..bc7560fd94c 100644 --- a/test/sql/compaction/rewrite_deletes_full_file_delete_after_flush.test +++ b/test/sql/compaction/rewrite_deletes_full_file_delete_after_flush.test @@ -28,14 +28,35 @@ SELECT COUNT(*) FROM ducklake_meta.ducklake_data_file WHERE end_snapshot IS NULL ---- 0 +statement ok +SET VARIABLE v_after_insert = (SELECT max(snapshot_id) FROM ducklake_snapshots('ducklake')); + +query I +SELECT COUNT(*) FROM t AT (VERSION => getvariable('v_after_insert')) +---- +20 + statement ok DELETE FROM t WHERE id < 20; +statement ok +SET VARIABLE v_after_delete = (SELECT max(snapshot_id) FROM ducklake_snapshots('ducklake')); + query I SELECT COUNT(*) FROM t; ---- 0 +query I +SELECT COUNT(*) FROM t AT (VERSION => getvariable('v_after_insert')) +---- +20 + +query I +SELECT COUNT(*) FROM t AT (VERSION => getvariable('v_after_delete')) +---- +0 + query I SELECT COUNT(*) FROM ducklake_flush_inlined_data('ducklake'); ---- @@ -56,6 +77,16 @@ SELECT COUNT(*) FROM ducklake_rewrite_data_files('ducklake', 't', delete_thresho ---- 0 +query I +SELECT COUNT(*) FROM t AT (VERSION => getvariable('v_after_insert')) +---- +20 + +query I +SELECT COUNT(*) FROM t AT (VERSION => getvariable('v_after_delete')) +---- +0 + query I SELECT COUNT(*) FROM t; ---- From 0f87923d0b7813583ed46d3814dc665c1cef47e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Rafael?= Date: Wed, 18 Mar 2026 09:43:34 +0000 Subject: [PATCH 24/59] Revert "Avoid flushing fully deleted inlined files" This reverts commit 644f808a443b884f2a03faff98c62e0928c9f38b. --- src/functions/ducklake_flush_inlined_data.cpp | 18 ------------------ ...e_deletes_full_file_delete_after_flush.test | 10 ++++++++++ 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/src/functions/ducklake_flush_inlined_data.cpp b/src/functions/ducklake_flush_inlined_data.cpp index be589daea1c..703ac1b4c62 100644 --- a/src/functions/ducklake_flush_inlined_data.cpp +++ b/src/functions/ducklake_flush_inlined_data.cpp @@ -108,23 +108,6 @@ SinkResultType DuckLakeFlushData::Sink(ExecutionContext &context, DataChunk &chu //===--------------------------------------------------------------------===// using DeletesPerFile = unordered_map>; -static void RemoveFullyDeletedWrittenFiles(ClientContext &context, vector &written_files, - DeletesPerFile &deletes_per_file) { - auto &fs = FileSystem::GetFileSystem(context); - vector remaining_files; - remaining_files.reserve(written_files.size()); - for (auto &written_file : written_files) { - auto delete_entry = deletes_per_file.find(written_file.file_name); - if (delete_entry != deletes_per_file.end() && delete_entry->second.size() >= written_file.row_count) { - fs.RemoveFile(written_file.file_name); - deletes_per_file.erase(delete_entry); - continue; - } - remaining_files.push_back(std::move(written_file)); - } - written_files = std::move(remaining_files); -} - static DeletesPerFile GroupDeletesByFile(QueryResult &deleted_rows_result, vector &written_files, vector &file_start_row_ids) { D_ASSERT(std::is_sorted(file_start_row_ids.begin(), file_start_row_ids.end())); @@ -184,7 +167,6 @@ SinkFinalizeType DuckLakeFlushData::Finalize(Pipeline &pipeline, Event &event, C auto deletes_per_file = GroupDeletesByFile(*deleted_rows_result, global_state.written_files, file_start_row_ids); - RemoveFullyDeletedWrittenFiles(context, global_state.written_files, deletes_per_file); if (!deletes_per_file.empty()) { auto &fs = FileSystem::GetFileSystem(context); diff --git a/test/sql/compaction/rewrite_deletes_full_file_delete_after_flush.test b/test/sql/compaction/rewrite_deletes_full_file_delete_after_flush.test index bc7560fd94c..be01a263f99 100644 --- a/test/sql/compaction/rewrite_deletes_full_file_delete_after_flush.test +++ b/test/sql/compaction/rewrite_deletes_full_file_delete_after_flush.test @@ -62,6 +62,16 @@ SELECT COUNT(*) FROM ducklake_flush_inlined_data('ducklake'); ---- 0 +query I +SELECT COUNT(*) FROM t AT (VERSION => getvariable('v_after_insert')) +---- +20 + +query I +SELECT COUNT(*) FROM t AT (VERSION => getvariable('v_after_delete')) +---- +0 + query II SELECT COUNT(*), COALESCE(SUM(record_count), 0) FROM ducklake_meta.ducklake_data_file WHERE end_snapshot IS NULL; ---- From 95f20cbc0dfe66c1dccd9a58dfc4a3c68c1018ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Rafael?= Date: Wed, 18 Mar 2026 10:43:23 +0000 Subject: [PATCH 25/59] Allow for delete compactions to return 0 new files --- .../ducklake_compaction_functions.cpp | 17 ++++- src/storage/ducklake_metadata_manager.cpp | 16 +++-- src/storage/ducklake_transaction.cpp | 71 ++++++++++++------- ..._deletes_full_file_delete_after_flush.test | 12 ++-- 4 files changed, 73 insertions(+), 43 deletions(-) diff --git a/src/functions/ducklake_compaction_functions.cpp b/src/functions/ducklake_compaction_functions.cpp index 571671a55dc..d63d75b1a07 100644 --- a/src/functions/ducklake_compaction_functions.cpp +++ b/src/functions/ducklake_compaction_functions.cpp @@ -93,11 +93,17 @@ SourceResultType DuckLakeCompaction::GetDataInternal(ExecutionContext &context, } source_state.returned_result = true; + if (!this->sink_state) { + throw InternalException("DuckLakeCompaction - missing sink state while producing result"); + } + auto &gstate = this->sink_state->Cast(); + auto files_created = gstate.written_files.size(); + chunk.SetCardinality(1); chunk.SetValue(0, 0, Value(table.schema.name)); chunk.SetValue(1, 0, Value(table.name)); chunk.SetValue(2, 0, Value::BIGINT(static_cast(source_files.size()))); - chunk.SetValue(3, 0, Value::BIGINT(1)); // Each compaction creates 1 output file + chunk.SetValue(3, 0, Value::BIGINT(static_cast(files_created))); return SourceResultType::FINISHED; } @@ -121,7 +127,10 @@ SinkFinalizeType DuckLakeCompaction::Finalize(Pipeline &pipeline, Event &event, OperatorSinkFinalizeInput &input) const { auto &global_state = input.global_state.Cast(); - if (global_state.written_files.size() != 1) { + if (global_state.written_files.size() > 1) { + throw InternalException("DuckLakeCompaction - expected at most a single output file"); + } + if (global_state.written_files.empty() && type != CompactionType::REWRITE_DELETES) { throw InternalException("DuckLakeCompaction - expected a single output file"); } // set the partition values correctly @@ -137,7 +146,9 @@ SinkFinalizeType DuckLakeCompaction::Finalize(Pipeline &pipeline, Event &event, DuckLakeCompactionEntry compaction_entry; compaction_entry.row_id_start = row_id_start; compaction_entry.source_files = source_files; - compaction_entry.written_file = global_state.written_files[0]; + if (!global_state.written_files.empty()) { + compaction_entry.written_file = global_state.written_files[0]; + } compaction_entry.type = type; auto &transaction = DuckLakeTransaction::Get(context, global_state.table.catalog); diff --git a/src/storage/ducklake_metadata_manager.cpp b/src/storage/ducklake_metadata_manager.cpp index 8c7e8ed1a4c..c64586b79ef 100644 --- a/src/storage/ducklake_metadata_manager.cpp +++ b/src/storage/ducklake_metadata_manager.cpp @@ -4049,13 +4049,15 @@ string DuckLakeMetadataManager::WriteDeleteRewrites(const vector merged_max_partial_snapshot.GetIndex()) { - merged_max_partial_snapshot = file_max_snapshot; + bool has_new_file = !compaction.written_file.file_name.empty(); + DuckLakeFileInfo new_file; + + if (!has_new_file) { + if (type != CompactionType::REWRITE_DELETES) { + throw InternalException("Compaction error - expected output file for non-rewrite compaction"); + } + } else { + new_file = GetNewDataFile(compaction.written_file, commit_state, table_id, compaction.row_id_start); + switch (type) { + case CompactionType::REWRITE_DELETES: + new_file.begin_snapshot = commit_snapshot.snapshot_id; + break; + case CompactionType::MERGE_ADJACENT_TABLES: { + // For MERGE_ADJACENT_TABLES, track the max partial snapshot across all source files + optional_idx merged_max_partial_snapshot; + idx_t first_begin_snapshot = compaction.source_files[0].file.begin_snapshot; + for (auto &compacted_file : compaction.source_files) { + idx_t file_max_snapshot = compacted_file.max_partial_file_snapshot.IsValid() + ? compacted_file.max_partial_file_snapshot.GetIndex() + : compacted_file.file.begin_snapshot; + if (!merged_max_partial_snapshot.IsValid() || + file_max_snapshot > merged_max_partial_snapshot.GetIndex()) { + merged_max_partial_snapshot = file_max_snapshot; + } } + // Use the first source file's begin_snapshot for proper time travel support + new_file.begin_snapshot = first_begin_snapshot; + if (compaction.source_files.size() > 1) { + new_file.max_partial_file_snapshot = merged_max_partial_snapshot; + } + break; } - // Use the first source file's begin_snapshot for proper time travel support - new_file.begin_snapshot = first_begin_snapshot; - if (compaction.source_files.size() > 1) { - new_file.max_partial_file_snapshot = merged_max_partial_snapshot; + default: + throw InternalException("DuckLakeTransaction::GetCompactionChanges Compaction type is invalid"); } - break; - } - default: - throw InternalException("DuckLakeTransaction::GetCompactionChanges Compaction type is invalid"); } idx_t row_id_limit = 0; @@ -2348,7 +2357,9 @@ CompactionInformation DuckLakeTransaction::GetCompactionChanges(DuckLakeCommitSt DuckLakeCompactedFileInfo file_info; file_info.path = compacted_file.file.data.path; file_info.source_id = compacted_file.file.id; - file_info.new_id = new_file.id; + if (has_new_file) { + file_info.new_id = new_file.id; + } if (!compacted_file.delete_files.empty()) { file_info.delete_file_path = compacted_file.delete_files.back().data.path; @@ -2358,12 +2369,18 @@ CompactionInformation DuckLakeTransaction::GetCompactionChanges(DuckLakeCommitSt file_info.delete_file_start_snapshot = commit_snapshot.snapshot_id; file_info.delete_file_end_snapshot = compacted_file.delete_files.back().end_snapshot; } - if (row_id_limit > new_file.row_count) { + if (has_new_file && row_id_limit > new_file.row_count) { throw InternalException("Compaction error - row id limit is larger than the row count of the file"); } result.compacted_files.push_back(std::move(file_info)); } - result.new_files.push_back(std::move(new_file)); + if (!has_new_file && row_id_limit != 0) { + throw InternalException( + "Compaction error - rewrite compaction without output file must fully delete source files"); + } + if (has_new_file) { + result.new_files.push_back(std::move(new_file)); + } } } return result; diff --git a/test/sql/compaction/rewrite_deletes_full_file_delete_after_flush.test b/test/sql/compaction/rewrite_deletes_full_file_delete_after_flush.test index be01a263f99..105fb8217bd 100644 --- a/test/sql/compaction/rewrite_deletes_full_file_delete_after_flush.test +++ b/test/sql/compaction/rewrite_deletes_full_file_delete_after_flush.test @@ -60,7 +60,7 @@ SELECT COUNT(*) FROM t AT (VERSION => getvariable('v_after_delete')) query I SELECT COUNT(*) FROM ducklake_flush_inlined_data('ducklake'); ---- -0 +1 query I SELECT COUNT(*) FROM t AT (VERSION => getvariable('v_after_insert')) @@ -75,17 +75,17 @@ SELECT COUNT(*) FROM t AT (VERSION => getvariable('v_after_delete')) query II SELECT COUNT(*), COALESCE(SUM(record_count), 0) FROM ducklake_meta.ducklake_data_file WHERE end_snapshot IS NULL; ---- -0 0 +1 20 query II SELECT COUNT(*), COALESCE(SUM(delete_count), 0) FROM ducklake_meta.ducklake_delete_file WHERE end_snapshot IS NULL; ---- -0 0 +1 20 -query I -SELECT COUNT(*) FROM ducklake_rewrite_data_files('ducklake', 't', delete_threshold => 0); +query II +SELECT COUNT(*), SUM(files_created) FROM ducklake_rewrite_data_files('ducklake', 't', delete_threshold => 0); ---- -0 +1 0 query I SELECT COUNT(*) FROM t AT (VERSION => getvariable('v_after_insert')) From 824329a797f1214afa5bffad7151a31022163c2c Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Wed, 18 Mar 2026 16:57:55 +0100 Subject: [PATCH 26/59] Add reinterpret from BYTEA to Varchar for postgres inlined data --- .../postgres_metadata_manager.hpp | 2 + .../postgres_metadata_manager.cpp | 41 +++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/src/include/metadata_manager/postgres_metadata_manager.hpp b/src/include/metadata_manager/postgres_metadata_manager.hpp index 187ac267229..cb1a221925a 100644 --- a/src/include/metadata_manager/postgres_metadata_manager.hpp +++ b/src/include/metadata_manager/postgres_metadata_manager.hpp @@ -27,6 +27,8 @@ class PostgresMetadataManager : public DuckLakeMetadataManager { } string GetColumnTypeInternal(const LogicalType &type) override; + shared_ptr TransformInlinedData(QueryResult &result, + const vector &expected_types) override; unique_ptr Execute(DuckLakeSnapshot snapshot, string &query) override; diff --git a/src/metadata_manager/postgres_metadata_manager.cpp b/src/metadata_manager/postgres_metadata_manager.cpp index 48a8380b01c..6c5150b4edf 100644 --- a/src/metadata_manager/postgres_metadata_manager.cpp +++ b/src/metadata_manager/postgres_metadata_manager.cpp @@ -127,4 +127,45 @@ string PostgresMetadataManager::GetLatestSnapshotQuery() const { )"; } +// We need a specialized function here to do a reinterpret for postgres from BLOB to VARCHAR +shared_ptr +PostgresMetadataManager::TransformInlinedData(QueryResult &result, const vector &expected_types) { + bool needs_reinterpret = false; + if (!expected_types.empty()) { + D_ASSERT(expected_types.size() == result.types.size()); + for (idx_t i = 0; i < expected_types.size(); i++) { + if (result.types[i] != expected_types[i]) { + D_ASSERT(result.types[i].id() == LogicalTypeId::BLOB && + expected_types[i].id() == LogicalTypeId::VARCHAR); + needs_reinterpret = true; + } + } + } + if (!needs_reinterpret) { + return DuckLakeMetadataManager::TransformInlinedData(result, expected_types); + } + + if (result.HasError()) { + result.GetErrorObject().Throw("Failed to read inlined data from DuckLake: "); + } + auto context = transaction.context.lock(); + auto data = make_uniq(*context, expected_types); + DataChunk reinterpret_chunk; + reinterpret_chunk.Initialize(*context, expected_types); + while (true) { + auto chunk = result.Fetch(); + if (!chunk) { + break; + } + for (idx_t i = 0; i < expected_types.size(); i++) { + reinterpret_chunk.data[i].Reinterpret(chunk->data[i]); + } + reinterpret_chunk.SetCardinality(chunk->size()); + data->Append(reinterpret_chunk); + } + auto inlined_data = make_shared_ptr(); + inlined_data->data = std::move(data); + return inlined_data; +} + } // namespace duckdb From 65c48cd2acfbd0baa474dc73f1f87200f4e8c683 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Wed, 18 Mar 2026 19:57:14 +0100 Subject: [PATCH 27/59] Do value comparison for boolean --- src/include/storage/ducklake_stats.hpp | 2 +- test/sql/transaction/update_null_column.test | 47 ++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 test/sql/transaction/update_null_column.test diff --git a/src/include/storage/ducklake_stats.hpp b/src/include/storage/ducklake_stats.hpp index 75fb45ef82b..04e9934535c 100644 --- a/src/include/storage/ducklake_stats.hpp +++ b/src/include/storage/ducklake_stats.hpp @@ -15,7 +15,7 @@ class BaseStatistics; //! Returns true for types that require value-based (not lexicographic string) comparison for min/max stats inline bool RequiresValueComparison(const LogicalType &type) { - return type.IsNumeric() || type.IsTemporal(); + return type.IsNumeric() || type.IsTemporal() || type.id() == LogicalTypeId::BOOLEAN; } struct DuckLakeColumnStats; diff --git a/test/sql/transaction/update_null_column.test b/test/sql/transaction/update_null_column.test new file mode 100644 index 00000000000..15ee48c69c1 --- /dev/null +++ b/test/sql/transaction/update_null_column.test @@ -0,0 +1,47 @@ +# name: test/sql/transaction/update_null_column.test +# description: test that boolean stats pruning works correctly after update/insert +# group: [transaction] + +require ducklake + +require parquet + +test-env DUCKLAKE_CONNECTION __TEST_DIR__/{UUID}.db + +test-env DATA_PATH __TEST_DIR__ + +statement ok +ATTACH 'ducklake:${DUCKLAKE_CONNECTION}' AS ducklake (DATA_PATH '${DATA_PATH}/update_null_column/', DATA_INLINING_ROW_LIMIT 0); + +statement ok +USE ducklake; + +statement ok +CREATE TABLE test (active BOOLEAN); + +statement ok +INSERT INTO test (active) VALUES (false); + +query I +SELECT COUNT(*) FROM test WHERE active = false; +---- +1 + +statement ok +CREATE TABLE t (id BIGINT, tag VARCHAR); + +statement ok +INSERT INTO t (id) VALUES (1); + +statement ok +UPDATE t SET tag = 'new'; + +query IT +SELECT * FROM t; +---- +1 new + +query IT +SELECT * FROM t WHERE tag = 'new'; +---- +1 new From 79d68868b72f807bdb593aa5872781ff2b391c2b Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 19 Mar 2026 11:23:08 +0100 Subject: [PATCH 28/59] Reset binding of views if binding fails --- src/include/storage/ducklake_view_entry.hpp | 2 + src/storage/ducklake_view_entry.cpp | 9 ++ test/sql/view/dangling_view_columns.test | 102 ++++++++++++++++++++ 3 files changed, 113 insertions(+) create mode 100644 test/sql/view/dangling_view_columns.test diff --git a/src/include/storage/ducklake_view_entry.hpp b/src/include/storage/ducklake_view_entry.hpp index 403bc2f12b0..7a116a46b25 100644 --- a/src/include/storage/ducklake_view_entry.hpp +++ b/src/include/storage/ducklake_view_entry.hpp @@ -45,6 +45,8 @@ class DuckLakeViewEntry : public ViewCatalogEntry { unique_ptr GetInfo() const override; string ToSQL() const override; + void BindView(ClientContext &context, BindViewAction action = BindViewAction::BIND_IF_UNBOUND) override; + string GetQuerySQL(); public: diff --git a/src/storage/ducklake_view_entry.cpp b/src/storage/ducklake_view_entry.cpp index 46cb079ff3c..f2ca9bcef6c 100644 --- a/src/storage/ducklake_view_entry.cpp +++ b/src/storage/ducklake_view_entry.cpp @@ -91,4 +91,13 @@ string DuckLakeViewEntry::GetQuerySQL() { return query_sql; } +void DuckLakeViewEntry::BindView(ClientContext &context, BindViewAction action) { + try { + ViewCatalogEntry::BindView(context, action); + } catch (...) { + // If binding fails, we reset the view + UpdateBinding({}, {}); + } +} + } // namespace duckdb diff --git a/test/sql/view/dangling_view_columns.test b/test/sql/view/dangling_view_columns.test new file mode 100644 index 00000000000..b0ec42c4f5c --- /dev/null +++ b/test/sql/view/dangling_view_columns.test @@ -0,0 +1,102 @@ +# name: test/sql/view/dangling_view_columns.test +# description: Test that dangling views don't error on inspection +# group: [view] + +require ducklake + +require parquet + +test-env DUCKLAKE_CONNECTION __TEST_DIR__/{UUID}.db + +test-env DATA_PATH __TEST_DIR__ + +statement ok +ATTACH 'ducklake:${DUCKLAKE_CONNECTION}' AS ducklake (DATA_PATH '${DATA_PATH}/dangling_view_files'); + +statement ok +USE ducklake + +statement ok +CREATE SCHEMA IF NOT EXISTS main + +statement ok +CREATE TABLE main.t1 (id INTEGER, label VARCHAR) + +statement ok +CREATE VIEW main.v1 AS SELECT 1 AS id, 'x' AS label + +statement ok +CREATE VIEW main.v2 AS SELECT id, label FROM main.v1 + +statement ok +CREATE VIEW main.v3 AS SELECT 100 AS score, 'hello' AS greeting + +statement ok +DROP VIEW main.v1 + +query I +SELECT view_name FROM duckdb_views() WHERE database_name = 'ducklake' AND schema_name = 'main' ORDER BY view_name +---- +v2 +v3 + +query III +SELECT table_name, column_name, data_type FROM information_schema.columns WHERE table_schema = 'main' AND table_name IN ('t1', 'v2', 'v3') ORDER BY table_name, ordinal_position +---- +t1 id INTEGER +t1 label VARCHAR +v3 score INTEGER +v3 greeting VARCHAR + +query III +SELECT table_name, column_name, data_type FROM duckdb_columns() WHERE database_name = 'ducklake' AND schema_name = 'main' AND table_name IN ('t1', 'v2', 'v3') ORDER BY table_name, column_index +---- +t1 id INTEGER +t1 label VARCHAR +v3 score INTEGER +v3 greeting VARCHAR + +statement error +SELECT * FROM main.v2 +---- +Table with name v1 does not exist + +statement error +SELECT * FROM main.v2 +---- +Table with name v1 does not exist + +statement ok +CREATE VIEW main.v1 AS SELECT 42 AS id, 'y' AS label + +query II +SELECT * FROM main.v2 +---- +42 y + +statement ok +DROP VIEW main.v1 + +query III +SELECT table_name, column_name, data_type FROM information_schema.columns WHERE table_schema = 'main' AND table_name IN ('t1', 'v2', 'v3') ORDER BY table_name, ordinal_position +---- +t1 id INTEGER +t1 label VARCHAR +v3 score INTEGER +v3 greeting VARCHAR + +query III +SELECT table_name, column_name, data_type FROM duckdb_columns() WHERE database_name = 'ducklake' AND schema_name = 'main' AND table_name IN ('t1', 'v2', 'v3') ORDER BY table_name, column_index +---- +t1 id INTEGER +t1 label VARCHAR +v3 score INTEGER +v3 greeting VARCHAR + +statement ok +CREATE VIEW main.v1 AS SELECT 1 AS a, 'x' AS b + +statement error +SELECT * FROM main.v2 +---- +Referenced column "id" not found in FROM clause From b980548fc695419e23008c43e62e31540b23dc69 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 19 Mar 2026 11:56:07 +0100 Subject: [PATCH 29/59] Make merge_adjacent max option global --- .../ducklake_compaction_functions.cpp | 3 ++ .../merge_adjacent_max_files_partitioned.test | 39 +++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 test/sql/compaction/merge_adjacent_max_files_partitioned.test diff --git a/src/functions/ducklake_compaction_functions.cpp b/src/functions/ducklake_compaction_functions.cpp index 6c2ba53be61..ac5350055e4 100644 --- a/src/functions/ducklake_compaction_functions.cpp +++ b/src/functions/ducklake_compaction_functions.cpp @@ -304,6 +304,9 @@ void DuckLakeCompactor::GenerateCompactions(DuckLakeTableEntry &table, break; } } + if (compacted_files >= options.max_files) { + break; + } } } diff --git a/test/sql/compaction/merge_adjacent_max_files_partitioned.test b/test/sql/compaction/merge_adjacent_max_files_partitioned.test new file mode 100644 index 00000000000..3b472a9e312 --- /dev/null +++ b/test/sql/compaction/merge_adjacent_max_files_partitioned.test @@ -0,0 +1,39 @@ +# name: test/sql/compaction/merge_adjacent_max_files_partitioned.test +# description: test that max_compacted_files respects global limit for partitioned tables +# group: [compaction] + +require ducklake + +require parquet + +test-env DUCKLAKE_CONNECTION __TEST_DIR__/{UUID}.db + +test-env DATA_PATH __TEST_DIR__ + +statement ok +ATTACH 'ducklake:${DUCKLAKE_CONNECTION}' AS ducklake (DATA_PATH '${DATA_PATH}/merge_adjacent_max_files_partitioned/', DATA_INLINING_ROW_LIMIT 0) + +statement ok +USE ducklake; + +statement ok +CREATE TABLE t (v INTEGER, p INTEGER); + +statement ok +ALTER TABLE t SET PARTITIONED BY (p); + +statement ok +INSERT INTO t VALUES (1, 0), (2, 1); + +statement ok +INSERT INTO t VALUES (3, 0), (4, 1); + +query I +SELECT count(*) FROM ducklake_merge_adjacent_files('ducklake', 't', max_compacted_files => 1) +---- +1 + +query I +SELECT count(*) FROM t +---- +4 From fb2f179aef234a771b247a3064356e19593c1fda Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 19 Mar 2026 12:04:09 +0100 Subject: [PATCH 30/59] WIP serialize ducklake_scan --- src/ducklake_extension.cpp | 9 +++ src/include/storage/ducklake_scan.hpp | 7 ++ src/storage/ducklake_scan.cpp | 67 +++++++++++++++---- .../macros/test_macro_window_function.test | 45 +++++++++++++ 4 files changed, 116 insertions(+), 12 deletions(-) create mode 100644 test/sql/macros/test_macro_window_function.test diff --git a/src/ducklake_extension.cpp b/src/ducklake_extension.cpp index 0c5214b6719..bc2cdf4496a 100644 --- a/src/ducklake_extension.cpp +++ b/src/ducklake_extension.cpp @@ -3,6 +3,7 @@ #include "duckdb/common/exception.hpp" #include "duckdb/common/string_util.hpp" #include "storage/ducklake_storage.hpp" +#include "storage/ducklake_scan.hpp" #include "functions/ducklake_table_functions.hpp" #include "storage/ducklake_secret.hpp" #include "duckdb/storage/storage_extension.hpp" @@ -83,6 +84,14 @@ static void LoadInternal(ExtensionLoader &loader) { DuckLakeSettingsFunction settings; loader.RegisterFunction(settings); + // Register ducklake_scan so it can be found during deserialization (e.g. for table macro Copy) + // We register a stub here because parquet may not be loaded yet at extension init time. + // The actual function is fully reconstructed in DuckLakeScanDeserialize. + TableFunction ducklake_scan_stub("ducklake_scan", {LogicalType::VARCHAR}, nullptr, nullptr); + ducklake_scan_stub.serialize = DuckLakeScanSerialize; + ducklake_scan_stub.deserialize = DuckLakeScanDeserialize; + loader.RegisterFunction(ducklake_scan_stub); + // secrets auto secret_type = DuckLakeSecret::GetSecretType(); loader.RegisterSecretType(secret_type); diff --git a/src/include/storage/ducklake_scan.hpp b/src/include/storage/ducklake_scan.hpp index 84963d10a51..83cbf57fed2 100644 --- a/src/include/storage/ducklake_scan.hpp +++ b/src/include/storage/ducklake_scan.hpp @@ -17,6 +17,8 @@ namespace duckdb { class DuckLakeMultiFileList; class DuckLakeTableEntry; class DuckLakeTransaction; +class Serializer; +class Deserializer; class DuckLakeFunctions { public: @@ -28,6 +30,11 @@ class DuckLakeFunctions { static CopyFunctionCatalogEntry &GetCopyFunction(ClientContext &context, const string &name); }; +//! Serialize/Deserialize callbacks for DuckLakeScan (used by table macro Copy) +void DuckLakeScanSerialize(Serializer &serializer, const optional_ptr bind_data, + const TableFunction &function); +unique_ptr DuckLakeScanDeserialize(Deserializer &deserializer, TableFunction &function); + enum class DuckLakeScanType { SCAN_TABLE, SCAN_INSERTIONS, SCAN_DELETIONS, SCAN_FOR_FLUSH }; struct DuckLakeFunctionInfo : public TableFunctionInfo { diff --git a/src/storage/ducklake_scan.cpp b/src/storage/ducklake_scan.cpp index 26aa29f80cc..6d1f5a7493e 100644 --- a/src/storage/ducklake_scan.cpp +++ b/src/storage/ducklake_scan.cpp @@ -9,6 +9,8 @@ #include "duckdb/catalog/catalog_entry/table_function_catalog_entry.hpp" #include "duckdb/common/multi_file/multi_file_data.hpp" #include "duckdb/common/string_util.hpp" +#include "duckdb/common/serializer/serializer.hpp" +#include "duckdb/common/serializer/deserializer.hpp" #include "duckdb/function/partition_stats.hpp" #include "duckdb/function/table_function.hpp" #include "duckdb/main/extension_helper.hpp" @@ -111,15 +113,6 @@ BindInfo DuckLakeBindInfo(const optional_ptr bind_data) { return BindInfo(file_list.GetTable()); } -void DuckLakeScanSerialize(Serializer &serializer, const optional_ptr bind_data, - const TableFunction &function) { - throw NotImplementedException("DuckLakeScan not implemented"); -} - -unique_ptr DuckLakeScanDeserialize(Deserializer &deserializer, TableFunction &function) { - throw NotImplementedException("DuckLakeScan not implemented"); -} - virtual_column_map_t DuckLakeVirtualColumns(ClientContext &context, optional_ptr bind_data_p) { auto &bind_data = bind_data_p->Cast(); auto &file_list = bind_data.file_list->Cast(); @@ -190,7 +183,7 @@ vector DuckLakeGetPartitionStats(ClientContext &context, Ge TableFunction DuckLakeFunctions::GetDuckLakeScanFunction(DatabaseInstance &instance) { // Parquet extension needs to be loaded for this to make sense - ExtensionHelper::AutoLoadExtension(instance, "parquet"); + ExtensionHelper::TryAutoLoadExtension(instance, "parquet"); // The ducklake_scan function is constructed by grabbing the parquet scan from the Catalog, then injecting the // DuckLakeMultiFileReader into it to create a DuckLake-based multi file read @@ -207,8 +200,6 @@ TableFunction DuckLakeFunctions::GetDuckLakeScanFunction(DatabaseInstance &insta function.get_row_id_columns = DuckLakeGetRowIdColumn; function.get_partition_stats = DuckLakeGetPartitionStats; - // Unset all of these: they are either broken, very inefficient. - // TODO: implement/fix these function.serialize = DuckLakeScanSerialize; function.deserialize = DuckLakeScanDeserialize; @@ -233,4 +224,56 @@ shared_ptr DuckLakeFunctionInfo::GetTransaction() { return result; } +void DuckLakeScanSerialize(Serializer &serializer, const optional_ptr bind_data, + const TableFunction &function) { + auto &func_info = function.function_info->Cast(); + auto &catalog = func_info.table.ParentCatalog(); + serializer.WriteProperty(100, "catalog_name", catalog.GetName()); + serializer.WriteProperty(101, "schema_name", func_info.table.ParentSchema().name); + serializer.WriteProperty(102, "table_name", func_info.table_name); + serializer.WriteProperty(103, "snapshot_id", func_info.snapshot.snapshot_id); + serializer.WriteProperty(104, "schema_version", func_info.snapshot.schema_version); + serializer.WriteProperty(105, "next_catalog_id", func_info.snapshot.next_catalog_id); + serializer.WriteProperty(106, "next_file_id", func_info.snapshot.next_file_id); +} + +unique_ptr DuckLakeScanDeserialize(Deserializer &deserializer, TableFunction &function) { + auto &context = deserializer.Get(); + auto catalog_name = deserializer.ReadProperty(100, "catalog_name"); + auto schema_name = deserializer.ReadProperty(101, "schema_name"); + auto table_name = deserializer.ReadProperty(102, "table_name"); + auto snapshot_id = deserializer.ReadProperty(103, "snapshot_id"); + auto schema_version = deserializer.ReadProperty(104, "schema_version"); + auto next_catalog_id = deserializer.ReadProperty(105, "next_catalog_id"); + auto next_file_id = deserializer.ReadProperty(106, "next_file_id"); + + // Look up the DuckLake catalog and table + auto &catalog = Catalog::GetCatalog(context, catalog_name); + auto &transaction = DuckLakeTransaction::Get(context, catalog); + DuckLakeSnapshot snapshot(snapshot_id, schema_version, next_catalog_id, next_file_id); + + auto &table_entry = + Catalog::GetEntry(context, catalog_name, schema_name, table_name).Cast(); + + // Reconstruct the full ducklake_scan function (the deserialized stub lacks parquet callbacks) + auto full_function = DuckLakeFunctions::GetDuckLakeScanFunction(*context.db); + + // Set up function info + auto function_info = make_shared_ptr(table_entry, transaction, snapshot); + function_info->table_name = table_name; + for (auto &col : table_entry.GetColumns().Logical()) { + function_info->column_names.push_back(col.Name()); + function_info->column_types.push_back(col.Type()); + } + function_info->table_id = table_entry.GetTableId(); + full_function.function_info = std::move(function_info); + + // Bind the scan using the full function + auto bind_data = DuckLakeFunctions::BindDuckLakeScan(context, full_function); + + // Copy the reconstructed function back so the LogicalGet uses it + function = std::move(full_function); + + return bind_data; +} } // namespace duckdb diff --git a/test/sql/macros/test_macro_window_function.test b/test/sql/macros/test_macro_window_function.test new file mode 100644 index 00000000000..f40f27dc165 --- /dev/null +++ b/test/sql/macros/test_macro_window_function.test @@ -0,0 +1,45 @@ +# name: test/sql/macros/test_macro_window_function.test +# description: test table macros with window functions +# group: [macros] + +require ducklake + +require parquet + +test-env DUCKLAKE_CONNECTION __TEST_DIR__/{UUID}.db + +test-env DATA_PATH __TEST_DIR__ + +statement ok +ATTACH 'ducklake:${DUCKLAKE_CONNECTION}' AS ducklake (DATA_PATH '${DATA_PATH}/test_macro_window'); + +statement ok +USE ducklake; + +statement ok +CREATE TABLE tbl (id INT, val REAL); + +statement ok +INSERT INTO tbl VALUES (1, 1.0), (2, 2.0), (3, 3.0), (4, 10.0); + +statement ok +CREATE MACRO macro_no_partition() AS TABLE (SELECT avg(val) OVER () AS avg_val FROM tbl); + +query I +FROM macro_no_partition(); +---- +4.0 +4.0 +4.0 +4.0 + +statement ok +CREATE MACRO macro_partition() AS TABLE (SELECT avg(val) OVER (PARTITION BY id) AS avg_val FROM tbl); + +query I +FROM macro_partition(); +---- +1.0 +2.0 +3.0 +10.0 From d1a598ab15dd7fd06fad208542b9de3f9bec60fb Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Thu, 19 Mar 2026 15:07:08 +0100 Subject: [PATCH 31/59] Defensively update default_value_type in migration after adding the column --- src/storage/ducklake_metadata_manager.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/storage/ducklake_metadata_manager.cpp b/src/storage/ducklake_metadata_manager.cpp index d355afbc1dd..b34dd4dc303 100644 --- a/src/storage/ducklake_metadata_manager.cpp +++ b/src/storage/ducklake_metadata_manager.cpp @@ -231,6 +231,7 @@ CREATE TABLE {IF_NOT_EXISTS} {METADATA_CATALOG}.ducklake_macro(schema_id BIGINT, CREATE TABLE {IF_NOT_EXISTS} {METADATA_CATALOG}.ducklake_macro_impl(macro_id BIGINT, impl_id BIGINT, dialect VARCHAR, sql VARCHAR, type VARCHAR); CREATE TABLE {IF_NOT_EXISTS} {METADATA_CATALOG}.ducklake_macro_parameters(macro_id BIGINT, impl_id BIGINT,column_id BIGINT, parameter_name VARCHAR, parameter_type VARCHAR, default_value VARCHAR, default_value_type VARCHAR); ALTER TABLE {METADATA_CATALOG}.ducklake_column ADD COLUMN {IF_NOT_EXISTS} default_value_type VARCHAR DEFAULT 'literal'; +UPDATE {METADATA_CATALOG}.ducklake_column SET default_value_type = 'literal' WHERE default_value_type IS NULL; ALTER TABLE {METADATA_CATALOG}.ducklake_column ADD COLUMN {IF_NOT_EXISTS} default_value_dialect VARCHAR DEFAULT NULL; CREATE TABLE {IF_NOT_EXISTS} {METADATA_CATALOG}.ducklake_sort_info(sort_id BIGINT, table_id BIGINT, begin_snapshot BIGINT, end_snapshot BIGINT); CREATE TABLE {IF_NOT_EXISTS} {METADATA_CATALOG}.ducklake_sort_expression(sort_id BIGINT, table_id BIGINT, sort_key_index BIGINT, expression VARCHAR, dialect VARCHAR, sort_direction VARCHAR, null_order VARCHAR); From be24898caf53bd89b17622805663eb0059709ed7 Mon Sep 17 00:00:00 2001 From: qsliu2017 Date: Wed, 18 Mar 2026 16:36:16 +0800 Subject: [PATCH 32/59] Fix inlined file deletes lost on concurrent commit retry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two bugs caused inlined file deletion metadata to silently disappear when a concurrent commit forced a retry in FlushChanges: 1. GetNewInlinedFileDeletes() used std::move on the source map, emptying it on the first attempt. On retry the data was gone. 2. The delete_inlined_table_cache was not cleared after rollback, so the retry skipped CREATE TABLE IF NOT EXISTS for the inlined deletion table — the subsequent INSERT failed on a non-existent table. Fix: copy instead of move in GetNewInlinedFileDeletes, and clear the inlined table caches before each retry. Also removes the unused inlined_table_name_cache field. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../storage/ducklake_metadata_manager.hpp | 3 +- src/storage/ducklake_metadata_manager.cpp | 5 ++ src/storage/ducklake_transaction.cpp | 5 +- ...st_deletion_inlining_concurrent_retry.test | 59 +++++++++++++++++++ 4 files changed, 70 insertions(+), 2 deletions(-) create mode 100644 test/sql/deletion_inlining/test_deletion_inlining_concurrent_retry.test diff --git a/src/include/storage/ducklake_metadata_manager.hpp b/src/include/storage/ducklake_metadata_manager.hpp index d955f5de346..5b3558eeea8 100644 --- a/src/include/storage/ducklake_metadata_manager.hpp +++ b/src/include/storage/ducklake_metadata_manager.hpp @@ -311,9 +311,10 @@ class DuckLakeMetadataManager { public: //! Read inlined file deletions for regular table scans (no snapshot info per row) map> ReadInlinedFileDeletions(TableIndex table_id, DuckLakeSnapshot snapshot); + //! Clear inlined table caches (needed after rollback so retry re-creates the tables) + void ClearInlinedTableCaches(); private: - unordered_map inlined_table_name_cache; static unordered_map metadata_managers; static mutex metadata_managers_lock; diff --git a/src/storage/ducklake_metadata_manager.cpp b/src/storage/ducklake_metadata_manager.cpp index 5cf9a1f0d61..d44948da0c0 100644 --- a/src/storage/ducklake_metadata_manager.cpp +++ b/src/storage/ducklake_metadata_manager.cpp @@ -2428,6 +2428,11 @@ string DuckLakeMetadataManager::WriteNewInlinedFileDeletes(DuckLakeSnapshot &com return batch_queries; } +void DuckLakeMetadataManager::ClearInlinedTableCaches() { + insert_inlined_table_name_cache.clear(); + delete_inlined_table_cache.clear(); +} + map> DuckLakeMetadataManager::ReadInlinedFileDeletions(TableIndex table_id, DuckLakeSnapshot snapshot) { map> result; diff --git a/src/storage/ducklake_transaction.cpp b/src/storage/ducklake_transaction.cpp index 76e5488fe11..27ba94eb03a 100644 --- a/src/storage/ducklake_transaction.cpp +++ b/src/storage/ducklake_transaction.cpp @@ -2495,6 +2495,8 @@ void DuckLakeTransaction::FlushChanges() { #endif // retry the transaction (with a new snapshot id) + // clear the inlined table caches - the rollback undid any table creation from the previous attempt + metadata_manager->ClearInlinedTableCaches(); connection->BeginTransaction(); snapshot.reset(); } @@ -2685,7 +2687,8 @@ DuckLakeTransaction::GetNewInlinedFileDeletes(DuckLakeCommitState &commit_state) } DuckLakeInlinedFileDeletionInfo info; info.table_id = table_id; - info.file_deletions.file_deletes = std::move(table_changes.new_inlined_file_deletes->file_deletes); + // copy, not move - data must survive commit retries in FlushChanges + info.file_deletions.file_deletes = table_changes.new_inlined_file_deletes->file_deletes; result.push_back(std::move(info)); } return result; diff --git a/test/sql/deletion_inlining/test_deletion_inlining_concurrent_retry.test b/test/sql/deletion_inlining/test_deletion_inlining_concurrent_retry.test new file mode 100644 index 00000000000..061a97c5114 --- /dev/null +++ b/test/sql/deletion_inlining/test_deletion_inlining_concurrent_retry.test @@ -0,0 +1,59 @@ +# name: test/sql/deletion_inlining/test_deletion_inlining_concurrent_retry.test +# description: test that inlined file deletes survive commit retries +# group: [deletion_inlining] + +require notwindows + +require ducklake + +require parquet + +test-env DUCKLAKE_CONNECTION __TEST_DIR__/{UUID}.db + +test-env DATA_PATH __TEST_DIR__ + +statement ok +ATTACH 'ducklake:${DUCKLAKE_CONNECTION}' AS ducklake (DATA_PATH '${DATA_PATH}/deletion_inlining_concurrent_retry', DATA_INLINING_ROW_LIMIT 10, METADATA_CATALOG 'ducklake_meta') + +statement ok +SET ducklake_retry_wait_ms=100 + +statement ok +SET ducklake_retry_backoff=2.0 + +# Create separate tables with data in parquet files (50 rows > inlining limit of 10). +# Using separate tables avoids table-level conflicts in CheckForConflicts — the only +# collision is on the snapshot primary key, which triggers the retry path. +loop i 0 3 + +statement ok +CREATE TABLE ducklake.t${i} AS SELECT range a FROM range(50) + +endloop + +# Each thread owns one table and deletes rows one at a time in a loop. +# Each single-row delete is its own transaction, causing many snapshot PK collisions +# across threads and forcing retries. With the original bugs (std::move emptying +# delete data, stale inlined table cache after rollback), retried deletes silently +# lose their inlined file deletion metadata. +concurrentloop tableid 0 3 + +loop rowid 0 5 + +statement maybe +DELETE FROM ducklake.t${tableid} WHERE a = ${rowid} +---- + +endloop + +endloop + +# Every table should have rows 0-4 deleted. +loop i 0 3 + +query I +SELECT COUNT(*) FROM ducklake.t${i} WHERE a < 5 +---- +0 + +endloop From e8f195fdc4bcc9c4d24d8609e6752e9bffb43e18 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Fri, 20 Mar 2026 13:05:31 +0100 Subject: [PATCH 33/59] Cleanup separator in DATA_PATH --- src/storage/ducklake_initializer.cpp | 8 +- .../macros/test_snapshots_after_macro.test | 2 +- .../remove_orphans/orphan_after_expire.test | 73 +++++++++++++++++++ 3 files changed, 79 insertions(+), 4 deletions(-) create mode 100644 test/sql/remove_orphans/orphan_after_expire.test diff --git a/src/storage/ducklake_initializer.cpp b/src/storage/ducklake_initializer.cpp index 5a4dbe9fb94..ae6457fed61 100644 --- a/src/storage/ducklake_initializer.cpp +++ b/src/storage/ducklake_initializer.cpp @@ -112,10 +112,12 @@ void DuckLakeInitializer::InitializeDataPath() { auto &fs = FileSystem::GetFileSystem(context); auto separator = fs.PathSeparator(data_path); - // ensure the paths we store always end in a path separator - if (!StringUtil::EndsWith(data_path, separator)) { - data_path += separator; + // pop trailing path separators + while (!data_path.empty() && (data_path.back() == '/' || data_path.back() == '\\')) { + data_path.pop_back(); } + // ensure the paths we store always end in a path separator + data_path += separator; catalog.Separator() = separator; } diff --git a/test/sql/macros/test_snapshots_after_macro.test b/test/sql/macros/test_snapshots_after_macro.test index d087b9da0ca..6e32219b001 100644 --- a/test/sql/macros/test_snapshots_after_macro.test +++ b/test/sql/macros/test_snapshots_after_macro.test @@ -1,6 +1,6 @@ # name: test/sql/macros/test_snapshots_after_macro.test # description: test that snapshots() works after creating a macro -# group: [macro] +# group: [macros] require ducklake diff --git a/test/sql/remove_orphans/orphan_after_expire.test b/test/sql/remove_orphans/orphan_after_expire.test new file mode 100644 index 00000000000..29e3a4f1d65 --- /dev/null +++ b/test/sql/remove_orphans/orphan_after_expire.test @@ -0,0 +1,73 @@ +# name: test/sql/remove_orphans/orphan_after_expire.test +# description: Test that active data files are NOT reported as orphaned after snapshot expiration +# group: [remove_orphans] + +require ducklake + +require parquet + +require icu + +test-env DUCKLAKE_CONNECTION __TEST_DIR__/{UUID}.db + +test-env DATA_PATH __TEST_DIR__ + +statement ok +ATTACH 'ducklake:${DUCKLAKE_CONNECTION}' AS lake (DATA_PATH '${DATA_PATH}/orphan_after_expire//', METADATA_CATALOG 'ducklake_meta') + +statement ok +USE lake + +statement ok +CREATE TABLE lake.t1 (id INTEGER, val VARCHAR) + +statement ok +INSERT INTO lake.t1 VALUES (1, 'Hello'), (2, 'World') + +statement ok +CALL ducklake_flush_inlined_data('lake') + +statement ok +CREATE TABLE lake.t2 (id INTEGER, val VARCHAR) + +statement ok +INSERT INTO lake.t2 VALUES (1, 'Foo'), (2, 'Bar') + +statement ok +CALL ducklake_flush_inlined_data('lake') + +statement ok +UPDATE lake.t1 SET val = 'DuckLake' WHERE id = 2 + +statement ok +ALTER TABLE lake.t1 ADD name VARCHAR + +statement ok +INSERT INTO lake.t1 VALUES (3, 'Test', 'Test') + +statement ok +CALL ducklake_flush_inlined_data('lake') + +statement ok +CALL ducklake_merge_adjacent_files('lake') + +statement ok +CALL ducklake_rewrite_data_files('lake') + +statement ok +CALL ducklake_expire_snapshots('lake', older_than => now() - INTERVAL '1 second') + +query I +SELECT COUNT(*) > 0 FROM lake.t1 +---- +true + +query I +SELECT COUNT(*) > 0 FROM lake.t2 +---- +true + +query I +SELECT count(*) FROM ducklake_delete_orphaned_files('lake', dry_run => true, cleanup_all => true) +---- +0 From 75828f3a118169818c4a40206a6d8c110d387292 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Fri, 20 Mar 2026 14:08:49 +0100 Subject: [PATCH 34/59] Change ducklake inlining code to handle chunks with non-physical columns --- src/storage/ducklake_inline_data.cpp | 28 +++++++++++++++++-- .../macros/test_snapshots_after_macro.test | 2 +- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/src/storage/ducklake_inline_data.cpp b/src/storage/ducklake_inline_data.cpp index 4fd2ecfb16c..1ca3c41301a 100644 --- a/src/storage/ducklake_inline_data.cpp +++ b/src/storage/ducklake_inline_data.cpp @@ -313,15 +313,18 @@ OperatorFinalResultType DuckLakeInlineData::OperatorFinalize(Pipeline &pipeline, auto &table = insert_gstate.table; auto result = make_uniq(); auto &inlined_data = *gstate.global_inlined_data; - result->data = std::move(gstate.global_inlined_data); // set the insert count to the total number of inlined rows insert_gstate.total_insert_count = inlined_data.Count(); + // use physical column count for stats + // If we are inlining from updates, we might have extra columms (e.g., row_id, partitions) + auto physical_col_count = table.GetColumns().PhysicalColumnCount(); + // compute the column stats for the data vector new_stats; auto &field_data = table.GetFieldData(); for (auto &chunk : inlined_data.Chunks()) { - for (idx_t c = 0; c < chunk.ColumnCount(); c++) { + for (idx_t c = 0; c < physical_col_count; c++) { UpdateStats(new_stats, c, chunk.data[c], chunk.size(), field_data.GetByRootIndex(PhysicalIndex(c))); } } @@ -339,6 +342,27 @@ OperatorFinalResultType DuckLakeInlineData::OperatorFinalize(Pipeline &pipeline, } } + if (inlined_data.Types().size() > physical_col_count) { + // If we have extra columns, we need to extract the pysical columns + vector phys_types(inlined_data.Types().begin(), inlined_data.Types().begin() + physical_col_count); + auto phys_data = make_uniq(context, phys_types); + ColumnDataAppendState append_state; + phys_data->InitializeAppend(append_state); + for (auto &chunk : inlined_data.Chunks()) { + DataChunk phys_chunk; + phys_chunk.InitializeEmpty(phys_types); + for (idx_t i = 0; i < physical_col_count; i++) { + phys_chunk.data[i].Reference(chunk.data[i]); + } + phys_chunk.SetCardinality(chunk.size()); + phys_data->Append(append_state, phys_chunk); + } + result->data = std::move(phys_data); + } else { + // Otherwise we just copy them + result->data = std::move(gstate.global_inlined_data); + } + // push the inlined data into the transaction auto &transaction = DuckLakeTransaction::Get(context, table.ParentCatalog()); transaction.AppendInlinedData(table.GetTableId(), std::move(result)); diff --git a/test/sql/macros/test_snapshots_after_macro.test b/test/sql/macros/test_snapshots_after_macro.test index d087b9da0ca..6e32219b001 100644 --- a/test/sql/macros/test_snapshots_after_macro.test +++ b/test/sql/macros/test_snapshots_after_macro.test @@ -1,6 +1,6 @@ # name: test/sql/macros/test_snapshots_after_macro.test # description: test that snapshots() works after creating a macro -# group: [macro] +# group: [macros] require ducklake From f56fb8e42fb3f2bfacdee774d64eae7f98b71378 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Fri, 20 Mar 2026 14:48:39 +0100 Subject: [PATCH 35/59] [WIP] Adds inline operator to update creation --- src/include/storage/ducklake_catalog.hpp | 2 ++ src/include/storage/ducklake_update.hpp | 3 ++ src/storage/ducklake_catalog.cpp | 21 ++++++++++++ src/storage/ducklake_insert.cpp | 9 ++--- src/storage/ducklake_update.cpp | 42 ++++++++++++++++++++++-- 5 files changed, 67 insertions(+), 10 deletions(-) diff --git a/src/include/storage/ducklake_catalog.hpp b/src/include/storage/ducklake_catalog.hpp index 8edcaa5f5a6..07ca436e2f5 100644 --- a/src/include/storage/ducklake_catalog.hpp +++ b/src/include/storage/ducklake_catalog.hpp @@ -57,6 +57,8 @@ class DuckLakeCatalog : public Catalog { return metadata_type; } idx_t DataInliningRowLimit(SchemaIndex schema_index, TableIndex table_index) const; + //! Returns the inlining limit (0 if the table is not eligible) + idx_t GetInliningLimit(ClientContext &context, DuckLakeTableEntry &table, const vector &types); string &Separator() { return separator; } diff --git a/src/include/storage/ducklake_update.hpp b/src/include/storage/ducklake_update.hpp index 4c960eb53f9..08f33b0ea83 100644 --- a/src/include/storage/ducklake_update.hpp +++ b/src/include/storage/ducklake_update.hpp @@ -11,6 +11,7 @@ #include "storage/ducklake_insert.hpp" namespace duckdb { +class DuckLakeInlineData; class DuckLakeUpdate : public PhysicalOperator { public: @@ -31,6 +32,8 @@ class DuckLakeUpdate : public PhysicalOperator { //! The row-id-index idx_t row_id_index; vector> expressions; + //! The (optional) inline data operator + optional_ptr inline_data_op; public: // // Source interface diff --git a/src/storage/ducklake_catalog.cpp b/src/storage/ducklake_catalog.cpp index 175385f6db9..44aa7b89b01 100644 --- a/src/storage/ducklake_catalog.cpp +++ b/src/storage/ducklake_catalog.cpp @@ -26,6 +26,7 @@ #include "duckdb/function/scalar_macro_function.hpp" #include "duckdb/function/table_macro_function.hpp" #include "storage/ducklake_macro_entry.hpp" +#include "common/ducklake_util.hpp" namespace duckdb { @@ -825,6 +826,26 @@ idx_t DuckLakeCatalog::DataInliningRowLimit(SchemaIndex schema_index, TableIndex return GetConfigOption("data_inlining_row_limit", schema_index, table_index, 10); } +idx_t DuckLakeCatalog::GetInliningLimit(ClientContext &context, DuckLakeTableEntry &table, + const vector &types) { + auto &schema = table.ParentSchema().Cast(); + idx_t limit = DataInliningRowLimit(schema.GetSchemaId(), table.GetTableId()); + if (limit == 0) { + return 0; + } + if (DuckLakeUtil::HasInlinedSystemColumnConflict(table.GetColumns())) { + // We also return 0 if we have inline column conflicts + return 0; + } + auto &transaction = DuckLakeTransaction::Get(context, *this); + auto &metadata_manager = transaction.GetMetadataManager(); + if (!metadata_manager.SupportsInliningTypes(types)) { + // Or of inline is not supported + return 0; + } + return limit; +} + unique_ptr DuckLakeCatalog::BindAlterAddIndex(Binder &binder, TableCatalogEntry &table_entry, unique_ptr plan, unique_ptr create_info, diff --git a/src/storage/ducklake_insert.cpp b/src/storage/ducklake_insert.cpp index f28fbe6d329..89349619bda 100644 --- a/src/storage/ducklake_insert.cpp +++ b/src/storage/ducklake_insert.cpp @@ -758,15 +758,10 @@ PhysicalOperator &DuckLakeCatalog::PlanInsert(ClientContext &context, PhysicalPl plan = planner.ResolveDefaultsProjection(op, *plan); } auto &ducklake_table = op.table.Cast(); - auto &ducklake_schema = ducklake_table.ParentSchema().Cast(); optional_ptr inline_data; - idx_t data_inlining_row_limit = DataInliningRowLimit(ducklake_schema.GetSchemaId(), ducklake_table.GetTableId()); - // FIXME: we are skipping columns that have conflicting names, we should resolve this - auto &duck_transaction = DuckLakeTransaction::Get(context, *this); - auto &metadata_manager = duck_transaction.GetMetadataManager(); - if (data_inlining_row_limit > 0 && !DuckLakeUtil::HasInlinedSystemColumnConflict(ducklake_table.GetColumns()) && - metadata_manager.SupportsInliningTypes(plan->types)) { + idx_t data_inlining_row_limit = GetInliningLimit(context, ducklake_table, plan->types); + if (data_inlining_row_limit > 0) { plan = planner.Make(*plan, data_inlining_row_limit); inline_data = plan->Cast(); } diff --git a/src/storage/ducklake_update.cpp b/src/storage/ducklake_update.cpp index 0a60c5dbfda..89eaf6aeba6 100644 --- a/src/storage/ducklake_update.cpp +++ b/src/storage/ducklake_update.cpp @@ -8,8 +8,12 @@ #include "duckdb/planner/expression.hpp" #include "duckdb/planner/expression/bound_reference_expression.hpp" #include "storage/ducklake_delete.hpp" +#include "storage/ducklake_inline_data.hpp" #include "storage/ducklake_table_entry.hpp" #include "storage/ducklake_catalog.hpp" +#include "storage/ducklake_schema_entry.hpp" +#include "storage/ducklake_transaction.hpp" +#include "common/ducklake_util.hpp" #include "duckdb/planner/operator/logical_update.hpp" #include "duckdb/parallel/thread_context.hpp" #include "duckdb/parser/expression/cast_expression.hpp" @@ -337,6 +341,31 @@ static unique_ptr GetPartitionExpressionForUpdate(ClientContext &con } } +static optional_ptr +PlanInlineDataForUpdate(ClientContext &context, DuckLakeCatalog &catalog, PhysicalPlanGenerator &planner, + DuckLakeTableEntry &table, const vector> &expressions, + idx_t physical_count, PhysicalOperator &child_plan, PhysicalOperator &insert_op) { + vector physical_types; + for (idx_t i = 0; i < physical_count; i++) { + physical_types.push_back(expressions[i]->return_type); + } + idx_t limit = catalog.GetInliningLimit(context, table, physical_types); + if (limit == 0) { + return nullptr; + } + // compute insert_chunk types: [physical (casted)] [BIGINT row_id] [partition (casted)] + vector insert_types; + for (auto &expr : expressions) { + insert_types.push_back(expr->return_type); + } + insert_types.insert(insert_types.begin() + physical_count, LogicalType::BIGINT); + + auto &inline_op = planner.Make(child_plan, limit).Cast(); + inline_op.types = insert_types; + inline_op.insert = insert_op.Cast(); + return &inline_op; +} + PhysicalOperator &DuckLakeCatalog::PlanUpdate(ClientContext &context, PhysicalPlanGenerator &planner, LogicalUpdate &op, PhysicalOperator &child_plan) { if (op.return_chunk) { @@ -348,8 +377,7 @@ PhysicalOperator &DuckLakeCatalog::PlanUpdate(ClientContext &context, PhysicalPl } } auto &table = op.table.Cast(); - // FIXME: we should take the inlining limit into account here and write new updates to the inline data tables if - // possible updates are executed as a delete + insert - generate the two nodes (delete and insert) plan the copy for + // updates are executed as a delete + insert - generate the two nodes (delete and insert) plan the copy for // the insert DuckLakeCopyInput copy_input(context, table); copy_input.virtual_columns = InsertVirtualColumns::WRITE_ROW_ID; @@ -401,7 +429,15 @@ PhysicalOperator &DuckLakeCatalog::PlanUpdate(ClientContext &context, PhysicalPl } } - return planner.Make(table, op.columns, child_plan, copy_op, delete_op, insert_op, expressions); + // plan inline data before creating update (update constructor moves expressions) + auto inline_data = + PlanInlineDataForUpdate(context, *this, planner, table, expressions, op.columns.size(), child_plan, insert_op); + + auto &update_op = + planner.Make(table, op.columns, child_plan, copy_op, delete_op, insert_op, expressions) + .Cast(); + update_op.inline_data_op = inline_data; + return update_op; } void DuckLakeTableEntry::BindUpdateConstraints(Binder &binder, LogicalGet &get, LogicalProjection &proj, From 88acd0f3267fa4a23f8e0aca789a8e30d2614c08 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Fri, 20 Mar 2026 15:05:33 +0100 Subject: [PATCH 36/59] Update executing inlining --- src/storage/ducklake_update.cpp | 77 ++++++++++++++++++++++++++++++--- 1 file changed, 72 insertions(+), 5 deletions(-) diff --git a/src/storage/ducklake_update.cpp b/src/storage/ducklake_update.cpp index 89eaf6aeba6..d048ab4263d 100644 --- a/src/storage/ducklake_update.cpp +++ b/src/storage/ducklake_update.cpp @@ -63,6 +63,9 @@ class DuckLakeUpdateGlobalState : public GlobalSinkState { //! Duplicate row detection (first-write-wins) mutex seen_rows_lock; unordered_set seen_rows; + + //! Inline data global state + unique_ptr inline_data_gstate; }; class DuckLakeUpdateLocalState : public LocalSinkState { @@ -75,12 +78,20 @@ class DuckLakeUpdateLocalState : public LocalSinkState { DataChunk insert_chunk; DataChunk delete_chunk; idx_t updated_count = 0; + + //! Inline data local state and output buffer + unique_ptr inline_data_lstate; + DataChunk inline_output_chunk; }; unique_ptr DuckLakeUpdate::GetGlobalSinkState(ClientContext &context) const { auto result = make_uniq(); copy_op.sink_state = copy_op.GetGlobalSinkState(context); delete_op.sink_state = delete_op.GetGlobalSinkState(context); + if (inline_data_op) { + // We need tto initialize data global state + result->inline_data_gstate = inline_data_op->GetGlobalOperatorState(context); + } return std::move(result); } @@ -115,6 +126,10 @@ unique_ptr DuckLakeUpdate::GetLocalSinkState(ExecutionContext &c result->insert_chunk.Initialize(context.client, insert_types); result->delete_chunk.Initialize(context.client, delete_types); + if (inline_data_op) { + result->inline_data_lstate = inline_data_op->GetOperatorState(context); + result->inline_output_chunk.Initialize(context.client, inline_data_op->types); + } return std::move(result); } @@ -189,8 +204,30 @@ SinkResultType DuckLakeUpdate::Sink(ExecutionContext &context, DataChunk &chunk, update_expression_chunk.data[physical_column_count + part_idx]); } - OperatorSinkInput copy_input {*copy_op.sink_state, *lstate.copy_local_state, input.interrupt_state}; - copy_op.Sink(context, insert_chunk, copy_input); + if (inline_data_op) { + // if we have an inline data operator, we need to execute through it, it will either inline it or pass it throuhg + auto &inline_output = lstate.inline_output_chunk; + inline_output.Reset(); + auto inline_result = inline_data_op->Execute(context, insert_chunk, inline_output, + *gstate.inline_data_gstate, *lstate.inline_data_lstate); + if (inline_output.size() > 0) { + OperatorSinkInput copy_input {*copy_op.sink_state, *lstate.copy_local_state, input.interrupt_state}; + copy_op.Sink(context, inline_output, copy_input); + } + // handle HAVE_MORE_OUTPUT + while (inline_result == OperatorResultType::HAVE_MORE_OUTPUT) { + inline_output.Reset(); + inline_result = inline_data_op->Execute(context, insert_chunk, inline_output, + *gstate.inline_data_gstate, *lstate.inline_data_lstate); + if (inline_output.size() > 0) { + OperatorSinkInput copy_input {*copy_op.sink_state, *lstate.copy_local_state, input.interrupt_state}; + copy_op.Sink(context, inline_output, copy_input); + } + } + } else { + OperatorSinkInput copy_input {*copy_op.sink_state, *lstate.copy_local_state, input.interrupt_state}; + copy_op.Sink(context, insert_chunk, copy_input); + } // push the rowids into the delete auto &delete_chunk = lstate.delete_chunk; @@ -212,6 +249,23 @@ SinkResultType DuckLakeUpdate::Sink(ExecutionContext &context, DataChunk &chunk, SinkCombineResultType DuckLakeUpdate::Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const { auto &global_state = input.global_state.Cast(); auto &local_state = input.local_state.Cast(); + // drain inline data FinalExecute — flushes local→global or emits on overflow + if (inline_data_op) { + auto &inline_output = local_state.inline_output_chunk; + while (true) { + inline_output.Reset(); + auto fresult = inline_data_op->FinalExecute(context, inline_output, *global_state.inline_data_gstate, + *local_state.inline_data_lstate); + if (inline_output.size() > 0) { + OperatorSinkInput copy_input {*copy_op.sink_state, *local_state.copy_local_state, + input.interrupt_state}; + copy_op.Sink(context, inline_output, copy_input); + } + if (fresult == OperatorFinalizeResultType::FINISHED) { + break; + } + } + } OperatorSinkCombineInput copy_combine_input {*copy_op.sink_state, *local_state.copy_local_state, input.interrupt_state}; auto result = copy_op.Combine(context, copy_combine_input); @@ -233,6 +287,15 @@ SinkCombineResultType DuckLakeUpdate::Combine(ExecutionContext &context, Operato //===--------------------------------------------------------------------===// SinkFinalizeType DuckLakeUpdate::Finalize(Pipeline &pipeline, Event &event, ClientContext &context, OperatorSinkFinalizeInput &input) const { + auto &gstate = input.global_state.Cast(); + + + if (inline_data_op) { + // call OperatorFinalize on inline data + OperatorFinalizeInput inline_finalize_input {*gstate.inline_data_gstate, input.interrupt_state}; + inline_data_op->OperatorFinalize(pipeline, event, context, inline_finalize_input); + } + OperatorSinkFinalizeInput copy_finalize_input {*copy_op.sink_state, input.interrupt_state}; auto result = copy_op.Finalize(pipeline, event, context, copy_finalize_input); if (result != SinkFinalizeType::READY) { @@ -253,11 +316,15 @@ SinkFinalizeType DuckLakeUpdate::Finalize(Pipeline &pipeline, Event &event, Clie DataChunk copy_source_chunk; copy_source_chunk.Initialize(context, copy_op.types); - auto global_sink = insert_op.GetGlobalSinkState(context); + + if (!insert_op.sink_state) { + // use existing sink_state if OperatorFinalize already created it + insert_op.sink_state = insert_op.GetGlobalSinkState(context); + } auto local_sink = insert_op.GetLocalSinkState(execution_context); OperatorSourceInput source_input {*global_source, *local_source, input.interrupt_state}; - OperatorSinkInput sink_input {*global_sink, *local_sink, input.interrupt_state}; + OperatorSinkInput sink_input {*insert_op.sink_state, *local_sink, input.interrupt_state}; while (true) { auto source_result = copy_op.GetData(execution_context, copy_source_chunk, source_input); if (copy_source_chunk.size() == 0) { @@ -276,7 +343,7 @@ SinkFinalizeType DuckLakeUpdate::Finalize(Pipeline &pipeline, Event &event, Clie } } - OperatorSinkFinalizeInput insert_finalize_input {*global_sink, input.interrupt_state}; + OperatorSinkFinalizeInput insert_finalize_input {*insert_op.sink_state, input.interrupt_state}; result = insert_op.Finalize(pipeline, event, context, insert_finalize_input); if (result != SinkFinalizeType::READY) { throw InternalException("DuckLakeUpdate::Finalize does not support async child operators"); From c25fbf951403cf71fb8880c26042d83eb6a0c1c5 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Fri, 20 Mar 2026 19:52:09 +0100 Subject: [PATCH 37/59] Preserve RowIds --- src/include/storage/ducklake_inlined_data.hpp | 2 ++ src/storage/ducklake_inline_data.cpp | 9 +++++++++ src/storage/ducklake_inlined_data_reader.cpp | 15 +++++++++++---- src/storage/ducklake_metadata_manager.cpp | 13 ++++++++++--- src/storage/ducklake_multi_file_list.cpp | 14 ++++++++++++-- src/storage/ducklake_transaction.cpp | 12 ++++++++++-- src/storage/ducklake_update.cpp | 15 +++++++-------- 7 files changed, 61 insertions(+), 19 deletions(-) diff --git a/src/include/storage/ducklake_inlined_data.hpp b/src/include/storage/ducklake_inlined_data.hpp index 99245360818..42571fb7059 100644 --- a/src/include/storage/ducklake_inlined_data.hpp +++ b/src/include/storage/ducklake_inlined_data.hpp @@ -17,6 +17,8 @@ namespace duckdb { struct DuckLakeInlinedData { unique_ptr data; map column_stats; + //! Row Ids for update inlining + vector row_ids; }; struct DuckLakeInlinedDataDeletes { diff --git a/src/storage/ducklake_inline_data.cpp b/src/storage/ducklake_inline_data.cpp index 1ca3c41301a..f6ab9a0a5ac 100644 --- a/src/storage/ducklake_inline_data.cpp +++ b/src/storage/ducklake_inline_data.cpp @@ -349,6 +349,15 @@ OperatorFinalResultType DuckLakeInlineData::OperatorFinalize(Pipeline &pipeline, ColumnDataAppendState append_state; phys_data->InitializeAppend(append_state); for (auto &chunk : inlined_data.Chunks()) { + // extract row_ids from the row_id column + auto &row_id_vec = chunk.data[physical_col_count]; + UnifiedVectorFormat row_id_format; + row_id_vec.ToUnifiedFormat(chunk.size(), row_id_format); + auto row_id_data = UnifiedVectorFormat::GetData(row_id_format); + for (idx_t r = 0; r < chunk.size(); r++) { + auto idx = row_id_format.sel->get_index(r); + result->row_ids.push_back(row_id_data[idx]); + } DataChunk phys_chunk; phys_chunk.InitializeEmpty(phys_types); for (idx_t i = 0; i < physical_col_count; i++) { diff --git a/src/storage/ducklake_inlined_data_reader.cpp b/src/storage/ducklake_inlined_data_reader.cpp index 405a5acf496..094ccdd457e 100644 --- a/src/storage/ducklake_inlined_data_reader.cpp +++ b/src/storage/ducklake_inlined_data_reader.cpp @@ -224,18 +224,25 @@ AsyncResult DuckLakeInlinedDataReader::Scan(ClientContext &context, GlobalTableF break; } case InlinedVirtualColumn::COLUMN_ROW_ID: { - // Generate ordinal data for row IDs Vector ordinal_vector(LogicalType::BIGINT); auto ordinal_data = FlatVector::GetData(ordinal_vector); - for (idx_t r = 0; r < scan_chunk.size(); r++) { - ordinal_data[r] = NumericCast(file_row_number + r); + if (!data->row_ids.empty()) { + // use preserved row_ids from update inlining + for (idx_t r = 0; r < scan_chunk.size(); r++) { + ordinal_data[r] = data->row_ids[file_row_number + r]; + } + } else { + // use general orginal row id + for (idx_t r = 0; r < scan_chunk.size(); r++) { + ordinal_data[r] = NumericCast(file_row_number + r); + } } if (TryEvaluateExpression(context, c, ordinal_vector, LogicalType::BIGINT, chunk.data[c])) { continue; } auto row_id_data = FlatVector::GetData(chunk.data[c]); for (idx_t r = 0; r < scan_chunk.size(); r++) { - row_id_data[r] = NumericCast(file_row_number + r); + row_id_data[r] = ordinal_data[r]; } continue; } diff --git a/src/storage/ducklake_metadata_manager.cpp b/src/storage/ducklake_metadata_manager.cpp index d355afbc1dd..781e39ae96b 100644 --- a/src/storage/ducklake_metadata_manager.cpp +++ b/src/storage/ducklake_metadata_manager.cpp @@ -2347,21 +2347,28 @@ WHERE table_id = %d AND schema_version=( // append the data // FIXME: we can do a much faster append than this string values; + bool has_preserved_row_ids = !entry.data->row_ids.empty(); idx_t row_id = entry.row_id_start; + idx_t global_row_idx = 0; for (auto &chunk : entry.data->data->Chunks()) { for (idx_t r = 0; r < chunk.size(); r++) { if (!values.empty()) { values += ", "; } values += "("; - values += to_string(row_id); + if (has_preserved_row_ids) { + values += to_string(entry.data->row_ids[global_row_idx]); + } else { + values += to_string(row_id); + row_id++; + } values += ", {SNAPSHOT_ID}, NULL"; for (idx_t c = 0; c < chunk.ColumnCount(); c++) { values += ", "; values += DuckLakeUtil::ValueToSQL(*this, context, chunk.GetValue(c, r)); } values += ")"; - row_id++; + global_row_idx++; } } if (!values.empty()) { @@ -2395,7 +2402,7 @@ VALUES %s UPDATE {METADATA_CATALOG}.%s SET end_snapshot = {SNAPSHOT_ID} FROM deleted_row_list -WHERE row_id=deleted_row_id; +WHERE row_id=deleted_row_id AND end_snapshot IS NULL AND begin_snapshot != {SNAPSHOT_ID}; )", row_id_list, entry.table_name); } diff --git a/src/storage/ducklake_multi_file_list.cpp b/src/storage/ducklake_multi_file_list.cpp index e3ddf53bdc6..edd08209ecf 100644 --- a/src/storage/ducklake_multi_file_list.cpp +++ b/src/storage/ducklake_multi_file_list.cpp @@ -278,7 +278,12 @@ vector DuckLakeMultiFileList::GetFilesExtended() file_entry.file_id = DataFileIndex(); file_entry.delete_file_id = DataFileIndex(); file_entry.row_count = transaction_local_data->data->Count(); - file_entry.row_id_start = transaction_row_start; + if (!transaction_local_data->row_ids.empty()) { + // preserved row_ids are absolute, so row_id_start must be 0 + file_entry.row_id_start = 0; + } else { + file_entry.row_id_start = transaction_row_start; + } file_entry.data_type = DuckLakeDataType::TRANSACTION_LOCAL_INLINED_DATA; result.push_back(std::move(file_entry)); } @@ -370,7 +375,12 @@ void DuckLakeMultiFileList::GetFilesForTable() const { // we have transaction local inlined data - create the dummy file entry DuckLakeFileListEntry file_entry; file_entry.file.path = DUCKLAKE_TRANSACTION_LOCAL_INLINED_FILENAME; - file_entry.row_id_start = transaction_row_start; + if (!transaction_local_data->row_ids.empty()) { + // preserved row_ids are absolute, so row_id_start must be 0 + file_entry.row_id_start = 0; + } else { + file_entry.row_id_start = transaction_row_start; + } file_entry.data_type = DuckLakeDataType::TRANSACTION_LOCAL_INLINED_DATA; files.push_back(std::move(file_entry)); } diff --git a/src/storage/ducklake_transaction.cpp b/src/storage/ducklake_transaction.cpp index f65e5a5930b..61c882480f9 100644 --- a/src/storage/ducklake_transaction.cpp +++ b/src/storage/ducklake_transaction.cpp @@ -126,6 +126,7 @@ shared_ptr LocalTableChanges::GetTransactionLocalInlinedDat for (auto &chunk : local_changes.data->Chunks()) { result->data->Append(chunk); } + result->row_ids = local_changes.row_ids; return result; } @@ -209,6 +210,11 @@ void LocalTableChanges::AppendInlinedData(ClientContext &context, TableIndex tab for (auto &chunk : new_data->data->Chunks()) { existing_data.data->Append(chunk); } + // merge preserved row_ids from update inlining + if (!new_data->row_ids.empty()) { + existing_data.row_ids.insert(existing_data.row_ids.end(), new_data->row_ids.begin(), + new_data->row_ids.end()); + } for (auto &entry : new_data->column_stats) { auto stats_entry = existing_data.column_stats.find(entry.first); if (stats_entry == existing_data.column_stats.end()) { @@ -2012,8 +2018,10 @@ NewDataInfo DuckLakeTransaction::GetNewDataFiles(string &batch_query, DuckLakeCo // update global stats new_stats.record_count += record_count; - new_stats.next_row_id += record_count; - + if (inlined_data.row_ids.empty()) { + // regular insert, we advance next_row_id + new_stats.next_row_id += record_count; + } // add the file to the to-be-written inlined data list new_inlined_data.data = table_changes.new_inlined_data.get(); result.new_inlined_data.push_back(new_inlined_data); diff --git a/src/storage/ducklake_update.cpp b/src/storage/ducklake_update.cpp index d048ab4263d..71e328c3a8e 100644 --- a/src/storage/ducklake_update.cpp +++ b/src/storage/ducklake_update.cpp @@ -89,7 +89,7 @@ unique_ptr DuckLakeUpdate::GetGlobalSinkState(ClientContext &co copy_op.sink_state = copy_op.GetGlobalSinkState(context); delete_op.sink_state = delete_op.GetGlobalSinkState(context); if (inline_data_op) { - // We need tto initialize data global state + // We need tto initialize data global state result->inline_data_gstate = inline_data_op->GetGlobalOperatorState(context); } return std::move(result); @@ -205,11 +205,12 @@ SinkResultType DuckLakeUpdate::Sink(ExecutionContext &context, DataChunk &chunk, } if (inline_data_op) { - // if we have an inline data operator, we need to execute through it, it will either inline it or pass it throuhg + // if we have an inline data operator, we need to execute through it, it will either inline it or pass it + // throuhg auto &inline_output = lstate.inline_output_chunk; inline_output.Reset(); - auto inline_result = inline_data_op->Execute(context, insert_chunk, inline_output, - *gstate.inline_data_gstate, *lstate.inline_data_lstate); + auto inline_result = inline_data_op->Execute(context, insert_chunk, inline_output, *gstate.inline_data_gstate, + *lstate.inline_data_lstate); if (inline_output.size() > 0) { OperatorSinkInput copy_input {*copy_op.sink_state, *lstate.copy_local_state, input.interrupt_state}; copy_op.Sink(context, inline_output, copy_input); @@ -217,8 +218,8 @@ SinkResultType DuckLakeUpdate::Sink(ExecutionContext &context, DataChunk &chunk, // handle HAVE_MORE_OUTPUT while (inline_result == OperatorResultType::HAVE_MORE_OUTPUT) { inline_output.Reset(); - inline_result = inline_data_op->Execute(context, insert_chunk, inline_output, - *gstate.inline_data_gstate, *lstate.inline_data_lstate); + inline_result = inline_data_op->Execute(context, insert_chunk, inline_output, *gstate.inline_data_gstate, + *lstate.inline_data_lstate); if (inline_output.size() > 0) { OperatorSinkInput copy_input {*copy_op.sink_state, *lstate.copy_local_state, input.interrupt_state}; copy_op.Sink(context, inline_output, copy_input); @@ -289,7 +290,6 @@ SinkFinalizeType DuckLakeUpdate::Finalize(Pipeline &pipeline, Event &event, Clie OperatorSinkFinalizeInput &input) const { auto &gstate = input.global_state.Cast(); - if (inline_data_op) { // call OperatorFinalize on inline data OperatorFinalizeInput inline_finalize_input {*gstate.inline_data_gstate, input.interrupt_state}; @@ -316,7 +316,6 @@ SinkFinalizeType DuckLakeUpdate::Finalize(Pipeline &pipeline, Event &event, Clie DataChunk copy_source_chunk; copy_source_chunk.Initialize(context, copy_op.types); - if (!insert_op.sink_state) { // use existing sink_state if OperatorFinalize already created it insert_op.sink_state = insert_op.GetGlobalSinkState(context); From 5fb96a22dd4b6777cda741e3b447e281c08dfbc1 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Fri, 20 Mar 2026 21:18:35 +0100 Subject: [PATCH 38/59] fix deletion over inlined insertions from updates --- src/storage/ducklake_metadata_manager.cpp | 3 ++- src/storage/ducklake_transaction.cpp | 18 ++++++++++++++++-- src/storage/ducklake_update.cpp | 13 +++++++------ 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/src/storage/ducklake_metadata_manager.cpp b/src/storage/ducklake_metadata_manager.cpp index 781e39ae96b..2b8d4845118 100644 --- a/src/storage/ducklake_metadata_manager.cpp +++ b/src/storage/ducklake_metadata_manager.cpp @@ -2585,7 +2585,8 @@ unique_ptr DuckLakeMetadataManager::ReadInlinedData(DuckLakeSnapsho auto result = transaction.Query(snapshot, StringUtil::Format(R"( SELECT %s FROM {METADATA_CATALOG}.%s inlined_data -WHERE {SNAPSHOT_ID} >= begin_snapshot AND ({SNAPSHOT_ID} < end_snapshot OR end_snapshot IS NULL);)", +WHERE {SNAPSHOT_ID} >= begin_snapshot AND ({SNAPSHOT_ID} < end_snapshot OR end_snapshot IS NULL) +ORDER BY row_id;)", projection, inlined_table_name)); return result; } diff --git a/src/storage/ducklake_transaction.cpp b/src/storage/ducklake_transaction.cpp index 61c882480f9..69059572e50 100644 --- a/src/storage/ducklake_transaction.cpp +++ b/src/storage/ducklake_transaction.cpp @@ -255,10 +255,13 @@ void LocalTableChanges::DeleteFromLocalInlinedData(ClientContext &context, Table } auto &table_changes = entry->second; auto &existing = *table_changes.new_inlined_data->data; + auto &preserved_row_ids = table_changes.new_inlined_data->row_ids; + bool has_preserved_row_ids = !preserved_row_ids.empty(); // construct a new collection from the existing data minus the deletes auto new_data = make_uniq(context, existing.Types()); idx_t base_row_id = 0; + vector new_row_ids; ColumnDataAppendState append_state; new_data->InitializeAppend(append_state); for (auto &chunk : existing.Chunks()) { @@ -267,12 +270,20 @@ void LocalTableChanges::DeleteFromLocalInlinedData(ClientContext &context, Table idx_t selected_rows = 0; for (idx_t r = 0; r < chunk.size(); r++) { - auto row_id = base_row_id + r; + idx_t row_id; + if (has_preserved_row_ids) { + row_id = NumericCast(preserved_row_ids[base_row_id + r]); + } else { + row_id = base_row_id + r; + } if (new_deletes.find(row_id) != new_deletes.end()) { // deleted - skip continue; } sel.set_index(selected_rows++, r); + if (has_preserved_row_ids) { + new_row_ids.push_back(preserved_row_ids[base_row_id + r]); + } } base_row_id += chunk.size(); if (selected_rows == 0) { @@ -282,8 +293,11 @@ void LocalTableChanges::DeleteFromLocalInlinedData(ClientContext &context, Table new_data->Append(append_state, chunk); } - // override the existing collection + // override the existing collection and row_ids table_changes.new_inlined_data->data = std::move(new_data); + if (has_preserved_row_ids) { + table_changes.new_inlined_data->row_ids = std::move(new_row_ids); + } } static void RemoveFieldStats(map &column_stats, const DuckLakeFieldId &field_id) { diff --git a/src/storage/ducklake_update.cpp b/src/storage/ducklake_update.cpp index 71e328c3a8e..1e6bc79c4f1 100644 --- a/src/storage/ducklake_update.cpp +++ b/src/storage/ducklake_update.cpp @@ -290,6 +290,12 @@ SinkFinalizeType DuckLakeUpdate::Finalize(Pipeline &pipeline, Event &event, Clie OperatorSinkFinalizeInput &input) const { auto &gstate = input.global_state.Cast(); + OperatorSinkFinalizeInput del_finalize_input {*delete_op.sink_state, input.interrupt_state}; + auto result = delete_op.Finalize(pipeline, event, context, del_finalize_input); + if (result != SinkFinalizeType::READY) { + throw InternalException("DuckLakeUpdate::Finalize does not support async child operators"); + } + if (inline_data_op) { // call OperatorFinalize on inline data OperatorFinalizeInput inline_finalize_input {*gstate.inline_data_gstate, input.interrupt_state}; @@ -297,12 +303,7 @@ SinkFinalizeType DuckLakeUpdate::Finalize(Pipeline &pipeline, Event &event, Clie } OperatorSinkFinalizeInput copy_finalize_input {*copy_op.sink_state, input.interrupt_state}; - auto result = copy_op.Finalize(pipeline, event, context, copy_finalize_input); - if (result != SinkFinalizeType::READY) { - throw InternalException("DuckLakeUpdate::Finalize does not support async child operators"); - } - OperatorSinkFinalizeInput del_finalize_input {*delete_op.sink_state, input.interrupt_state}; - result = delete_op.Finalize(pipeline, event, context, del_finalize_input); + result = copy_op.Finalize(pipeline, event, context, copy_finalize_input); if (result != SinkFinalizeType::READY) { throw InternalException("DuckLakeUpdate::Finalize does not support async child operators"); } From f670ca2b7645f4271007106bb10d6f5127fb7bce Mon Sep 17 00:00:00 2001 From: Robert Gernhardt Date: Fri, 20 Mar 2026 15:09:03 -0700 Subject: [PATCH 39/59] Cache existence of GetInlinedDeletionTableName --- src/include/storage/ducklake_catalog.hpp | 13 +++++ src/storage/ducklake_catalog.cpp | 22 +++++++++ src/storage/ducklake_metadata_manager.cpp | 58 +++++++++++++++-------- 3 files changed, 73 insertions(+), 20 deletions(-) diff --git a/src/include/storage/ducklake_catalog.hpp b/src/include/storage/ducklake_catalog.hpp index 8edcaa5f5a6..9d408b251cb 100644 --- a/src/include/storage/ducklake_catalog.hpp +++ b/src/include/storage/ducklake_catalog.hpp @@ -171,6 +171,12 @@ class DuckLakeCatalog : public Catalog { //! Return the schema for the given snapshot - loading it if it is not yet loaded DuckLakeCatalogSet &GetSchemaForSnapshot(DuckLakeTransaction &transaction, DuckLakeSnapshot snapshot); + //! Check if an inlined deletion table is known to exist or not exist for the given table and snapshot + //! Returns: 1 = known to exist, -1 = known to not exist, 0 = unknown (need to query) + int CheckInlinedDeletionTableCache(TableIndex table_id, DuckLakeSnapshot snapshot); + //! Cache the result of an inlined deletion table existence check + void CacheInlinedDeletionTableResult(TableIndex table_id, DuckLakeSnapshot snapshot, bool exists); + private: void DropSchema(ClientContext &context, DropInfo &info) override; unique_ptr LoadSchemaForSnapshot(DuckLakeTransaction &transaction, DuckLakeSnapshot snapshot); @@ -201,6 +207,13 @@ class DuckLakeCatalog : public Catalog { string metadata_type; //! Whether or not the catalog is initialized bool initialized = false; + //! Cache for inlined deletion table existence checks + mutex inlined_deletion_cache_lock; + //! Table IDs where the inlined deletion table is known to exist (permanent - never invalidated) + unordered_set inlined_deletion_exists; + //! Table IDs where the inlined deletion table is known to NOT exist, with the snapshot_id at which we checked + //! Valid as long as current snapshot.snapshot_id <= cached snapshot_id + unordered_map inlined_deletion_not_exists; //! The id of the last committed snapshot, set at FlushChanges on a successful commit mutable mutex commit_lock; optional_idx last_committed_snapshot; diff --git a/src/storage/ducklake_catalog.cpp b/src/storage/ducklake_catalog.cpp index 175385f6db9..015fe727b5a 100644 --- a/src/storage/ducklake_catalog.cpp +++ b/src/storage/ducklake_catalog.cpp @@ -832,4 +832,26 @@ unique_ptr DuckLakeCatalog::BindAlterAddIndex(Binder &binder, T throw NotImplementedException("Adding indexes or constraints is not supported in DuckLake"); } +int DuckLakeCatalog::CheckInlinedDeletionTableCache(TableIndex table_id, DuckLakeSnapshot snapshot) { + lock_guard guard(inlined_deletion_cache_lock); + if (inlined_deletion_exists.find(table_id.index) != inlined_deletion_exists.end()) { + return 1; // known to exist + } + auto it = inlined_deletion_not_exists.find(table_id.index); + if (it != inlined_deletion_not_exists.end() && snapshot.snapshot_id <= it->second) { + return -1; // known to not exist at this or earlier snapshot + } + return 0; // unknown +} + +void DuckLakeCatalog::CacheInlinedDeletionTableResult(TableIndex table_id, DuckLakeSnapshot snapshot, bool exists) { + lock_guard guard(inlined_deletion_cache_lock); + if (exists) { + inlined_deletion_exists.insert(table_id.index); + inlined_deletion_not_exists.erase(table_id.index); + } else { + inlined_deletion_not_exists[table_id.index] = snapshot.snapshot_id; + } +} + } // namespace duckdb diff --git a/src/storage/ducklake_metadata_manager.cpp b/src/storage/ducklake_metadata_manager.cpp index 2bc8a457259..e3c2de00cdf 100644 --- a/src/storage/ducklake_metadata_manager.cpp +++ b/src/storage/ducklake_metadata_manager.cpp @@ -2445,12 +2445,13 @@ map> DuckLakeMetadataManager::ReadInlinedFileDeletions(TableIn "{SNAPSHOT_ID}", inlined_table_name); auto query_result = transaction.Query(snapshot, query); - if (!query_result->HasError()) { - for (auto &row : *query_result) { - auto file_id = row.GetValue(0); - auto row_id = row.GetValue(1); - result[file_id].insert(row_id); - } + if (query_result->HasError()) { + query_result->GetErrorObject().Throw("Failed to read inlined file deletions from DuckLake: "); + } + for (auto &row : *query_result) { + auto file_id = row.GetValue(0); + auto row_id = row.GetValue(1); + result[file_id].insert(row_id); } return result; } @@ -2479,10 +2480,11 @@ unordered_set DuckLakeMetadataManager::GetFileIdsWithInlinedDeletions(Tab "begin_snapshot <= {SNAPSHOT_ID}", inlined_table_name, file_id_list); auto query_result = transaction.Query(snapshot, query); - if (!query_result->HasError()) { - for (auto &row : *query_result) { - result.insert(row.GetValue(0)); - } + if (query_result->HasError()) { + query_result->GetErrorObject().Throw("Failed to read inlined file deletion IDs from DuckLake: "); + } + for (auto &row : *query_result) { + result.insert(row.GetValue(0)); } return result; } @@ -2499,13 +2501,14 @@ DuckLakeMetadataManager::ReadInlinedFileDeletionsForRange(TableIndex table_id, D "WHERE begin_snapshot >= %d AND begin_snapshot <= {SNAPSHOT_ID}", inlined_table_name, start_snapshot.snapshot_id); auto query_result = transaction.Query(end_snapshot, query); - if (!query_result->HasError()) { - for (auto &row : *query_result) { - auto file_id = row.GetValue(0); - auto row_id = row.GetValue(1); - auto snapshot_id = row.GetValue(2); - result[file_id][row_id] = snapshot_id; - } + if (query_result->HasError()) { + query_result->GetErrorObject().Throw("Failed to read inlined file deletions for range from DuckLake: "); + } + for (auto &row : *query_result) { + auto file_id = row.GetValue(0); + auto row_id = row.GetValue(1); + auto snapshot_id = row.GetValue(2); + result[file_id][row_id] = snapshot_id; } return result; } @@ -2515,13 +2518,22 @@ string DuckLakeMetadataManager::GetInlinedDeletionTableName(TableIndex table_id, // The table name is always deterministic string table_name = StringUtil::Format("ducklake_inlined_delete_%d", table_id.index); - // Check the cache first (tracks whether table exists) + // Check per-transaction cache first (covers tables created in this transaction) if (delete_inlined_table_cache.find(table_id.index) != delete_inlined_table_cache.end()) { return table_name; } + // Check catalog-level cache (persists across transactions) + auto &catalog = transaction.GetCatalog(); + auto cache_result = catalog.CheckInlinedDeletionTableCache(table_id, snapshot); + if (cache_result == 1) { + return table_name; // known to exist (committed) + } + if (cache_result == -1 && !create_if_not_exists) { + return string(); // known to not exist + } + if (create_if_not_exists) { - // Create the table if it doesn't exist auto create_query = StringUtil::Format( "CREATE TABLE IF NOT EXISTS {METADATA_CATALOG}.%s(file_id BIGINT, row_id BIGINT, begin_snapshot BIGINT);", table_name); @@ -2529,17 +2541,23 @@ string DuckLakeMetadataManager::GetInlinedDeletionTableName(TableIndex table_id, if (create_result->HasError()) { create_result->GetErrorObject().Throw("Failed to create inlined deletion table: "); } + // Only cache per-transaction — the CREATE is transactional and may be rolled back delete_inlined_table_cache.insert(table_id.index); return table_name; } - // Check if the table exists by querying duckdb_tables() + // Read path: table visibility implies it was committed, safe to cache at catalog level auto query = StringUtil::Format("SELECT NULL FROM {METADATA_CATALOG}.%s LIMIT 1", table_name); auto result = transaction.Query(snapshot, query); + // TODO: Using the error state to check for existence here is fragile. + // Even if the table exists, a transient error in the catalog query would lead us to assume it does not exist. + // Maybe persist the existence of the deletion inlining table on the table metadata instead? if (!result->HasError()) { delete_inlined_table_cache.insert(table_id.index); + catalog.CacheInlinedDeletionTableResult(table_id, snapshot, true); return table_name; } + catalog.CacheInlinedDeletionTableResult(table_id, snapshot, false); return string(); } From 9bb64b4c8b6ef6343f0d0a35542abe4916dac717 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Sat, 21 Mar 2026 00:11:26 +0100 Subject: [PATCH 40/59] in deletes always generate explicit row_ids --- src/storage/ducklake_transaction.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/storage/ducklake_transaction.cpp b/src/storage/ducklake_transaction.cpp index 69059572e50..4739a062ca0 100644 --- a/src/storage/ducklake_transaction.cpp +++ b/src/storage/ducklake_transaction.cpp @@ -283,6 +283,8 @@ void LocalTableChanges::DeleteFromLocalInlinedData(ClientContext &context, Table sel.set_index(selected_rows++, r); if (has_preserved_row_ids) { new_row_ids.push_back(preserved_row_ids[base_row_id + r]); + } else { + new_row_ids.push_back(NumericCast(row_id)); } } base_row_id += chunk.size(); @@ -295,9 +297,7 @@ void LocalTableChanges::DeleteFromLocalInlinedData(ClientContext &context, Table // override the existing collection and row_ids table_changes.new_inlined_data->data = std::move(new_data); - if (has_preserved_row_ids) { - table_changes.new_inlined_data->row_ids = std::move(new_row_ids); - } + table_changes.new_inlined_data->row_ids = std::move(new_row_ids); } static void RemoveFieldStats(map &column_stats, const DuckLakeFieldId &field_id) { From 3a8f2279c8701ab492efff99792c2cfebb950372 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Sat, 21 Mar 2026 00:35:19 +0100 Subject: [PATCH 41/59] we now have to flush test/sql/merge/merge_partition_update.test --- test/sql/merge/merge_partition_update.test | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/sql/merge/merge_partition_update.test b/test/sql/merge/merge_partition_update.test index bc6f7476d5b..fd7929c13b0 100644 --- a/test/sql/merge/merge_partition_update.test +++ b/test/sql/merge/merge_partition_update.test @@ -39,6 +39,10 @@ MERGE INTO my_timeseries ON my_timeseries.ts = timeseries_updates.ts WHEN MATCHED THEN UPDATE; +# flush inlined data from the MERGE update +statement ok +CALL ducklake_flush_inlined_data('ducklake') + query I SELECT COUNT(*) FROM GLOB('${DATA_PATH}/merge_partition_update/**/year=2025/*') ---- @@ -51,6 +55,10 @@ UPDATE my_timeseries SET x = 43::DOUBLE PRECISION WHERE ts = '2025-09-17'::TIMESTAMP; +# flush inlined data from the UPDATE +statement ok +CALL ducklake_flush_inlined_data('ducklake') + query I SELECT COUNT(*) FROM GLOB('${DATA_PATH}/merge_partition_update/**/year=2025/*') ---- From c20ee038e140941e647fe35b00aea244c6d2ea8e Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Sat, 21 Mar 2026 11:45:11 +0100 Subject: [PATCH 42/59] Fix for inserts and updates in the same transaction, as they mix row_ids in the same tsx --- .../storage/ducklake_multi_file_list.hpp | 2 +- src/storage/ducklake_metadata_manager.cpp | 11 +++++- src/storage/ducklake_transaction.cpp | 38 +++++++++++++++++-- .../data_inlining_interleaved_update.test | 9 +++-- 4 files changed, 50 insertions(+), 10 deletions(-) diff --git a/src/include/storage/ducklake_multi_file_list.hpp b/src/include/storage/ducklake_multi_file_list.hpp index 22c92b19045..28dec286d33 100644 --- a/src/include/storage/ducklake_multi_file_list.hpp +++ b/src/include/storage/ducklake_multi_file_list.hpp @@ -20,11 +20,11 @@ namespace duckdb { //! The DuckLakeMultiFileList implements the MultiFileList API to allow injecting it into the regular DuckDB parquet //! scan class DuckLakeMultiFileList : public MultiFileList { - static constexpr const idx_t TRANSACTION_LOCAL_ID_START = 1000000000000000000ULL; static constexpr const char *DUCKLAKE_TRANSACTION_LOCAL_INLINED_FILENAME = "__ducklake_inlined_transaction_local_data"; public: + static constexpr const idx_t TRANSACTION_LOCAL_ID_START = 1000000000000000000ULL; DuckLakeMultiFileList(DuckLakeFunctionInfo &read_info, vector transaction_local_files, shared_ptr transaction_local_data, unique_ptr filter_info = nullptr); diff --git a/src/storage/ducklake_metadata_manager.cpp b/src/storage/ducklake_metadata_manager.cpp index 2b8d4845118..8373df894c7 100644 --- a/src/storage/ducklake_metadata_manager.cpp +++ b/src/storage/ducklake_metadata_manager.cpp @@ -6,6 +6,7 @@ #include "duckdb/common/types/blob.hpp" #include "duckdb/common/type_visitor.hpp" #include "storage/ducklake_catalog.hpp" +#include "storage/ducklake_multi_file_list.hpp" #include "common/ducklake_types.hpp" #include "storage/ducklake_schema_entry.hpp" #include "storage/ducklake_table_entry.hpp" @@ -2357,7 +2358,15 @@ WHERE table_id = %d AND schema_version=( } values += "("; if (has_preserved_row_ids) { - values += to_string(entry.data->row_ids[global_row_idx]); + auto rid = entry.data->row_ids[global_row_idx]; + if (NumericCast(rid) >= DuckLakeMultiFileList::TRANSACTION_LOCAL_ID_START) { + // This is a INSERT row w a placeholder id, we assign sequential row_id + values += to_string(row_id); + row_id++; + } else { + // This is a UPDATE row, we use preserved row_id + values += to_string(rid); + } } else { values += to_string(row_id); row_id++; diff --git a/src/storage/ducklake_transaction.cpp b/src/storage/ducklake_transaction.cpp index 4739a062ca0..1c32b428b9d 100644 --- a/src/storage/ducklake_transaction.cpp +++ b/src/storage/ducklake_transaction.cpp @@ -16,6 +16,7 @@ #include "storage/ducklake_macro_entry.hpp" #include "storage/ducklake_schema_entry.hpp" #include "storage/ducklake_table_entry.hpp" +#include "storage/ducklake_multi_file_list.hpp" #include "storage/ducklake_transaction_changes.hpp" #include "storage/ducklake_transaction_manager.hpp" #include "storage/ducklake_view_entry.hpp" @@ -211,9 +212,29 @@ void LocalTableChanges::AppendInlinedData(ClientContext &context, TableIndex tab existing_data.data->Append(chunk); } // merge preserved row_ids from update inlining - if (!new_data->row_ids.empty()) { - existing_data.row_ids.insert(existing_data.row_ids.end(), new_data->row_ids.begin(), - new_data->row_ids.end()); + if (!new_data->row_ids.empty() || !existing_data.row_ids.empty()) { + if (existing_data.row_ids.empty()) { + idx_t existing_count = existing_data.data->Count() - new_data->data->Count(); + existing_data.row_ids.reserve(existing_count + new_data->row_ids.size()); + auto next_id = NumericCast(DuckLakeMultiFileList::TRANSACTION_LOCAL_ID_START); + for (idx_t i = 0; i < existing_count; i++) { + existing_data.row_ids.push_back(next_id + NumericCast(i)); + } + } + if (!new_data->row_ids.empty()) { + existing_data.row_ids.insert(existing_data.row_ids.end(), new_data->row_ids.begin(), + new_data->row_ids.end()); + } else { + int64_t next_id = NumericCast(DuckLakeMultiFileList::TRANSACTION_LOCAL_ID_START); + for (auto &rid : existing_data.row_ids) { + if (rid >= next_id) { + next_id = rid + 1; + } + } + for (idx_t i = 0; i < new_data->data->Count(); i++) { + existing_data.row_ids.push_back(next_id + NumericCast(i)); + } + } } for (auto &entry : new_data->column_stats) { auto stats_entry = existing_data.column_stats.find(entry.first); @@ -284,7 +305,9 @@ void LocalTableChanges::DeleteFromLocalInlinedData(ClientContext &context, Table if (has_preserved_row_ids) { new_row_ids.push_back(preserved_row_ids[base_row_id + r]); } else { - new_row_ids.push_back(NumericCast(row_id)); + // generate transaction local row ids w a placeholder + new_row_ids.push_back( + NumericCast(DuckLakeMultiFileList::TRANSACTION_LOCAL_ID_START + row_id)); } } base_row_id += chunk.size(); @@ -2035,6 +2058,13 @@ NewDataInfo DuckLakeTransaction::GetNewDataFiles(string &batch_query, DuckLakeCo if (inlined_data.row_ids.empty()) { // regular insert, we advance next_row_id new_stats.next_row_id += record_count; + } else { + // mixed insert and updates, we only advance inserted row_ids + for (auto &rid : inlined_data.row_ids) { + if (NumericCast(rid) >= DuckLakeMultiFileList::TRANSACTION_LOCAL_ID_START) { + new_stats.next_row_id++; + } + } } // add the file to the to-be-written inlined data list new_inlined_data.data = table_changes.new_inlined_data.get(); diff --git a/test/sql/data_inlining/data_inlining_interleaved_update.test b/test/sql/data_inlining/data_inlining_interleaved_update.test index 9592c87d92b..b5babea7eda 100644 --- a/test/sql/data_inlining/data_inlining_interleaved_update.test +++ b/test/sql/data_inlining/data_inlining_interleaved_update.test @@ -55,13 +55,14 @@ SELECT * FROM ducklake.test ORDER BY id 4 d # verify rowids - the updated row (id=1) should preserve its original rowid (0) +# INSERT rows get sequential row_ids from next_row_id (2), UPDATE rows keep preserved row_ids query III SELECT rowid, * FROM ducklake.test ORDER BY id ---- 0 1 aa 1 2 b -3 3 c -4 4 d +2 3 c +3 4 d # verify table_changes shows the update correctly (not as separate delete+insert) query IIIII @@ -69,5 +70,5 @@ FROM ducklake.table_changes('test', 3, 3) ORDER BY ALL ---- 3 0 update_postimage 1 aa 3 0 update_preimage 1 a -3 3 insert 3 c -3 4 insert 4 d +3 2 insert 3 c +3 3 insert 4 d From e6a60a7a83ccbfc54ab756bf611d0d613eafefb0 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Sat, 21 Mar 2026 12:46:12 +0100 Subject: [PATCH 43/59] Cleanup --- src/include/common/index.hpp | 5 ++ src/include/storage/ducklake_inlined_data.hpp | 4 + .../storage/ducklake_multi_file_list.hpp | 3 +- src/include/storage/ducklake_update.hpp | 6 ++ src/storage/ducklake_catalog.cpp | 2 +- src/storage/ducklake_inline_data.cpp | 12 +-- src/storage/ducklake_inlined_data_reader.cpp | 4 +- src/storage/ducklake_metadata_manager.cpp | 5 +- src/storage/ducklake_multi_file_list.cpp | 26 +++---- src/storage/ducklake_transaction.cpp | 73 +++++++++---------- src/storage/ducklake_update.cpp | 47 +++++++----- 11 files changed, 104 insertions(+), 83 deletions(-) diff --git a/src/include/common/index.hpp b/src/include/common/index.hpp index acf03ca4b1e..161af856fa4 100644 --- a/src/include/common/index.hpp +++ b/src/include/common/index.hpp @@ -15,6 +15,11 @@ namespace duckdb { struct DuckLakeConstants { static constexpr const idx_t TRANSACTION_LOCAL_ID_START = 9223372036854775808ULL; + static constexpr const idx_t TRANSACTION_LOCAL_ROW_ID_START = 1000000000000000000ULL; + + static bool IsTransactionLocalRowId(int64_t rid) { + return rid >= 0 && static_cast(rid) >= TRANSACTION_LOCAL_ROW_ID_START; + } }; struct SchemaIndex { diff --git a/src/include/storage/ducklake_inlined_data.hpp b/src/include/storage/ducklake_inlined_data.hpp index 42571fb7059..8a7f9ffed88 100644 --- a/src/include/storage/ducklake_inlined_data.hpp +++ b/src/include/storage/ducklake_inlined_data.hpp @@ -19,6 +19,10 @@ struct DuckLakeInlinedData { map column_stats; //! Row Ids for update inlining vector row_ids; + + bool HasPreservedRowIds() const { + return !row_ids.empty(); + } }; struct DuckLakeInlinedDataDeletes { diff --git a/src/include/storage/ducklake_multi_file_list.hpp b/src/include/storage/ducklake_multi_file_list.hpp index 28dec286d33..ff103ead7ba 100644 --- a/src/include/storage/ducklake_multi_file_list.hpp +++ b/src/include/storage/ducklake_multi_file_list.hpp @@ -24,7 +24,6 @@ class DuckLakeMultiFileList : public MultiFileList { "__ducklake_inlined_transaction_local_data"; public: - static constexpr const idx_t TRANSACTION_LOCAL_ID_START = 1000000000000000000ULL; DuckLakeMultiFileList(DuckLakeFunctionInfo &read_info, vector transaction_local_files, shared_ptr transaction_local_data, unique_ptr filter_info = nullptr); @@ -63,6 +62,8 @@ class DuckLakeMultiFileList : public MultiFileList { void GetFilesForTable() const; void GetTableInsertions() const; void GetTableDeletions() const; + //! Get the row_id_start for transaction-local inlined data. + idx_t GetTransactionLocalRowIdStart(idx_t transaction_row_start) const; private: mutable mutex file_lock; diff --git a/src/include/storage/ducklake_update.hpp b/src/include/storage/ducklake_update.hpp index 08f33b0ea83..e769bb9b464 100644 --- a/src/include/storage/ducklake_update.hpp +++ b/src/include/storage/ducklake_update.hpp @@ -65,6 +65,12 @@ class DuckLakeUpdate : public PhysicalOperator { string GetName() const override; InsertionOrderPreservingMap ParamsToString() const override; + +private: + //! Forward output from the inline data operator to the copy sink, draining HAVE_MORE_OUTPUT as needed. + void ForwardInlineOutputToCopy(ExecutionContext &context, DataChunk &inline_output, + GlobalOperatorState &inline_gstate, OperatorState &inline_lstate, DataChunk &input, + LocalSinkState ©_lstate, InterruptState &interrupt_state) const; }; } // namespace duckdb diff --git a/src/storage/ducklake_catalog.cpp b/src/storage/ducklake_catalog.cpp index 44aa7b89b01..516b23b76c9 100644 --- a/src/storage/ducklake_catalog.cpp +++ b/src/storage/ducklake_catalog.cpp @@ -840,7 +840,7 @@ idx_t DuckLakeCatalog::GetInliningLimit(ClientContext &context, DuckLakeTableEnt auto &transaction = DuckLakeTransaction::Get(context, *this); auto &metadata_manager = transaction.GetMetadataManager(); if (!metadata_manager.SupportsInliningTypes(types)) { - // Or of inline is not supported + // Or if inlining is not supported return 0; } return limit; diff --git a/src/storage/ducklake_inline_data.cpp b/src/storage/ducklake_inline_data.cpp index f6ab9a0a5ac..9067626a006 100644 --- a/src/storage/ducklake_inline_data.cpp +++ b/src/storage/ducklake_inline_data.cpp @@ -300,10 +300,10 @@ OperatorFinalResultType DuckLakeInlineData::OperatorFinalize(Pipeline &pipeline, return OperatorFinalResultType::FINISHED; } { - auto cinsert = const_cast(insert.get()); - lock_guard lock(cinsert->lock); - if (!cinsert->sink_state) { - cinsert->sink_state = insert->GetGlobalSinkState(context); + auto mutable_insert = insert.get_mutable(); + lock_guard lock(mutable_insert->lock); + if (!mutable_insert->sink_state) { + mutable_insert->sink_state = insert->GetGlobalSinkState(context); } } auto &insert_gstate = insert->sink_state->Cast(); @@ -317,7 +317,7 @@ OperatorFinalResultType DuckLakeInlineData::OperatorFinalize(Pipeline &pipeline, insert_gstate.total_insert_count = inlined_data.Count(); // use physical column count for stats - // If we are inlining from updates, we might have extra columms (e.g., row_id, partitions) + // If we are inlining from updates, we might have extra columns (e.g., row_id, partitions) auto physical_col_count = table.GetColumns().PhysicalColumnCount(); // compute the column stats for the data @@ -343,7 +343,7 @@ OperatorFinalResultType DuckLakeInlineData::OperatorFinalize(Pipeline &pipeline, } if (inlined_data.Types().size() > physical_col_count) { - // If we have extra columns, we need to extract the pysical columns + // If we have extra columns, we need to extract the physical columns vector phys_types(inlined_data.Types().begin(), inlined_data.Types().begin() + physical_col_count); auto phys_data = make_uniq(context, phys_types); ColumnDataAppendState append_state; diff --git a/src/storage/ducklake_inlined_data_reader.cpp b/src/storage/ducklake_inlined_data_reader.cpp index 094ccdd457e..ce2e3bcbf45 100644 --- a/src/storage/ducklake_inlined_data_reader.cpp +++ b/src/storage/ducklake_inlined_data_reader.cpp @@ -226,13 +226,13 @@ AsyncResult DuckLakeInlinedDataReader::Scan(ClientContext &context, GlobalTableF case InlinedVirtualColumn::COLUMN_ROW_ID: { Vector ordinal_vector(LogicalType::BIGINT); auto ordinal_data = FlatVector::GetData(ordinal_vector); - if (!data->row_ids.empty()) { + if (data->HasPreservedRowIds()) { // use preserved row_ids from update inlining for (idx_t r = 0; r < scan_chunk.size(); r++) { ordinal_data[r] = data->row_ids[file_row_number + r]; } } else { - // use general orginal row id + // use general ordinal row id for (idx_t r = 0; r < scan_chunk.size(); r++) { ordinal_data[r] = NumericCast(file_row_number + r); } diff --git a/src/storage/ducklake_metadata_manager.cpp b/src/storage/ducklake_metadata_manager.cpp index c697e38319d..d32166100d7 100644 --- a/src/storage/ducklake_metadata_manager.cpp +++ b/src/storage/ducklake_metadata_manager.cpp @@ -6,7 +6,6 @@ #include "duckdb/common/types/blob.hpp" #include "duckdb/common/type_visitor.hpp" #include "storage/ducklake_catalog.hpp" -#include "storage/ducklake_multi_file_list.hpp" #include "common/ducklake_types.hpp" #include "storage/ducklake_schema_entry.hpp" #include "storage/ducklake_table_entry.hpp" @@ -2349,7 +2348,7 @@ WHERE table_id = %d AND schema_version=( // append the data // FIXME: we can do a much faster append than this string values; - bool has_preserved_row_ids = !entry.data->row_ids.empty(); + bool has_preserved_row_ids = entry.data->HasPreservedRowIds(); idx_t row_id = entry.row_id_start; idx_t global_row_idx = 0; for (auto &chunk : entry.data->data->Chunks()) { @@ -2360,7 +2359,7 @@ WHERE table_id = %d AND schema_version=( values += "("; if (has_preserved_row_ids) { auto rid = entry.data->row_ids[global_row_idx]; - if (NumericCast(rid) >= DuckLakeMultiFileList::TRANSACTION_LOCAL_ID_START) { + if (DuckLakeConstants::IsTransactionLocalRowId(rid)) { // This is a INSERT row w a placeholder id, we assign sequential row_id values += to_string(row_id); row_id++; diff --git a/src/storage/ducklake_multi_file_list.cpp b/src/storage/ducklake_multi_file_list.cpp index edd08209ecf..d3e94e2c8de 100644 --- a/src/storage/ducklake_multi_file_list.cpp +++ b/src/storage/ducklake_multi_file_list.cpp @@ -248,7 +248,7 @@ vector DuckLakeMultiFileList::GetFilesExtended() transaction.GetLocalDeleteForFile(read_info.table_id, file_entry.file.path, file_entry.delete_file); } } - idx_t transaction_row_start = TRANSACTION_LOCAL_ID_START; + idx_t transaction_row_start = DuckLakeConstants::TRANSACTION_LOCAL_ROW_ID_START; for (auto &file : transaction_local_files) { DuckLakeFileListExtendedEntry file_entry; file_entry.file_id = DataFileIndex(); @@ -278,12 +278,7 @@ vector DuckLakeMultiFileList::GetFilesExtended() file_entry.file_id = DataFileIndex(); file_entry.delete_file_id = DataFileIndex(); file_entry.row_count = transaction_local_data->data->Count(); - if (!transaction_local_data->row_ids.empty()) { - // preserved row_ids are absolute, so row_id_start must be 0 - file_entry.row_id_start = 0; - } else { - file_entry.row_id_start = transaction_row_start; - } + file_entry.row_id_start = GetTransactionLocalRowIdStart(transaction_row_start); file_entry.data_type = DuckLakeDataType::TRANSACTION_LOCAL_INLINED_DATA; result.push_back(std::move(file_entry)); } @@ -353,7 +348,7 @@ void DuckLakeMultiFileList::GetFilesForTable() const { } } } - idx_t transaction_row_start = TRANSACTION_LOCAL_ID_START; + idx_t transaction_row_start = DuckLakeConstants::TRANSACTION_LOCAL_ROW_ID_START; for (auto &file : transaction_local_files) { DuckLakeFileListEntry file_entry; file_entry.file = GetFileData(file); @@ -375,12 +370,7 @@ void DuckLakeMultiFileList::GetFilesForTable() const { // we have transaction local inlined data - create the dummy file entry DuckLakeFileListEntry file_entry; file_entry.file.path = DUCKLAKE_TRANSACTION_LOCAL_INLINED_FILENAME; - if (!transaction_local_data->row_ids.empty()) { - // preserved row_ids are absolute, so row_id_start must be 0 - file_entry.row_id_start = 0; - } else { - file_entry.row_id_start = transaction_row_start; - } + file_entry.row_id_start = GetTransactionLocalRowIdStart(transaction_row_start); file_entry.data_type = DuckLakeDataType::TRANSACTION_LOCAL_INLINED_DATA; files.push_back(std::move(file_entry)); } @@ -462,4 +452,12 @@ const vector &DuckLakeMultiFileList::GetFiles() const { return files; } +idx_t DuckLakeMultiFileList::GetTransactionLocalRowIdStart(idx_t transaction_row_start) const { + if (transaction_local_data && transaction_local_data->HasPreservedRowIds()) { + // preserved row_ids are absolute, so row_id_start must be 0 + return 0; + } + return transaction_row_start; +} + } // namespace duckdb diff --git a/src/storage/ducklake_transaction.cpp b/src/storage/ducklake_transaction.cpp index 59fb2263b3d..b0c845becfc 100644 --- a/src/storage/ducklake_transaction.cpp +++ b/src/storage/ducklake_transaction.cpp @@ -16,7 +16,6 @@ #include "storage/ducklake_macro_entry.hpp" #include "storage/ducklake_schema_entry.hpp" #include "storage/ducklake_table_entry.hpp" -#include "storage/ducklake_multi_file_list.hpp" #include "storage/ducklake_transaction_changes.hpp" #include "storage/ducklake_transaction_manager.hpp" #include "storage/ducklake_view_entry.hpp" @@ -127,7 +126,7 @@ shared_ptr LocalTableChanges::GetTransactionLocalInlinedDat for (auto &chunk : local_changes.data->Chunks()) { result->data->Append(chunk); } - result->row_ids = local_changes.row_ids; + result->row_ids = local_changes.row_ids; // propagate preserved row_ids if any return result; } @@ -212,20 +211,21 @@ void LocalTableChanges::AppendInlinedData(ClientContext &context, TableIndex tab existing_data.data->Append(chunk); } // merge preserved row_ids from update inlining - if (!new_data->row_ids.empty() || !existing_data.row_ids.empty()) { - if (existing_data.row_ids.empty()) { + if (new_data->HasPreservedRowIds() || existing_data.HasPreservedRowIds()) { + if (!existing_data.HasPreservedRowIds()) { idx_t existing_count = existing_data.data->Count() - new_data->data->Count(); existing_data.row_ids.reserve(existing_count + new_data->row_ids.size()); - auto next_id = NumericCast(DuckLakeMultiFileList::TRANSACTION_LOCAL_ID_START); + auto next_id = NumericCast(DuckLakeConstants::TRANSACTION_LOCAL_ROW_ID_START); for (idx_t i = 0; i < existing_count; i++) { existing_data.row_ids.push_back(next_id + NumericCast(i)); } } - if (!new_data->row_ids.empty()) { + if (new_data->HasPreservedRowIds()) { existing_data.row_ids.insert(existing_data.row_ids.end(), new_data->row_ids.begin(), new_data->row_ids.end()); } else { - int64_t next_id = NumericCast(DuckLakeMultiFileList::TRANSACTION_LOCAL_ID_START); + // new data is insert-only — assign sequential placeholder ids after the max existing id + int64_t next_id = NumericCast(DuckLakeConstants::TRANSACTION_LOCAL_ROW_ID_START); for (auto &rid : existing_data.row_ids) { if (rid >= next_id) { next_id = rid + 1; @@ -277,7 +277,7 @@ void LocalTableChanges::DeleteFromLocalInlinedData(ClientContext &context, Table auto &table_changes = entry->second; auto &existing = *table_changes.new_inlined_data->data; auto &preserved_row_ids = table_changes.new_inlined_data->row_ids; - bool has_preserved_row_ids = !preserved_row_ids.empty(); + bool has_preserved_row_ids = table_changes.new_inlined_data->HasPreservedRowIds(); // construct a new collection from the existing data minus the deletes auto new_data = make_uniq(context, existing.Types()); @@ -305,9 +305,8 @@ void LocalTableChanges::DeleteFromLocalInlinedData(ClientContext &context, Table if (has_preserved_row_ids) { new_row_ids.push_back(preserved_row_ids[base_row_id + r]); } else { - // generate transaction local row ids w a placeholder - new_row_ids.push_back( - NumericCast(DuckLakeMultiFileList::TRANSACTION_LOCAL_ID_START + row_id)); + // generate transaction-local placeholder row_ids + new_row_ids.push_back(NumericCast(DuckLakeConstants::TRANSACTION_LOCAL_ROW_ID_START + row_id)); } } base_row_id += chunk.size(); @@ -2055,13 +2054,13 @@ NewDataInfo DuckLakeTransaction::GetNewDataFiles(string &batch_query, DuckLakeCo // update global stats new_stats.record_count += record_count; - if (inlined_data.row_ids.empty()) { + if (!inlined_data.HasPreservedRowIds()) { // regular insert, we advance next_row_id new_stats.next_row_id += record_count; } else { // mixed insert and updates, we only advance inserted row_ids for (auto &rid : inlined_data.row_ids) { - if (NumericCast(rid) >= DuckLakeMultiFileList::TRANSACTION_LOCAL_ID_START) { + if (DuckLakeConstants::IsTransactionLocalRowId(rid)) { new_stats.next_row_id++; } } @@ -2362,31 +2361,31 @@ CompactionInformation DuckLakeTransaction::GetCompactionChanges(DuckLakeCommitSt if (type != compaction.type) { continue; } - bool has_new_file = !compaction.written_file.file_name.empty(); - DuckLakeFileInfo new_file; + bool has_new_file = !compaction.written_file.file_name.empty(); + DuckLakeFileInfo new_file; - if (!has_new_file) { - if (type != CompactionType::REWRITE_DELETES) { - throw InternalException("Compaction error - expected output file for non-rewrite compaction"); - } - } else { - new_file = GetNewDataFile(compaction.written_file, commit_state, table_id, compaction.row_id_start); - switch (type) { - case CompactionType::REWRITE_DELETES: - new_file.begin_snapshot = commit_snapshot.snapshot_id; - break; - case CompactionType::MERGE_ADJACENT_TABLES: { - // For MERGE_ADJACENT_TABLES, track the max partial snapshot across all source files - optional_idx merged_max_partial_snapshot; - idx_t first_begin_snapshot = compaction.source_files[0].file.begin_snapshot; - for (auto &compacted_file : compaction.source_files) { - idx_t file_max_snapshot = compacted_file.max_partial_file_snapshot.IsValid() - ? compacted_file.max_partial_file_snapshot.GetIndex() - : compacted_file.file.begin_snapshot; - if (!merged_max_partial_snapshot.IsValid() || - file_max_snapshot > merged_max_partial_snapshot.GetIndex()) { - merged_max_partial_snapshot = file_max_snapshot; - } + if (!has_new_file) { + if (type != CompactionType::REWRITE_DELETES) { + throw InternalException("Compaction error - expected output file for non-rewrite compaction"); + } + } else { + new_file = GetNewDataFile(compaction.written_file, commit_state, table_id, compaction.row_id_start); + switch (type) { + case CompactionType::REWRITE_DELETES: + new_file.begin_snapshot = commit_snapshot.snapshot_id; + break; + case CompactionType::MERGE_ADJACENT_TABLES: { + // For MERGE_ADJACENT_TABLES, track the max partial snapshot across all source files + optional_idx merged_max_partial_snapshot; + idx_t first_begin_snapshot = compaction.source_files[0].file.begin_snapshot; + for (auto &compacted_file : compaction.source_files) { + idx_t file_max_snapshot = compacted_file.max_partial_file_snapshot.IsValid() + ? compacted_file.max_partial_file_snapshot.GetIndex() + : compacted_file.file.begin_snapshot; + if (!merged_max_partial_snapshot.IsValid() || + file_max_snapshot > merged_max_partial_snapshot.GetIndex()) { + merged_max_partial_snapshot = file_max_snapshot; + } } // Use the first source file's begin_snapshot for proper time travel support new_file.begin_snapshot = first_begin_snapshot; diff --git a/src/storage/ducklake_update.cpp b/src/storage/ducklake_update.cpp index 1e6bc79c4f1..641072abb41 100644 --- a/src/storage/ducklake_update.cpp +++ b/src/storage/ducklake_update.cpp @@ -89,7 +89,7 @@ unique_ptr DuckLakeUpdate::GetGlobalSinkState(ClientContext &co copy_op.sink_state = copy_op.GetGlobalSinkState(context); delete_op.sink_state = delete_op.GetGlobalSinkState(context); if (inline_data_op) { - // We need tto initialize data global state + // We need to initialize the inline data global state result->inline_data_gstate = inline_data_op->GetGlobalOperatorState(context); } return std::move(result); @@ -133,6 +133,29 @@ unique_ptr DuckLakeUpdate::GetLocalSinkState(ExecutionContext &c return std::move(result); } +//===--------------------------------------------------------------------===// +// Inline data helpers +//===--------------------------------------------------------------------===// +void DuckLakeUpdate::ForwardInlineOutputToCopy(ExecutionContext &context, DataChunk &inline_output, + GlobalOperatorState &inline_gstate, OperatorState &inline_lstate, + DataChunk &input, LocalSinkState ©_lstate, + InterruptState &interrupt_state) const { + inline_output.Reset(); + auto result = inline_data_op->Execute(context, input, inline_output, inline_gstate, inline_lstate); + if (inline_output.size() > 0) { + OperatorSinkInput copy_input {*copy_op.sink_state, copy_lstate, interrupt_state}; + copy_op.Sink(context, inline_output, copy_input); + } + while (result == OperatorResultType::HAVE_MORE_OUTPUT) { + inline_output.Reset(); + result = inline_data_op->Execute(context, input, inline_output, inline_gstate, inline_lstate); + if (inline_output.size() > 0) { + OperatorSinkInput copy_input {*copy_op.sink_state, copy_lstate, interrupt_state}; + copy_op.Sink(context, inline_output, copy_input); + } + } +} + //===--------------------------------------------------------------------===// // Sink //===--------------------------------------------------------------------===// @@ -207,24 +230,9 @@ SinkResultType DuckLakeUpdate::Sink(ExecutionContext &context, DataChunk &chunk, if (inline_data_op) { // if we have an inline data operator, we need to execute through it, it will either inline it or pass it // throuhg - auto &inline_output = lstate.inline_output_chunk; - inline_output.Reset(); - auto inline_result = inline_data_op->Execute(context, insert_chunk, inline_output, *gstate.inline_data_gstate, - *lstate.inline_data_lstate); - if (inline_output.size() > 0) { - OperatorSinkInput copy_input {*copy_op.sink_state, *lstate.copy_local_state, input.interrupt_state}; - copy_op.Sink(context, inline_output, copy_input); - } - // handle HAVE_MORE_OUTPUT - while (inline_result == OperatorResultType::HAVE_MORE_OUTPUT) { - inline_output.Reset(); - inline_result = inline_data_op->Execute(context, insert_chunk, inline_output, *gstate.inline_data_gstate, - *lstate.inline_data_lstate); - if (inline_output.size() > 0) { - OperatorSinkInput copy_input {*copy_op.sink_state, *lstate.copy_local_state, input.interrupt_state}; - copy_op.Sink(context, inline_output, copy_input); - } - } + ForwardInlineOutputToCopy(context, lstate.inline_output_chunk, *gstate.inline_data_gstate, + *lstate.inline_data_lstate, insert_chunk, *lstate.copy_local_state, + input.interrupt_state); } else { OperatorSinkInput copy_input {*copy_op.sink_state, *lstate.copy_local_state, input.interrupt_state}; copy_op.Sink(context, insert_chunk, copy_input); @@ -267,6 +275,7 @@ SinkCombineResultType DuckLakeUpdate::Combine(ExecutionContext &context, Operato } } } + OperatorSinkCombineInput copy_combine_input {*copy_op.sink_state, *local_state.copy_local_state, input.interrupt_state}; auto result = copy_op.Combine(context, copy_combine_input); From cc91e4c88ba31577a20f4df2a4e89ef3322813dd Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Sat, 21 Mar 2026 13:04:00 +0100 Subject: [PATCH 44/59] Fix issue with macro and schema being created in same transaction and multiple macros in same transaction --- src/storage/ducklake_metadata_manager.cpp | 4 +- src/storage/ducklake_transaction.cpp | 50 +++++++++++------------ 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/src/storage/ducklake_metadata_manager.cpp b/src/storage/ducklake_metadata_manager.cpp index 2bc8a457259..e2e77cb5e72 100644 --- a/src/storage/ducklake_metadata_manager.cpp +++ b/src/storage/ducklake_metadata_manager.cpp @@ -2187,10 +2187,10 @@ string DuckLakeMetadataManager::WriteNewMacros(const vector & string batch_query; for (auto ¯o : new_macros) { // Insert in the macro table - batch_query = StringUtil::Format(R"( + batch_query += StringUtil::Format(R"( INSERT INTO {METADATA_CATALOG}.ducklake_macro values(%llu,%llu,'%s',{SNAPSHOT_ID}, NULL); )", - macro.schema_id.index, macro.macro_id.index, macro.macro_name); + macro.schema_id.index, macro.macro_id.index, macro.macro_name); // Insert in the implementation table for (idx_t impl_id = 0; impl_id < macro.implementations.size(); ++impl_id) { auto &impl = macro.implementations[impl_id]; diff --git a/src/storage/ducklake_transaction.cpp b/src/storage/ducklake_transaction.cpp index 27ba94eb03a..e8912340882 100644 --- a/src/storage/ducklake_transaction.cpp +++ b/src/storage/ducklake_transaction.cpp @@ -1684,7 +1684,7 @@ void DuckLakeTransaction::GetNewMacroInfo(DuckLakeCommitState &commit_state, ref new_macro_info.macro_id = MacroIndex(commit_state.commit_snapshot.next_catalog_id++); new_macro_info.macro_name = macro_entry.name; - new_macro_info.schema_id = ducklake_schema.GetSchemaId(); + new_macro_info.schema_id = commit_state.GetSchemaId(ducklake_schema); // Let's do the implementations for (const auto &impl : macro_entry.macros) { DuckLakeMacroImplementation macro_impl; @@ -2310,31 +2310,31 @@ CompactionInformation DuckLakeTransaction::GetCompactionChanges(DuckLakeCommitSt if (type != compaction.type) { continue; } - bool has_new_file = !compaction.written_file.file_name.empty(); - DuckLakeFileInfo new_file; + bool has_new_file = !compaction.written_file.file_name.empty(); + DuckLakeFileInfo new_file; - if (!has_new_file) { - if (type != CompactionType::REWRITE_DELETES) { - throw InternalException("Compaction error - expected output file for non-rewrite compaction"); - } - } else { - new_file = GetNewDataFile(compaction.written_file, commit_state, table_id, compaction.row_id_start); - switch (type) { - case CompactionType::REWRITE_DELETES: - new_file.begin_snapshot = commit_snapshot.snapshot_id; - break; - case CompactionType::MERGE_ADJACENT_TABLES: { - // For MERGE_ADJACENT_TABLES, track the max partial snapshot across all source files - optional_idx merged_max_partial_snapshot; - idx_t first_begin_snapshot = compaction.source_files[0].file.begin_snapshot; - for (auto &compacted_file : compaction.source_files) { - idx_t file_max_snapshot = compacted_file.max_partial_file_snapshot.IsValid() - ? compacted_file.max_partial_file_snapshot.GetIndex() - : compacted_file.file.begin_snapshot; - if (!merged_max_partial_snapshot.IsValid() || - file_max_snapshot > merged_max_partial_snapshot.GetIndex()) { - merged_max_partial_snapshot = file_max_snapshot; - } + if (!has_new_file) { + if (type != CompactionType::REWRITE_DELETES) { + throw InternalException("Compaction error - expected output file for non-rewrite compaction"); + } + } else { + new_file = GetNewDataFile(compaction.written_file, commit_state, table_id, compaction.row_id_start); + switch (type) { + case CompactionType::REWRITE_DELETES: + new_file.begin_snapshot = commit_snapshot.snapshot_id; + break; + case CompactionType::MERGE_ADJACENT_TABLES: { + // For MERGE_ADJACENT_TABLES, track the max partial snapshot across all source files + optional_idx merged_max_partial_snapshot; + idx_t first_begin_snapshot = compaction.source_files[0].file.begin_snapshot; + for (auto &compacted_file : compaction.source_files) { + idx_t file_max_snapshot = compacted_file.max_partial_file_snapshot.IsValid() + ? compacted_file.max_partial_file_snapshot.GetIndex() + : compacted_file.file.begin_snapshot; + if (!merged_max_partial_snapshot.IsValid() || + file_max_snapshot > merged_max_partial_snapshot.GetIndex()) { + merged_max_partial_snapshot = file_max_snapshot; + } } // Use the first source file's begin_snapshot for proper time travel support new_file.begin_snapshot = first_begin_snapshot; From 487cfeed7b5ca64e703b1fbf5df450ebda8057d5 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Sat, 21 Mar 2026 13:43:52 +0100 Subject: [PATCH 45/59] small unrelated fix for alter --- src/storage/ducklake_table_entry.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/storage/ducklake_table_entry.cpp b/src/storage/ducklake_table_entry.cpp index 5f8f4d7d363..4cf67b36d21 100644 --- a/src/storage/ducklake_table_entry.cpp +++ b/src/storage/ducklake_table_entry.cpp @@ -1208,7 +1208,8 @@ unique_ptr DuckLakeTableEntry::Alter(DuckLakeTransaction &transact info.alter_table_type != AlterTableType::RENAME_COLUMN && info.alter_table_type != AlterTableType::ALTER_COLUMN_TYPE && info.alter_table_type != AlterTableType::SET_NOT_NULL && - info.alter_table_type != AlterTableType::DROP_NOT_NULL) { + info.alter_table_type != AlterTableType::DROP_NOT_NULL && + info.alter_table_type != AlterTableType::SET_DEFAULT) { throw NotImplementedException("ALTER on a table with transaction-local inlined data is not supported %s", EnumUtil::ToString(info.alter_table_type)); } From ccc9177b2beff0e30f17e7ac40b675969f4f7c6d Mon Sep 17 00:00:00 2001 From: Ilia Ablamonov Date: Fri, 20 Mar 2026 15:14:27 +0100 Subject: [PATCH 46/59] fix: compaction should not inherit per_thread_output from lake config --- .../ducklake_compaction_functions.cpp | 2 +- .../compaction_per_thread_output.test | 65 +++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 test/sql/compaction/compaction_per_thread_output.test diff --git a/src/functions/ducklake_compaction_functions.cpp b/src/functions/ducklake_compaction_functions.cpp index 8a39eb33e33..7e16234d8e8 100644 --- a/src/functions/ducklake_compaction_functions.cpp +++ b/src/functions/ducklake_compaction_functions.cpp @@ -574,7 +574,7 @@ DuckLakeCompactor::GenerateCompactionCommand(vector copy->filename_pattern = std::move(copy_options.filename_pattern); copy->file_extension = std::move(copy_options.file_extension); copy->overwrite_mode = copy_options.overwrite_mode; - copy->per_thread_output = copy_options.per_thread_output; + copy->per_thread_output = false; copy->file_size_bytes = copy_options.file_size_bytes; copy->rotate = copy_options.rotate; copy->return_type = copy_options.return_type; diff --git a/test/sql/compaction/compaction_per_thread_output.test b/test/sql/compaction/compaction_per_thread_output.test new file mode 100644 index 00000000000..d853d04df2d --- /dev/null +++ b/test/sql/compaction/compaction_per_thread_output.test @@ -0,0 +1,65 @@ +# name: test/sql/compaction/compaction_per_thread_output.test +# description: test compaction with per_thread_output enabled +# group: [compaction] + +require ducklake + +require parquet + +test-env DUCKLAKE_CONNECTION __TEST_DIR__/{UUID}.db + +test-env DATA_PATH __TEST_DIR__ + +statement ok +ATTACH 'ducklake:${DUCKLAKE_CONNECTION}' AS ducklake (DATA_PATH '${DATA_PATH}/ducklake_compaction_per_thread_output') + +statement ok +CALL ducklake.set_option('per_thread_output', 'true'); + +statement ok +CREATE TABLE ducklake.test(i INTEGER); + +statement ok +INSERT INTO ducklake.test VALUES (1); + +statement ok +CALL ducklake_flush_inlined_data('ducklake') + +statement ok +INSERT INTO ducklake.test VALUES (2); + +statement ok +CALL ducklake_flush_inlined_data('ducklake') + +statement ok +INSERT INTO ducklake.test VALUES (3); + +statement ok +CALL ducklake_flush_inlined_data('ducklake') + +# merge_adjacent_files should work with per_thread_output enabled +statement ok +CALL ducklake_merge_adjacent_files('ducklake'); + +# verify data integrity after merge +query I +SELECT * FROM ducklake.test ORDER BY ALL +---- +1 +2 +3 + +# delete from the merged file to create a delete file +statement ok +DELETE FROM ducklake.test WHERE i = 3; + +# rewrite_data_files should work with per_thread_output enabled +statement ok +CALL ducklake_rewrite_data_files('ducklake', delete_threshold => 0); + +# verify data integrity after rewrite +query I +SELECT * FROM ducklake.test ORDER BY ALL +---- +1 +2 From afa97f5f1f350aa0d138745ef0c4b76e8dcf1652 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Mon, 23 Mar 2026 13:56:34 +0100 Subject: [PATCH 47/59] Fix for merge on update-only existing data then inserting --- src/storage/ducklake_transaction.cpp | 13 +++++++--- .../data_inlining_interleaved_update.test | 26 +++++++++++++++++++ 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/src/storage/ducklake_transaction.cpp b/src/storage/ducklake_transaction.cpp index b0c845becfc..527933a8786 100644 --- a/src/storage/ducklake_transaction.cpp +++ b/src/storage/ducklake_transaction.cpp @@ -213,6 +213,8 @@ void LocalTableChanges::AppendInlinedData(ClientContext &context, TableIndex tab // merge preserved row_ids from update inlining if (new_data->HasPreservedRowIds() || existing_data.HasPreservedRowIds()) { if (!existing_data.HasPreservedRowIds()) { + // if the existing data doesnt have preserved row ids, we sign them off sequentially from + // transaction local id idx_t existing_count = existing_data.data->Count() - new_data->data->Count(); existing_data.row_ids.reserve(existing_count + new_data->row_ids.size()); auto next_id = NumericCast(DuckLakeConstants::TRANSACTION_LOCAL_ROW_ID_START); @@ -221,14 +223,17 @@ void LocalTableChanges::AppendInlinedData(ClientContext &context, TableIndex tab } } if (new_data->HasPreservedRowIds()) { + // if this is already preserved we can just insert it existing_data.row_ids.insert(existing_data.row_ids.end(), new_data->row_ids.begin(), new_data->row_ids.end()); } else { - // new data is insert-only — assign sequential placeholder ids after the max existing id + // new data doesnt preserve row_ids, we need to use the TRANSACTION_LOCAL_ROW_ID_START int64_t next_id = NumericCast(DuckLakeConstants::TRANSACTION_LOCAL_ROW_ID_START); - for (auto &rid : existing_data.row_ids) { - if (rid >= next_id) { - next_id = rid + 1; + if (!existing_data.row_ids.empty()) { + auto max_id = *std::max_element(existing_data.row_ids.begin(), existing_data.row_ids.end()); + if (max_id >= next_id) { + // we only use max_id if it's guaranteed to be transactional (over next_id) + next_id = max_id + 1; } } for (idx_t i = 0; i < new_data->data->Count(); i++) { diff --git a/test/sql/data_inlining/data_inlining_interleaved_update.test b/test/sql/data_inlining/data_inlining_interleaved_update.test index b5babea7eda..b85b8d07313 100644 --- a/test/sql/data_inlining/data_inlining_interleaved_update.test +++ b/test/sql/data_inlining/data_inlining_interleaved_update.test @@ -72,3 +72,29 @@ FROM ducklake.table_changes('test', 3, 3) ORDER BY ALL 3 0 update_preimage 1 a 3 2 insert 3 c 3 3 insert 4 d + +# UPDATE -> INSERT in one transaction (update-only existing data, then insert) +# this tests the case where existing_data has only preserved row_ids (from the update) +# and the new insert data needs TRANSACTION_LOCAL_ROW_ID_START-based ids +statement ok +BEGIN + +query I +UPDATE ducklake.test SET val='bb' WHERE id=2 +---- +1 + +statement ok +INSERT INTO ducklake.test VALUES (5, 'e') + +statement ok +COMMIT + +query III +SELECT rowid, * FROM ducklake.test ORDER BY id +---- +0 1 aa +1 2 bb +2 3 c +3 4 d +4 5 e From 7ac4b2d55fe5f02c5eb37975521c1262b9ac2f6c Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Mon, 23 Mar 2026 14:31:26 +0100 Subject: [PATCH 48/59] Some cleanup --- src/include/storage/ducklake_inlined_data.hpp | 10 +++- src/storage/CMakeLists.txt | 1 + src/storage/ducklake_inlined_data.cpp | 58 +++++++++++++++++++ src/storage/ducklake_transaction.cpp | 55 +++--------------- 4 files changed, 74 insertions(+), 50 deletions(-) create mode 100644 src/storage/ducklake_inlined_data.cpp diff --git a/src/include/storage/ducklake_inlined_data.hpp b/src/include/storage/ducklake_inlined_data.hpp index 8a7f9ffed88..0ce107fcdc8 100644 --- a/src/include/storage/ducklake_inlined_data.hpp +++ b/src/include/storage/ducklake_inlined_data.hpp @@ -20,9 +20,13 @@ struct DuckLakeInlinedData { //! Row Ids for update inlining vector row_ids; - bool HasPreservedRowIds() const { - return !row_ids.empty(); - } + bool HasPreservedRowIds() const; + //! Get the row_id for a given position in the data collection + idx_t GetRowId(idx_t position) const; + //! Get the output row_id for a surviving (non-deleted) row + int64_t GetOutputRowId(idx_t position) const; + //! Merge preserved row_ids from update inlining + void MergeRowIds(const DuckLakeInlinedData &new_data, idx_t new_data_count); }; struct DuckLakeInlinedDataDeletes { diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt index 41ef909178c..26187f5861f 100644 --- a/src/storage/CMakeLists.txt +++ b/src/storage/CMakeLists.txt @@ -7,6 +7,7 @@ add_library( ducklake_delete_filter.cpp ducklake_field_data.cpp ducklake_inline_data.cpp + ducklake_inlined_data.cpp ducklake_inlined_data_reader.cpp ducklake_insert.cpp ducklake_merge_into.cpp diff --git a/src/storage/ducklake_inlined_data.cpp b/src/storage/ducklake_inlined_data.cpp new file mode 100644 index 00000000000..cb4d40c2372 --- /dev/null +++ b/src/storage/ducklake_inlined_data.cpp @@ -0,0 +1,58 @@ +#include "storage/ducklake_inlined_data.hpp" + +#include + +namespace duckdb { + +bool DuckLakeInlinedData::HasPreservedRowIds() const { + return !row_ids.empty(); +} + +idx_t DuckLakeInlinedData::GetRowId(idx_t position) const { + if (HasPreservedRowIds()) { + return NumericCast(row_ids[position]); + } + return position; +} + +int64_t DuckLakeInlinedData::GetOutputRowId(idx_t position) const { + if (HasPreservedRowIds()) { + return row_ids[position]; + } + return NumericCast(DuckLakeConstants::TRANSACTION_LOCAL_ROW_ID_START + position); +} + +void DuckLakeInlinedData::MergeRowIds(const DuckLakeInlinedData &new_data, idx_t new_data_count) { + if (!new_data.HasPreservedRowIds() && !HasPreservedRowIds()) { + return; + } + if (!HasPreservedRowIds()) { + // if the existing data doesnt have preserved row ids, we assign them sequentially from + // transaction local id + idx_t existing_count = data->Count() - new_data_count; + row_ids.reserve(existing_count + new_data.row_ids.size()); + auto next_id = NumericCast(DuckLakeConstants::TRANSACTION_LOCAL_ROW_ID_START); + for (idx_t i = 0; i < existing_count; i++) { + row_ids.push_back(next_id + NumericCast(i)); + } + } + if (new_data.HasPreservedRowIds()) { + // if this is already preserved we can just insert it + row_ids.insert(row_ids.end(), new_data.row_ids.begin(), new_data.row_ids.end()); + } else { + // new data doesnt preserve row_ids, we need to use the TRANSACTION_LOCAL_ROW_ID_START + int64_t next_id = NumericCast(DuckLakeConstants::TRANSACTION_LOCAL_ROW_ID_START); + if (!row_ids.empty()) { + // we only use max_id if it's guaranteed to be transactional (over next_id) + auto max_id = *std::max_element(row_ids.begin(), row_ids.end()); + if (max_id >= next_id) { + next_id = max_id + 1; + } + } + for (idx_t i = 0; i < new_data_count; i++) { + row_ids.push_back(next_id + NumericCast(i)); + } + } +} + +} // namespace duckdb diff --git a/src/storage/ducklake_transaction.cpp b/src/storage/ducklake_transaction.cpp index 527933a8786..f1224d8e329 100644 --- a/src/storage/ducklake_transaction.cpp +++ b/src/storage/ducklake_transaction.cpp @@ -211,36 +211,7 @@ void LocalTableChanges::AppendInlinedData(ClientContext &context, TableIndex tab existing_data.data->Append(chunk); } // merge preserved row_ids from update inlining - if (new_data->HasPreservedRowIds() || existing_data.HasPreservedRowIds()) { - if (!existing_data.HasPreservedRowIds()) { - // if the existing data doesnt have preserved row ids, we sign them off sequentially from - // transaction local id - idx_t existing_count = existing_data.data->Count() - new_data->data->Count(); - existing_data.row_ids.reserve(existing_count + new_data->row_ids.size()); - auto next_id = NumericCast(DuckLakeConstants::TRANSACTION_LOCAL_ROW_ID_START); - for (idx_t i = 0; i < existing_count; i++) { - existing_data.row_ids.push_back(next_id + NumericCast(i)); - } - } - if (new_data->HasPreservedRowIds()) { - // if this is already preserved we can just insert it - existing_data.row_ids.insert(existing_data.row_ids.end(), new_data->row_ids.begin(), - new_data->row_ids.end()); - } else { - // new data doesnt preserve row_ids, we need to use the TRANSACTION_LOCAL_ROW_ID_START - int64_t next_id = NumericCast(DuckLakeConstants::TRANSACTION_LOCAL_ROW_ID_START); - if (!existing_data.row_ids.empty()) { - auto max_id = *std::max_element(existing_data.row_ids.begin(), existing_data.row_ids.end()); - if (max_id >= next_id) { - // we only use max_id if it's guaranteed to be transactional (over next_id) - next_id = max_id + 1; - } - } - for (idx_t i = 0; i < new_data->data->Count(); i++) { - existing_data.row_ids.push_back(next_id + NumericCast(i)); - } - } - } + existing_data.MergeRowIds(*new_data, new_data->data->Count()); for (auto &entry : new_data->column_stats) { auto stats_entry = existing_data.column_stats.find(entry.first); if (stats_entry == existing_data.column_stats.end()) { @@ -280,9 +251,8 @@ void LocalTableChanges::DeleteFromLocalInlinedData(ClientContext &context, Table throw InternalException("DeleteFromLocalInlinedData called but no transaction-local data exists for table"); } auto &table_changes = entry->second; - auto &existing = *table_changes.new_inlined_data->data; - auto &preserved_row_ids = table_changes.new_inlined_data->row_ids; - bool has_preserved_row_ids = table_changes.new_inlined_data->HasPreservedRowIds(); + auto &inlined_data = *table_changes.new_inlined_data; + auto &existing = *inlined_data.data; // construct a new collection from the existing data minus the deletes auto new_data = make_uniq(context, existing.Types()); @@ -296,23 +266,14 @@ void LocalTableChanges::DeleteFromLocalInlinedData(ClientContext &context, Table idx_t selected_rows = 0; for (idx_t r = 0; r < chunk.size(); r++) { - idx_t row_id; - if (has_preserved_row_ids) { - row_id = NumericCast(preserved_row_ids[base_row_id + r]); - } else { - row_id = base_row_id + r; - } + idx_t position = base_row_id + r; + auto row_id = inlined_data.GetRowId(position); if (new_deletes.find(row_id) != new_deletes.end()) { // deleted - skip continue; } sel.set_index(selected_rows++, r); - if (has_preserved_row_ids) { - new_row_ids.push_back(preserved_row_ids[base_row_id + r]); - } else { - // generate transaction-local placeholder row_ids - new_row_ids.push_back(NumericCast(DuckLakeConstants::TRANSACTION_LOCAL_ROW_ID_START + row_id)); - } + new_row_ids.push_back(inlined_data.GetOutputRowId(position)); } base_row_id += chunk.size(); if (selected_rows == 0) { @@ -323,8 +284,8 @@ void LocalTableChanges::DeleteFromLocalInlinedData(ClientContext &context, Table } // override the existing collection and row_ids - table_changes.new_inlined_data->data = std::move(new_data); - table_changes.new_inlined_data->row_ids = std::move(new_row_ids); + inlined_data.data = std::move(new_data); + inlined_data.row_ids = std::move(new_row_ids); } static void RemoveFieldStats(map &column_stats, const DuckLakeFieldId &field_id) { From a71ad8cc3fb6c17891d1c43c3822751bf8181260 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Mon, 23 Mar 2026 16:07:57 +0100 Subject: [PATCH 49/59] Make update output insert chunks downstream so we can use the insert logic directly --- src/include/storage/ducklake_update.hpp | 52 ++- src/storage/ducklake_update.cpp | 399 +++++------------------- 2 files changed, 98 insertions(+), 353 deletions(-) diff --git a/src/include/storage/ducklake_update.hpp b/src/include/storage/ducklake_update.hpp index e769bb9b464..818f256e403 100644 --- a/src/include/storage/ducklake_update.hpp +++ b/src/include/storage/ducklake_update.hpp @@ -11,66 +11,50 @@ #include "storage/ducklake_insert.hpp" namespace duckdb { -class DuckLakeInlineData; class DuckLakeUpdate : public PhysicalOperator { public: DuckLakeUpdate(PhysicalPlan &physical_plan, DuckLakeTableEntry &table, vector columns, - PhysicalOperator &child, PhysicalOperator ©_op, PhysicalOperator &delete_op, - PhysicalOperator &insert_op, vector> &expressions); + PhysicalOperator &child, PhysicalOperator &delete_op, + vector> &expressions); //! The table to update DuckLakeTableEntry &table; //! The order of to-be-inserted columns vector columns; - //! The copy operator for writing new data to files - PhysicalOperator ©_op; //! The delete operator for deleting the old data PhysicalOperator &delete_op; - //! The (final) insert operator that registers inserted data - PhysicalOperator &insert_op; //! The row-id-index idx_t row_id_index; vector> expressions; - //! The (optional) inline data operator - optional_ptr inline_data_op; - -public: - // // Source interface - SourceResultType GetDataInternal(ExecutionContext &context, DataChunk &chunk, - OperatorSourceInput &input) const override; - - bool IsSource() const override { - return true; - } static constexpr uint8_t DELETION_INFO_SIZE = 3; public: - // Sink interface - SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override; - SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override; - SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context, - OperatorSinkFinalizeInput &input) const override; - unique_ptr GetGlobalSinkState(ClientContext &context) const override; - unique_ptr GetLocalSinkState(ExecutionContext &context) const override; + // Operator interface + unique_ptr GetGlobalOperatorState(ClientContext &context) const override; + unique_ptr GetOperatorState(ExecutionContext &context) const override; + OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk, + GlobalOperatorState &gstate, OperatorState &state) const override; + OperatorFinalizeResultType FinalExecute(ExecutionContext &context, DataChunk &chunk, + GlobalOperatorState &gstate, OperatorState &state) const override; + OperatorFinalResultType OperatorFinalize(Pipeline &pipeline, Event &event, ClientContext &context, + OperatorFinalizeInput &input) const override; + + bool ParallelOperator() const override { + return true; + } - bool IsSink() const override { + bool RequiresFinalExecute() const override { return true; } - bool ParallelSink() const override { + bool RequiresOperatorFinalize() const override { return true; } string GetName() const override; InsertionOrderPreservingMap ParamsToString() const override; - -private: - //! Forward output from the inline data operator to the copy sink, draining HAVE_MORE_OUTPUT as needed. - void ForwardInlineOutputToCopy(ExecutionContext &context, DataChunk &inline_output, - GlobalOperatorState &inline_gstate, OperatorState &inline_lstate, DataChunk &input, - LocalSinkState ©_lstate, InterruptState &interrupt_state) const; }; -} // namespace duckdb +} // namespace duckdb \ No newline at end of file diff --git a/src/storage/ducklake_update.cpp b/src/storage/ducklake_update.cpp index 641072abb41..9e4418fb89c 100644 --- a/src/storage/ducklake_update.cpp +++ b/src/storage/ducklake_update.cpp @@ -41,11 +41,10 @@ struct FileRowIdHash { }; DuckLakeUpdate::DuckLakeUpdate(PhysicalPlan &physical_plan, DuckLakeTableEntry &table, vector columns_p, - PhysicalOperator &child, PhysicalOperator ©_op, PhysicalOperator &delete_op, - PhysicalOperator &insert_op, vector> &expressions) - : PhysicalOperator(physical_plan, PhysicalOperatorType::EXTENSION, {LogicalType::BIGINT}, 1), table(table), - columns(std::move(columns_p)), copy_op(copy_op), delete_op(delete_op), insert_op(insert_op), - expressions(std::move(expressions)) { + PhysicalOperator &child, PhysicalOperator &delete_op, + vector> &expressions) + : PhysicalOperator(physical_plan, PhysicalOperatorType::EXTENSION, {}, 1), table(table), + columns(std::move(columns_p)), delete_op(delete_op), expressions(std::move(expressions)) { children.push_back(child); row_id_index = columns.size(); } @@ -53,7 +52,7 @@ DuckLakeUpdate::DuckLakeUpdate(PhysicalPlan &physical_plan, DuckLakeTableEntry & //===--------------------------------------------------------------------===// // States //===--------------------------------------------------------------------===// -class DuckLakeUpdateGlobalState : public GlobalSinkState { +class DuckLakeUpdateGlobalState : public GlobalOperatorState { public: DuckLakeUpdateGlobalState() : total_updated_count(0) { } @@ -63,14 +62,10 @@ class DuckLakeUpdateGlobalState : public GlobalSinkState { //! Duplicate row detection (first-write-wins) mutex seen_rows_lock; unordered_set seen_rows; - - //! Inline data global state - unique_ptr inline_data_gstate; }; -class DuckLakeUpdateLocalState : public LocalSinkState { +class DuckLakeUpdateLocalState : public OperatorState { public: - unique_ptr copy_local_state; unique_ptr delete_local_state; unique_ptr expression_executor; //! Chunk where the updated expressions are executed. @@ -78,26 +73,17 @@ class DuckLakeUpdateLocalState : public LocalSinkState { DataChunk insert_chunk; DataChunk delete_chunk; idx_t updated_count = 0; - - //! Inline data local state and output buffer - unique_ptr inline_data_lstate; - DataChunk inline_output_chunk; }; -unique_ptr DuckLakeUpdate::GetGlobalSinkState(ClientContext &context) const { +unique_ptr DuckLakeUpdate::GetGlobalOperatorState(ClientContext &context) const { auto result = make_uniq(); - copy_op.sink_state = copy_op.GetGlobalSinkState(context); + // init delete_op sink state delete_op.sink_state = delete_op.GetGlobalSinkState(context); - if (inline_data_op) { - // We need to initialize the inline data global state - result->inline_data_gstate = inline_data_op->GetGlobalOperatorState(context); - } return std::move(result); } -unique_ptr DuckLakeUpdate::GetLocalSinkState(ExecutionContext &context) const { +unique_ptr DuckLakeUpdate::GetOperatorState(ExecutionContext &context) const { auto result = make_uniq(); - result->copy_local_state = copy_op.GetLocalSinkState(context); result->delete_local_state = delete_op.GetLocalSinkState(context); vector delete_types; @@ -111,74 +97,37 @@ unique_ptr DuckLakeUpdate::GetLocalSinkState(ExecutionContext &c expression_types.push_back(expr->return_type); } - for (auto &type : expression_types) { - if (DuckLakeTypes::RequiresCast(type)) { - type = DuckLakeTypes::GetCastedType(type); - } - } result->update_expression_chunk.Initialize(context.client, expression_types); - // updates also write the row id to the file, so the final version needs the row_id placed - // right after the physical columns (before computed partition columns). - vector insert_types = expression_types; - auto physical_column_count = columns.size(); - D_ASSERT(physical_column_count <= insert_types.size()); - insert_types.insert(insert_types.begin() + physical_column_count, LogicalType::BIGINT); - result->insert_chunk.Initialize(context.client, insert_types); + result->insert_chunk.Initialize(context.client, types); result->delete_chunk.Initialize(context.client, delete_types); - if (inline_data_op) { - result->inline_data_lstate = inline_data_op->GetOperatorState(context); - result->inline_output_chunk.Initialize(context.client, inline_data_op->types); - } return std::move(result); } //===--------------------------------------------------------------------===// -// Inline data helpers -//===--------------------------------------------------------------------===// -void DuckLakeUpdate::ForwardInlineOutputToCopy(ExecutionContext &context, DataChunk &inline_output, - GlobalOperatorState &inline_gstate, OperatorState &inline_lstate, - DataChunk &input, LocalSinkState ©_lstate, - InterruptState &interrupt_state) const { - inline_output.Reset(); - auto result = inline_data_op->Execute(context, input, inline_output, inline_gstate, inline_lstate); - if (inline_output.size() > 0) { - OperatorSinkInput copy_input {*copy_op.sink_state, copy_lstate, interrupt_state}; - copy_op.Sink(context, inline_output, copy_input); - } - while (result == OperatorResultType::HAVE_MORE_OUTPUT) { - inline_output.Reset(); - result = inline_data_op->Execute(context, input, inline_output, inline_gstate, inline_lstate); - if (inline_output.size() > 0) { - OperatorSinkInput copy_input {*copy_op.sink_state, copy_lstate, interrupt_state}; - copy_op.Sink(context, inline_output, copy_input); - } - } -} - -//===--------------------------------------------------------------------===// -// Sink +// Execute //===--------------------------------------------------------------------===// -SinkResultType DuckLakeUpdate::Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const { - auto &gstate = input.global_state.Cast(); - auto &lstate = input.local_state.Cast(); +OperatorResultType DuckLakeUpdate::Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk, + GlobalOperatorState &gstate_p, OperatorState &state_p) const { + auto &gstate = gstate_p.Cast(); + auto &lstate = state_p.Cast(); // filter duplicate row IDs using deletion info (last 3 columns) - idx_t delete_idx_start = chunk.ColumnCount() - DELETION_INFO_SIZE; - auto &file_index_vec = chunk.data[delete_idx_start + 1]; - auto &row_number_vec = chunk.data[delete_idx_start + 2]; + idx_t delete_idx_start = input.ColumnCount() - DELETION_INFO_SIZE; + auto &file_index_vec = input.data[delete_idx_start + 1]; + auto &row_number_vec = input.data[delete_idx_start + 2]; UnifiedVectorFormat file_index_data, row_number_data; - file_index_vec.ToUnifiedFormat(chunk.size(), file_index_data); - row_number_vec.ToUnifiedFormat(chunk.size(), row_number_data); + file_index_vec.ToUnifiedFormat(input.size(), file_index_data); + row_number_vec.ToUnifiedFormat(input.size(), row_number_data); auto file_indices = UnifiedVectorFormat::GetData(file_index_data); auto row_numbers = UnifiedVectorFormat::GetData(row_number_data); - SelectionVector sel(chunk.size()); + SelectionVector sel(input.size()); idx_t sel_count = 0; { lock_guard guard(gstate.seen_rows_lock); - for (idx_t i = 0; i < chunk.size(); i++) { + for (idx_t i = 0; i < input.size(); i++) { auto file_idx = file_index_data.sel->get_index(i); auto row_idx = row_number_data.sel->get_index(i); FileRowId key {file_indices[file_idx], row_numbers[row_idx]}; @@ -190,186 +139,68 @@ SinkResultType DuckLakeUpdate::Sink(ExecutionContext &context, DataChunk &chunk, if (sel_count == 0) { // all rows were duplicates - return SinkResultType::NEED_MORE_INPUT; + return OperatorResultType::NEED_MORE_INPUT; } // slice to non-duplicate rows only - chunk.Slice(sel, sel_count); + input.Slice(sel, sel_count); - // push the to-be-inserted data into the copy + // evaluate update expressions auto &update_expression_chunk = lstate.update_expression_chunk; auto &insert_chunk = lstate.insert_chunk; - update_expression_chunk.SetCardinality(chunk.size()); - insert_chunk.SetCardinality(chunk.size()); - lstate.expression_executor->Execute(chunk, update_expression_chunk); + update_expression_chunk.SetCardinality(input.size()); + insert_chunk.SetCardinality(input.size()); + lstate.expression_executor->Execute(input, update_expression_chunk); const idx_t physical_column_count = columns.size(); - const idx_t expression_column_count = update_expression_chunk.ColumnCount(); - D_ASSERT(expression_column_count >= physical_column_count); - const idx_t partition_column_count = expression_column_count - physical_column_count; - const idx_t insert_column_count = insert_chunk.ColumnCount(); - D_ASSERT(insert_column_count >= expression_column_count); - // virtual columns (PlanUpdate sets WRITE_ROW_ID, so there is exactly one: the row id) must sit between the physical - // and partition columns - const idx_t virtual_column_count = insert_column_count - expression_column_count; - D_ASSERT(virtual_column_count == 1); - - // copy the physical columns directly + + // build output, physical columns + row_id for (idx_t i = 0; i < physical_column_count; i++) { insert_chunk.data[i].Reference(update_expression_chunk.data[i]); } - // reference the row id right after the physical columns - insert_chunk.data[physical_column_count].Reference(chunk.data[row_id_index]); - // place computed partition columns after the virtual columns - for (idx_t part_idx = 0; part_idx < partition_column_count; part_idx++) { - insert_chunk.data[physical_column_count + virtual_column_count + part_idx].Reference( - update_expression_chunk.data[physical_column_count + part_idx]); - } + // we place row_id right after physical columns + insert_chunk.data[physical_column_count].Reference(input.data[row_id_index]); - if (inline_data_op) { - // if we have an inline data operator, we need to execute through it, it will either inline it or pass it - // throuhg - ForwardInlineOutputToCopy(context, lstate.inline_output_chunk, *gstate.inline_data_gstate, - *lstate.inline_data_lstate, insert_chunk, *lstate.copy_local_state, - input.interrupt_state); - } else { - OperatorSinkInput copy_input {*copy_op.sink_state, *lstate.copy_local_state, input.interrupt_state}; - copy_op.Sink(context, insert_chunk, copy_input); - } + chunk.Reference(insert_chunk); - // push the rowids into the delete auto &delete_chunk = lstate.delete_chunk; - delete_chunk.SetCardinality(chunk.size()); + delete_chunk.SetCardinality(input.size()); for (idx_t i = 0; i < DELETION_INFO_SIZE; i++) { - delete_chunk.data[i].Reference(chunk.data[delete_idx_start + i]); + delete_chunk.data[i].Reference(input.data[delete_idx_start + i]); } - OperatorSinkInput delete_input {*delete_op.sink_state, *lstate.delete_local_state, input.interrupt_state}; + InterruptState interrupt_state; + OperatorSinkInput delete_input {*delete_op.sink_state, *lstate.delete_local_state, interrupt_state}; delete_op.Sink(context, delete_chunk, delete_input); - lstate.updated_count += chunk.size(); - return SinkResultType::NEED_MORE_INPUT; + lstate.updated_count += input.size(); + return OperatorResultType::NEED_MORE_INPUT; } -//===--------------------------------------------------------------------===// -// Combine -//===--------------------------------------------------------------------===// -SinkCombineResultType DuckLakeUpdate::Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const { - auto &global_state = input.global_state.Cast(); - auto &local_state = input.local_state.Cast(); - // drain inline data FinalExecute — flushes local→global or emits on overflow - if (inline_data_op) { - auto &inline_output = local_state.inline_output_chunk; - while (true) { - inline_output.Reset(); - auto fresult = inline_data_op->FinalExecute(context, inline_output, *global_state.inline_data_gstate, - *local_state.inline_data_lstate); - if (inline_output.size() > 0) { - OperatorSinkInput copy_input {*copy_op.sink_state, *local_state.copy_local_state, - input.interrupt_state}; - copy_op.Sink(context, inline_output, copy_input); - } - if (fresult == OperatorFinalizeResultType::FINISHED) { - break; - } - } - } +OperatorFinalizeResultType DuckLakeUpdate::FinalExecute(ExecutionContext &context, DataChunk &chunk, + GlobalOperatorState &gstate_p, OperatorState &state_p) const { + auto &gstate = gstate_p.Cast(); + auto &lstate = state_p.Cast(); - OperatorSinkCombineInput copy_combine_input {*copy_op.sink_state, *local_state.copy_local_state, - input.interrupt_state}; - auto result = copy_op.Combine(context, copy_combine_input); - if (result != SinkCombineResultType::FINISHED) { - throw InternalException("DuckLakeUpdate::Combine does not support async child operators"); - } - OperatorSinkCombineInput del_combine_input {*delete_op.sink_state, *local_state.delete_local_state, - input.interrupt_state}; - result = delete_op.Combine(context, del_combine_input); + InterruptState interrupt_state; + OperatorSinkCombineInput del_combine_input {*delete_op.sink_state, *lstate.delete_local_state, interrupt_state}; + auto result = delete_op.Combine(context, del_combine_input); if (result != SinkCombineResultType::FINISHED) { - throw InternalException("DuckLakeUpdate::Combine does not support async child operators"); + throw InternalException("DuckLakeUpdate::FinalExecute does not support async child operators"); } - global_state.total_updated_count += local_state.updated_count; - return SinkCombineResultType::FINISHED; + gstate.total_updated_count += lstate.updated_count; + return OperatorFinalizeResultType::FINISHED; } -//===--------------------------------------------------------------------===// -// Finalize -//===--------------------------------------------------------------------===// -SinkFinalizeType DuckLakeUpdate::Finalize(Pipeline &pipeline, Event &event, ClientContext &context, - OperatorSinkFinalizeInput &input) const { - auto &gstate = input.global_state.Cast(); - +OperatorFinalResultType DuckLakeUpdate::OperatorFinalize(Pipeline &pipeline, Event &event, ClientContext &context, + OperatorFinalizeInput &input) const { OperatorSinkFinalizeInput del_finalize_input {*delete_op.sink_state, input.interrupt_state}; auto result = delete_op.Finalize(pipeline, event, context, del_finalize_input); if (result != SinkFinalizeType::READY) { - throw InternalException("DuckLakeUpdate::Finalize does not support async child operators"); - } - - if (inline_data_op) { - // call OperatorFinalize on inline data - OperatorFinalizeInput inline_finalize_input {*gstate.inline_data_gstate, input.interrupt_state}; - inline_data_op->OperatorFinalize(pipeline, event, context, inline_finalize_input); + throw InternalException("DuckLakeUpdate::OperatorFinalize does not support async child operators"); } - - OperatorSinkFinalizeInput copy_finalize_input {*copy_op.sink_state, input.interrupt_state}; - result = copy_op.Finalize(pipeline, event, context, copy_finalize_input); - if (result != SinkFinalizeType::READY) { - throw InternalException("DuckLakeUpdate::Finalize does not support async child operators"); - } - - // scan the copy operator and sink into the insert operator - ThreadContext thread_context(context); - ExecutionContext execution_context(context, thread_context, nullptr); - auto global_source = copy_op.GetGlobalSourceState(context); - auto local_source = copy_op.GetLocalSourceState(execution_context, *global_source); - - DataChunk copy_source_chunk; - copy_source_chunk.Initialize(context, copy_op.types); - - if (!insert_op.sink_state) { - // use existing sink_state if OperatorFinalize already created it - insert_op.sink_state = insert_op.GetGlobalSinkState(context); - } - auto local_sink = insert_op.GetLocalSinkState(execution_context); - - OperatorSourceInput source_input {*global_source, *local_source, input.interrupt_state}; - OperatorSinkInput sink_input {*insert_op.sink_state, *local_sink, input.interrupt_state}; - while (true) { - auto source_result = copy_op.GetData(execution_context, copy_source_chunk, source_input); - if (copy_source_chunk.size() == 0) { - break; - } - if (source_result == SourceResultType::BLOCKED) { - throw InternalException("DuckLakeUpdate::Finalize does not support async child operators"); - } - - auto sink_result = insert_op.Sink(execution_context, copy_source_chunk, sink_input); - if (sink_result == SinkResultType::BLOCKED) { - throw InternalException("DuckLakeUpdate::Finalize does not support async child operators"); - } - if (source_result == SourceResultType::FINISHED) { - break; - } - } - - OperatorSinkFinalizeInput insert_finalize_input {*insert_op.sink_state, input.interrupt_state}; - result = insert_op.Finalize(pipeline, event, context, insert_finalize_input); - if (result != SinkFinalizeType::READY) { - throw InternalException("DuckLakeUpdate::Finalize does not support async child operators"); - } - return SinkFinalizeType::READY; -} - -//===--------------------------------------------------------------------===// -// GetData -//===--------------------------------------------------------------------===// -SourceResultType DuckLakeUpdate::GetDataInternal(ExecutionContext &context, DataChunk &chunk, - OperatorSourceInput &input) const { - auto &global_state = sink_state->Cast(); - auto value = Value::BIGINT(NumericCast(global_state.total_updated_count.load())); - chunk.SetCardinality(1); - chunk.SetValue(0, 0, value); - return SourceResultType::FINISHED; + return OperatorFinalResultType::FINISHED; } //===--------------------------------------------------------------------===// @@ -385,63 +216,6 @@ InsertionOrderPreservingMap DuckLakeUpdate::ParamsToString() const { return result; } -static unique_ptr GetFunction(ClientContext &context, unique_ptr column_reference, - const string &function_name) { - vector> children; - children.emplace_back(std::move(column_reference)); - ErrorData error; - FunctionBinder binder(context); - auto function = binder.BindScalarFunction(DEFAULT_SCHEMA, function_name, std::move(children), error, false); - if (!function) { - error.Throw(); - } - return function; -} - -static unique_ptr GetPartitionExpressionForUpdate(ClientContext &context, - unique_ptr column_reference, - const DuckLakePartitionField &field) { - switch (field.transform.type) { - case DuckLakeTransformType::IDENTITY: - return std::move(column_reference); - case DuckLakeTransformType::YEAR: - return GetFunction(context, std::move(column_reference), "year"); - case DuckLakeTransformType::MONTH: - return GetFunction(context, std::move(column_reference), "month"); - case DuckLakeTransformType::DAY: - return GetFunction(context, std::move(column_reference), "day"); - case DuckLakeTransformType::HOUR: - return GetFunction(context, std::move(column_reference), "hour"); - default: - throw NotImplementedException("Unsupported partition transform type in GetPartitionExpressionForUpdate"); - } -} - -static optional_ptr -PlanInlineDataForUpdate(ClientContext &context, DuckLakeCatalog &catalog, PhysicalPlanGenerator &planner, - DuckLakeTableEntry &table, const vector> &expressions, - idx_t physical_count, PhysicalOperator &child_plan, PhysicalOperator &insert_op) { - vector physical_types; - for (idx_t i = 0; i < physical_count; i++) { - physical_types.push_back(expressions[i]->return_type); - } - idx_t limit = catalog.GetInliningLimit(context, table, physical_types); - if (limit == 0) { - return nullptr; - } - // compute insert_chunk types: [physical (casted)] [BIGINT row_id] [partition (casted)] - vector insert_types; - for (auto &expr : expressions) { - insert_types.push_back(expr->return_type); - } - insert_types.insert(insert_types.begin() + physical_count, LogicalType::BIGINT); - - auto &inline_op = planner.Make(child_plan, limit).Cast(); - inline_op.types = insert_types; - inline_op.insert = insert_op.Cast(); - return &inline_op; -} - PhysicalOperator &DuckLakeCatalog::PlanUpdate(ClientContext &context, PhysicalPlanGenerator &planner, LogicalUpdate &op, PhysicalOperator &child_plan) { if (op.return_chunk) { @@ -453,67 +227,54 @@ PhysicalOperator &DuckLakeCatalog::PlanUpdate(ClientContext &context, PhysicalPl } } auto &table = op.table.Cast(); - // updates are executed as a delete + insert - generate the two nodes (delete and insert) plan the copy for - // the insert + DuckLakeCopyInput copy_input(context, table); copy_input.virtual_columns = InsertVirtualColumns::WRITE_ROW_ID; - auto ©_op = DuckLakeInsert::PlanCopyForInsert(context, planner, copy_input, nullptr); - // plan the delete vector row_id_indexes; for (idx_t i = 0; i < DuckLakeUpdate::DELETION_INFO_SIZE; i++) { row_id_indexes.push_back(i); } auto &delete_op = DuckLakeDelete::PlanDelete(context, planner, table, child_plan, std::move(row_id_indexes), copy_input.encryption_key, false); - // plan the actual insert - auto &insert_op = DuckLakeInsert::PlanInsert(context, planner, table, copy_input.encryption_key); + // build update expressions (physical columns only, no partition cols, no casts) vector> expressions; unordered_map expression_map; - for (idx_t i = 0; i < op.columns.size(); i++) { expression_map[op.columns[i].index] = i; } for (idx_t i = 0; i < op.columns.size(); i++) { expressions.push_back(op.expressions[expression_map[i]]->Copy()); } - if (copy_input.partition_data) { - bool all_identity = true; - for (auto &field : copy_input.partition_data->fields) { - if (field.transform.type != DuckLakeTransformType::IDENTITY) { - all_identity = false; - break; - } - } - if (!all_identity) { - for (auto &field : copy_input.partition_data->fields) { - optional_idx col_idx; - DuckLakeInsert::GetTopLevelColumn(copy_input, field.field_id, col_idx); - D_ASSERT(col_idx.IsValid()); - auto &child_expression = expressions[col_idx.GetIndex()]->Cast(); - auto column_reference = - make_uniq(child_expression.return_type, child_expression.index); - expressions.push_back(GetPartitionExpressionForUpdate(context, std::move(column_reference), field)); - } - } - } - for (auto &expr : expressions) { - if (DuckLakeTypes::RequiresCast(expr->return_type)) { - auto target_type = DuckLakeTypes::GetCastedType(expr->return_type); - expr = BoundCastExpression::AddCastToType(context, std::move(expr), std::move(target_type)); - } + auto &update_op = + planner.Make(table, op.columns, child_plan, delete_op, expressions).Cast(); + + // set output types we use physical column types + BIGINT row_id + vector update_output_types; + for (auto &expr : update_op.expressions) { + update_output_types.push_back(expr->return_type); } + update_output_types.push_back(LogicalType::BIGINT); + update_op.types = std::move(update_output_types); - // plan inline data before creating update (update constructor moves expressions) - auto inline_data = - PlanInlineDataForUpdate(context, *this, planner, table, expressions, op.columns.size(), child_plan, insert_op); + // follow the insert path for inlining: + optional_ptr plan = &update_op; + optional_ptr inline_data; - auto &update_op = - planner.Make(table, op.columns, child_plan, copy_op, delete_op, insert_op, expressions) - .Cast(); - update_op.inline_data_op = inline_data; - return update_op; + idx_t data_inlining_row_limit = GetInliningLimit(context, table, plan->types); + if (data_inlining_row_limit > 0) { + plan = planner.Make(*plan, data_inlining_row_limit); + inline_data = plan->Cast(); + } + + auto &physical_copy = DuckLakeInsert::PlanCopyForInsert(context, planner, copy_input, plan); + auto &insert_op = DuckLakeInsert::PlanInsert(context, planner, table, std::move(copy_input.encryption_key)); + if (inline_data) { + inline_data->insert = insert_op.Cast(); + } + insert_op.children.push_back(physical_copy); + return insert_op; } void DuckLakeTableEntry::BindUpdateConstraints(Binder &binder, LogicalGet &get, LogicalProjection &proj, From 29cfdd7a703003af4aaa416e30ba78c00a554a87 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Mon, 23 Mar 2026 16:35:14 +0100 Subject: [PATCH 50/59] Create PlanUpdateOperator so we can call in both update and merge into --- src/include/storage/ducklake_update.hpp | 12 +++++---- src/storage/ducklake_update.cpp | 35 ++++++++++++++++--------- 2 files changed, 29 insertions(+), 18 deletions(-) diff --git a/src/include/storage/ducklake_update.hpp b/src/include/storage/ducklake_update.hpp index 818f256e403..453f80585dc 100644 --- a/src/include/storage/ducklake_update.hpp +++ b/src/include/storage/ducklake_update.hpp @@ -15,8 +15,7 @@ namespace duckdb { class DuckLakeUpdate : public PhysicalOperator { public: DuckLakeUpdate(PhysicalPlan &physical_plan, DuckLakeTableEntry &table, vector columns, - PhysicalOperator &child, PhysicalOperator &delete_op, - vector> &expressions); + PhysicalOperator &child, PhysicalOperator &delete_op, vector> &expressions); //! The table to update DuckLakeTableEntry &table; @@ -36,8 +35,8 @@ class DuckLakeUpdate : public PhysicalOperator { unique_ptr GetOperatorState(ExecutionContext &context) const override; OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk, GlobalOperatorState &gstate, OperatorState &state) const override; - OperatorFinalizeResultType FinalExecute(ExecutionContext &context, DataChunk &chunk, - GlobalOperatorState &gstate, OperatorState &state) const override; + OperatorFinalizeResultType FinalExecute(ExecutionContext &context, DataChunk &chunk, GlobalOperatorState &gstate, + OperatorState &state) const override; OperatorFinalResultType OperatorFinalize(Pipeline &pipeline, Event &event, ClientContext &context, OperatorFinalizeInput &input) const override; @@ -55,6 +54,9 @@ class DuckLakeUpdate : public PhysicalOperator { string GetName() const override; InsertionOrderPreservingMap ParamsToString() const override; + + static DuckLakeUpdate &PlanUpdateOperator(ClientContext &context, PhysicalPlanGenerator &planner, LogicalUpdate &op, + PhysicalOperator &child_plan, DuckLakeCopyInput ©_input); }; -} // namespace duckdb \ No newline at end of file +} // namespace duckdb diff --git a/src/storage/ducklake_update.cpp b/src/storage/ducklake_update.cpp index 9e4418fb89c..ee8234f1926 100644 --- a/src/storage/ducklake_update.cpp +++ b/src/storage/ducklake_update.cpp @@ -216,20 +216,11 @@ InsertionOrderPreservingMap DuckLakeUpdate::ParamsToString() const { return result; } -PhysicalOperator &DuckLakeCatalog::PlanUpdate(ClientContext &context, PhysicalPlanGenerator &planner, LogicalUpdate &op, - PhysicalOperator &child_plan) { - if (op.return_chunk) { - throw BinderException("RETURNING clause not yet supported for updates of a DuckLake table"); - } - for (auto &expr : op.expressions) { - if (expr->type == ExpressionType::VALUE_DEFAULT) { - throw BinderException("SET DEFAULT is not yet supported for updates of a DuckLake table"); - } - } +DuckLakeUpdate &DuckLakeUpdate::PlanUpdateOperator(ClientContext &context, PhysicalPlanGenerator &planner, + LogicalUpdate &op, PhysicalOperator &child_plan, + DuckLakeCopyInput ©_input) { auto &table = op.table.Cast(); - DuckLakeCopyInput copy_input(context, table); - copy_input.virtual_columns = InsertVirtualColumns::WRITE_ROW_ID; vector row_id_indexes; for (idx_t i = 0; i < DuckLakeUpdate::DELETION_INFO_SIZE; i++) { row_id_indexes.push_back(i); @@ -257,8 +248,26 @@ PhysicalOperator &DuckLakeCatalog::PlanUpdate(ClientContext &context, PhysicalPl } update_output_types.push_back(LogicalType::BIGINT); update_op.types = std::move(update_output_types); + return update_op; +} + +PhysicalOperator &DuckLakeCatalog::PlanUpdate(ClientContext &context, PhysicalPlanGenerator &planner, LogicalUpdate &op, + PhysicalOperator &child_plan) { + if (op.return_chunk) { + throw BinderException("RETURNING clause not yet supported for updates of a DuckLake table"); + } + for (auto &expr : op.expressions) { + if (expr->type == ExpressionType::VALUE_DEFAULT) { + throw BinderException("SET DEFAULT is not yet supported for updates of a DuckLake table"); + } + } + auto &table = op.table.Cast(); + + DuckLakeCopyInput copy_input(context, table); + copy_input.virtual_columns = InsertVirtualColumns::WRITE_ROW_ID; + auto &update_op = DuckLakeUpdate::PlanUpdateOperator(context, planner, op, child_plan, copy_input); - // follow the insert path for inlining: + // follow the insert path for inlining optional_ptr plan = &update_op; optional_ptr inline_data; From c89438ffc3b2e9f258674256aec745327ee1d065 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Mon, 23 Mar 2026 16:39:18 +0100 Subject: [PATCH 51/59] Create merge update, its basically the old logic we had, copied and pasted to handle the sink, since merge gotta do that --- src/storage/ducklake_merge_into.cpp | 239 +++++++++++++++++++++++++++- 1 file changed, 234 insertions(+), 5 deletions(-) diff --git a/src/storage/ducklake_merge_into.cpp b/src/storage/ducklake_merge_into.cpp index 65adf801cf6..8018a5aebf4 100644 --- a/src/storage/ducklake_merge_into.cpp +++ b/src/storage/ducklake_merge_into.cpp @@ -4,6 +4,7 @@ #include "storage/ducklake_update.hpp" #include "storage/ducklake_delete.hpp" #include "storage/ducklake_insert.hpp" +#include "storage/ducklake_inline_data.hpp" #include "storage/ducklake_table_entry.hpp" #include "duckdb/planner/operator/logical_update.hpp" #include "duckdb/planner/operator/logical_dummy_scan.hpp" @@ -184,6 +185,207 @@ unique_ptr DuckLakeMergeInsert::GetLocalSinkState(ExecutionConte return std::move(result); } +//===--------------------------------------------------------------------===// +// Merge Update +//===--------------------------------------------------------------------===// +class DuckLakeMergeUpdate : public PhysicalOperator { +public: + DuckLakeMergeUpdate(PhysicalPlan &physical_plan, const vector &types, DuckLakeUpdate &update_op, + optional_ptr inline_data_op, PhysicalOperator ©_op, + PhysicalOperator &insert_op) + : PhysicalOperator(physical_plan, PhysicalOperatorType::EXTENSION, types, 1), update_op(update_op), + inline_data_op(inline_data_op), copy_op(copy_op), insert_op(insert_op) { + } + + DuckLakeUpdate &update_op; + optional_ptr inline_data_op; + PhysicalOperator ©_op; + PhysicalOperator &insert_op; + +public: + SourceResultType GetDataInternal(ExecutionContext &context, DataChunk &chunk, + OperatorSourceInput &input) const override { + return SourceResultType::FINISHED; + } + bool IsSource() const override { + return true; + } + bool IsSink() const override { + return true; + } + bool ParallelSink() const override { + return true; + } + string GetName() const override { + return "DUCKLAKE_MERGE_UPDATE"; + } + + unique_ptr GetGlobalSinkState(ClientContext &context) const override; + unique_ptr GetLocalSinkState(ExecutionContext &context) const override; + SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override; + SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override; + SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context, + OperatorSinkFinalizeInput &input) const override; +}; + +class DuckLakeMergeUpdateGlobalState : public GlobalSinkState { +public: + unique_ptr update_gstate; + unique_ptr inline_data_gstate; +}; + +class DuckLakeMergeUpdateLocalState : public LocalSinkState { +public: + unique_ptr update_lstate; + unique_ptr inline_data_lstate; + unique_ptr copy_lstate; + DataChunk update_output; + DataChunk inline_output; +}; + +unique_ptr DuckLakeMergeUpdate::GetGlobalSinkState(ClientContext &context) const { + auto result = make_uniq(); + result->update_gstate = update_op.GetGlobalOperatorState(context); + if (inline_data_op) { + result->inline_data_gstate = inline_data_op->GetGlobalOperatorState(context); + } + copy_op.sink_state = copy_op.GetGlobalSinkState(context); + return std::move(result); +} + +unique_ptr DuckLakeMergeUpdate::GetLocalSinkState(ExecutionContext &context) const { + auto result = make_uniq(); + result->update_lstate = update_op.GetOperatorState(context); + if (inline_data_op) { + result->inline_data_lstate = inline_data_op->GetOperatorState(context); + result->inline_output.Initialize(context.client, inline_data_op->types); + } + result->copy_lstate = copy_op.GetLocalSinkState(context); + result->update_output.Initialize(context.client, update_op.types); + return std::move(result); +} + +// Forward output from inline data operator to copy sink, draining HAVE_MORE_OUTPUT +static void ForwardInlineToCopy(ExecutionContext &context, const DuckLakeInlineData &inline_op, + GlobalOperatorState &inline_gstate, OperatorState &inline_lstate, + DataChunk &inline_output, DataChunk &input, PhysicalOperator ©_op, + LocalSinkState ©_lstate, InterruptState &interrupt_state) { + inline_output.Reset(); + auto result = inline_op.Execute(context, input, inline_output, inline_gstate, inline_lstate); + if (inline_output.size() > 0) { + OperatorSinkInput copy_input {*copy_op.sink_state, copy_lstate, interrupt_state}; + copy_op.Sink(context, inline_output, copy_input); + } + while (result == OperatorResultType::HAVE_MORE_OUTPUT) { + inline_output.Reset(); + result = inline_op.Execute(context, input, inline_output, inline_gstate, inline_lstate); + if (inline_output.size() > 0) { + OperatorSinkInput copy_input {*copy_op.sink_state, copy_lstate, interrupt_state}; + copy_op.Sink(context, inline_output, copy_input); + } + } +} + +SinkResultType DuckLakeMergeUpdate::Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const { + auto &gstate = input.global_state.Cast(); + auto &lstate = input.local_state.Cast(); + + lstate.update_output.Reset(); + update_op.Execute(context, chunk, lstate.update_output, *gstate.update_gstate, *lstate.update_lstate); + + if (lstate.update_output.size() == 0) { + return SinkResultType::NEED_MORE_INPUT; + } + + // if we have an inline data operator, we need to execute through it, it will either inline it or pass it + // through to the CopyToFile sink + if (inline_data_op) { + ForwardInlineToCopy(context, *inline_data_op, *gstate.inline_data_gstate, *lstate.inline_data_lstate, + lstate.inline_output, lstate.update_output, copy_op, *lstate.copy_lstate, + input.interrupt_state); + } else { + OperatorSinkInput copy_input {*copy_op.sink_state, *lstate.copy_lstate, input.interrupt_state}; + copy_op.Sink(context, lstate.update_output, copy_input); + } + return SinkResultType::NEED_MORE_INPUT; +} + +SinkCombineResultType DuckLakeMergeUpdate::Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const { + auto &gstate = input.global_state.Cast(); + auto &lstate = input.local_state.Cast(); + + // drain inline data + if (inline_data_op) { + auto &inline_output = lstate.inline_output; + while (true) { + inline_output.Reset(); + auto fresult = inline_data_op->FinalExecute(context, inline_output, *gstate.inline_data_gstate, + *lstate.inline_data_lstate); + if (inline_output.size() > 0) { + OperatorSinkInput copy_input {*copy_op.sink_state, *lstate.copy_lstate, input.interrupt_state}; + copy_op.Sink(context, inline_output, copy_input); + } + if (fresult == OperatorFinalizeResultType::FINISHED) { + break; + } + } + } + + DataChunk dummy; + update_op.FinalExecute(context, dummy, *gstate.update_gstate, *lstate.update_lstate); + + OperatorSinkCombineInput copy_combine {*copy_op.sink_state, *lstate.copy_lstate, input.interrupt_state}; + copy_op.Combine(context, copy_combine); + return SinkCombineResultType::FINISHED; +} + +SinkFinalizeType DuckLakeMergeUpdate::Finalize(Pipeline &pipeline, Event &event, ClientContext &context, + OperatorSinkFinalizeInput &input) const { + auto &gstate = input.global_state.Cast(); + + OperatorFinalizeInput update_finalize {*gstate.update_gstate, input.interrupt_state}; + update_op.OperatorFinalize(pipeline, event, context, update_finalize); + + if (inline_data_op) { + OperatorFinalizeInput inline_finalize {*gstate.inline_data_gstate, input.interrupt_state}; + inline_data_op->OperatorFinalize(pipeline, event, context, inline_finalize); + } + + OperatorSinkFinalizeInput copy_finalize {*copy_op.sink_state, input.interrupt_state}; + copy_op.Finalize(pipeline, event, context, copy_finalize); + + DataChunk chunk; + chunk.Initialize(context, copy_op.types); + ThreadContext thread(context); + ExecutionContext exec_context(context, thread, nullptr); + + auto copy_global = copy_op.GetGlobalSourceState(context); + auto copy_local = copy_op.GetLocalSourceState(exec_context, *copy_global); + OperatorSourceInput source_input {*copy_global, *copy_local, input.interrupt_state}; + + auto insert_global = insert_op.GetGlobalSinkState(context); + auto insert_local = insert_op.GetLocalSinkState(exec_context); + OperatorSinkInput sink_input {*insert_global, *insert_local, input.interrupt_state}; + + while (true) { + chunk.Reset(); + auto source_res = copy_op.GetData(exec_context, chunk, source_input); + if (chunk.size() > 0) { + insert_op.Sink(exec_context, chunk, sink_input); + } + if (source_res == SourceResultType::FINISHED || chunk.size() == 0) { + break; + } + } + + OperatorSinkCombineInput insert_combine {*insert_global, *insert_local, input.interrupt_state}; + insert_op.Combine(exec_context, insert_combine); + OperatorSinkFinalizeInput insert_finalize {*insert_global, input.interrupt_state}; + insert_op.Finalize(pipeline, event, context, insert_finalize); + + return SinkFinalizeType::READY; +} + //===--------------------------------------------------------------------===// // Plan Merge Into //===--------------------------------------------------------------------===// @@ -211,11 +413,38 @@ static unique_ptr DuckLakePlanMergeIntoAction(DuckLakeCatalog update.expressions = std::move(action.expressions); update.columns = std::move(action.columns); update.update_is_del_and_insert = action.update_is_del_and_insert; - auto &update_plan = catalog.PlanUpdate(context, planner, update, child_plan); - result->op = update_plan; - auto &dl_update = result->op->Cast(); - // The row_id comes before the deletion information, which is always the 3 last column of the chunk. - dl_update.row_id_index = child_plan.types.size() - DuckLakeUpdate::DELETION_INFO_SIZE - 1; + + auto &ducklake_table = op.table.Cast(); + DuckLakeCopyInput copy_input(context, ducklake_table); + copy_input.virtual_columns = InsertVirtualColumns::WRITE_ROW_ID; + + auto &update_op = DuckLakeUpdate::PlanUpdateOperator(context, planner, update, child_plan, copy_input); + + // The row_id comes before the deletion information, that is always the 3 last column of the chunk. + update_op.row_id_index = child_plan.types.size() - DuckLakeUpdate::DELETION_INFO_SIZE - 1; + + // maybe wrap with InlineData if we hit the row limit + optional_ptr inline_data; + idx_t data_inlining_row_limit = catalog.GetInliningLimit(context, ducklake_table, update_op.types); + if (data_inlining_row_limit > 0) { + auto &inline_op = + planner.Make(update_op, data_inlining_row_limit).Cast(); + inline_data = &inline_op; + } + + // plan copy and insert + auto ©_op = DuckLakeInsert::PlanCopyForInsert(context, planner, copy_input, nullptr); + auto &insert_op = + DuckLakeInsert::PlanInsert(context, planner, ducklake_table, std::move(copy_input.encryption_key)); + if (inline_data) { + inline_data->insert = insert_op.Cast(); + } + insert_op.children.push_back(copy_op); + + // wrap in DuckLakeMergeUpdate + auto &merge_update = planner.Make(return_types, update_op, inline_data, copy_op, insert_op) + .Cast(); + result->op = merge_update; break; } case MergeActionType::MERGE_DELETE: { From e3912088fa5bf0e9e4b48fe95a608b09042078ab Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Mon, 23 Mar 2026 16:46:25 +0100 Subject: [PATCH 52/59] Fix for s3 and SET disabled_filesystems = 'LocalFileSystem' --- .../storage/ducklake_metadata_manager.hpp | 1 + src/storage/ducklake_metadata_manager.cpp | 16 ++++++++++++---- test/sql/settings/disabled_filesystems.test | 19 +++++++++++++++++++ 3 files changed, 32 insertions(+), 4 deletions(-) create mode 100644 test/sql/settings/disabled_filesystems.test diff --git a/src/include/storage/ducklake_metadata_manager.hpp b/src/include/storage/ducklake_metadata_manager.hpp index 5b3558eeea8..30cc19d0c14 100644 --- a/src/include/storage/ducklake_metadata_manager.hpp +++ b/src/include/storage/ducklake_metadata_manager.hpp @@ -243,6 +243,7 @@ class DuckLakeMetadataManager { string LoadPath(string path); string StorePath(string path); + string GetPathSeparator(const string &path); protected: virtual string GetLatestSnapshotQuery() const; diff --git a/src/storage/ducklake_metadata_manager.cpp b/src/storage/ducklake_metadata_manager.cpp index e2e77cb5e72..7029f1bfe1a 100644 --- a/src/storage/ducklake_metadata_manager.cpp +++ b/src/storage/ducklake_metadata_manager.cpp @@ -2806,9 +2806,18 @@ DuckLakePath DuckLakeMetadataManager::GetRelativePath(const string &path, const return result; } +string DuckLakeMetadataManager::GetPathSeparator(const string &path) { + auto &catalog = transaction.GetCatalog(); + if (!catalog.DataPath().empty()) { + // use the cached separator from the catalog + return catalog.Separator(); + } + // if catalog is not loaded, use the file system + return GetFileSystem().PathSeparator(path); +} + string DuckLakeMetadataManager::StorePath(string path) { - auto &fs = GetFileSystem(); - auto separator = fs.PathSeparator(path); + auto separator = GetPathSeparator(path); if (separator == "/") { return path; } @@ -2816,8 +2825,7 @@ string DuckLakeMetadataManager::StorePath(string path) { } string DuckLakeMetadataManager::LoadPath(string path) { - auto &fs = GetFileSystem(); - auto separator = fs.PathSeparator(path); + auto separator = GetPathSeparator(path); if (separator == "/") { return path; } diff --git a/test/sql/settings/disabled_filesystems.test b/test/sql/settings/disabled_filesystems.test new file mode 100644 index 00000000000..4b88f852d94 --- /dev/null +++ b/test/sql/settings/disabled_filesystems.test @@ -0,0 +1,19 @@ +# name: test/sql/settings/disabled_filesystems.test +# description: StorePath/LoadPath breaks when LocalFileSystem is disabled +# group: [settings] + +require ducklake + +require httpfs + +statement ok +ATTACH 'ducklake::memory:' AS ducklake (DATA_PATH 's3://any-bucket/data/') + +statement ok +USE ducklake + +statement ok +SET disabled_filesystems = 'LocalFileSystem' + +statement ok +CREATE TABLE test (id INTEGER) From 486bdd1eda5fe9817f59aca59e12c27d0919963f Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Mon, 23 Mar 2026 17:10:19 +0100 Subject: [PATCH 53/59] Make FinalizeCopyToInsert reusable --- src/storage/ducklake_merge_into.cpp | 88 +++++++++++------------------ 1 file changed, 33 insertions(+), 55 deletions(-) diff --git a/src/storage/ducklake_merge_into.cpp b/src/storage/ducklake_merge_into.cpp index 8018a5aebf4..419b8383546 100644 --- a/src/storage/ducklake_merge_into.cpp +++ b/src/storage/ducklake_merge_into.cpp @@ -113,54 +113,60 @@ SinkCombineResultType DuckLakeMergeInsert::Combine(ExecutionContext &context, Op return copy.Combine(context, combine_input); } -SinkFinalizeType DuckLakeMergeInsert::Finalize(Pipeline &pipeline, Event &event, ClientContext &context, - OperatorSinkFinalizeInput &input) const { - OperatorSinkFinalizeInput copy_finalize {*copy.sink_state, input.interrupt_state}; - auto finalize_result = copy.Finalize(pipeline, event, context, copy_finalize); - if (finalize_result == SinkFinalizeType::BLOCKED) { - return SinkFinalizeType::BLOCKED; - } - - // now scan the copy +// Scan copy operator source and sink into insert operator — shared by MergeInsert and MergeUpdate +static void FinalizeCopyToInsert(Pipeline &pipeline, Event &event, ClientContext &context, + PhysicalOperator ©_op, PhysicalOperator &insert_op, + InterruptState &interrupt_state) { DataChunk chunk; - chunk.Initialize(context, copy.types); + chunk.Initialize(context, copy_op.types); ThreadContext thread(context); ExecutionContext exec_context(context, thread, nullptr); - auto copy_global = copy.GetGlobalSourceState(context); - auto copy_local = copy.GetLocalSourceState(exec_context, *copy_global); - OperatorSourceInput source_input {*copy_global, *copy_local, input.interrupt_state}; + auto copy_global = copy_op.GetGlobalSourceState(context); + auto copy_local = copy_op.GetLocalSourceState(exec_context, *copy_global); + OperatorSourceInput source_input {*copy_global, *copy_local, interrupt_state}; - auto insert_global = insert.GetGlobalSinkState(context); - auto insert_local = insert.GetLocalSinkState(exec_context); - OperatorSinkInput sink_input {*insert_global, *insert_local, input.interrupt_state}; + auto insert_global = insert_op.GetGlobalSinkState(context); + auto insert_local = insert_op.GetLocalSinkState(exec_context); + OperatorSinkInput sink_input {*insert_global, *insert_local, interrupt_state}; SourceResultType source_res = SourceResultType::HAVE_MORE_OUTPUT; while (source_res == SourceResultType::HAVE_MORE_OUTPUT) { chunk.Reset(); - source_res = copy.GetData(exec_context, chunk, source_input); + source_res = copy_op.GetData(exec_context, chunk, source_input); if (chunk.size() == 0) { continue; } if (source_res == SourceResultType::BLOCKED) { - throw InternalException("BLOCKED not supported in DuckLakeMergeInsert"); + throw InternalException("BLOCKED not supported in DuckLakeMerge"); } - auto sink_result = insert.Sink(exec_context, chunk, sink_input); + auto sink_result = insert_op.Sink(exec_context, chunk, sink_input); if (sink_result != SinkResultType::NEED_MORE_INPUT) { - throw InternalException("BLOCKED not supported in DuckLakeMergeInsert"); + throw InternalException("BLOCKED not supported in DuckLakeMerge"); } } - OperatorSinkCombineInput combine_input {*insert_global, *insert_local, input.interrupt_state}; - auto combine_res = insert.Combine(exec_context, combine_input); + OperatorSinkCombineInput combine_input {*insert_global, *insert_local, interrupt_state}; + auto combine_res = insert_op.Combine(exec_context, combine_input); if (combine_res == SinkCombineResultType::BLOCKED) { - throw InternalException("BLOCKED not supported in DuckLakeMergeInsert"); + throw InternalException("BLOCKED not supported in DuckLakeMerge"); } - OperatorSinkFinalizeInput finalize_input {*insert_global, input.interrupt_state}; - auto finalize_res = insert.Finalize(pipeline, event, context, finalize_input); + OperatorSinkFinalizeInput finalize_input {*insert_global, interrupt_state}; + auto finalize_res = insert_op.Finalize(pipeline, event, context, finalize_input); if (finalize_res == SinkFinalizeType::BLOCKED) { - throw InternalException("BLOCKED not supported in DuckLakeMergeInsert"); + throw InternalException("BLOCKED not supported in DuckLakeMerge"); + } +} + +SinkFinalizeType DuckLakeMergeInsert::Finalize(Pipeline &pipeline, Event &event, ClientContext &context, + OperatorSinkFinalizeInput &input) const { + OperatorSinkFinalizeInput copy_finalize {*copy.sink_state, input.interrupt_state}; + auto finalize_result = copy.Finalize(pipeline, event, context, copy_finalize); + if (finalize_result == SinkFinalizeType::BLOCKED) { + return SinkFinalizeType::BLOCKED; } + + FinalizeCopyToInsert(pipeline, event, context, copy, insert, input.interrupt_state); return SinkFinalizeType::READY; } @@ -354,35 +360,7 @@ SinkFinalizeType DuckLakeMergeUpdate::Finalize(Pipeline &pipeline, Event &event, OperatorSinkFinalizeInput copy_finalize {*copy_op.sink_state, input.interrupt_state}; copy_op.Finalize(pipeline, event, context, copy_finalize); - DataChunk chunk; - chunk.Initialize(context, copy_op.types); - ThreadContext thread(context); - ExecutionContext exec_context(context, thread, nullptr); - - auto copy_global = copy_op.GetGlobalSourceState(context); - auto copy_local = copy_op.GetLocalSourceState(exec_context, *copy_global); - OperatorSourceInput source_input {*copy_global, *copy_local, input.interrupt_state}; - - auto insert_global = insert_op.GetGlobalSinkState(context); - auto insert_local = insert_op.GetLocalSinkState(exec_context); - OperatorSinkInput sink_input {*insert_global, *insert_local, input.interrupt_state}; - - while (true) { - chunk.Reset(); - auto source_res = copy_op.GetData(exec_context, chunk, source_input); - if (chunk.size() > 0) { - insert_op.Sink(exec_context, chunk, sink_input); - } - if (source_res == SourceResultType::FINISHED || chunk.size() == 0) { - break; - } - } - - OperatorSinkCombineInput insert_combine {*insert_global, *insert_local, input.interrupt_state}; - insert_op.Combine(exec_context, insert_combine); - OperatorSinkFinalizeInput insert_finalize {*insert_global, input.interrupt_state}; - insert_op.Finalize(pipeline, event, context, insert_finalize); - + FinalizeCopyToInsert(pipeline, event, context, copy_op, insert_op, input.interrupt_state); return SinkFinalizeType::READY; } From 6ab7d674aef387aa171f6afc7fe1cc21622a71f6 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Mon, 23 Mar 2026 20:49:09 +0100 Subject: [PATCH 54/59] Fix merge into issues related to projections (similar to what we have in the insertion) --- src/storage/ducklake_merge_into.cpp | 120 +++++++++++++++++----------- src/storage/ducklake_update.cpp | 10 +-- 2 files changed, 77 insertions(+), 53 deletions(-) diff --git a/src/storage/ducklake_merge_into.cpp b/src/storage/ducklake_merge_into.cpp index 419b8383546..c090c445b23 100644 --- a/src/storage/ducklake_merge_into.cpp +++ b/src/storage/ducklake_merge_into.cpp @@ -69,6 +69,31 @@ SourceResultType DuckLakeMergeInsert::GetDataInternal(ExecutionContext &context, return SourceResultType::FINISHED; } +//===--------------------------------------------------------------------===// +// Shared helpers +//===--------------------------------------------------------------------===// + +// Apply extra projections and casts before feeding a chunk to CopyToFile — shared by MergeInsert and MergeUpdate +static void ProjectAndCastForCopy(ClientContext &context, DataChunk &input_chunk, PhysicalOperator ©_op, + ExpressionExecutor *expression_executor, DataChunk &projected_chunk, + DataChunk &cast_chunk) { + reference chunk_ref = input_chunk; + if (expression_executor) { + projected_chunk.Reset(); + expression_executor->Execute(input_chunk, projected_chunk); + chunk_ref = projected_chunk; + } + auto ©_types = copy_op.Cast().expected_types; + for (idx_t i = 0; i < chunk_ref.get().ColumnCount(); i++) { + if (chunk_ref.get().data[i].GetType() != copy_types[i]) { + VectorOperations::Cast(context, chunk_ref.get().data[i], cast_chunk.data[i], chunk_ref.get().size()); + } else { + cast_chunk.data[i].Reference(chunk_ref.get().data[i]); + } + } + cast_chunk.SetCardinality(chunk_ref.get().size()); +} + //===--------------------------------------------------------------------===// // Sink //===--------------------------------------------------------------------===// @@ -83,27 +108,9 @@ class DuckLakeMergeIntoLocalState : public LocalSinkState { SinkResultType DuckLakeMergeInsert::Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const { auto &local_state = input.local_state.Cast(); + ProjectAndCastForCopy(context.client, chunk, copy, local_state.expression_executor.get(), local_state.chunk, + local_state.cast_chunk); OperatorSinkInput sink_input {*copy.sink_state, *local_state.copy_sink_state, input.interrupt_state}; - reference chunk_ref = chunk; - if (!extra_projections.empty()) { - // We have extra projections, we need to execute them - local_state.chunk.Reset(); - local_state.expression_executor->Execute(chunk, local_state.chunk); - chunk_ref = local_state.chunk; - } - - // Cast the chunk if needed - auto ©_types = copy.Cast().expected_types; - for (idx_t i = 0; i < chunk_ref.get().ColumnCount(); i++) { - if (chunk_ref.get().data[i].GetType() != copy_types[i]) { - VectorOperations::Cast(context.client, chunk_ref.get().data[i], local_state.cast_chunk.data[i], - chunk_ref.get().size()); - } else { - local_state.cast_chunk.data[i].Reference(chunk_ref.get().data[i]); - } - } - local_state.cast_chunk.SetCardinality(chunk_ref.get().size()); - return copy.Sink(context, local_state.cast_chunk, sink_input); } @@ -114,9 +121,8 @@ SinkCombineResultType DuckLakeMergeInsert::Combine(ExecutionContext &context, Op } // Scan copy operator source and sink into insert operator — shared by MergeInsert and MergeUpdate -static void FinalizeCopyToInsert(Pipeline &pipeline, Event &event, ClientContext &context, - PhysicalOperator ©_op, PhysicalOperator &insert_op, - InterruptState &interrupt_state) { +static void FinalizeCopyToInsert(Pipeline &pipeline, Event &event, ClientContext &context, PhysicalOperator ©_op, + PhysicalOperator &insert_op, InterruptState &interrupt_state) { DataChunk chunk; chunk.Initialize(context, copy_op.types); @@ -207,6 +213,8 @@ class DuckLakeMergeUpdate : public PhysicalOperator { optional_ptr inline_data_op; PhysicalOperator ©_op; PhysicalOperator &insert_op; + //! Extra projections for partition columns + vector> extra_projections; public: SourceResultType GetDataInternal(ExecutionContext &context, DataChunk &chunk, @@ -247,6 +255,10 @@ class DuckLakeMergeUpdateLocalState : public LocalSinkState { unique_ptr copy_lstate; DataChunk update_output; DataChunk inline_output; + //! Projection and cast chunks for partition columns (same pattern as DuckLakeMergeInsert) + unique_ptr expression_executor; + DataChunk projected_chunk; + DataChunk cast_chunk; }; unique_ptr DuckLakeMergeUpdate::GetGlobalSinkState(ClientContext &context) const { @@ -268,28 +280,16 @@ unique_ptr DuckLakeMergeUpdate::GetLocalSinkState(ExecutionConte } result->copy_lstate = copy_op.GetLocalSinkState(context); result->update_output.Initialize(context.client, update_op.types); - return std::move(result); -} - -// Forward output from inline data operator to copy sink, draining HAVE_MORE_OUTPUT -static void ForwardInlineToCopy(ExecutionContext &context, const DuckLakeInlineData &inline_op, - GlobalOperatorState &inline_gstate, OperatorState &inline_lstate, - DataChunk &inline_output, DataChunk &input, PhysicalOperator ©_op, - LocalSinkState ©_lstate, InterruptState &interrupt_state) { - inline_output.Reset(); - auto result = inline_op.Execute(context, input, inline_output, inline_gstate, inline_lstate); - if (inline_output.size() > 0) { - OperatorSinkInput copy_input {*copy_op.sink_state, copy_lstate, interrupt_state}; - copy_op.Sink(context, inline_output, copy_input); - } - while (result == OperatorResultType::HAVE_MORE_OUTPUT) { - inline_output.Reset(); - result = inline_op.Execute(context, input, inline_output, inline_gstate, inline_lstate); - if (inline_output.size() > 0) { - OperatorSinkInput copy_input {*copy_op.sink_state, copy_lstate, interrupt_state}; - copy_op.Sink(context, inline_output, copy_input); + if (!extra_projections.empty()) { + result->expression_executor = make_uniq(context.client, extra_projections); + vector projected_types; + for (auto &expr : result->expression_executor->expressions) { + projected_types.push_back(expr->return_type); } + result->projected_chunk.Initialize(context.client, projected_types); } + result->cast_chunk.Initialize(context.client, copy_op.Cast().expected_types); + return std::move(result); } SinkResultType DuckLakeMergeUpdate::Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const { @@ -306,12 +306,32 @@ SinkResultType DuckLakeMergeUpdate::Sink(ExecutionContext &context, DataChunk &c // if we have an inline data operator, we need to execute through it, it will either inline it or pass it // through to the CopyToFile sink if (inline_data_op) { - ForwardInlineToCopy(context, *inline_data_op, *gstate.inline_data_gstate, *lstate.inline_data_lstate, - lstate.inline_output, lstate.update_output, copy_op, *lstate.copy_lstate, - input.interrupt_state); + auto &inline_output = lstate.inline_output; + inline_output.Reset(); + auto result = inline_data_op->Execute(context, lstate.update_output, inline_output, *gstate.inline_data_gstate, + *lstate.inline_data_lstate); + if (inline_output.size() > 0) { + ProjectAndCastForCopy(context.client, inline_output, copy_op, lstate.expression_executor.get(), + lstate.projected_chunk, lstate.cast_chunk); + OperatorSinkInput copy_input {*copy_op.sink_state, *lstate.copy_lstate, input.interrupt_state}; + copy_op.Sink(context, lstate.cast_chunk, copy_input); + } + while (result == OperatorResultType::HAVE_MORE_OUTPUT) { + inline_output.Reset(); + result = inline_data_op->Execute(context, lstate.update_output, inline_output, *gstate.inline_data_gstate, + *lstate.inline_data_lstate); + if (inline_output.size() > 0) { + ProjectAndCastForCopy(context.client, inline_output, copy_op, lstate.expression_executor.get(), + lstate.projected_chunk, lstate.cast_chunk); + OperatorSinkInput copy_input {*copy_op.sink_state, *lstate.copy_lstate, input.interrupt_state}; + copy_op.Sink(context, lstate.cast_chunk, copy_input); + } + } } else { + ProjectAndCastForCopy(context.client, lstate.update_output, copy_op, lstate.expression_executor.get(), + lstate.projected_chunk, lstate.cast_chunk); OperatorSinkInput copy_input {*copy_op.sink_state, *lstate.copy_lstate, input.interrupt_state}; - copy_op.Sink(context, lstate.update_output, copy_input); + copy_op.Sink(context, lstate.cast_chunk, copy_input); } return SinkResultType::NEED_MORE_INPUT; } @@ -328,8 +348,10 @@ SinkCombineResultType DuckLakeMergeUpdate::Combine(ExecutionContext &context, Op auto fresult = inline_data_op->FinalExecute(context, inline_output, *gstate.inline_data_gstate, *lstate.inline_data_lstate); if (inline_output.size() > 0) { + ProjectAndCastForCopy(context.client, inline_output, copy_op, lstate.expression_executor.get(), + lstate.projected_chunk, lstate.cast_chunk); OperatorSinkInput copy_input {*copy_op.sink_state, *lstate.copy_lstate, input.interrupt_state}; - copy_op.Sink(context, inline_output, copy_input); + copy_op.Sink(context, lstate.cast_chunk, copy_input); } if (fresult == OperatorFinalizeResultType::FINISHED) { break; @@ -411,6 +433,7 @@ static unique_ptr DuckLakePlanMergeIntoAction(DuckLakeCatalog } // plan copy and insert + auto copy_options = DuckLakeInsert::GetCopyOptions(context, copy_input); auto ©_op = DuckLakeInsert::PlanCopyForInsert(context, planner, copy_input, nullptr); auto &insert_op = DuckLakeInsert::PlanInsert(context, planner, ducklake_table, std::move(copy_input.encryption_key)); @@ -422,6 +445,7 @@ static unique_ptr DuckLakePlanMergeIntoAction(DuckLakeCatalog // wrap in DuckLakeMergeUpdate auto &merge_update = planner.Make(return_types, update_op, inline_data, copy_op, insert_op) .Cast(); + merge_update.extra_projections = std::move(copy_options.projection_list); result->op = merge_update; break; } diff --git a/src/storage/ducklake_update.cpp b/src/storage/ducklake_update.cpp index ee8234f1926..cd4ff531224 100644 --- a/src/storage/ducklake_update.cpp +++ b/src/storage/ducklake_update.cpp @@ -219,6 +219,11 @@ InsertionOrderPreservingMap DuckLakeUpdate::ParamsToString() const { DuckLakeUpdate &DuckLakeUpdate::PlanUpdateOperator(ClientContext &context, PhysicalPlanGenerator &planner, LogicalUpdate &op, PhysicalOperator &child_plan, DuckLakeCopyInput ©_input) { + for (auto &expr : op.expressions) { + if (expr->type == ExpressionType::VALUE_DEFAULT) { + throw BinderException("SET DEFAULT is not yet supported for updates of a DuckLake table"); + } + } auto &table = op.table.Cast(); vector row_id_indexes; @@ -256,11 +261,6 @@ PhysicalOperator &DuckLakeCatalog::PlanUpdate(ClientContext &context, PhysicalPl if (op.return_chunk) { throw BinderException("RETURNING clause not yet supported for updates of a DuckLake table"); } - for (auto &expr : op.expressions) { - if (expr->type == ExpressionType::VALUE_DEFAULT) { - throw BinderException("SET DEFAULT is not yet supported for updates of a DuckLake table"); - } - } auto &table = op.table.Cast(); DuckLakeCopyInput copy_input(context, table); From 430f02c348cd4b169a6793d4b696ec4795478a48 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Mon, 23 Mar 2026 23:25:03 +0100 Subject: [PATCH 55/59] Add sanity test --- ...a_inlining_update_inline_verification.test | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 test/sql/data_inlining/data_inlining_update_inline_verification.test diff --git a/test/sql/data_inlining/data_inlining_update_inline_verification.test b/test/sql/data_inlining/data_inlining_update_inline_verification.test new file mode 100644 index 00000000000..bd7071277d9 --- /dev/null +++ b/test/sql/data_inlining/data_inlining_update_inline_verification.test @@ -0,0 +1,109 @@ +# name: test/sql/data_inlining/data_inlining_update_inline_verification.test +# description: verify that a small update on file-backed data gets inlined (insert + delete tables, no new file) +# group: [data_inlining] + +require ducklake + +require parquet + +test-env DUCKLAKE_CONNECTION __TEST_DIR__/{UUID}.db + +test-env DATA_PATH __TEST_DIR__ + +statement ok +ATTACH 'ducklake:${DUCKLAKE_CONNECTION}' AS ducklake (DATA_PATH '${DATA_PATH}/ducklake_update_inline_verify_files', METADATA_CATALOG 'ducklake_meta', DATA_INLINING_ROW_LIMIT 10) + +statement ok +CREATE TABLE ducklake.test AS SELECT i, 'val_' || i::VARCHAR AS j FROM range(20) t(i) + +statement ok +SELECT COUNT(*) cnt FROM GLOB('${DATA_PATH}/ducklake_update_inline_verify_files/**') + +query I +SELECT COUNT(*) = 1 cnt FROM GLOB('${DATA_PATH}/ducklake_update_inline_verify_files/**') +---- +true + +query I +UPDATE ducklake.test SET j='updated' WHERE i=5 +---- +1 + +query I +SELECT COUNT(*) = 1 FROM GLOB('${DATA_PATH}/ducklake_update_inline_verify_files/**') +---- +true + +# verify the data is correct +query II +SELECT * FROM ducklake.test WHERE i=5 +---- +5 updated + +# the inlined insert table should have the new row from the update +query II +SELECT i, j FROM ducklake_meta.ducklake_inlined_data_1_1 +---- +5 updated + +# the inlined delete table should track the old row that was removed from the file +query I +SELECT COUNT(*) FROM ducklake_meta.ducklake_inlined_delete_1 +---- +1 + +# verify rowids are preserved +query III +SELECT rowid, * FROM ducklake.test WHERE i=5 +---- +5 5 updated + +# do another update +query I +UPDATE ducklake.test SET j='changed' WHERE i=10 +---- +1 + +# still no new files +query I +SELECT COUNT(*) = 1FROM GLOB('${DATA_PATH}/ducklake_update_inline_verify_files/**') +---- +true + +# inlined insert table now has 2 rows +query II +SELECT i, j FROM ducklake_meta.ducklake_inlined_data_1_1 ORDER BY i +---- +5 updated +10 changed + +# inlined delete table now has 2 entries +query I +SELECT COUNT(*) FROM ducklake_meta.ducklake_inlined_delete_1 +---- +2 + +# verify all data is correct +query II +SELECT * FROM ducklake.test ORDER BY i +---- +0 val_0 +1 val_1 +2 val_2 +3 val_3 +4 val_4 +5 updated +6 val_6 +7 val_7 +8 val_8 +9 val_9 +10 changed +11 val_11 +12 val_12 +13 val_13 +14 val_14 +15 val_15 +16 val_16 +17 val_17 +18 val_18 +19 val_19 \ No newline at end of file From 7266870d0dc4de5224c80393a364ebadf456411e Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Mon, 23 Mar 2026 23:25:48 +0100 Subject: [PATCH 56/59] woopsie --- .../data_inlining_update_inline_verification.test | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/sql/data_inlining/data_inlining_update_inline_verification.test b/test/sql/data_inlining/data_inlining_update_inline_verification.test index bd7071277d9..9b714fc8536 100644 --- a/test/sql/data_inlining/data_inlining_update_inline_verification.test +++ b/test/sql/data_inlining/data_inlining_update_inline_verification.test @@ -16,9 +16,6 @@ ATTACH 'ducklake:${DUCKLAKE_CONNECTION}' AS ducklake (DATA_PATH '${DATA_PATH}/du statement ok CREATE TABLE ducklake.test AS SELECT i, 'val_' || i::VARCHAR AS j FROM range(20) t(i) -statement ok -SELECT COUNT(*) cnt FROM GLOB('${DATA_PATH}/ducklake_update_inline_verify_files/**') - query I SELECT COUNT(*) = 1 cnt FROM GLOB('${DATA_PATH}/ducklake_update_inline_verify_files/**') ---- From b8c32d8ca08dd93eee55e466ce02fdc6a165f5b1 Mon Sep 17 00:00:00 2001 From: Robert Gernhardt Date: Mon, 23 Mar 2026 20:54:08 -0700 Subject: [PATCH 57/59] Use enum for CheckInlinedDeletionTableCache return value --- src/include/storage/ducklake_catalog.hpp | 5 +++-- src/storage/ducklake_catalog.cpp | 9 +++++---- src/storage/ducklake_metadata_manager.cpp | 4 ++-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/include/storage/ducklake_catalog.hpp b/src/include/storage/ducklake_catalog.hpp index 9d408b251cb..864ff524b33 100644 --- a/src/include/storage/ducklake_catalog.hpp +++ b/src/include/storage/ducklake_catalog.hpp @@ -25,6 +25,8 @@ struct DuckLakeConfigOption; struct DeleteFileMap; class LogicalGet; +enum class InlinedDeletionCacheResult { EXISTS, DOES_NOT_EXIST, UNKNOWN }; + class DuckLakeCatalog : public Catalog { public: // default target file size: 512MB @@ -172,8 +174,7 @@ class DuckLakeCatalog : public Catalog { DuckLakeCatalogSet &GetSchemaForSnapshot(DuckLakeTransaction &transaction, DuckLakeSnapshot snapshot); //! Check if an inlined deletion table is known to exist or not exist for the given table and snapshot - //! Returns: 1 = known to exist, -1 = known to not exist, 0 = unknown (need to query) - int CheckInlinedDeletionTableCache(TableIndex table_id, DuckLakeSnapshot snapshot); + InlinedDeletionCacheResult CheckInlinedDeletionTableCache(TableIndex table_id, DuckLakeSnapshot snapshot); //! Cache the result of an inlined deletion table existence check void CacheInlinedDeletionTableResult(TableIndex table_id, DuckLakeSnapshot snapshot, bool exists); diff --git a/src/storage/ducklake_catalog.cpp b/src/storage/ducklake_catalog.cpp index 015fe727b5a..d504ec90701 100644 --- a/src/storage/ducklake_catalog.cpp +++ b/src/storage/ducklake_catalog.cpp @@ -832,16 +832,17 @@ unique_ptr DuckLakeCatalog::BindAlterAddIndex(Binder &binder, T throw NotImplementedException("Adding indexes or constraints is not supported in DuckLake"); } -int DuckLakeCatalog::CheckInlinedDeletionTableCache(TableIndex table_id, DuckLakeSnapshot snapshot) { +InlinedDeletionCacheResult DuckLakeCatalog::CheckInlinedDeletionTableCache(TableIndex table_id, + DuckLakeSnapshot snapshot) { lock_guard guard(inlined_deletion_cache_lock); if (inlined_deletion_exists.find(table_id.index) != inlined_deletion_exists.end()) { - return 1; // known to exist + return InlinedDeletionCacheResult::EXISTS; } auto it = inlined_deletion_not_exists.find(table_id.index); if (it != inlined_deletion_not_exists.end() && snapshot.snapshot_id <= it->second) { - return -1; // known to not exist at this or earlier snapshot + return InlinedDeletionCacheResult::DOES_NOT_EXIST; } - return 0; // unknown + return InlinedDeletionCacheResult::UNKNOWN; } void DuckLakeCatalog::CacheInlinedDeletionTableResult(TableIndex table_id, DuckLakeSnapshot snapshot, bool exists) { diff --git a/src/storage/ducklake_metadata_manager.cpp b/src/storage/ducklake_metadata_manager.cpp index e3c2de00cdf..fb6761712ee 100644 --- a/src/storage/ducklake_metadata_manager.cpp +++ b/src/storage/ducklake_metadata_manager.cpp @@ -2526,10 +2526,10 @@ string DuckLakeMetadataManager::GetInlinedDeletionTableName(TableIndex table_id, // Check catalog-level cache (persists across transactions) auto &catalog = transaction.GetCatalog(); auto cache_result = catalog.CheckInlinedDeletionTableCache(table_id, snapshot); - if (cache_result == 1) { + if (cache_result == InlinedDeletionCacheResult::EXISTS) { return table_name; // known to exist (committed) } - if (cache_result == -1 && !create_if_not_exists) { + if (cache_result == InlinedDeletionCacheResult::DOES_NOT_EXIST && !create_if_not_exists) { return string(); // known to not exist } From f689aa3bbbf1f122d275adb8f0ac525283574844 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Tue, 24 Mar 2026 13:40:26 +0100 Subject: [PATCH 58/59] Cleanup, more of serialization, remove dummy registration --- src/common/CMakeLists.txt | 4 +- src/common/ducklake_snapshot.cpp | 23 ++++++++ src/ducklake_extension.cpp | 10 +--- src/include/common/ducklake_snapshot.hpp | 6 ++ src/include/storage/ducklake_scan.hpp | 3 + src/storage/ducklake_scan.cpp | 70 ++++++++++++------------ src/storage/ducklake_table_entry.cpp | 10 +--- 7 files changed, 74 insertions(+), 52 deletions(-) create mode 100644 src/common/ducklake_snapshot.cpp diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index c432e6761b0..91ccb393631 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -1,7 +1,7 @@ add_library( ducklake_common OBJECT - ducklake_data_file.cpp ducklake_name_map.cpp ducklake_types.cpp - ducklake_util.cpp parquet_file_scanner.cpp) + ducklake_data_file.cpp ducklake_name_map.cpp ducklake_snapshot.cpp + ducklake_types.cpp ducklake_util.cpp parquet_file_scanner.cpp) set(ALL_OBJECT_FILES ${ALL_OBJECT_FILES} $ PARENT_SCOPE) diff --git a/src/common/ducklake_snapshot.cpp b/src/common/ducklake_snapshot.cpp new file mode 100644 index 00000000000..33eaa21090a --- /dev/null +++ b/src/common/ducklake_snapshot.cpp @@ -0,0 +1,23 @@ +#include "common/ducklake_snapshot.hpp" +#include "duckdb/common/serializer/serializer.hpp" +#include "duckdb/common/serializer/deserializer.hpp" + +namespace duckdb { + +void DuckLakeSnapshot::Serialize(Serializer &serializer) const { + serializer.WriteProperty(100, "snapshot_id", snapshot_id); + serializer.WriteProperty(101, "schema_version", schema_version); + serializer.WriteProperty(102, "next_catalog_id", next_catalog_id); + serializer.WriteProperty(103, "next_file_id", next_file_id); +} + +DuckLakeSnapshot DuckLakeSnapshot::Deserialize(Deserializer &deserializer) { + DuckLakeSnapshot result; + result.snapshot_id = deserializer.ReadProperty(100, "snapshot_id"); + result.schema_version = deserializer.ReadProperty(101, "schema_version"); + result.next_catalog_id = deserializer.ReadProperty(102, "next_catalog_id"); + result.next_file_id = deserializer.ReadProperty(103, "next_file_id"); + return result; +} + +} // namespace duckdb diff --git a/src/ducklake_extension.cpp b/src/ducklake_extension.cpp index bc2cdf4496a..69f6b6218f9 100644 --- a/src/ducklake_extension.cpp +++ b/src/ducklake_extension.cpp @@ -84,13 +84,9 @@ static void LoadInternal(ExtensionLoader &loader) { DuckLakeSettingsFunction settings; loader.RegisterFunction(settings); - // Register ducklake_scan so it can be found during deserialization (e.g. for table macro Copy) - // We register a stub here because parquet may not be loaded yet at extension init time. - // The actual function is fully reconstructed in DuckLakeScanDeserialize. - TableFunction ducklake_scan_stub("ducklake_scan", {LogicalType::VARCHAR}, nullptr, nullptr); - ducklake_scan_stub.serialize = DuckLakeScanSerialize; - ducklake_scan_stub.deserialize = DuckLakeScanDeserialize; - loader.RegisterFunction(ducklake_scan_stub); + // Register ducklake_scan so it can be found during deserialization + auto ducklake_scan = DuckLakeFunctions::GetDuckLakeScanFunction(loader.GetDatabaseInstance()); + loader.RegisterFunction(ducklake_scan); // secrets auto secret_type = DuckLakeSecret::GetSecretType(); diff --git a/src/include/common/ducklake_snapshot.hpp b/src/include/common/ducklake_snapshot.hpp index b2828049cba..b931f1ee3c3 100644 --- a/src/include/common/ducklake_snapshot.hpp +++ b/src/include/common/ducklake_snapshot.hpp @@ -12,6 +12,9 @@ namespace duckdb { +class Serializer; +class Deserializer; + struct DuckLakeSnapshot { DuckLakeSnapshot(idx_t snapshot_id, idx_t schema_version, idx_t next_catalog_id, idx_t next_file_id) : snapshot_id(snapshot_id), schema_version(schema_version), next_catalog_id(next_catalog_id), @@ -27,6 +30,9 @@ struct DuckLakeSnapshot { idx_t schema_version; idx_t next_catalog_id; idx_t next_file_id; + + void Serialize(Serializer &serializer) const; + static DuckLakeSnapshot Deserialize(Deserializer &deserializer); }; } // namespace duckdb diff --git a/src/include/storage/ducklake_scan.hpp b/src/include/storage/ducklake_scan.hpp index 83cbf57fed2..331ad798e36 100644 --- a/src/include/storage/ducklake_scan.hpp +++ b/src/include/storage/ducklake_scan.hpp @@ -40,6 +40,9 @@ enum class DuckLakeScanType { SCAN_TABLE, SCAN_INSERTIONS, SCAN_DELETIONS, SCAN_ struct DuckLakeFunctionInfo : public TableFunctionInfo { DuckLakeFunctionInfo(DuckLakeTableEntry &table, DuckLakeTransaction &transaction, DuckLakeSnapshot snapshot); + static shared_ptr Create(DuckLakeTableEntry &table, DuckLakeTransaction &transaction, + DuckLakeSnapshot snapshot); + DuckLakeTableEntry &table; weak_ptr transaction; string table_name; diff --git a/src/storage/ducklake_scan.cpp b/src/storage/ducklake_scan.cpp index 6d1f5a7493e..f329e15ebc3 100644 --- a/src/storage/ducklake_scan.cpp +++ b/src/storage/ducklake_scan.cpp @@ -182,17 +182,18 @@ vector DuckLakeGetPartitionStats(ClientContext &context, Ge } TableFunction DuckLakeFunctions::GetDuckLakeScanFunction(DatabaseInstance &instance) { - // Parquet extension needs to be loaded for this to make sense - ExtensionHelper::TryAutoLoadExtension(instance, "parquet"); - // The ducklake_scan function is constructed by grabbing the parquet scan from the Catalog, then injecting the // DuckLakeMultiFileReader into it to create a DuckLake-based multi file read + ExtensionHelper::TryAutoLoadExtension(instance, "parquet"); ExtensionLoader loader(instance, "ducklake"); - auto &parquet_scan = loader.GetTableFunction("parquet_scan"); - auto function = parquet_scan.functions.GetFunctionByOffset(0); - // Register the MultiFileReader as the driver for reads - function.get_multi_file_reader = DuckLakeMultiFileReader::CreateInstance; + TableFunction function("ducklake_scan", {LogicalType::VARCHAR}, nullptr, nullptr); + auto parquet_entry = loader.TryGetTableFunction("parquet_scan"); + if (parquet_entry) { + auto &parquet_scan = parquet_entry->Cast(); + function = parquet_scan.functions.GetFunctionByOffset(0); + function.get_multi_file_reader = DuckLakeMultiFileReader::CreateInstance; + } function.statistics = DuckLakeStatistics; function.get_bind_info = DuckLakeBindInfo; @@ -215,6 +216,18 @@ DuckLakeFunctionInfo::DuckLakeFunctionInfo(DuckLakeTableEntry &table, DuckLakeTr : table(table), transaction(transaction_p.shared_from_this()), snapshot(snapshot) { } +shared_ptr +DuckLakeFunctionInfo::Create(DuckLakeTableEntry &table, DuckLakeTransaction &transaction, DuckLakeSnapshot snapshot) { + auto result = make_shared_ptr(table, transaction, snapshot); + result->table_name = table.name; + for (auto &col : table.GetColumns().Logical()) { + result->column_names.push_back(col.Name()); + result->column_types.push_back(col.Type()); + } + result->table_id = table.GetTableId(); + return result; +} + shared_ptr DuckLakeFunctionInfo::GetTransaction() { auto result = transaction.lock(); if (!result) { @@ -227,14 +240,12 @@ shared_ptr DuckLakeFunctionInfo::GetTransaction() { void DuckLakeScanSerialize(Serializer &serializer, const optional_ptr bind_data, const TableFunction &function) { auto &func_info = function.function_info->Cast(); + D_ASSERT(func_info.scan_type == DuckLakeScanType::SCAN_TABLE); auto &catalog = func_info.table.ParentCatalog(); serializer.WriteProperty(100, "catalog_name", catalog.GetName()); serializer.WriteProperty(101, "schema_name", func_info.table.ParentSchema().name); serializer.WriteProperty(102, "table_name", func_info.table_name); - serializer.WriteProperty(103, "snapshot_id", func_info.snapshot.snapshot_id); - serializer.WriteProperty(104, "schema_version", func_info.snapshot.schema_version); - serializer.WriteProperty(105, "next_catalog_id", func_info.snapshot.next_catalog_id); - serializer.WriteProperty(106, "next_file_id", func_info.snapshot.next_file_id); + serializer.WriteObject(103, "snapshot", [&](Serializer &obj) { func_info.snapshot.Serialize(obj); }); } unique_ptr DuckLakeScanDeserialize(Deserializer &deserializer, TableFunction &function) { @@ -242,38 +253,27 @@ unique_ptr DuckLakeScanDeserialize(Deserializer &deserializer, Tab auto catalog_name = deserializer.ReadProperty(100, "catalog_name"); auto schema_name = deserializer.ReadProperty(101, "schema_name"); auto table_name = deserializer.ReadProperty(102, "table_name"); - auto snapshot_id = deserializer.ReadProperty(103, "snapshot_id"); - auto schema_version = deserializer.ReadProperty(104, "schema_version"); - auto next_catalog_id = deserializer.ReadProperty(105, "next_catalog_id"); - auto next_file_id = deserializer.ReadProperty(106, "next_file_id"); + DuckLakeSnapshot snapshot; + deserializer.ReadObject(103, "snapshot", [&](Deserializer &obj) { snapshot = DuckLakeSnapshot::Deserialize(obj); }); + + // If ducklake_scan was registered before parquet was loaded, we set it now + if (!function.bind) { + function = DuckLakeFunctions::GetDuckLakeScanFunction(*context.db); + if (!function.bind) { + throw InvalidInputException("ducklake_scan requires the parquet extension to be loaded"); + } + } // Look up the DuckLake catalog and table auto &catalog = Catalog::GetCatalog(context, catalog_name); auto &transaction = DuckLakeTransaction::Get(context, catalog); - DuckLakeSnapshot snapshot(snapshot_id, schema_version, next_catalog_id, next_file_id); auto &table_entry = Catalog::GetEntry(context, catalog_name, schema_name, table_name).Cast(); - // Reconstruct the full ducklake_scan function (the deserialized stub lacks parquet callbacks) - auto full_function = DuckLakeFunctions::GetDuckLakeScanFunction(*context.db); - - // Set up function info - auto function_info = make_shared_ptr(table_entry, transaction, snapshot); - function_info->table_name = table_name; - for (auto &col : table_entry.GetColumns().Logical()) { - function_info->column_names.push_back(col.Name()); - function_info->column_types.push_back(col.Type()); - } - function_info->table_id = table_entry.GetTableId(); - full_function.function_info = std::move(function_info); + function.function_info = DuckLakeFunctionInfo::Create(table_entry, transaction, snapshot); - // Bind the scan using the full function - auto bind_data = DuckLakeFunctions::BindDuckLakeScan(context, full_function); - - // Copy the reconstructed function back so the LogicalGet uses it - function = std::move(full_function); - - return bind_data; + return DuckLakeFunctions::BindDuckLakeScan(context, function); } + } // namespace duckdb diff --git a/src/storage/ducklake_table_entry.cpp b/src/storage/ducklake_table_entry.cpp index 5f8f4d7d363..639cae56690 100644 --- a/src/storage/ducklake_table_entry.cpp +++ b/src/storage/ducklake_table_entry.cpp @@ -260,14 +260,8 @@ TableFunction DuckLakeTableEntry::GetScanFunction(ClientContext &context, unique auto function = DuckLakeFunctions::GetDuckLakeScanFunction(*context.db); auto &transaction = DuckLakeTransaction::Get(context, ParentCatalog()); auto function_info = - make_shared_ptr(*this, transaction, transaction.GetSnapshot(lookup_info.GetAtClause())); - function_info->table_name = name; - for (auto &col : columns.Logical()) { - function_info->column_names.push_back(col.Name()); - function_info->column_types.push_back(col.Type()); - } - auto table_id = GetTableId(); - function_info->table_id = table_id; + DuckLakeFunctionInfo::Create(*this, transaction, transaction.GetSnapshot(lookup_info.GetAtClause())); + auto table_id = function_info->table_id; function.function_info = std::move(function_info); auto &dropped_tables = transaction.GetDroppedTables(); auto &renamed_tables = transaction.GetRenamedTables(); From 0711faefdf1fe37e6d0ed5b17aae7aa4585875e2 Mon Sep 17 00:00:00 2001 From: Pedro Holanda Date: Tue, 24 Mar 2026 15:21:24 +0100 Subject: [PATCH 59/59] Fix for #865 --- src/storage/ducklake_multi_file_reader.cpp | 8 +-- .../issues/issue_865_update_wrong_result.test | 60 +++++++++++++++++++ 2 files changed, 64 insertions(+), 4 deletions(-) create mode 100644 test/sql/issues/issue_865_update_wrong_result.test diff --git a/src/storage/ducklake_multi_file_reader.cpp b/src/storage/ducklake_multi_file_reader.cpp index 710110ac742..ade2e4cec59 100644 --- a/src/storage/ducklake_multi_file_reader.cpp +++ b/src/storage/ducklake_multi_file_reader.cpp @@ -253,6 +253,10 @@ ReaderInitializeType DuckLakeMultiFileReader::InitializeReader(MultiFileReaderDa if (!file_entry.delete_file.path.empty()) { delete_filter->Initialize(context, file_entry.delete_file); } + if (delete_map && !file_entry.delete_file.path.empty()) { + auto delete_data_copy = make_shared_ptr(*delete_filter->delete_data); + delete_map->AddDeleteData(reader.GetFileName(), std::move(delete_data_copy)); + } // Apply inlined file deletions (stored in metadata database instead of delete file) if (!file_entry.inlined_file_deletions.empty()) { DuckLakeInlinedDataDeletes inlined_deletes; @@ -264,10 +268,6 @@ ReaderInitializeType DuckLakeMultiFileReader::InitializeReader(MultiFileReaderDa } // set the snapshot id so we know what to skip from deletion files delete_filter->SetSnapshotFilter(read_info.snapshot.snapshot_id); - // Only add to delete_map if there's an actual delete file and not just inlined deletions - if (delete_map && !file_entry.delete_file.path.empty()) { - delete_map->AddDeleteData(reader.GetFileName(), delete_filter->delete_data); - } reader.deletion_filter = std::move(delete_filter); } } else { diff --git a/test/sql/issues/issue_865_update_wrong_result.test b/test/sql/issues/issue_865_update_wrong_result.test new file mode 100644 index 00000000000..589b4600dac --- /dev/null +++ b/test/sql/issues/issue_865_update_wrong_result.test @@ -0,0 +1,60 @@ +# name: test/sql/issues/issue_865_update_wrong_result.test +# description: UPDATE on ducklake table produces duplicate rows when a data file has both a committed delete file and committed inlined file deletions +# group: [issues] + +require ducklake + +require parquet + +test-env DUCKLAKE_CONNECTION __TEST_DIR__/{UUID}.db + +test-env DATA_PATH __TEST_DIR__ + +statement ok +ATTACH 'ducklake:${DUCKLAKE_CONNECTION}' AS ducklake (DATA_PATH '${DATA_PATH}/issue_865', DATA_INLINING_ROW_LIMIT 10) + +# Create table with enough rows to NOT be inlined +statement ok +CREATE TABLE ducklake.test AS SELECT i AS id, 'original' AS val FROM range(100) t(i) + +# Create delete file +statement ok +DELETE FROM ducklake.test WHERE id >= 80 + +query I +SELECT count(*) FROM ducklake.test +---- +80 + +# Inline deletes +statement ok +DELETE FROM ducklake.test WHERE id >= 75 + +query I +SELECT count(*) FROM ducklake.test +---- +75 + +# If we update more than the inline value, inlined deletion merge will corrupt de delete data +query I +UPDATE ducklake.test SET val = 'updated' WHERE id < 20 +---- +20 + +# After update: should still have 75 rows +query I +SELECT count(*) FROM ducklake.test +---- +75 + +# The 20 updated rows should exist exactly once with 'updated' +query I +SELECT count(*) FROM ducklake.test WHERE val = 'updated' +---- +20 + +# No duplicate ids +query I +SELECT count(*) FROM (SELECT id, count(*) cnt FROM ducklake.test GROUP BY id HAVING count(*) > 1) +---- +0