diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index c432e6761b0..91ccb393631 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -1,7 +1,7 @@ add_library( ducklake_common OBJECT - ducklake_data_file.cpp ducklake_name_map.cpp ducklake_types.cpp - ducklake_util.cpp parquet_file_scanner.cpp) + ducklake_data_file.cpp ducklake_name_map.cpp ducklake_snapshot.cpp + ducklake_types.cpp ducklake_util.cpp parquet_file_scanner.cpp) set(ALL_OBJECT_FILES ${ALL_OBJECT_FILES} $ PARENT_SCOPE) diff --git a/src/common/ducklake_snapshot.cpp b/src/common/ducklake_snapshot.cpp new file mode 100644 index 00000000000..33eaa21090a --- /dev/null +++ b/src/common/ducklake_snapshot.cpp @@ -0,0 +1,23 @@ +#include "common/ducklake_snapshot.hpp" +#include "duckdb/common/serializer/serializer.hpp" +#include "duckdb/common/serializer/deserializer.hpp" + +namespace duckdb { + +void DuckLakeSnapshot::Serialize(Serializer &serializer) const { + serializer.WriteProperty(100, "snapshot_id", snapshot_id); + serializer.WriteProperty(101, "schema_version", schema_version); + serializer.WriteProperty(102, "next_catalog_id", next_catalog_id); + serializer.WriteProperty(103, "next_file_id", next_file_id); +} + +DuckLakeSnapshot DuckLakeSnapshot::Deserialize(Deserializer &deserializer) { + DuckLakeSnapshot result; + result.snapshot_id = deserializer.ReadProperty(100, "snapshot_id"); + result.schema_version = deserializer.ReadProperty(101, "schema_version"); + result.next_catalog_id = deserializer.ReadProperty(102, "next_catalog_id"); + result.next_file_id = deserializer.ReadProperty(103, "next_file_id"); + return result; +} + +} // namespace duckdb diff --git a/src/ducklake_extension.cpp b/src/ducklake_extension.cpp index 67d4a1e349d..6307187db69 100644 --- a/src/ducklake_extension.cpp +++ b/src/ducklake_extension.cpp @@ -3,6 +3,7 @@ #include "duckdb/common/exception.hpp" #include "duckdb/common/string_util.hpp" #include "storage/ducklake_storage.hpp" +#include "storage/ducklake_scan.hpp" #include "functions/ducklake_table_functions.hpp" #include "storage/ducklake_secret.hpp" #include "duckdb/storage/storage_extension.hpp" @@ -86,6 +87,10 @@ static void LoadInternal(ExtensionLoader &loader) { DuckLakeSettingsFunction settings; loader.RegisterFunction(settings); + // Register ducklake_scan so it can be found during deserialization + auto ducklake_scan = DuckLakeFunctions::GetDuckLakeScanFunction(loader.GetDatabaseInstance()); + loader.RegisterFunction(ducklake_scan); + // secrets auto secret_type = DuckLakeSecret::GetSecretType(); loader.RegisterSecretType(secret_type); diff --git a/src/functions/ducklake_add_data_files.cpp b/src/functions/ducklake_add_data_files.cpp index 89c911bfd53..809a198c6c1 100644 --- a/src/functions/ducklake_add_data_files.cpp +++ b/src/functions/ducklake_add_data_files.cpp @@ -130,9 +130,8 @@ struct DuckLakeFileProcessor { public: DuckLakeFileProcessor(DuckLakeTransaction &transaction, ClientContext &context, const DuckLakeAddDataFilesData &bind_data) - : transaction(transaction), context(context), table(bind_data.table), - allow_missing(bind_data.allow_missing), ignore_extra_columns(bind_data.ignore_extra_columns), - hive_partitioning(bind_data.hive_partitioning) { + : transaction(transaction), context(context), table(bind_data.table), allow_missing(bind_data.allow_missing), + ignore_extra_columns(bind_data.ignore_extra_columns), hive_partitioning(bind_data.hive_partitioning) { } vector AddFiles(const vector &globs); @@ -1167,7 +1166,8 @@ void DuckLakeFileProcessor::MapPartitionColumns(ParquetFileMetadata &file) { // For YEAR/MONTH/DAY/HOUR transforms, the type is BIGINT, not the source column type auto partition_key_type = DuckLakePartitionUtils::GetPartitionKeyType(partition_field.transform.type, field_id->Type()); - auto hive_value = HivePartitioning::GetValue(context, partition_key_name, hive_entry->second, partition_key_type); + auto hive_value = + HivePartitioning::GetValue(context, partition_key_name, hive_entry->second, partition_key_type); file.hive_partition_values.emplace_back( HivePartition {partition_field.field_id, partition_key_type, hive_value}); } diff --git a/src/functions/ducklake_compaction_functions.cpp b/src/functions/ducklake_compaction_functions.cpp index 6c2ba53be61..7e16234d8e8 100644 --- a/src/functions/ducklake_compaction_functions.cpp +++ b/src/functions/ducklake_compaction_functions.cpp @@ -93,11 +93,17 @@ SourceResultType DuckLakeCompaction::GetDataInternal(ExecutionContext &context, } source_state.returned_result = true; + if (!this->sink_state) { + throw InternalException("DuckLakeCompaction - missing sink state while producing result"); + } + auto &gstate = this->sink_state->Cast(); + auto files_created = gstate.written_files.size(); + chunk.SetCardinality(1); chunk.SetValue(0, 0, Value(table.schema.name)); chunk.SetValue(1, 0, Value(table.name)); chunk.SetValue(2, 0, Value::BIGINT(static_cast(source_files.size()))); - chunk.SetValue(3, 0, Value::BIGINT(1)); // Each compaction creates 1 output file + chunk.SetValue(3, 0, Value::BIGINT(static_cast(files_created))); return SourceResultType::FINISHED; } @@ -121,7 +127,10 @@ SinkFinalizeType DuckLakeCompaction::Finalize(Pipeline &pipeline, Event &event, OperatorSinkFinalizeInput &input) const { auto &global_state = input.global_state.Cast(); - if (global_state.written_files.size() != 1) { + if (global_state.written_files.size() > 1) { + throw InternalException("DuckLakeCompaction - expected at most a single output file"); + } + if (global_state.written_files.empty() && type != CompactionType::REWRITE_DELETES) { throw InternalException("DuckLakeCompaction - expected a single output file"); } // set the partition values correctly @@ -137,7 +146,9 @@ SinkFinalizeType DuckLakeCompaction::Finalize(Pipeline &pipeline, Event &event, DuckLakeCompactionEntry compaction_entry; compaction_entry.row_id_start = row_id_start; compaction_entry.source_files = source_files; - compaction_entry.written_file = global_state.written_files[0]; + if (!global_state.written_files.empty()) { + compaction_entry.written_file = global_state.written_files[0]; + } compaction_entry.type = type; auto &transaction = DuckLakeTransaction::Get(context, global_state.table.catalog); @@ -222,8 +233,9 @@ void DuckLakeCompactor::GenerateCompactions(DuckLakeTableEntry &table, compaction_map_t candidates; for (idx_t file_idx = 0; file_idx < files.size(); file_idx++) { auto &candidate = files[file_idx]; - if (candidate.file.data.file_size_bytes >= target_file_size) { + if (candidate.file.data.file_size_bytes >= target_file_size && type != CompactionType::REWRITE_DELETES) { // this file by itself exceeds the threshold - skip merging + // (does not apply to REWRITE_DELETES - delete files must be rewritten regardless of data file size) continue; } if ((!candidate.delete_files.empty() && type == CompactionType::MERGE_ADJACENT_TABLES) || @@ -304,6 +316,9 @@ void DuckLakeCompactor::GenerateCompactions(DuckLakeTableEntry &table, break; } } + if (compacted_files >= options.max_files) { + break; + } } } @@ -559,7 +574,7 @@ DuckLakeCompactor::GenerateCompactionCommand(vector copy->filename_pattern = std::move(copy_options.filename_pattern); copy->file_extension = std::move(copy_options.file_extension); copy->overwrite_mode = copy_options.overwrite_mode; - copy->per_thread_output = copy_options.per_thread_output; + copy->per_thread_output = false; copy->file_size_bytes = copy_options.file_size_bytes; copy->rotate = copy_options.rotate; copy->return_type = copy_options.return_type; diff --git a/src/functions/ducklake_flush_inlined_data.cpp b/src/functions/ducklake_flush_inlined_data.cpp index eab6be6e8b6..97d84561124 100644 --- a/src/functions/ducklake_flush_inlined_data.cpp +++ b/src/functions/ducklake_flush_inlined_data.cpp @@ -278,8 +278,9 @@ DuckLakeDataFlusher::DuckLakeDataFlusher(ClientContext &context, DuckLakeCatalog unique_ptr DuckLakeDataFlusher::GenerateFlushCommand() { // get the table entry at the specified snapshot - DuckLakeSnapshot snapshot(catalog.GetBeginSnapshotForTable(table_id, transaction), inlined_table.schema_version, 0, - 0); + DuckLakeSnapshot snapshot( + catalog.GetBeginSnapshotForSchemaVersion(table_id, inlined_table.schema_version, transaction), + inlined_table.schema_version, 0, 0); auto entry = catalog.GetEntryById(transaction, snapshot, table_id); if (!entry) { diff --git a/src/include/common/ducklake_snapshot.hpp b/src/include/common/ducklake_snapshot.hpp index b2828049cba..b931f1ee3c3 100644 --- a/src/include/common/ducklake_snapshot.hpp +++ b/src/include/common/ducklake_snapshot.hpp @@ -12,6 +12,9 @@ namespace duckdb { +class Serializer; +class Deserializer; + struct DuckLakeSnapshot { DuckLakeSnapshot(idx_t snapshot_id, idx_t schema_version, idx_t next_catalog_id, idx_t next_file_id) : snapshot_id(snapshot_id), schema_version(schema_version), next_catalog_id(next_catalog_id), @@ -27,6 +30,9 @@ struct DuckLakeSnapshot { idx_t schema_version; idx_t next_catalog_id; idx_t next_file_id; + + void Serialize(Serializer &serializer) const; + static DuckLakeSnapshot Deserialize(Deserializer &deserializer); }; } // namespace duckdb diff --git a/src/include/common/index.hpp b/src/include/common/index.hpp index acf03ca4b1e..161af856fa4 100644 --- a/src/include/common/index.hpp +++ b/src/include/common/index.hpp @@ -15,6 +15,11 @@ namespace duckdb { struct DuckLakeConstants { static constexpr const idx_t TRANSACTION_LOCAL_ID_START = 9223372036854775808ULL; + static constexpr const idx_t TRANSACTION_LOCAL_ROW_ID_START = 1000000000000000000ULL; + + static bool IsTransactionLocalRowId(int64_t rid) { + return rid >= 0 && static_cast(rid) >= TRANSACTION_LOCAL_ROW_ID_START; + } }; struct SchemaIndex { diff --git a/src/include/storage/ducklake_catalog.hpp b/src/include/storage/ducklake_catalog.hpp index b95374d68e4..af460f12fd5 100644 --- a/src/include/storage/ducklake_catalog.hpp +++ b/src/include/storage/ducklake_catalog.hpp @@ -25,6 +25,8 @@ struct DuckLakeConfigOption; struct DeleteFileMap; class LogicalGet; +enum class InlinedDeletionCacheResult { EXISTS, DOES_NOT_EXIST, UNKNOWN }; + class DuckLakeCatalog : public Catalog { public: // default target file size: 512MB @@ -57,6 +59,8 @@ class DuckLakeCatalog : public Catalog { return metadata_type; } idx_t DataInliningRowLimit(SchemaIndex schema_index, TableIndex table_index) const; + //! Returns the inlining limit (0 if the table is not eligible) + idx_t GetInliningLimit(ClientContext &context, DuckLakeTableEntry &table, const vector &types); string &Separator() { return separator; } @@ -164,12 +168,18 @@ class DuckLakeCatalog : public Catalog { optional_ptr TryGetMappingById(DuckLakeTransaction &transaction, MappingIndex mapping_id); MappingIndex TryGetCompatibleNameMap(DuckLakeTransaction &transaction, const DuckLakeNameMap &name_map); idx_t GetBeginSnapshotForTable(TableIndex table_id, DuckLakeTransaction &transaction); + idx_t GetBeginSnapshotForSchemaVersion(TableIndex table_id, idx_t schema_version, DuckLakeTransaction &transaction); static unique_ptr ConstructStatsMap(vector &global_stats, DuckLakeCatalogSet &schema); //! Return the schema for the given snapshot - loading it if it is not yet loaded DuckLakeCatalogSet &GetSchemaForSnapshot(DuckLakeTransaction &transaction, DuckLakeSnapshot snapshot); + //! Check if an inlined deletion table is known to exist or not exist for the given table and snapshot + InlinedDeletionCacheResult CheckInlinedDeletionTableCache(TableIndex table_id, DuckLakeSnapshot snapshot); + //! Cache the result of an inlined deletion table existence check + void CacheInlinedDeletionTableResult(TableIndex table_id, DuckLakeSnapshot snapshot, bool exists); + private: void DropSchema(ClientContext &context, DropInfo &info) override; unique_ptr LoadSchemaForSnapshot(DuckLakeTransaction &transaction, DuckLakeSnapshot snapshot); @@ -200,6 +210,13 @@ class DuckLakeCatalog : public Catalog { string metadata_type; //! Whether or not the catalog is initialized bool initialized = false; + //! Cache for inlined deletion table existence checks + mutex inlined_deletion_cache_lock; + //! Table IDs where the inlined deletion table is known to exist (permanent - never invalidated) + unordered_set inlined_deletion_exists; + //! Table IDs where the inlined deletion table is known to NOT exist, with the snapshot_id at which we checked + //! Valid as long as current snapshot.snapshot_id <= cached snapshot_id + unordered_map inlined_deletion_not_exists; //! The id of the last committed snapshot, set at FlushChanges on a successful commit mutable mutex commit_lock; optional_idx last_committed_snapshot; diff --git a/src/include/storage/ducklake_inlined_data.hpp b/src/include/storage/ducklake_inlined_data.hpp index 99245360818..0ce107fcdc8 100644 --- a/src/include/storage/ducklake_inlined_data.hpp +++ b/src/include/storage/ducklake_inlined_data.hpp @@ -17,6 +17,16 @@ namespace duckdb { struct DuckLakeInlinedData { unique_ptr data; map column_stats; + //! Row Ids for update inlining + vector row_ids; + + bool HasPreservedRowIds() const; + //! Get the row_id for a given position in the data collection + idx_t GetRowId(idx_t position) const; + //! Get the output row_id for a surviving (non-deleted) row + int64_t GetOutputRowId(idx_t position) const; + //! Merge preserved row_ids from update inlining + void MergeRowIds(const DuckLakeInlinedData &new_data, idx_t new_data_count); }; struct DuckLakeInlinedDataDeletes { diff --git a/src/include/storage/ducklake_metadata_manager.hpp b/src/include/storage/ducklake_metadata_manager.hpp index 73ea5cd7563..de26c522d17 100644 --- a/src/include/storage/ducklake_metadata_manager.hpp +++ b/src/include/storage/ducklake_metadata_manager.hpp @@ -159,6 +159,7 @@ class DuckLakeMetadataManager { DuckLakeSnapshot snapshot, DuckLakeFileSizeOptions options); virtual idx_t GetBeginSnapshotForTable(TableIndex table_id); + virtual idx_t GetBeginSnapshotForSchemaVersion(TableIndex table_id, idx_t schema_version); virtual idx_t GetNetDataFileRowCount(TableIndex table_id, DuckLakeSnapshot snapshot); virtual idx_t GetNetInlinedRowCount(const string &inlined_table_name, DuckLakeSnapshot snapshot); virtual vector GetOldFilesForCleanup(const string &filter); @@ -259,6 +260,7 @@ class DuckLakeMetadataManager { string LoadPath(string path); string StorePath(string path); + string GetPathSeparator(const string &path); protected: virtual string GetLatestSnapshotQuery() const; diff --git a/src/include/storage/ducklake_multi_file_list.hpp b/src/include/storage/ducklake_multi_file_list.hpp index 22c92b19045..ff103ead7ba 100644 --- a/src/include/storage/ducklake_multi_file_list.hpp +++ b/src/include/storage/ducklake_multi_file_list.hpp @@ -20,7 +20,6 @@ namespace duckdb { //! The DuckLakeMultiFileList implements the MultiFileList API to allow injecting it into the regular DuckDB parquet //! scan class DuckLakeMultiFileList : public MultiFileList { - static constexpr const idx_t TRANSACTION_LOCAL_ID_START = 1000000000000000000ULL; static constexpr const char *DUCKLAKE_TRANSACTION_LOCAL_INLINED_FILENAME = "__ducklake_inlined_transaction_local_data"; @@ -63,6 +62,8 @@ class DuckLakeMultiFileList : public MultiFileList { void GetFilesForTable() const; void GetTableInsertions() const; void GetTableDeletions() const; + //! Get the row_id_start for transaction-local inlined data. + idx_t GetTransactionLocalRowIdStart(idx_t transaction_row_start) const; private: mutable mutex file_lock; diff --git a/src/include/storage/ducklake_scan.hpp b/src/include/storage/ducklake_scan.hpp index 84963d10a51..331ad798e36 100644 --- a/src/include/storage/ducklake_scan.hpp +++ b/src/include/storage/ducklake_scan.hpp @@ -17,6 +17,8 @@ namespace duckdb { class DuckLakeMultiFileList; class DuckLakeTableEntry; class DuckLakeTransaction; +class Serializer; +class Deserializer; class DuckLakeFunctions { public: @@ -28,11 +30,19 @@ class DuckLakeFunctions { static CopyFunctionCatalogEntry &GetCopyFunction(ClientContext &context, const string &name); }; +//! Serialize/Deserialize callbacks for DuckLakeScan (used by table macro Copy) +void DuckLakeScanSerialize(Serializer &serializer, const optional_ptr bind_data, + const TableFunction &function); +unique_ptr DuckLakeScanDeserialize(Deserializer &deserializer, TableFunction &function); + enum class DuckLakeScanType { SCAN_TABLE, SCAN_INSERTIONS, SCAN_DELETIONS, SCAN_FOR_FLUSH }; struct DuckLakeFunctionInfo : public TableFunctionInfo { DuckLakeFunctionInfo(DuckLakeTableEntry &table, DuckLakeTransaction &transaction, DuckLakeSnapshot snapshot); + static shared_ptr Create(DuckLakeTableEntry &table, DuckLakeTransaction &transaction, + DuckLakeSnapshot snapshot); + DuckLakeTableEntry &table; weak_ptr transaction; string table_name; diff --git a/src/include/storage/ducklake_stats.hpp b/src/include/storage/ducklake_stats.hpp index 75fb45ef82b..04e9934535c 100644 --- a/src/include/storage/ducklake_stats.hpp +++ b/src/include/storage/ducklake_stats.hpp @@ -15,7 +15,7 @@ class BaseStatistics; //! Returns true for types that require value-based (not lexicographic string) comparison for min/max stats inline bool RequiresValueComparison(const LogicalType &type) { - return type.IsNumeric() || type.IsTemporal(); + return type.IsNumeric() || type.IsTemporal() || type.id() == LogicalTypeId::BOOLEAN; } struct DuckLakeColumnStats; diff --git a/src/include/storage/ducklake_transaction.hpp b/src/include/storage/ducklake_transaction.hpp index b7df185f995..6fd1e1969bf 100644 --- a/src/include/storage/ducklake_transaction.hpp +++ b/src/include/storage/ducklake_transaction.hpp @@ -38,6 +38,7 @@ struct CompactionInformation; struct DuckLakePath; struct DuckLakeCommitState; class DuckLakeFieldId; +class LocalTableChangeIterationHelper; struct LocalTableDataChanges { vector new_data_files; @@ -49,6 +50,92 @@ struct LocalTableDataChanges { bool IsEmpty() const; }; +struct LocalTableChanges { +public: + void Clear(); + bool HasChanges() const; + LocalTableChangeIterationHelper Changes() const; + void CleanupFiles(DatabaseInstance &db); + void CleanupFiles(ClientContext &context, TableIndex table_id); + bool HasTransactionLocalInserts(TableIndex table_id) const; + bool HasTransactionInlinedData(TableIndex table_id) const; + vector GetTransactionLocalFiles(TableIndex table_id) const; + shared_ptr GetTransactionLocalInlinedData(ClientContext &context, TableIndex table_id) const; + void DropTransactionLocalFile(ClientContext &context, TableIndex table_id, const string &path); + void AppendFiles(TableIndex table_id, vector files); + void AppendInlinedData(ClientContext &context, TableIndex table_id, unique_ptr new_data); + void AddNewInlinedDeletes(TableIndex table_id, const string &table_name, set new_deletes); + void DeleteFromLocalInlinedData(ClientContext &context, TableIndex table_id, set new_deletes); + void AddColumnToLocalInlinedData(ClientContext &context, TableIndex table_id, const LogicalType &new_column_type, + FieldIndex new_field_index, const Value &default_value); + void RemoveColumnFromLocalInlinedData(ClientContext &context, TableIndex table_id, + LogicalIndex removed_column_index, const DuckLakeFieldId &field_id); + optional_ptr GetInlinedDeletes(TableIndex table_id, const string &table_name) const; + void AddNewInlinedFileDeletes(TableIndex table_id, idx_t file_id, set new_deletes); + void AddCompaction(TableIndex table_id, DuckLakeCompactionEntry entry); + bool HasLocalDeletes(TableIndex table_id) const; + bool HasAnyLocalChanges(TableIndex table_id) const; + + void GetLocalDeleteForFile(TableIndex table_id, const string &path, DuckLakeFileData &result) const; + bool HasLocalInlinedFileDeletes(TableIndex table_id) const; + + void GetLocalInlinedFileDeletesForFile(TableIndex table_id, idx_t file_id, set &result) const; + + void TransactionLocalDelete(ClientContext &context, TableIndex table_id, const string &data_file_path, + DuckLakeDeleteFile delete_file); + void AddDeletes(ClientContext &context, TableIndex table_id, vector files); + static void AddDeletesToMap(ClientContext &context, vector new_deletes, + unordered_map> &delete_file_map); + +private: + mutable mutex lock; + map changes; +}; + +class LocalTableChangeIterationHelper { +public: + LocalTableChangeIterationHelper(mutex &local_changes_lock, const map &changes); + +private: + unique_lock lock; + const map &changes; + +private: + struct LocalTableChangeIteratorEntry { + friend class LocalTableChangeIterationHelper; + + public: + LocalTableChangeIteratorEntry(); + TableIndex GetTableIndex() const; + const LocalTableDataChanges &GetTableChanges() const; + + private: + TableIndex table_id; + optional_ptr changes; + }; + class LocalTableChangeIterator { + public: + explicit LocalTableChangeIterator(map::const_iterator it, + map::const_iterator end_it); + map::const_iterator it; + map::const_iterator end_it; + LocalTableChangeIteratorEntry entry; + + public: + LocalTableChangeIterator &operator++(); + bool operator!=(const LocalTableChangeIterator &other) const; + const LocalTableChangeIteratorEntry &operator*() const; + }; + +public: + LocalTableChangeIterator begin() { // NOLINT: match stl API + return LocalTableChangeIterator(changes.begin(), changes.end()); + } + LocalTableChangeIterator end() { // NOLINT: match stl API + return LocalTableChangeIterator(changes.end(), changes.end()); + } +}; + struct SnapshotAndStats { vector stats; DuckLakeSnapshot snapshot; @@ -96,8 +183,8 @@ class DuckLakeTransaction : public Transaction, public enable_shared_from_this GetTransactionLocalEntries(CatalogType type, const string &schema_name); optional_ptr GetTransactionLocalEntry(CatalogType catalog_type, const string &schema_name, const string &entry_name); - vector GetTransactionLocalFiles(TableIndex table_id); - shared_ptr GetTransactionLocalInlinedData(TableIndex table_id); + vector GetTransactionLocalFiles(TableIndex table_id) const; + shared_ptr GetTransactionLocalInlinedData(TableIndex table_id) const; void DropTransactionLocalFile(TableIndex table_id, const string &path); bool HasTransactionLocalInserts(TableIndex table_id) const; bool HasTransactionInlinedData(TableIndex table_id) const; @@ -116,8 +203,8 @@ class DuckLakeTransaction : public Transaction, public enable_shared_from_this GetInlinedDeletes(TableIndex table_id, const string &table_name); - vector GetNewInlinedDeletes(DuckLakeCommitState &commit_state); + optional_ptr GetInlinedDeletes(TableIndex table_id, const string &table_name) const; + vector GetNewInlinedDeletes(DuckLakeCommitState &commit_state) const; //! Add inlined file deletions (deletions from parquet files stored in metadata) void AddNewInlinedFileDeletes(TableIndex table_id, idx_t file_id, set new_deletes); @@ -134,8 +221,8 @@ class DuckLakeTransaction : public Transaction, public enable_shared_from_this &snapshots); void DeleteInlinedData(const DuckLakeInlinedTableInfo &inlined_table); - bool SchemaChangesMade(); - bool ChangesMade(); + bool SchemaChangesMade() const; + bool ChangesMade() const; idx_t GetLocalCatalogId(); static bool IsTransactionLocal(idx_t id) { return id >= DuckLakeConstants::TRANSACTION_LOCAL_ID_START; @@ -146,17 +233,17 @@ class DuckLakeTransaction : public Transaction, public enable_shared_from_this &result); + bool HasLocalInlinedFileDeletes(TableIndex table_id) const; + void GetLocalInlinedFileDeletesForFile(TableIndex table_id, idx_t file_id, set &result) const; bool HasDroppedFiles() const; bool FileIsDropped(const string &path) const; //! Check if there are any uncommitted changes for this table (inserts, deletes, or dropped files) - bool HasAnyLocalChanges(TableIndex table_id); + bool HasAnyLocalChanges(TableIndex table_id) const; string GenerateUUID() const; static string GenerateUUIDv7(); @@ -203,21 +290,23 @@ class DuckLakeTransaction : public Transaction, public enable_shared_from_this> stats); vector GetNewDeleteFiles(const DuckLakeCommitState &commit_state, vector &overwritten_delete_files) const; + DuckLakeDeleteFileInfo GetNewDeleteFile(TableIndex table_id, const DuckLakeCommitState &commit_state, + const DuckLakeDeleteFile &file) const; string UpdateGlobalTableStats(TableIndex table_id, const DuckLakeNewGlobalStats &new_stats); SnapshotAndStats CheckForConflicts(DuckLakeSnapshot transaction_snapshot, const TransactionChangeInformation &changes); void CheckForConflicts(const TransactionChangeInformation &changes, const SnapshotChangeInformation &other_changes, - DuckLakeSnapshot transaction_snapshot); - string WriteSnapshotChanges(DuckLakeCommitState &commit_state, TransactionChangeInformation &changes); + DuckLakeSnapshot transaction_snapshot) const; + string WriteSnapshotChanges(DuckLakeCommitState &commit_state, TransactionChangeInformation &changes) const; //! Return the set of changes made by this transaction - TransactionChangeInformation GetTransactionChanges(); + TransactionChangeInformation GetTransactionChanges() const; void GetNewTableInfo(DuckLakeCommitState &commit_state, DuckLakeCatalogSet &catalog_set, reference table_entry, NewTableInfo &result, TransactionChangeInformation &transaction_changes); @@ -225,12 +314,12 @@ class DuckLakeTransaction : public Transaction, public enable_shared_from_this table_entry, NewTableInfo &result, TransactionChangeInformation &transaction_changes); - CompactionInformation GetCompactionChanges(DuckLakeSnapshot &commit_snapshot, CompactionType type); + CompactionInformation GetCompactionChanges(DuckLakeCommitState &commit_state, CompactionType type); void AlterEntryInternal(DuckLakeTableEntry &old_entry, unique_ptr new_entry); void AlterEntryInternal(DuckLakeViewEntry &old_entry, unique_ptr new_entry); void AddTableChanges(TableIndex table_id, const LocalTableDataChanges &table_changes, - TransactionChangeInformation &changes); + TransactionChangeInformation &changes) const; private: DuckLakeCatalog &ducklake_catalog; @@ -260,8 +349,7 @@ class DuckLakeTransaction : public Transaction, public enable_shared_from_this new_schemas; map> dropped_schemas; //! Local changes made to tables - mutex table_data_changes_lock; - map table_data_changes; + LocalTableChanges local_changes; //! Snapshot cache for the AT (...) conditions that are referenced in the transaction value_map_t snapshot_cache; //! New set of transaction-local name maps diff --git a/src/include/storage/ducklake_update.hpp b/src/include/storage/ducklake_update.hpp index 4c960eb53f9..453f80585dc 100644 --- a/src/include/storage/ducklake_update.hpp +++ b/src/include/storage/ducklake_update.hpp @@ -15,53 +15,48 @@ namespace duckdb { class DuckLakeUpdate : public PhysicalOperator { public: DuckLakeUpdate(PhysicalPlan &physical_plan, DuckLakeTableEntry &table, vector columns, - PhysicalOperator &child, PhysicalOperator ©_op, PhysicalOperator &delete_op, - PhysicalOperator &insert_op, vector> &expressions); + PhysicalOperator &child, PhysicalOperator &delete_op, vector> &expressions); //! The table to update DuckLakeTableEntry &table; //! The order of to-be-inserted columns vector columns; - //! The copy operator for writing new data to files - PhysicalOperator ©_op; //! The delete operator for deleting the old data PhysicalOperator &delete_op; - //! The (final) insert operator that registers inserted data - PhysicalOperator &insert_op; //! The row-id-index idx_t row_id_index; vector> expressions; -public: - // // Source interface - SourceResultType GetDataInternal(ExecutionContext &context, DataChunk &chunk, - OperatorSourceInput &input) const override; - - bool IsSource() const override { - return true; - } - static constexpr uint8_t DELETION_INFO_SIZE = 3; public: - // Sink interface - SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override; - SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override; - SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context, - OperatorSinkFinalizeInput &input) const override; - unique_ptr GetGlobalSinkState(ClientContext &context) const override; - unique_ptr GetLocalSinkState(ExecutionContext &context) const override; + // Operator interface + unique_ptr GetGlobalOperatorState(ClientContext &context) const override; + unique_ptr GetOperatorState(ExecutionContext &context) const override; + OperatorResultType Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk, + GlobalOperatorState &gstate, OperatorState &state) const override; + OperatorFinalizeResultType FinalExecute(ExecutionContext &context, DataChunk &chunk, GlobalOperatorState &gstate, + OperatorState &state) const override; + OperatorFinalResultType OperatorFinalize(Pipeline &pipeline, Event &event, ClientContext &context, + OperatorFinalizeInput &input) const override; + + bool ParallelOperator() const override { + return true; + } - bool IsSink() const override { + bool RequiresFinalExecute() const override { return true; } - bool ParallelSink() const override { + bool RequiresOperatorFinalize() const override { return true; } string GetName() const override; InsertionOrderPreservingMap ParamsToString() const override; + + static DuckLakeUpdate &PlanUpdateOperator(ClientContext &context, PhysicalPlanGenerator &planner, LogicalUpdate &op, + PhysicalOperator &child_plan, DuckLakeCopyInput ©_input); }; } // namespace duckdb diff --git a/src/include/storage/ducklake_view_entry.hpp b/src/include/storage/ducklake_view_entry.hpp index 403bc2f12b0..7a116a46b25 100644 --- a/src/include/storage/ducklake_view_entry.hpp +++ b/src/include/storage/ducklake_view_entry.hpp @@ -45,6 +45,8 @@ class DuckLakeViewEntry : public ViewCatalogEntry { unique_ptr GetInfo() const override; string ToSQL() const override; + void BindView(ClientContext &context, BindViewAction action = BindViewAction::BIND_IF_UNBOUND) override; + string GetQuerySQL(); public: diff --git a/src/metadata_manager/postgres_metadata_manager.cpp b/src/metadata_manager/postgres_metadata_manager.cpp index c841520fe44..0b159fc4280 100644 --- a/src/metadata_manager/postgres_metadata_manager.cpp +++ b/src/metadata_manager/postgres_metadata_manager.cpp @@ -86,7 +86,8 @@ bool PostgresMetadataManager::InlinedDeletionTableExists(const string &table_nam auto query = StringUtil::Format(R"( SELECT 1 FROM information_schema.tables WHERE table_schema = {METADATA_SCHEMA_NAME_LITERAL} AND table_name = '%s' -)", table_name); +)", + table_name); auto result = Query(snapshot, query); if (result->HasError()) { return false; @@ -271,13 +272,30 @@ vector PostgresMetadataManager::LoadInlinedDataTables( return result; } +// Postgres metadata manager needs specialized TransformInlinedData: +// 1. BLOB->VARCHAR reinterpret (Postgres stores VARCHAR as BYTEA) +// 2. VARCHAR->DuckDB type cast (non-native types stored as VARCHAR in Postgres) shared_ptr PostgresMetadataManager::TransformInlinedData(QueryResult &result, const vector &expected_types) { + bool needs_transform = false; + if (!expected_types.empty()) { + D_ASSERT(expected_types.size() == result.types.size()); + for (idx_t i = 0; i < expected_types.size(); i++) { + if (result.types[i] != expected_types[i]) { + needs_transform = true; + break; + } + } + } + if (!needs_transform) { + return DuckLakeMetadataManager::TransformInlinedData(result, expected_types); + } + if (result.HasError()) { result.GetErrorObject().Throw("Failed to read inlined data from DuckLake: "); } - // Transform the result by casting VARCHAR vectors to their expected types + // Transform the result by casting vectors to their expected types auto context = transaction.context.lock(); auto data = make_uniq(*context, expected_types); @@ -292,7 +310,7 @@ PostgresMetadataManager::TransformInlinedData(QueryResult &result, const vector< casted_chunk.Initialize(*context, expected_types, chunk->size()); casted_chunk.SetCardinality(chunk->size()); - // Cast each column from VARCHAR to its expected type + // Cast each column to its expected type for (idx_t col_idx = 0; col_idx < chunk->ColumnCount(); col_idx++) { auto &source_vector = chunk->data[col_idx]; auto &target_vector = casted_chunk.data[col_idx]; @@ -300,8 +318,12 @@ PostgresMetadataManager::TransformInlinedData(QueryResult &result, const vector< if (source_vector.GetType() == target_vector.GetType()) { // No casting needed, just copy target_vector.Reference(source_vector); + } else if (source_vector.GetType().id() == LogicalTypeId::BLOB && + target_vector.GetType().id() == LogicalTypeId::VARCHAR) { + // BLOB->VARCHAR: same binary representation, use Reinterpret + target_vector.Reinterpret(source_vector); } else { - // Cast from VARCHAR to the expected type + // General case: cast from source to expected type VectorOperations::Cast(*context, source_vector, target_vector, chunk->size()); } } diff --git a/src/storage/CMakeLists.txt b/src/storage/CMakeLists.txt index 41ef909178c..26187f5861f 100644 --- a/src/storage/CMakeLists.txt +++ b/src/storage/CMakeLists.txt @@ -7,6 +7,7 @@ add_library( ducklake_delete_filter.cpp ducklake_field_data.cpp ducklake_inline_data.cpp + ducklake_inlined_data.cpp ducklake_inlined_data_reader.cpp ducklake_insert.cpp ducklake_merge_into.cpp diff --git a/src/storage/ducklake_catalog.cpp b/src/storage/ducklake_catalog.cpp index 8f76567fc2d..43f94e145a5 100644 --- a/src/storage/ducklake_catalog.cpp +++ b/src/storage/ducklake_catalog.cpp @@ -26,6 +26,7 @@ #include "duckdb/function/scalar_macro_function.hpp" #include "duckdb/function/table_macro_function.hpp" #include "storage/ducklake_macro_entry.hpp" +#include "common/ducklake_util.hpp" namespace duckdb { @@ -185,6 +186,12 @@ idx_t DuckLakeCatalog::GetBeginSnapshotForTable(TableIndex table_id, DuckLakeTra return metadata_manager.GetBeginSnapshotForTable(table_id); } +idx_t DuckLakeCatalog::GetBeginSnapshotForSchemaVersion(TableIndex table_id, idx_t schema_version, + DuckLakeTransaction &transaction) { + auto &metadata_manager = transaction.GetMetadataManager(); + return metadata_manager.GetBeginSnapshotForSchemaVersion(table_id, schema_version); +} + DuckLakeCatalogSet &DuckLakeCatalog::GetSchemaForSnapshot(DuckLakeTransaction &transaction, DuckLakeSnapshot snapshot) { lock_guard guard(schemas_lock); auto entry = schemas.find(snapshot.schema_version); @@ -819,6 +826,26 @@ idx_t DuckLakeCatalog::DataInliningRowLimit(SchemaIndex schema_index, TableIndex return GetConfigOption("data_inlining_row_limit", schema_index, table_index, 10); } +idx_t DuckLakeCatalog::GetInliningLimit(ClientContext &context, DuckLakeTableEntry &table, + const vector &types) { + auto &schema = table.ParentSchema().Cast(); + idx_t limit = DataInliningRowLimit(schema.GetSchemaId(), table.GetTableId()); + if (limit == 0) { + return 0; + } + if (DuckLakeUtil::HasInlinedSystemColumnConflict(table.GetColumns())) { + // We also return 0 if we have inline column conflicts + return 0; + } + auto &transaction = DuckLakeTransaction::Get(context, *this); + auto &metadata_manager = transaction.GetMetadataManager(); + if (!metadata_manager.SupportsInliningTypes(types)) { + // Or if inlining is not supported + return 0; + } + return limit; +} + unique_ptr DuckLakeCatalog::BindAlterAddIndex(Binder &binder, TableCatalogEntry &table_entry, unique_ptr plan, unique_ptr create_info, @@ -826,4 +853,27 @@ unique_ptr DuckLakeCatalog::BindAlterAddIndex(Binder &binder, T throw NotImplementedException("Adding indexes or constraints is not supported in DuckLake"); } +InlinedDeletionCacheResult DuckLakeCatalog::CheckInlinedDeletionTableCache(TableIndex table_id, + DuckLakeSnapshot snapshot) { + lock_guard guard(inlined_deletion_cache_lock); + if (inlined_deletion_exists.find(table_id.index) != inlined_deletion_exists.end()) { + return InlinedDeletionCacheResult::EXISTS; + } + auto it = inlined_deletion_not_exists.find(table_id.index); + if (it != inlined_deletion_not_exists.end() && snapshot.snapshot_id <= it->second) { + return InlinedDeletionCacheResult::DOES_NOT_EXIST; + } + return InlinedDeletionCacheResult::UNKNOWN; +} + +void DuckLakeCatalog::CacheInlinedDeletionTableResult(TableIndex table_id, DuckLakeSnapshot snapshot, bool exists) { + lock_guard guard(inlined_deletion_cache_lock); + if (exists) { + inlined_deletion_exists.insert(table_id.index); + inlined_deletion_not_exists.erase(table_id.index); + } else { + inlined_deletion_not_exists[table_id.index] = snapshot.snapshot_id; + } +} + } // namespace duckdb diff --git a/src/storage/ducklake_initializer.cpp b/src/storage/ducklake_initializer.cpp index 0dbc063ed61..35d398957c1 100644 --- a/src/storage/ducklake_initializer.cpp +++ b/src/storage/ducklake_initializer.cpp @@ -56,10 +56,12 @@ void DuckLakeInitializer::InitializeDataPath() { auto &fs = FileSystem::GetFileSystem(context); auto separator = fs.PathSeparator(data_path); - // ensure the paths we store always end in a path separator - if (!StringUtil::EndsWith(data_path, separator)) { - data_path += separator; + // pop trailing path separators + while (!data_path.empty() && (data_path.back() == '/' || data_path.back() == '\\')) { + data_path.pop_back(); } + // ensure the paths we store always end in a path separator + data_path += separator; catalog.Separator() = separator; } diff --git a/src/storage/ducklake_inline_data.cpp b/src/storage/ducklake_inline_data.cpp index 4fd2ecfb16c..9067626a006 100644 --- a/src/storage/ducklake_inline_data.cpp +++ b/src/storage/ducklake_inline_data.cpp @@ -300,10 +300,10 @@ OperatorFinalResultType DuckLakeInlineData::OperatorFinalize(Pipeline &pipeline, return OperatorFinalResultType::FINISHED; } { - auto cinsert = const_cast(insert.get()); - lock_guard lock(cinsert->lock); - if (!cinsert->sink_state) { - cinsert->sink_state = insert->GetGlobalSinkState(context); + auto mutable_insert = insert.get_mutable(); + lock_guard lock(mutable_insert->lock); + if (!mutable_insert->sink_state) { + mutable_insert->sink_state = insert->GetGlobalSinkState(context); } } auto &insert_gstate = insert->sink_state->Cast(); @@ -313,15 +313,18 @@ OperatorFinalResultType DuckLakeInlineData::OperatorFinalize(Pipeline &pipeline, auto &table = insert_gstate.table; auto result = make_uniq(); auto &inlined_data = *gstate.global_inlined_data; - result->data = std::move(gstate.global_inlined_data); // set the insert count to the total number of inlined rows insert_gstate.total_insert_count = inlined_data.Count(); + // use physical column count for stats + // If we are inlining from updates, we might have extra columns (e.g., row_id, partitions) + auto physical_col_count = table.GetColumns().PhysicalColumnCount(); + // compute the column stats for the data vector new_stats; auto &field_data = table.GetFieldData(); for (auto &chunk : inlined_data.Chunks()) { - for (idx_t c = 0; c < chunk.ColumnCount(); c++) { + for (idx_t c = 0; c < physical_col_count; c++) { UpdateStats(new_stats, c, chunk.data[c], chunk.size(), field_data.GetByRootIndex(PhysicalIndex(c))); } } @@ -339,6 +342,36 @@ OperatorFinalResultType DuckLakeInlineData::OperatorFinalize(Pipeline &pipeline, } } + if (inlined_data.Types().size() > physical_col_count) { + // If we have extra columns, we need to extract the physical columns + vector phys_types(inlined_data.Types().begin(), inlined_data.Types().begin() + physical_col_count); + auto phys_data = make_uniq(context, phys_types); + ColumnDataAppendState append_state; + phys_data->InitializeAppend(append_state); + for (auto &chunk : inlined_data.Chunks()) { + // extract row_ids from the row_id column + auto &row_id_vec = chunk.data[physical_col_count]; + UnifiedVectorFormat row_id_format; + row_id_vec.ToUnifiedFormat(chunk.size(), row_id_format); + auto row_id_data = UnifiedVectorFormat::GetData(row_id_format); + for (idx_t r = 0; r < chunk.size(); r++) { + auto idx = row_id_format.sel->get_index(r); + result->row_ids.push_back(row_id_data[idx]); + } + DataChunk phys_chunk; + phys_chunk.InitializeEmpty(phys_types); + for (idx_t i = 0; i < physical_col_count; i++) { + phys_chunk.data[i].Reference(chunk.data[i]); + } + phys_chunk.SetCardinality(chunk.size()); + phys_data->Append(append_state, phys_chunk); + } + result->data = std::move(phys_data); + } else { + // Otherwise we just copy them + result->data = std::move(gstate.global_inlined_data); + } + // push the inlined data into the transaction auto &transaction = DuckLakeTransaction::Get(context, table.ParentCatalog()); transaction.AppendInlinedData(table.GetTableId(), std::move(result)); diff --git a/src/storage/ducklake_inlined_data.cpp b/src/storage/ducklake_inlined_data.cpp new file mode 100644 index 00000000000..cb4d40c2372 --- /dev/null +++ b/src/storage/ducklake_inlined_data.cpp @@ -0,0 +1,58 @@ +#include "storage/ducklake_inlined_data.hpp" + +#include + +namespace duckdb { + +bool DuckLakeInlinedData::HasPreservedRowIds() const { + return !row_ids.empty(); +} + +idx_t DuckLakeInlinedData::GetRowId(idx_t position) const { + if (HasPreservedRowIds()) { + return NumericCast(row_ids[position]); + } + return position; +} + +int64_t DuckLakeInlinedData::GetOutputRowId(idx_t position) const { + if (HasPreservedRowIds()) { + return row_ids[position]; + } + return NumericCast(DuckLakeConstants::TRANSACTION_LOCAL_ROW_ID_START + position); +} + +void DuckLakeInlinedData::MergeRowIds(const DuckLakeInlinedData &new_data, idx_t new_data_count) { + if (!new_data.HasPreservedRowIds() && !HasPreservedRowIds()) { + return; + } + if (!HasPreservedRowIds()) { + // if the existing data doesnt have preserved row ids, we assign them sequentially from + // transaction local id + idx_t existing_count = data->Count() - new_data_count; + row_ids.reserve(existing_count + new_data.row_ids.size()); + auto next_id = NumericCast(DuckLakeConstants::TRANSACTION_LOCAL_ROW_ID_START); + for (idx_t i = 0; i < existing_count; i++) { + row_ids.push_back(next_id + NumericCast(i)); + } + } + if (new_data.HasPreservedRowIds()) { + // if this is already preserved we can just insert it + row_ids.insert(row_ids.end(), new_data.row_ids.begin(), new_data.row_ids.end()); + } else { + // new data doesnt preserve row_ids, we need to use the TRANSACTION_LOCAL_ROW_ID_START + int64_t next_id = NumericCast(DuckLakeConstants::TRANSACTION_LOCAL_ROW_ID_START); + if (!row_ids.empty()) { + // we only use max_id if it's guaranteed to be transactional (over next_id) + auto max_id = *std::max_element(row_ids.begin(), row_ids.end()); + if (max_id >= next_id) { + next_id = max_id + 1; + } + } + for (idx_t i = 0; i < new_data_count; i++) { + row_ids.push_back(next_id + NumericCast(i)); + } + } +} + +} // namespace duckdb diff --git a/src/storage/ducklake_inlined_data_reader.cpp b/src/storage/ducklake_inlined_data_reader.cpp index 405a5acf496..ce2e3bcbf45 100644 --- a/src/storage/ducklake_inlined_data_reader.cpp +++ b/src/storage/ducklake_inlined_data_reader.cpp @@ -224,18 +224,25 @@ AsyncResult DuckLakeInlinedDataReader::Scan(ClientContext &context, GlobalTableF break; } case InlinedVirtualColumn::COLUMN_ROW_ID: { - // Generate ordinal data for row IDs Vector ordinal_vector(LogicalType::BIGINT); auto ordinal_data = FlatVector::GetData(ordinal_vector); - for (idx_t r = 0; r < scan_chunk.size(); r++) { - ordinal_data[r] = NumericCast(file_row_number + r); + if (data->HasPreservedRowIds()) { + // use preserved row_ids from update inlining + for (idx_t r = 0; r < scan_chunk.size(); r++) { + ordinal_data[r] = data->row_ids[file_row_number + r]; + } + } else { + // use general ordinal row id + for (idx_t r = 0; r < scan_chunk.size(); r++) { + ordinal_data[r] = NumericCast(file_row_number + r); + } } if (TryEvaluateExpression(context, c, ordinal_vector, LogicalType::BIGINT, chunk.data[c])) { continue; } auto row_id_data = FlatVector::GetData(chunk.data[c]); for (idx_t r = 0; r < scan_chunk.size(); r++) { - row_id_data[r] = NumericCast(file_row_number + r); + row_id_data[r] = ordinal_data[r]; } continue; } diff --git a/src/storage/ducklake_insert.cpp b/src/storage/ducklake_insert.cpp index f28fbe6d329..89349619bda 100644 --- a/src/storage/ducklake_insert.cpp +++ b/src/storage/ducklake_insert.cpp @@ -758,15 +758,10 @@ PhysicalOperator &DuckLakeCatalog::PlanInsert(ClientContext &context, PhysicalPl plan = planner.ResolveDefaultsProjection(op, *plan); } auto &ducklake_table = op.table.Cast(); - auto &ducklake_schema = ducklake_table.ParentSchema().Cast(); optional_ptr inline_data; - idx_t data_inlining_row_limit = DataInliningRowLimit(ducklake_schema.GetSchemaId(), ducklake_table.GetTableId()); - // FIXME: we are skipping columns that have conflicting names, we should resolve this - auto &duck_transaction = DuckLakeTransaction::Get(context, *this); - auto &metadata_manager = duck_transaction.GetMetadataManager(); - if (data_inlining_row_limit > 0 && !DuckLakeUtil::HasInlinedSystemColumnConflict(ducklake_table.GetColumns()) && - metadata_manager.SupportsInliningTypes(plan->types)) { + idx_t data_inlining_row_limit = GetInliningLimit(context, ducklake_table, plan->types); + if (data_inlining_row_limit > 0) { plan = planner.Make(*plan, data_inlining_row_limit); inline_data = plan->Cast(); } diff --git a/src/storage/ducklake_merge_into.cpp b/src/storage/ducklake_merge_into.cpp index 65adf801cf6..c090c445b23 100644 --- a/src/storage/ducklake_merge_into.cpp +++ b/src/storage/ducklake_merge_into.cpp @@ -4,6 +4,7 @@ #include "storage/ducklake_update.hpp" #include "storage/ducklake_delete.hpp" #include "storage/ducklake_insert.hpp" +#include "storage/ducklake_inline_data.hpp" #include "storage/ducklake_table_entry.hpp" #include "duckdb/planner/operator/logical_update.hpp" #include "duckdb/planner/operator/logical_dummy_scan.hpp" @@ -68,6 +69,31 @@ SourceResultType DuckLakeMergeInsert::GetDataInternal(ExecutionContext &context, return SourceResultType::FINISHED; } +//===--------------------------------------------------------------------===// +// Shared helpers +//===--------------------------------------------------------------------===// + +// Apply extra projections and casts before feeding a chunk to CopyToFile — shared by MergeInsert and MergeUpdate +static void ProjectAndCastForCopy(ClientContext &context, DataChunk &input_chunk, PhysicalOperator ©_op, + ExpressionExecutor *expression_executor, DataChunk &projected_chunk, + DataChunk &cast_chunk) { + reference chunk_ref = input_chunk; + if (expression_executor) { + projected_chunk.Reset(); + expression_executor->Execute(input_chunk, projected_chunk); + chunk_ref = projected_chunk; + } + auto ©_types = copy_op.Cast().expected_types; + for (idx_t i = 0; i < chunk_ref.get().ColumnCount(); i++) { + if (chunk_ref.get().data[i].GetType() != copy_types[i]) { + VectorOperations::Cast(context, chunk_ref.get().data[i], cast_chunk.data[i], chunk_ref.get().size()); + } else { + cast_chunk.data[i].Reference(chunk_ref.get().data[i]); + } + } + cast_chunk.SetCardinality(chunk_ref.get().size()); +} + //===--------------------------------------------------------------------===// // Sink //===--------------------------------------------------------------------===// @@ -82,27 +108,9 @@ class DuckLakeMergeIntoLocalState : public LocalSinkState { SinkResultType DuckLakeMergeInsert::Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const { auto &local_state = input.local_state.Cast(); + ProjectAndCastForCopy(context.client, chunk, copy, local_state.expression_executor.get(), local_state.chunk, + local_state.cast_chunk); OperatorSinkInput sink_input {*copy.sink_state, *local_state.copy_sink_state, input.interrupt_state}; - reference chunk_ref = chunk; - if (!extra_projections.empty()) { - // We have extra projections, we need to execute them - local_state.chunk.Reset(); - local_state.expression_executor->Execute(chunk, local_state.chunk); - chunk_ref = local_state.chunk; - } - - // Cast the chunk if needed - auto ©_types = copy.Cast().expected_types; - for (idx_t i = 0; i < chunk_ref.get().ColumnCount(); i++) { - if (chunk_ref.get().data[i].GetType() != copy_types[i]) { - VectorOperations::Cast(context.client, chunk_ref.get().data[i], local_state.cast_chunk.data[i], - chunk_ref.get().size()); - } else { - local_state.cast_chunk.data[i].Reference(chunk_ref.get().data[i]); - } - } - local_state.cast_chunk.SetCardinality(chunk_ref.get().size()); - return copy.Sink(context, local_state.cast_chunk, sink_input); } @@ -112,54 +120,59 @@ SinkCombineResultType DuckLakeMergeInsert::Combine(ExecutionContext &context, Op return copy.Combine(context, combine_input); } -SinkFinalizeType DuckLakeMergeInsert::Finalize(Pipeline &pipeline, Event &event, ClientContext &context, - OperatorSinkFinalizeInput &input) const { - OperatorSinkFinalizeInput copy_finalize {*copy.sink_state, input.interrupt_state}; - auto finalize_result = copy.Finalize(pipeline, event, context, copy_finalize); - if (finalize_result == SinkFinalizeType::BLOCKED) { - return SinkFinalizeType::BLOCKED; - } - - // now scan the copy +// Scan copy operator source and sink into insert operator — shared by MergeInsert and MergeUpdate +static void FinalizeCopyToInsert(Pipeline &pipeline, Event &event, ClientContext &context, PhysicalOperator ©_op, + PhysicalOperator &insert_op, InterruptState &interrupt_state) { DataChunk chunk; - chunk.Initialize(context, copy.types); + chunk.Initialize(context, copy_op.types); ThreadContext thread(context); ExecutionContext exec_context(context, thread, nullptr); - auto copy_global = copy.GetGlobalSourceState(context); - auto copy_local = copy.GetLocalSourceState(exec_context, *copy_global); - OperatorSourceInput source_input {*copy_global, *copy_local, input.interrupt_state}; + auto copy_global = copy_op.GetGlobalSourceState(context); + auto copy_local = copy_op.GetLocalSourceState(exec_context, *copy_global); + OperatorSourceInput source_input {*copy_global, *copy_local, interrupt_state}; - auto insert_global = insert.GetGlobalSinkState(context); - auto insert_local = insert.GetLocalSinkState(exec_context); - OperatorSinkInput sink_input {*insert_global, *insert_local, input.interrupt_state}; + auto insert_global = insert_op.GetGlobalSinkState(context); + auto insert_local = insert_op.GetLocalSinkState(exec_context); + OperatorSinkInput sink_input {*insert_global, *insert_local, interrupt_state}; SourceResultType source_res = SourceResultType::HAVE_MORE_OUTPUT; while (source_res == SourceResultType::HAVE_MORE_OUTPUT) { chunk.Reset(); - source_res = copy.GetData(exec_context, chunk, source_input); + source_res = copy_op.GetData(exec_context, chunk, source_input); if (chunk.size() == 0) { continue; } if (source_res == SourceResultType::BLOCKED) { - throw InternalException("BLOCKED not supported in DuckLakeMergeInsert"); + throw InternalException("BLOCKED not supported in DuckLakeMerge"); } - auto sink_result = insert.Sink(exec_context, chunk, sink_input); + auto sink_result = insert_op.Sink(exec_context, chunk, sink_input); if (sink_result != SinkResultType::NEED_MORE_INPUT) { - throw InternalException("BLOCKED not supported in DuckLakeMergeInsert"); + throw InternalException("BLOCKED not supported in DuckLakeMerge"); } } - OperatorSinkCombineInput combine_input {*insert_global, *insert_local, input.interrupt_state}; - auto combine_res = insert.Combine(exec_context, combine_input); + OperatorSinkCombineInput combine_input {*insert_global, *insert_local, interrupt_state}; + auto combine_res = insert_op.Combine(exec_context, combine_input); if (combine_res == SinkCombineResultType::BLOCKED) { - throw InternalException("BLOCKED not supported in DuckLakeMergeInsert"); + throw InternalException("BLOCKED not supported in DuckLakeMerge"); } - OperatorSinkFinalizeInput finalize_input {*insert_global, input.interrupt_state}; - auto finalize_res = insert.Finalize(pipeline, event, context, finalize_input); + OperatorSinkFinalizeInput finalize_input {*insert_global, interrupt_state}; + auto finalize_res = insert_op.Finalize(pipeline, event, context, finalize_input); if (finalize_res == SinkFinalizeType::BLOCKED) { - throw InternalException("BLOCKED not supported in DuckLakeMergeInsert"); + throw InternalException("BLOCKED not supported in DuckLakeMerge"); + } +} + +SinkFinalizeType DuckLakeMergeInsert::Finalize(Pipeline &pipeline, Event &event, ClientContext &context, + OperatorSinkFinalizeInput &input) const { + OperatorSinkFinalizeInput copy_finalize {*copy.sink_state, input.interrupt_state}; + auto finalize_result = copy.Finalize(pipeline, event, context, copy_finalize); + if (finalize_result == SinkFinalizeType::BLOCKED) { + return SinkFinalizeType::BLOCKED; } + + FinalizeCopyToInsert(pipeline, event, context, copy, insert, input.interrupt_state); return SinkFinalizeType::READY; } @@ -184,6 +197,195 @@ unique_ptr DuckLakeMergeInsert::GetLocalSinkState(ExecutionConte return std::move(result); } +//===--------------------------------------------------------------------===// +// Merge Update +//===--------------------------------------------------------------------===// +class DuckLakeMergeUpdate : public PhysicalOperator { +public: + DuckLakeMergeUpdate(PhysicalPlan &physical_plan, const vector &types, DuckLakeUpdate &update_op, + optional_ptr inline_data_op, PhysicalOperator ©_op, + PhysicalOperator &insert_op) + : PhysicalOperator(physical_plan, PhysicalOperatorType::EXTENSION, types, 1), update_op(update_op), + inline_data_op(inline_data_op), copy_op(copy_op), insert_op(insert_op) { + } + + DuckLakeUpdate &update_op; + optional_ptr inline_data_op; + PhysicalOperator ©_op; + PhysicalOperator &insert_op; + //! Extra projections for partition columns + vector> extra_projections; + +public: + SourceResultType GetDataInternal(ExecutionContext &context, DataChunk &chunk, + OperatorSourceInput &input) const override { + return SourceResultType::FINISHED; + } + bool IsSource() const override { + return true; + } + bool IsSink() const override { + return true; + } + bool ParallelSink() const override { + return true; + } + string GetName() const override { + return "DUCKLAKE_MERGE_UPDATE"; + } + + unique_ptr GetGlobalSinkState(ClientContext &context) const override; + unique_ptr GetLocalSinkState(ExecutionContext &context) const override; + SinkResultType Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const override; + SinkCombineResultType Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const override; + SinkFinalizeType Finalize(Pipeline &pipeline, Event &event, ClientContext &context, + OperatorSinkFinalizeInput &input) const override; +}; + +class DuckLakeMergeUpdateGlobalState : public GlobalSinkState { +public: + unique_ptr update_gstate; + unique_ptr inline_data_gstate; +}; + +class DuckLakeMergeUpdateLocalState : public LocalSinkState { +public: + unique_ptr update_lstate; + unique_ptr inline_data_lstate; + unique_ptr copy_lstate; + DataChunk update_output; + DataChunk inline_output; + //! Projection and cast chunks for partition columns (same pattern as DuckLakeMergeInsert) + unique_ptr expression_executor; + DataChunk projected_chunk; + DataChunk cast_chunk; +}; + +unique_ptr DuckLakeMergeUpdate::GetGlobalSinkState(ClientContext &context) const { + auto result = make_uniq(); + result->update_gstate = update_op.GetGlobalOperatorState(context); + if (inline_data_op) { + result->inline_data_gstate = inline_data_op->GetGlobalOperatorState(context); + } + copy_op.sink_state = copy_op.GetGlobalSinkState(context); + return std::move(result); +} + +unique_ptr DuckLakeMergeUpdate::GetLocalSinkState(ExecutionContext &context) const { + auto result = make_uniq(); + result->update_lstate = update_op.GetOperatorState(context); + if (inline_data_op) { + result->inline_data_lstate = inline_data_op->GetOperatorState(context); + result->inline_output.Initialize(context.client, inline_data_op->types); + } + result->copy_lstate = copy_op.GetLocalSinkState(context); + result->update_output.Initialize(context.client, update_op.types); + if (!extra_projections.empty()) { + result->expression_executor = make_uniq(context.client, extra_projections); + vector projected_types; + for (auto &expr : result->expression_executor->expressions) { + projected_types.push_back(expr->return_type); + } + result->projected_chunk.Initialize(context.client, projected_types); + } + result->cast_chunk.Initialize(context.client, copy_op.Cast().expected_types); + return std::move(result); +} + +SinkResultType DuckLakeMergeUpdate::Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const { + auto &gstate = input.global_state.Cast(); + auto &lstate = input.local_state.Cast(); + + lstate.update_output.Reset(); + update_op.Execute(context, chunk, lstate.update_output, *gstate.update_gstate, *lstate.update_lstate); + + if (lstate.update_output.size() == 0) { + return SinkResultType::NEED_MORE_INPUT; + } + + // if we have an inline data operator, we need to execute through it, it will either inline it or pass it + // through to the CopyToFile sink + if (inline_data_op) { + auto &inline_output = lstate.inline_output; + inline_output.Reset(); + auto result = inline_data_op->Execute(context, lstate.update_output, inline_output, *gstate.inline_data_gstate, + *lstate.inline_data_lstate); + if (inline_output.size() > 0) { + ProjectAndCastForCopy(context.client, inline_output, copy_op, lstate.expression_executor.get(), + lstate.projected_chunk, lstate.cast_chunk); + OperatorSinkInput copy_input {*copy_op.sink_state, *lstate.copy_lstate, input.interrupt_state}; + copy_op.Sink(context, lstate.cast_chunk, copy_input); + } + while (result == OperatorResultType::HAVE_MORE_OUTPUT) { + inline_output.Reset(); + result = inline_data_op->Execute(context, lstate.update_output, inline_output, *gstate.inline_data_gstate, + *lstate.inline_data_lstate); + if (inline_output.size() > 0) { + ProjectAndCastForCopy(context.client, inline_output, copy_op, lstate.expression_executor.get(), + lstate.projected_chunk, lstate.cast_chunk); + OperatorSinkInput copy_input {*copy_op.sink_state, *lstate.copy_lstate, input.interrupt_state}; + copy_op.Sink(context, lstate.cast_chunk, copy_input); + } + } + } else { + ProjectAndCastForCopy(context.client, lstate.update_output, copy_op, lstate.expression_executor.get(), + lstate.projected_chunk, lstate.cast_chunk); + OperatorSinkInput copy_input {*copy_op.sink_state, *lstate.copy_lstate, input.interrupt_state}; + copy_op.Sink(context, lstate.cast_chunk, copy_input); + } + return SinkResultType::NEED_MORE_INPUT; +} + +SinkCombineResultType DuckLakeMergeUpdate::Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const { + auto &gstate = input.global_state.Cast(); + auto &lstate = input.local_state.Cast(); + + // drain inline data + if (inline_data_op) { + auto &inline_output = lstate.inline_output; + while (true) { + inline_output.Reset(); + auto fresult = inline_data_op->FinalExecute(context, inline_output, *gstate.inline_data_gstate, + *lstate.inline_data_lstate); + if (inline_output.size() > 0) { + ProjectAndCastForCopy(context.client, inline_output, copy_op, lstate.expression_executor.get(), + lstate.projected_chunk, lstate.cast_chunk); + OperatorSinkInput copy_input {*copy_op.sink_state, *lstate.copy_lstate, input.interrupt_state}; + copy_op.Sink(context, lstate.cast_chunk, copy_input); + } + if (fresult == OperatorFinalizeResultType::FINISHED) { + break; + } + } + } + + DataChunk dummy; + update_op.FinalExecute(context, dummy, *gstate.update_gstate, *lstate.update_lstate); + + OperatorSinkCombineInput copy_combine {*copy_op.sink_state, *lstate.copy_lstate, input.interrupt_state}; + copy_op.Combine(context, copy_combine); + return SinkCombineResultType::FINISHED; +} + +SinkFinalizeType DuckLakeMergeUpdate::Finalize(Pipeline &pipeline, Event &event, ClientContext &context, + OperatorSinkFinalizeInput &input) const { + auto &gstate = input.global_state.Cast(); + + OperatorFinalizeInput update_finalize {*gstate.update_gstate, input.interrupt_state}; + update_op.OperatorFinalize(pipeline, event, context, update_finalize); + + if (inline_data_op) { + OperatorFinalizeInput inline_finalize {*gstate.inline_data_gstate, input.interrupt_state}; + inline_data_op->OperatorFinalize(pipeline, event, context, inline_finalize); + } + + OperatorSinkFinalizeInput copy_finalize {*copy_op.sink_state, input.interrupt_state}; + copy_op.Finalize(pipeline, event, context, copy_finalize); + + FinalizeCopyToInsert(pipeline, event, context, copy_op, insert_op, input.interrupt_state); + return SinkFinalizeType::READY; +} + //===--------------------------------------------------------------------===// // Plan Merge Into //===--------------------------------------------------------------------===// @@ -211,11 +413,40 @@ static unique_ptr DuckLakePlanMergeIntoAction(DuckLakeCatalog update.expressions = std::move(action.expressions); update.columns = std::move(action.columns); update.update_is_del_and_insert = action.update_is_del_and_insert; - auto &update_plan = catalog.PlanUpdate(context, planner, update, child_plan); - result->op = update_plan; - auto &dl_update = result->op->Cast(); - // The row_id comes before the deletion information, which is always the 3 last column of the chunk. - dl_update.row_id_index = child_plan.types.size() - DuckLakeUpdate::DELETION_INFO_SIZE - 1; + + auto &ducklake_table = op.table.Cast(); + DuckLakeCopyInput copy_input(context, ducklake_table); + copy_input.virtual_columns = InsertVirtualColumns::WRITE_ROW_ID; + + auto &update_op = DuckLakeUpdate::PlanUpdateOperator(context, planner, update, child_plan, copy_input); + + // The row_id comes before the deletion information, that is always the 3 last column of the chunk. + update_op.row_id_index = child_plan.types.size() - DuckLakeUpdate::DELETION_INFO_SIZE - 1; + + // maybe wrap with InlineData if we hit the row limit + optional_ptr inline_data; + idx_t data_inlining_row_limit = catalog.GetInliningLimit(context, ducklake_table, update_op.types); + if (data_inlining_row_limit > 0) { + auto &inline_op = + planner.Make(update_op, data_inlining_row_limit).Cast(); + inline_data = &inline_op; + } + + // plan copy and insert + auto copy_options = DuckLakeInsert::GetCopyOptions(context, copy_input); + auto ©_op = DuckLakeInsert::PlanCopyForInsert(context, planner, copy_input, nullptr); + auto &insert_op = + DuckLakeInsert::PlanInsert(context, planner, ducklake_table, std::move(copy_input.encryption_key)); + if (inline_data) { + inline_data->insert = insert_op.Cast(); + } + insert_op.children.push_back(copy_op); + + // wrap in DuckLakeMergeUpdate + auto &merge_update = planner.Make(return_types, update_op, inline_data, copy_op, insert_op) + .Cast(); + merge_update.extra_projections = std::move(copy_options.projection_list); + result->op = merge_update; break; } case MergeActionType::MERGE_DELETE: { diff --git a/src/storage/ducklake_metadata_manager.cpp b/src/storage/ducklake_metadata_manager.cpp index e0bdffb6313..a3a2ad3160a 100644 --- a/src/storage/ducklake_metadata_manager.cpp +++ b/src/storage/ducklake_metadata_manager.cpp @@ -191,7 +191,8 @@ void DuckLakeMetadataManager::InitializeDuckLake(bool has_explicit_schema, DuckL string data_path = StorePath(base_data_path); string encryption_str = encryption == DuckLakeEncryption::ENCRYPTED ? "true" : "false"; string initial_schema_uuid = transaction.GenerateUUID(); - initialize_query += StringUtil::Format(R"( + initialize_query += + StringUtil::Format(R"( CREATE TABLE {METADATA_CATALOG}.ducklake_metadata(key VARCHAR NOT NULL, value VARCHAR NOT NULL, scope VARCHAR, scope_id BIGINT); CREATE TABLE {METADATA_CATALOG}.ducklake_snapshot(snapshot_id BIGINT PRIMARY KEY, snapshot_time TIMESTAMPTZ, schema_version BIGINT, next_catalog_id BIGINT, next_file_id BIGINT); CREATE TABLE {METADATA_CATALOG}.ducklake_snapshot_changes(snapshot_id BIGINT PRIMARY KEY, changes_made VARCHAR, author VARCHAR, commit_message VARCHAR, commit_extra_info VARCHAR); @@ -225,7 +226,7 @@ INSERT INTO {METADATA_CATALOG}.ducklake_snapshot_changes VALUES (0, 'created_sch INSERT INTO {METADATA_CATALOG}.ducklake_metadata (key, value) VALUES ('version', '0.4'), ('created_by', 'DuckDB %s'), ('data_path', %s), ('encrypted', '%s'); INSERT INTO {METADATA_CATALOG}.ducklake_schema VALUES (0, '%s'::UUID, 0, NULL, 'main', 'main/', true); )", - DuckDB::SourceID(), SQLString(data_path), encryption_str, initial_schema_uuid); + DuckDB::SourceID(), SQLString(data_path), encryption_str, initial_schema_uuid); auto result = Execute(initialize_query); if (result->HasError()) { result->GetErrorObject().Throw("Failed to initialize DuckLake: "); @@ -293,6 +294,7 @@ CREATE TABLE {IF_NOT_EXISTS} {METADATA_CATALOG}.ducklake_macro(schema_id BIGINT, CREATE TABLE {IF_NOT_EXISTS} {METADATA_CATALOG}.ducklake_macro_impl(macro_id BIGINT, impl_id BIGINT, dialect VARCHAR, sql VARCHAR, type VARCHAR); CREATE TABLE {IF_NOT_EXISTS} {METADATA_CATALOG}.ducklake_macro_parameters(macro_id BIGINT, impl_id BIGINT,column_id BIGINT, parameter_name VARCHAR, parameter_type VARCHAR, default_value VARCHAR, default_value_type VARCHAR); ALTER TABLE {METADATA_CATALOG}.ducklake_column ADD COLUMN {IF_NOT_EXISTS} default_value_type VARCHAR DEFAULT 'literal'; +UPDATE {METADATA_CATALOG}.ducklake_column SET default_value_type = 'literal' WHERE default_value_type IS NULL; ALTER TABLE {METADATA_CATALOG}.ducklake_column ADD COLUMN {IF_NOT_EXISTS} default_value_dialect VARCHAR DEFAULT NULL; CREATE TABLE {IF_NOT_EXISTS} {METADATA_CATALOG}.ducklake_sort_info(sort_id BIGINT, table_id BIGINT, begin_snapshot BIGINT, end_snapshot BIGINT); CREATE TABLE {IF_NOT_EXISTS} {METADATA_CATALOG}.ducklake_sort_expression(sort_id BIGINT, table_id BIGINT, sort_key_index BIGINT, expression VARCHAR, dialect VARCHAR, sort_direction VARCHAR, null_order VARCHAR); @@ -465,6 +467,21 @@ WHERE table_id = {TABLE_ID})"; throw InternalException("Table %llu does not exist", table_id.index); } +idx_t DuckLakeMetadataManager::GetBeginSnapshotForSchemaVersion(TableIndex table_id, idx_t schema_version) { + string query = R"( +SELECT begin_snapshot +FROM {METADATA_CATALOG}.ducklake_schema_versions +WHERE table_id = {TABLE_ID} AND schema_version = {SCHEMA_VERSION})"; + query = StringUtil::Replace(query, "{TABLE_ID}", to_string(table_id.index)); + query = StringUtil::Replace(query, "{SCHEMA_VERSION}", to_string(schema_version)); + auto result = transaction.Query(query); + for (auto &row : *result) { + return row.GetValue(0); + } + // We need to fallback to GetBeginSnapshotForTable if this table doesnt have an alter yet + return GetBeginSnapshotForTable(table_id); +} + idx_t DuckLakeMetadataManager::GetNetDataFileRowCount(TableIndex table_id, DuckLakeSnapshot snapshot) { // Compute sum(record_count) - sum(delete_count) - inlined_deletions in a single query // Delete files are only counted if their corresponding data file is still visible @@ -569,7 +586,8 @@ WHERE {SNAPSHOT_ID} >= begin_snapshot AND ({SNAPSHOT_ID} < end_snapshot OR end_s }; // load the table information - result = Query(snapshot, StringUtil::Format(R"( + result = + Query(snapshot, StringUtil::Format(R"( SELECT schema_id, tbl.table_id, table_uuid::VARCHAR, table_name, ( SELECT %s @@ -596,9 +614,8 @@ WHERE {SNAPSHOT_ID} >= tbl.begin_snapshot AND ({SNAPSHOT_ID} < tbl.end_snapshot AND (({SNAPSHOT_ID} >= col.begin_snapshot AND ({SNAPSHOT_ID} < col.end_snapshot OR col.end_snapshot IS NULL)) OR column_id IS NULL) ORDER BY table_id, parent_column NULLS FIRST, column_order )", - ListAggregation(TAG_FIELDS), - ListAggregation(INLINED_DATA_TABLES_FIELDS), - ListAggregation(TAG_FIELDS))); + ListAggregation(TAG_FIELDS), ListAggregation(INLINED_DATA_TABLES_FIELDS), + ListAggregation(TAG_FIELDS))); if (result->HasError()) { result->GetErrorObject().Throw("Failed to get table information from DuckLake: "); } @@ -696,7 +713,7 @@ SELECT view_id, view_uuid, schema_id, view_name, dialect, sql, column_aliases, FROM {METADATA_CATALOG}.ducklake_view view WHERE {SNAPSHOT_ID} >= begin_snapshot AND ({SNAPSHOT_ID} < view.end_snapshot OR view.end_snapshot IS NULL) )", - ListAggregation(TAG_FIELDS))); + ListAggregation(TAG_FIELDS))); if (result->HasError()) { result->GetErrorObject().Throw("Failed to get partition information from DuckLake: "); } @@ -743,7 +760,7 @@ SELECT schema_id, ducklake_macro.macro_id, macro_name, ( FROM {METADATA_CATALOG}.ducklake_macro WHERE {SNAPSHOT_ID} >= ducklake_macro.begin_snapshot AND ({SNAPSHOT_ID} < ducklake_macro.end_snapshot OR ducklake_macro.end_snapshot IS NULL) )", - ListAggregation(MACRO_IMPL_FIELDS))); + ListAggregation(MACRO_IMPL_FIELDS))); if (result->HasError()) { result->GetErrorObject().Throw("Failed to get macro information from DuckLake: "); } @@ -1644,7 +1661,8 @@ USING (data_file_id), ( select_list, table_id.index, start_snapshot.snapshot_id, table_id.index); if (has_inlined_table) { - string null_file_cols = "NULL::VARCHAR AS path, NULL::BOOLEAN AS path_is_relative, NULL::BIGINT AS file_size_bytes, NULL::BIGINT AS footer_size"; + string null_file_cols = "NULL::VARCHAR AS path, NULL::BOOLEAN AS path_is_relative, NULL::BIGINT AS " + "file_size_bytes, NULL::BIGINT AS footer_size"; if (IsEncrypted()) { null_file_cols += ", NULL::VARCHAR AS encryption_key"; } @@ -2290,10 +2308,10 @@ string DuckLakeMetadataManager::WriteNewMacros(const vector & string batch_query; for (auto ¯o : new_macros) { // Insert in the macro table - batch_query = StringUtil::Format(R"( + batch_query += StringUtil::Format(R"( INSERT INTO {METADATA_CATALOG}.ducklake_macro values(%llu,%llu,'%s',{SNAPSHOT_ID}, NULL); )", - macro.schema_id.index, macro.macro_id.index, macro.macro_name); + macro.schema_id.index, macro.macro_id.index, macro.macro_name); // Insert in the implementation table for (idx_t impl_id = 0; impl_id < macro.implementations.size(); ++impl_id) { auto &impl = macro.implementations[impl_id]; @@ -2451,21 +2469,36 @@ WHERE table_id = %d AND schema_version=( // append the data // FIXME: we can do a much faster append than this string values; + bool has_preserved_row_ids = entry.data->HasPreservedRowIds(); idx_t row_id = entry.row_id_start; + idx_t global_row_idx = 0; for (auto &chunk : entry.data->data->Chunks()) { for (idx_t r = 0; r < chunk.size(); r++) { if (!values.empty()) { values += ", "; } values += "("; - values += to_string(row_id); + if (has_preserved_row_ids) { + auto rid = entry.data->row_ids[global_row_idx]; + if (DuckLakeConstants::IsTransactionLocalRowId(rid)) { + // This is a INSERT row w a placeholder id, we assign sequential row_id + values += to_string(row_id); + row_id++; + } else { + // This is a UPDATE row, we use preserved row_id + values += to_string(rid); + } + } else { + values += to_string(row_id); + row_id++; + } values += ", {SNAPSHOT_ID}, NULL"; for (idx_t c = 0; c < chunk.ColumnCount(); c++) { values += ", "; values += DuckLakeUtil::ValueToSQL(*this, context, chunk.GetValue(c, r)); } values += ")"; - row_id++; + global_row_idx++; } } if (!values.empty()) { @@ -2499,7 +2532,7 @@ VALUES %s UPDATE {METADATA_CATALOG}.%s SET end_snapshot = {SNAPSHOT_ID} FROM deleted_row_list -WHERE row_id=deleted_row_id; +WHERE row_id=deleted_row_id AND end_snapshot IS NULL AND begin_snapshot != {SNAPSHOT_ID}; )", row_id_list, entry.table_name); } @@ -2548,12 +2581,13 @@ map> DuckLakeMetadataManager::ReadInlinedFileDeletions(TableIn "{SNAPSHOT_ID}", inlined_table_name); auto query_result = Query(snapshot, query); - if (!query_result->HasError()) { - for (auto &row : *query_result) { - auto file_id = row.GetValue(0); - auto row_id = row.GetValue(1); - result[file_id].insert(row_id); - } + if (query_result->HasError()) { + query_result->GetErrorObject().Throw("Failed to read inlined file deletions from DuckLake: "); + } + for (auto &row : *query_result) { + auto file_id = row.GetValue(0); + auto row_id = row.GetValue(1); + result[file_id].insert(row_id); } return result; } @@ -2582,10 +2616,11 @@ unordered_set DuckLakeMetadataManager::GetFileIdsWithInlinedDeletions(Tab "begin_snapshot <= {SNAPSHOT_ID}", inlined_table_name, file_id_list); auto query_result = Query(snapshot, query); - if (!query_result->HasError()) { - for (auto &row : *query_result) { - result.insert(row.GetValue(0)); - } + if (query_result->HasError()) { + query_result->GetErrorObject().Throw("Failed to read inlined file deletion IDs from DuckLake: "); + } + for (auto &row : *query_result) { + result.insert(row.GetValue(0)); } return result; } @@ -2602,13 +2637,14 @@ DuckLakeMetadataManager::ReadInlinedFileDeletionsForRange(TableIndex table_id, D "WHERE begin_snapshot >= %d AND begin_snapshot <= {SNAPSHOT_ID}", inlined_table_name, start_snapshot.snapshot_id); auto query_result = Query(end_snapshot, query); - if (!query_result->HasError()) { - for (auto &row : *query_result) { - auto file_id = row.GetValue(0); - auto row_id = row.GetValue(1); - auto snapshot_id = row.GetValue(2); - result[file_id][row_id] = snapshot_id; - } + if (query_result->HasError()) { + query_result->GetErrorObject().Throw("Failed to read inlined file deletions for range from DuckLake: "); + } + for (auto &row : *query_result) { + auto file_id = row.GetValue(0); + auto row_id = row.GetValue(1); + auto snapshot_id = row.GetValue(2); + result[file_id][row_id] = snapshot_id; } return result; } @@ -2618,13 +2654,22 @@ string DuckLakeMetadataManager::GetInlinedDeletionTableName(TableIndex table_id, // The table name is always deterministic string table_name = StringUtil::Format("ducklake_inlined_delete_%d", table_id.index); - // Check the cache first (tracks whether table exists) + // Check per-transaction cache first (covers tables created in this transaction) if (delete_inlined_table_cache.find(table_id.index) != delete_inlined_table_cache.end()) { return table_name; } + // Check catalog-level cache (persists across transactions) + auto &catalog = transaction.GetCatalog(); + auto cache_result = catalog.CheckInlinedDeletionTableCache(table_id, snapshot); + if (cache_result == InlinedDeletionCacheResult::EXISTS) { + return table_name; // known to exist (committed) + } + if (cache_result == InlinedDeletionCacheResult::DOES_NOT_EXIST && !create_if_not_exists) { + return string(); // known to not exist + } + if (create_if_not_exists) { - // Create the table if it doesn't exist auto create_query = StringUtil::Format( "CREATE TABLE IF NOT EXISTS {METADATA_CATALOG}.%s(file_id BIGINT, row_id BIGINT, begin_snapshot BIGINT);", table_name); @@ -2632,14 +2677,17 @@ string DuckLakeMetadataManager::GetInlinedDeletionTableName(TableIndex table_id, if (create_result->HasError()) { create_result->GetErrorObject().Throw("Failed to create inlined deletion table: "); } + // Only cache per-transaction — the CREATE is transactional and may be rolled back delete_inlined_table_cache.insert(table_id.index); return table_name; } if (InlinedDeletionTableExists(table_name, snapshot)) { delete_inlined_table_cache.insert(table_id.index); + catalog.CacheInlinedDeletionTableResult(table_id, snapshot, true); return table_name; } + catalog.CacheInlinedDeletionTableResult(table_id, snapshot, false); return string(); } @@ -2690,8 +2738,9 @@ unique_ptr DuckLakeMetadataManager::ReadInlinedData(DuckLakeSnapsho auto result = Query(snapshot, StringUtil::Format(R"( SELECT %s FROM {METADATA_CATALOG}.%s inlined_data -WHERE {SNAPSHOT_ID} >= begin_snapshot AND ({SNAPSHOT_ID} < end_snapshot OR end_snapshot IS NULL);)", - projection, inlined_table_name)); +WHERE {SNAPSHOT_ID} >= begin_snapshot AND ({SNAPSHOT_ID} < end_snapshot OR end_snapshot IS NULL) +ORDER BY row_id;)", + projection, inlined_table_name)); return result; } @@ -2700,12 +2749,11 @@ unique_ptr DuckLakeMetadataManager::ReadInlinedDataInsertions(DuckL const string &inlined_table_name, const vector &columns_to_read) { auto projection = GetProjection(columns_to_read); - auto result = - Query(end_snapshot, StringUtil::Format(R"( + auto result = Query(end_snapshot, StringUtil::Format(R"( SELECT %s FROM {METADATA_CATALOG}.%s inlined_data WHERE inlined_data.begin_snapshot >= %d AND inlined_data.begin_snapshot <= {SNAPSHOT_ID};)", - projection, inlined_table_name, start_snapshot.snapshot_id)); + projection, inlined_table_name, start_snapshot.snapshot_id)); return result; } @@ -2714,12 +2762,11 @@ unique_ptr DuckLakeMetadataManager::ReadInlinedDataDeletions(DuckLa const string &inlined_table_name, const vector &columns_to_read) { auto projection = GetProjection(columns_to_read); - auto result = - Query(end_snapshot, StringUtil::Format(R"( + auto result = Query(end_snapshot, StringUtil::Format(R"( SELECT %s FROM {METADATA_CATALOG}.%s inlined_data WHERE inlined_data.end_snapshot >= %d AND inlined_data.end_snapshot <= {SNAPSHOT_ID};)", - projection, inlined_table_name, start_snapshot.snapshot_id)); + projection, inlined_table_name, start_snapshot.snapshot_id)); return result; } @@ -2732,7 +2779,7 @@ SELECT %s FROM {METADATA_CATALOG}.%s inlined_data WHERE {SNAPSHOT_ID} >= begin_snapshot ORDER BY row_id;)", - projection, inlined_table_name)); + projection, inlined_table_name)); return result; } @@ -2750,7 +2797,7 @@ string DuckLakeMetadataManager::GetPathForSchema(SchemaIndex schema_id, SELECT path, path_is_relative FROM {METADATA_CATALOG}.ducklake_schema WHERE schema_id = %d;)", - schema_id.index)); + schema_id.index)); for (auto &row : *result) { DuckLakePath path; path.path = row.GetValue(0); @@ -2770,7 +2817,7 @@ INNER JOIN {METADATA_CATALOG}.ducklake_column c WHERE c.column_name = '%s' AND t.table_name = '%s' AND c.begin_snapshot = t.begin_snapshot AND c.end_snapshot IS NULL; )", - column_name, table_name)); + column_name, table_name)); if (result->HasError()) { result->GetErrorObject().Throw("Failed to get schema information from DuckLake: "); } @@ -2787,7 +2834,7 @@ string DuckLakeMetadataManager::GetPathForTable(TableIndex table_id, const vecto SELECT s.path, s.path_is_relative FROM {METADATA_CATALOG}.ducklake_schema s WHERE schema_id = %d;)", - new_table.schema_id.index)); + new_table.schema_id.index)); for (auto &row : *result) { DuckLakePath schema_path; schema_path.path = row.GetValue(0); @@ -2824,7 +2871,7 @@ FROM {METADATA_CATALOG}.ducklake_schema s JOIN {METADATA_CATALOG}.ducklake_table t USING (schema_id) WHERE table_id = %d;)", - table_id.index)); + table_id.index)); for (auto &row : *result) { DuckLakePath schema_path; schema_path.path = row.GetValue(0); @@ -2912,9 +2959,18 @@ DuckLakePath DuckLakeMetadataManager::GetRelativePath(const string &path, const return result; } +string DuckLakeMetadataManager::GetPathSeparator(const string &path) { + auto &catalog = transaction.GetCatalog(); + if (!catalog.DataPath().empty()) { + // use the cached separator from the catalog + return catalog.Separator(); + } + // if catalog is not loaded, use the file system + return GetFileSystem().PathSeparator(path); +} + string DuckLakeMetadataManager::StorePath(string path) { - auto &fs = GetFileSystem(); - auto separator = fs.PathSeparator(path); + auto separator = GetPathSeparator(path); if (separator == "/") { return path; } @@ -2922,8 +2978,7 @@ string DuckLakeMetadataManager::StorePath(string path) { } string DuckLakeMetadataManager::LoadPath(string path) { - auto &fs = GetFileSystem(); - auto separator = fs.PathSeparator(path); + auto separator = GetPathSeparator(path); if (separator == "/") { return path; } @@ -2945,7 +3000,6 @@ string DuckLakeMetadataManager::FromRelativePath(TableIndex table_id, const Duck return FromRelativePath(path, GetPath(table_id, {}, {})); } - // Optimized version using DuckDB Appender API for much faster inserts string DuckLakeMetadataManager::WriteNewDataFilesWithAppender(DuckLakeSnapshot &commit_snapshot, const vector &new_files, @@ -2969,8 +3023,8 @@ string DuckLakeMetadataManager::WriteNewDataFilesWithAppender(DuckLakeSnapshot & auto data_file_index = static_cast(file.id.index); auto table_id = static_cast(file.table_id.index); int64_t begin_snapshot_val = file.begin_snapshot.IsValid() - ? static_cast(file.begin_snapshot.GetIndex()) - : static_cast(commit_snapshot.snapshot_id); + ? static_cast(file.begin_snapshot.GetIndex()) + : static_cast(commit_snapshot.snapshot_id); auto path = GetRelativePath(file.table_id, file.file_name, new_tables, new_schemas_result); // ducklake_data_file columns: @@ -2978,15 +3032,15 @@ string DuckLakeMetadataManager::WriteNewDataFilesWithAppender(DuckLakeSnapshot & // file_format, record_count, file_size_bytes, footer_size, row_id_start, partition_id, // encryption_key, mapping_id, partial_max data_file_appender.BeginRow(); - data_file_appender.Append(data_file_index); // data_file_id - data_file_appender.Append(table_id); // table_id - data_file_appender.Append(begin_snapshot_val); // begin_snapshot - data_file_appender.Append(Value()); // end_snapshot (NULL) - data_file_appender.Append(Value()); // file_order (NULL) - data_file_appender.Append(string_t(path.path)); // path - data_file_appender.Append(path.path_is_relative); // path_is_relative - data_file_appender.Append(string_t("parquet")); // file_format - data_file_appender.Append(static_cast(file.row_count)); // record_count + data_file_appender.Append(data_file_index); // data_file_id + data_file_appender.Append(table_id); // table_id + data_file_appender.Append(begin_snapshot_val); // begin_snapshot + data_file_appender.Append(Value()); // end_snapshot (NULL) + data_file_appender.Append(Value()); // file_order (NULL) + data_file_appender.Append(string_t(path.path)); // path + data_file_appender.Append(path.path_is_relative); // path_is_relative + data_file_appender.Append(string_t("parquet")); // file_format + data_file_appender.Append(static_cast(file.row_count)); // record_count data_file_appender.Append(static_cast(file.file_size_bytes)); // file_size_bytes if (file.footer_size.IsValid()) { data_file_appender.Append(static_cast(file.footer_size.GetIndex())); // footer_size @@ -3004,7 +3058,8 @@ string DuckLakeMetadataManager::WriteNewDataFilesWithAppender(DuckLakeSnapshot & data_file_appender.Append(Value()); } if (!file.encryption_key.empty()) { - data_file_appender.Append(string_t(Blob::ToBase64(string_t(file.encryption_key)))); // encryption_key + data_file_appender.Append( + string_t(Blob::ToBase64(string_t(file.encryption_key)))); // encryption_key } else { data_file_appender.Append(Value()); } @@ -3014,7 +3069,8 @@ string DuckLakeMetadataManager::WriteNewDataFilesWithAppender(DuckLakeSnapshot & data_file_appender.Append(Value()); } if (file.max_partial_file_snapshot.IsValid()) { - data_file_appender.Append(static_cast(file.max_partial_file_snapshot.GetIndex())); // partial_max + data_file_appender.Append( + static_cast(file.max_partial_file_snapshot.GetIndex())); // partial_max } else { data_file_appender.Append(Value()); } @@ -3089,11 +3145,14 @@ string DuckLakeMetadataManager::WriteNewDataFilesWithAppender(DuckLakeSnapshot & variant_stats_appender.Append(table_id); variant_stats_appender.Append(column_id); variant_stats_appender.Append(string_t(variant_entry.first)); - variant_stats_appender.Append(string_t(DuckLakeTypes::ToString(variant_entry.second.shredded_type))); + variant_stats_appender.Append( + string_t(DuckLakeTypes::ToString(variant_entry.second.shredded_type))); variant_stats_appender.Append(static_cast(field_stats.column_size_bytes)); - if (field_stats.has_null_count && field_stats.has_num_values && field_stats.null_count <= field_stats.num_values) { - variant_stats_appender.Append(static_cast(field_stats.num_values - field_stats.null_count)); + if (field_stats.has_null_count && field_stats.has_num_values && + field_stats.null_count <= field_stats.num_values) { + variant_stats_appender.Append( + static_cast(field_stats.num_values - field_stats.null_count)); variant_stats_appender.Append(static_cast(field_stats.null_count)); } else { variant_stats_appender.Append(Value()); @@ -3120,7 +3179,8 @@ string DuckLakeMetadataManager::WriteNewDataFilesWithAppender(DuckLakeSnapshot & string field_extra_stats_str; if (field_stats.extra_stats && field_stats.extra_stats->TrySerialize(field_extra_stats_str)) { // TrySerialize wraps the JSON in single quotes for SQL - strip them for Appender - if (field_extra_stats_str.size() >= 2 && field_extra_stats_str.front() == '\'' && field_extra_stats_str.back() == '\'') { + if (field_extra_stats_str.size() >= 2 && field_extra_stats_str.front() == '\'' && + field_extra_stats_str.back() == '\'') { field_extra_stats_str = field_extra_stats_str.substr(1, field_extra_stats_str.size() - 2); } variant_stats_appender.Append(string_t(field_extra_stats_str)); @@ -3349,7 +3409,7 @@ JOIN {METADATA_CATALOG}.ducklake_name_mapping USING (mapping_id) %s ORDER BY mapping_id, parent_column NULLS FIRST )", - filter)); + filter)); vector column_maps; for (auto &row : *result) { MappingIndex mapping_id(row.GetValue(0)); @@ -3565,7 +3625,7 @@ unique_ptr DuckLakeMetadataManager::GetSnapshot(BoundAtClause SELECT snapshot_id, schema_version, next_catalog_id, next_file_id FROM {METADATA_CATALOG}.ducklake_snapshot WHERE snapshot_id = %llu;)", - val.DefaultCastAs(LogicalType::UBIGINT).GetValue())); + val.DefaultCastAs(LogicalType::UBIGINT).GetValue())); } else if (StringUtil::CIEquals(unit, "timestamp")) { result = Query(StringUtil::Format( R"( @@ -3951,7 +4011,7 @@ LEFT JOIN {METADATA_CATALOG}.ducklake_snapshot_changes USING (snapshot_id) %s %s ORDER BY snapshot_id )", - filter.empty() ? "" : "WHERE", filter)); + filter.empty() ? "" : "WHERE", filter)); if (res->HasError()) { res->GetErrorObject().Throw("Failed to get snapshot information from DuckLake: "); } @@ -4095,7 +4155,7 @@ void DuckLakeMetadataManager::RemoveFilesScheduledForCleanup(const vectorHasError()) { result->GetErrorObject().Throw("Failed to delete scheduled cleanup files in DuckLake: "); } @@ -4107,7 +4167,7 @@ idx_t DuckLakeMetadataManager::GetNextColumnId(TableIndex table_id) { FROM {METADATA_CATALOG}.ducklake_column WHERE table_id=%d )", - table_id.index)); + table_id.index)); if (result->HasError()) { result->GetErrorObject().Throw("Failed to get next column id in DuckLake: "); } @@ -4190,13 +4250,15 @@ string DuckLakeMetadataManager::WriteDeleteRewrites(const vector DELETE FROM {METADATA_CATALOG}.%s WHERE snapshot_id IN (%s); )", - delete_tbl, snapshot_ids)); + delete_tbl, snapshot_ids)); if (result->HasError()) { result->GetErrorObject().Throw("Failed to delete snapshots in DuckLake: "); } @@ -4278,7 +4340,7 @@ WHERE %s (end_snapshot IS NOT NULL AND NOT EXISTS( FROM {METADATA_CATALOG}.ducklake_snapshot WHERE snapshot_id >= begin_snapshot AND snapshot_id < end_snapshot ));)", - table_id_filter)); + table_id_filter)); vector cleanup_files; for (auto &row : *result) { DuckLakeFileForCleanup info; @@ -4315,7 +4377,7 @@ WHERE %s (end_snapshot IS NOT NULL AND NOT EXISTS( DELETE FROM {METADATA_CATALOG}.%s WHERE data_file_id IN (%s); )", - delete_tbl, deleted_file_ids)); + delete_tbl, deleted_file_ids)); if (result->HasError()) { result->GetErrorObject().Throw("Failed to delete old data file information in DuckLake: "); } @@ -4325,7 +4387,7 @@ WHERE data_file_id IN (%s); INSERT INTO {METADATA_CATALOG}.ducklake_files_scheduled_for_deletion VALUES %s; )", - files_scheduled_for_cleanup)); + files_scheduled_for_cleanup)); if (result->HasError()) { result->GetErrorObject().Throw("Failed to schedule files for clean-up in DuckLake: "); } @@ -4345,7 +4407,7 @@ WHERE %s %s (end_snapshot IS NOT NULL AND NOT EXISTS( FROM {METADATA_CATALOG}.ducklake_snapshot WHERE snapshot_id >= begin_snapshot AND snapshot_id < end_snapshot ));)", - table_id_filter, file_id_filter)); + table_id_filter, file_id_filter)); vector cleanup_deletes; for (auto &row : *result) { DuckLakeFileForCleanup info; @@ -4380,7 +4442,7 @@ WHERE %s %s (end_snapshot IS NOT NULL AND NOT EXISTS( DELETE FROM {METADATA_CATALOG}.ducklake_delete_file WHERE delete_file_id IN (%s); )", - deleted_delete_ids)); + deleted_delete_ids)); if (result->HasError()) { result->GetErrorObject().Throw("Failed to delete old delete file information in DuckLake: "); } @@ -4389,7 +4451,7 @@ WHERE delete_file_id IN (%s); INSERT INTO {METADATA_CATALOG}.ducklake_files_scheduled_for_deletion VALUES %s; )", - files_scheduled_for_cleanup)); + files_scheduled_for_cleanup)); if (result->HasError()) { result->GetErrorObject().Throw("Failed to schedule files for clean-up in DuckLake: "); } @@ -4404,7 +4466,7 @@ VALUES %s; auto result = Execute(StringUtil::Format(R"( DELETE FROM {METADATA_CATALOG}.%s WHERE table_id IN (%s);)", - delete_tbl, deleted_table_ids)); + delete_tbl, deleted_table_ids)); if (result->HasError()) { result->GetErrorObject().Throw("Failed to delete from " + delete_tbl + " in DuckLake: "); } @@ -4421,7 +4483,7 @@ WHERE end_snapshot IS NOT NULL AND NOT EXISTS( FROM {METADATA_CATALOG}.ducklake_snapshot WHERE snapshot_id >= begin_snapshot AND snapshot_id < end_snapshot );)", - delete_tbl)); + delete_tbl)); if (result->HasError()) { result->GetErrorObject().Throw("Failed to delete from " + delete_tbl + " in DuckLake: "); } @@ -4432,7 +4494,7 @@ void DuckLakeMetadataManager::DeleteInlinedData(const DuckLakeInlinedTableInfo & auto result = Execute(StringUtil::Format(R"( DELETE FROM {METADATA_CATALOG}.%s )", - SQLIdentifier(inlined_table.table_name))); + SQLIdentifier(inlined_table.table_name))); if (result->HasError()) { result->GetErrorObject().Throw("Failed to delete inlined data in DuckLake from table " + inlined_table.table_name + ": "); @@ -4519,7 +4581,7 @@ SELECT COUNT(*) FROM {METADATA_CATALOG}.ducklake_metadata WHERE key = %s AND %s )", - SQLString(option_key), scope_filter)); + SQLString(option_key), scope_filter)); auto count = result->Fetch()->GetValue(0, 0).GetValue(); if (count == 0) { @@ -4527,13 +4589,13 @@ WHERE key = %s AND %s result = Execute(StringUtil::Format(R"( INSERT INTO {METADATA_CATALOG}.ducklake_metadata VALUES (%s, %s, %s, %s) )", - SQLString(option_key), SQLString(option_value), scope, scope_id)); + SQLString(option_key), SQLString(option_value), scope, scope_id)); } else { // option already exists - update it result = Execute(StringUtil::Format(R"( UPDATE {METADATA_CATALOG}.ducklake_metadata SET value=%s WHERE key=%s AND %s )", - SQLString(option_value), SQLString(option_key), scope_filter)); + SQLString(option_value), SQLString(option_key), scope_filter)); } if (result->HasError()) { result->GetErrorObject().Throw("Failed to insert config option in DuckLake: "); diff --git a/src/storage/ducklake_multi_file_list.cpp b/src/storage/ducklake_multi_file_list.cpp index e3ddf53bdc6..d3e94e2c8de 100644 --- a/src/storage/ducklake_multi_file_list.cpp +++ b/src/storage/ducklake_multi_file_list.cpp @@ -248,7 +248,7 @@ vector DuckLakeMultiFileList::GetFilesExtended() transaction.GetLocalDeleteForFile(read_info.table_id, file_entry.file.path, file_entry.delete_file); } } - idx_t transaction_row_start = TRANSACTION_LOCAL_ID_START; + idx_t transaction_row_start = DuckLakeConstants::TRANSACTION_LOCAL_ROW_ID_START; for (auto &file : transaction_local_files) { DuckLakeFileListExtendedEntry file_entry; file_entry.file_id = DataFileIndex(); @@ -278,7 +278,7 @@ vector DuckLakeMultiFileList::GetFilesExtended() file_entry.file_id = DataFileIndex(); file_entry.delete_file_id = DataFileIndex(); file_entry.row_count = transaction_local_data->data->Count(); - file_entry.row_id_start = transaction_row_start; + file_entry.row_id_start = GetTransactionLocalRowIdStart(transaction_row_start); file_entry.data_type = DuckLakeDataType::TRANSACTION_LOCAL_INLINED_DATA; result.push_back(std::move(file_entry)); } @@ -348,7 +348,7 @@ void DuckLakeMultiFileList::GetFilesForTable() const { } } } - idx_t transaction_row_start = TRANSACTION_LOCAL_ID_START; + idx_t transaction_row_start = DuckLakeConstants::TRANSACTION_LOCAL_ROW_ID_START; for (auto &file : transaction_local_files) { DuckLakeFileListEntry file_entry; file_entry.file = GetFileData(file); @@ -370,7 +370,7 @@ void DuckLakeMultiFileList::GetFilesForTable() const { // we have transaction local inlined data - create the dummy file entry DuckLakeFileListEntry file_entry; file_entry.file.path = DUCKLAKE_TRANSACTION_LOCAL_INLINED_FILENAME; - file_entry.row_id_start = transaction_row_start; + file_entry.row_id_start = GetTransactionLocalRowIdStart(transaction_row_start); file_entry.data_type = DuckLakeDataType::TRANSACTION_LOCAL_INLINED_DATA; files.push_back(std::move(file_entry)); } @@ -452,4 +452,12 @@ const vector &DuckLakeMultiFileList::GetFiles() const { return files; } +idx_t DuckLakeMultiFileList::GetTransactionLocalRowIdStart(idx_t transaction_row_start) const { + if (transaction_local_data && transaction_local_data->HasPreservedRowIds()) { + // preserved row_ids are absolute, so row_id_start must be 0 + return 0; + } + return transaction_row_start; +} + } // namespace duckdb diff --git a/src/storage/ducklake_multi_file_reader.cpp b/src/storage/ducklake_multi_file_reader.cpp index 245c7f7b0ad..ade2e4cec59 100644 --- a/src/storage/ducklake_multi_file_reader.cpp +++ b/src/storage/ducklake_multi_file_reader.cpp @@ -253,6 +253,10 @@ ReaderInitializeType DuckLakeMultiFileReader::InitializeReader(MultiFileReaderDa if (!file_entry.delete_file.path.empty()) { delete_filter->Initialize(context, file_entry.delete_file); } + if (delete_map && !file_entry.delete_file.path.empty()) { + auto delete_data_copy = make_shared_ptr(*delete_filter->delete_data); + delete_map->AddDeleteData(reader.GetFileName(), std::move(delete_data_copy)); + } // Apply inlined file deletions (stored in metadata database instead of delete file) if (!file_entry.inlined_file_deletions.empty()) { DuckLakeInlinedDataDeletes inlined_deletes; @@ -264,10 +268,6 @@ ReaderInitializeType DuckLakeMultiFileReader::InitializeReader(MultiFileReaderDa } // set the snapshot id so we know what to skip from deletion files delete_filter->SetSnapshotFilter(read_info.snapshot.snapshot_id); - // Only add to delete_map if there's an actual delete file and not just inlined deletions - if (delete_map && !file_entry.delete_file.path.empty()) { - delete_map->AddDeleteData(reader.GetFileName(), delete_filter->delete_data); - } reader.deletion_filter = std::move(delete_filter); } } else { @@ -362,7 +362,8 @@ shared_ptr DuckLakeMultiFileReader::TryCreateInlinedDataReader(c // read the table at the specified version auto transaction = read_info.GetTransaction(); auto &catalog = transaction->GetCatalog(); - DuckLakeSnapshot snapshot(catalog.GetBeginSnapshotForTable(read_info.table.GetTableId(), *transaction), + DuckLakeSnapshot snapshot(catalog.GetBeginSnapshotForSchemaVersion(read_info.table.GetTableId(), + schema_version.GetIndex(), *transaction), schema_version.GetIndex(), 0, 0); auto entry = catalog.GetEntryById(*transaction, snapshot, read_info.table.GetTableId()); if (!entry) { @@ -443,7 +444,8 @@ vector MapColumns(ClientContext &context, MultiFileRe column_map->source_name, reader_data.reader->file.path); } // Use GetValue to handle NULL values (__HIVE_DEFAULT_PARTITION__) and type casting - Value partition_val = HivePartitioning::GetValue(context, column_map->source_name, entry->second, result_col.type); + Value partition_val = + HivePartitioning::GetValue(context, column_map->source_name, entry->second, result_col.type); result_col.default_expression = make_uniq(std::move(partition_val)); continue; } @@ -457,7 +459,8 @@ vector MapColumns(ClientContext &context, MultiFileRe // recursively process any child nodes if (!column_map->child_entries.empty()) { bool is_list = result_col.type.id() == LogicalTypeId::LIST; - result_col.children = MapColumns(context, reader_data, result_col.children, column_map->child_entries, is_list); + result_col.children = + MapColumns(context, reader_data, result_col.children, column_map->child_entries, is_list); } } return result; diff --git a/src/storage/ducklake_scan.cpp b/src/storage/ducklake_scan.cpp index 26aa29f80cc..f329e15ebc3 100644 --- a/src/storage/ducklake_scan.cpp +++ b/src/storage/ducklake_scan.cpp @@ -9,6 +9,8 @@ #include "duckdb/catalog/catalog_entry/table_function_catalog_entry.hpp" #include "duckdb/common/multi_file/multi_file_data.hpp" #include "duckdb/common/string_util.hpp" +#include "duckdb/common/serializer/serializer.hpp" +#include "duckdb/common/serializer/deserializer.hpp" #include "duckdb/function/partition_stats.hpp" #include "duckdb/function/table_function.hpp" #include "duckdb/main/extension_helper.hpp" @@ -111,15 +113,6 @@ BindInfo DuckLakeBindInfo(const optional_ptr bind_data) { return BindInfo(file_list.GetTable()); } -void DuckLakeScanSerialize(Serializer &serializer, const optional_ptr bind_data, - const TableFunction &function) { - throw NotImplementedException("DuckLakeScan not implemented"); -} - -unique_ptr DuckLakeScanDeserialize(Deserializer &deserializer, TableFunction &function) { - throw NotImplementedException("DuckLakeScan not implemented"); -} - virtual_column_map_t DuckLakeVirtualColumns(ClientContext &context, optional_ptr bind_data_p) { auto &bind_data = bind_data_p->Cast(); auto &file_list = bind_data.file_list->Cast(); @@ -189,17 +182,18 @@ vector DuckLakeGetPartitionStats(ClientContext &context, Ge } TableFunction DuckLakeFunctions::GetDuckLakeScanFunction(DatabaseInstance &instance) { - // Parquet extension needs to be loaded for this to make sense - ExtensionHelper::AutoLoadExtension(instance, "parquet"); - // The ducklake_scan function is constructed by grabbing the parquet scan from the Catalog, then injecting the // DuckLakeMultiFileReader into it to create a DuckLake-based multi file read + ExtensionHelper::TryAutoLoadExtension(instance, "parquet"); ExtensionLoader loader(instance, "ducklake"); - auto &parquet_scan = loader.GetTableFunction("parquet_scan"); - auto function = parquet_scan.functions.GetFunctionByOffset(0); - // Register the MultiFileReader as the driver for reads - function.get_multi_file_reader = DuckLakeMultiFileReader::CreateInstance; + TableFunction function("ducklake_scan", {LogicalType::VARCHAR}, nullptr, nullptr); + auto parquet_entry = loader.TryGetTableFunction("parquet_scan"); + if (parquet_entry) { + auto &parquet_scan = parquet_entry->Cast(); + function = parquet_scan.functions.GetFunctionByOffset(0); + function.get_multi_file_reader = DuckLakeMultiFileReader::CreateInstance; + } function.statistics = DuckLakeStatistics; function.get_bind_info = DuckLakeBindInfo; @@ -207,8 +201,6 @@ TableFunction DuckLakeFunctions::GetDuckLakeScanFunction(DatabaseInstance &insta function.get_row_id_columns = DuckLakeGetRowIdColumn; function.get_partition_stats = DuckLakeGetPartitionStats; - // Unset all of these: they are either broken, very inefficient. - // TODO: implement/fix these function.serialize = DuckLakeScanSerialize; function.deserialize = DuckLakeScanDeserialize; @@ -224,6 +216,18 @@ DuckLakeFunctionInfo::DuckLakeFunctionInfo(DuckLakeTableEntry &table, DuckLakeTr : table(table), transaction(transaction_p.shared_from_this()), snapshot(snapshot) { } +shared_ptr +DuckLakeFunctionInfo::Create(DuckLakeTableEntry &table, DuckLakeTransaction &transaction, DuckLakeSnapshot snapshot) { + auto result = make_shared_ptr(table, transaction, snapshot); + result->table_name = table.name; + for (auto &col : table.GetColumns().Logical()) { + result->column_names.push_back(col.Name()); + result->column_types.push_back(col.Type()); + } + result->table_id = table.GetTableId(); + return result; +} + shared_ptr DuckLakeFunctionInfo::GetTransaction() { auto result = transaction.lock(); if (!result) { @@ -233,4 +237,43 @@ shared_ptr DuckLakeFunctionInfo::GetTransaction() { return result; } +void DuckLakeScanSerialize(Serializer &serializer, const optional_ptr bind_data, + const TableFunction &function) { + auto &func_info = function.function_info->Cast(); + D_ASSERT(func_info.scan_type == DuckLakeScanType::SCAN_TABLE); + auto &catalog = func_info.table.ParentCatalog(); + serializer.WriteProperty(100, "catalog_name", catalog.GetName()); + serializer.WriteProperty(101, "schema_name", func_info.table.ParentSchema().name); + serializer.WriteProperty(102, "table_name", func_info.table_name); + serializer.WriteObject(103, "snapshot", [&](Serializer &obj) { func_info.snapshot.Serialize(obj); }); +} + +unique_ptr DuckLakeScanDeserialize(Deserializer &deserializer, TableFunction &function) { + auto &context = deserializer.Get(); + auto catalog_name = deserializer.ReadProperty(100, "catalog_name"); + auto schema_name = deserializer.ReadProperty(101, "schema_name"); + auto table_name = deserializer.ReadProperty(102, "table_name"); + DuckLakeSnapshot snapshot; + deserializer.ReadObject(103, "snapshot", [&](Deserializer &obj) { snapshot = DuckLakeSnapshot::Deserialize(obj); }); + + // If ducklake_scan was registered before parquet was loaded, we set it now + if (!function.bind) { + function = DuckLakeFunctions::GetDuckLakeScanFunction(*context.db); + if (!function.bind) { + throw InvalidInputException("ducklake_scan requires the parquet extension to be loaded"); + } + } + + // Look up the DuckLake catalog and table + auto &catalog = Catalog::GetCatalog(context, catalog_name); + auto &transaction = DuckLakeTransaction::Get(context, catalog); + + auto &table_entry = + Catalog::GetEntry(context, catalog_name, schema_name, table_name).Cast(); + + function.function_info = DuckLakeFunctionInfo::Create(table_entry, transaction, snapshot); + + return DuckLakeFunctions::BindDuckLakeScan(context, function); +} + } // namespace duckdb diff --git a/src/storage/ducklake_table_entry.cpp b/src/storage/ducklake_table_entry.cpp index 5f8f4d7d363..bc9ec201c91 100644 --- a/src/storage/ducklake_table_entry.cpp +++ b/src/storage/ducklake_table_entry.cpp @@ -260,14 +260,8 @@ TableFunction DuckLakeTableEntry::GetScanFunction(ClientContext &context, unique auto function = DuckLakeFunctions::GetDuckLakeScanFunction(*context.db); auto &transaction = DuckLakeTransaction::Get(context, ParentCatalog()); auto function_info = - make_shared_ptr(*this, transaction, transaction.GetSnapshot(lookup_info.GetAtClause())); - function_info->table_name = name; - for (auto &col : columns.Logical()) { - function_info->column_names.push_back(col.Name()); - function_info->column_types.push_back(col.Type()); - } - auto table_id = GetTableId(); - function_info->table_id = table_id; + DuckLakeFunctionInfo::Create(*this, transaction, transaction.GetSnapshot(lookup_info.GetAtClause())); + auto table_id = function_info->table_id; function.function_info = std::move(function_info); auto &dropped_tables = transaction.GetDroppedTables(); auto &renamed_tables = transaction.GetRenamedTables(); @@ -1208,7 +1202,8 @@ unique_ptr DuckLakeTableEntry::Alter(DuckLakeTransaction &transact info.alter_table_type != AlterTableType::RENAME_COLUMN && info.alter_table_type != AlterTableType::ALTER_COLUMN_TYPE && info.alter_table_type != AlterTableType::SET_NOT_NULL && - info.alter_table_type != AlterTableType::DROP_NOT_NULL) { + info.alter_table_type != AlterTableType::DROP_NOT_NULL && + info.alter_table_type != AlterTableType::SET_DEFAULT) { throw NotImplementedException("ALTER on a table with transaction-local inlined data is not supported %s", EnumUtil::ToString(info.alter_table_type)); } diff --git a/src/storage/ducklake_transaction.cpp b/src/storage/ducklake_transaction.cpp index 801c21178f1..5f8240d5e33 100644 --- a/src/storage/ducklake_transaction.cpp +++ b/src/storage/ducklake_transaction.cpp @@ -48,6 +48,607 @@ bool LocalTableDataChanges::IsEmpty() const { return true; } +void LocalTableChanges::Clear() { + lock_guard guard(lock); + changes.clear(); +} + +bool LocalTableChanges::HasChanges() const { + lock_guard guard(lock); + return !changes.empty(); +} + +void LocalTableChanges::CleanupFiles(DatabaseInstance &db) { + auto &fs = FileSystem::GetFileSystem(db); + lock_guard guard(lock); + for (auto &entry : changes) { + auto &table_changes = entry.second; + for (auto &file : table_changes.new_data_files) { + if (file.created_by_ducklake) { + fs.TryRemoveFile(file.file_name); + } + for (auto &del_file : file.delete_files) { + fs.TryRemoveFile(del_file.file_name); + } + } + for (auto &file : table_changes.new_delete_files) { + for (auto &delete_files : file.second) { + fs.TryRemoveFile(delete_files.file_name); + } + } + table_changes.new_data_files.clear(); + table_changes.new_delete_files.clear(); + } +} + +bool LocalTableChanges::HasTransactionLocalInserts(TableIndex table_id) const { + lock_guard guard(lock); + auto entry = changes.find(table_id); + if (entry == changes.end()) { + return false; + } + auto &table_changes = entry->second; + return !table_changes.new_data_files.empty() || table_changes.new_inlined_data; +} + +bool LocalTableChanges::HasTransactionInlinedData(TableIndex table_id) const { + lock_guard guard(lock); + auto entry = changes.find(table_id); + if (entry == changes.end()) { + return false; + } + auto &table_changes = entry->second; + return table_changes.new_inlined_data != nullptr; +} + +vector LocalTableChanges::GetTransactionLocalFiles(TableIndex table_id) const { + lock_guard guard(lock); + auto entry = changes.find(table_id); + if (entry == changes.end()) { + return vector(); + } + return entry->second.new_data_files; +} + +shared_ptr LocalTableChanges::GetTransactionLocalInlinedData(ClientContext &context, + TableIndex table_id) const { + lock_guard guard(lock); + auto entry = changes.find(table_id); + if (entry == changes.end()) { + return nullptr; + } + auto &table_changes = entry->second; + if (!table_changes.new_inlined_data) { + return nullptr; + } + auto &local_changes = *table_changes.new_inlined_data; + auto result = make_shared_ptr(); + result->data = make_uniq(context, local_changes.data->Types()); + for (auto &chunk : local_changes.data->Chunks()) { + result->data->Append(chunk); + } + result->row_ids = local_changes.row_ids; // propagate preserved row_ids if any + return result; +} + +void LocalTableChanges::DropTransactionLocalFile(ClientContext &context, TableIndex table_id, const string &path) { + lock_guard guard(lock); + auto entry = changes.find(table_id); + if (entry == changes.end()) { + throw InternalException( + "DropTransactionLocalFile called for a table for which no transaction-local files exist"); + } + auto &table_changes = entry->second; + auto &table_files = table_changes.new_data_files; + auto &fs = FileSystem::GetFileSystem(context); + for (idx_t i = 0; i < table_files.size(); i++) { + auto &file = table_files[i]; + if (file.file_name == path) { + for (auto &del_file : file.delete_files) { + fs.RemoveFile(del_file.file_name); + } + file.delete_files.clear(); + // found the file - delete it from the table list and from disk + table_files.erase_at(i); + fs.RemoveFile(path); + if (table_changes.IsEmpty()) { + // no more files remaining + changes.erase(entry); + } + return; + } + } + throw InternalException("Failed to find matching transaction-local file for DropTransactionLocalFile"); +} + +void LocalTableChanges::AppendFiles(TableIndex table_id, vector files) { + lock_guard guard(lock); + auto &table_changes = changes[table_id]; + if (table_changes.new_data_files.empty()) { + // If empty, just move the entire vector + table_changes.new_data_files = std::move(files); + } else { + // Reserve to avoid reallocations during insertion + table_changes.new_data_files.reserve(table_changes.new_data_files.size() + files.size()); + // Use move_iterator for efficient batch move + table_changes.new_data_files.insert(table_changes.new_data_files.end(), std::make_move_iterator(files.begin()), + std::make_move_iterator(files.end())); + } +} + +void LocalTableChanges::AppendInlinedData(ClientContext &context, TableIndex table_id, + unique_ptr new_data) { + lock_guard guard(lock); + auto &table_changes = changes[table_id]; + if (table_changes.new_inlined_data) { + // already exists - append + auto &existing_data = *table_changes.new_inlined_data; + auto &existing_types = existing_data.data->Types(); + auto &new_types = new_data->data->Types(); + // check if types changed (e.g. due to ALTER COLUMN TYPE) + if (existing_types != new_types) { + // if types differ we gotta add a cast. + auto casted_data = make_uniq(context, new_types); + ColumnDataAppendState append_state; + casted_data->InitializeAppend(append_state); + for (auto &chunk : existing_data.data->Chunks()) { + DataChunk casted_chunk; + casted_chunk.Initialize(context, new_types); + for (idx_t col_idx = 0; col_idx < chunk.ColumnCount(); col_idx++) { + if (existing_types[col_idx] != new_types[col_idx]) { + VectorOperations::Cast(context, chunk.data[col_idx], casted_chunk.data[col_idx], chunk.size()); + } else { + casted_chunk.data[col_idx].Reference(chunk.data[col_idx]); + } + } + casted_chunk.SetCardinality(chunk.size()); + casted_data->Append(append_state, casted_chunk); + } + existing_data.data = std::move(casted_data); + } + ColumnDataAppendState append_state; + existing_data.data->InitializeAppend(append_state); + for (auto &chunk : new_data->data->Chunks()) { + existing_data.data->Append(chunk); + } + // merge preserved row_ids from update inlining + existing_data.MergeRowIds(*new_data, new_data->data->Count()); + for (auto &entry : new_data->column_stats) { + auto stats_entry = existing_data.column_stats.find(entry.first); + if (stats_entry == existing_data.column_stats.end()) { + throw InternalException("Missing stats when merging inlined data"); + } + stats_entry->second.MergeStats(entry.second); + } + } else { + // does not exist yet - set it + table_changes.new_inlined_data = std::move(new_data); + } +} + +void LocalTableChanges::AddNewInlinedDeletes(TableIndex table_id, const string &table_name, set new_deletes) { + lock_guard guard(lock); + auto &table_changes = changes[table_id]; + auto &table_deletes = table_changes.new_inlined_data_deletes; + auto entry = table_deletes.find(table_name); + if (entry != table_deletes.end()) { + // merge deletes + auto &existing_rows = entry->second->rows; + for (auto &row_idx : new_deletes) { + existing_rows.insert(row_idx); + } + } else { + auto new_data = make_uniq(); + new_data->rows = std::move(new_deletes); + table_deletes.emplace(table_name, std::move(new_data)); + } +} + +void LocalTableChanges::DeleteFromLocalInlinedData(ClientContext &context, TableIndex table_id, + set new_deletes) { + lock_guard guard(lock); + auto entry = changes.find(table_id); + if (entry == changes.end()) { + throw InternalException("DeleteFromLocalInlinedData called but no transaction-local data exists for table"); + } + auto &table_changes = entry->second; + auto &inlined_data = *table_changes.new_inlined_data; + auto &existing = *inlined_data.data; + // construct a new collection from the existing data minus the deletes + auto new_data = make_uniq(context, existing.Types()); + + idx_t base_row_id = 0; + vector new_row_ids; + ColumnDataAppendState append_state; + new_data->InitializeAppend(append_state); + for (auto &chunk : existing.Chunks()) { + // slice out non-deleted rows + SelectionVector sel(chunk.size()); + idx_t selected_rows = 0; + + for (idx_t r = 0; r < chunk.size(); r++) { + idx_t position = base_row_id + r; + auto row_id = inlined_data.GetRowId(position); + if (new_deletes.find(row_id) != new_deletes.end()) { + // deleted - skip + continue; + } + sel.set_index(selected_rows++, r); + new_row_ids.push_back(inlined_data.GetOutputRowId(position)); + } + base_row_id += chunk.size(); + if (selected_rows == 0) { + continue; + } + chunk.Slice(sel, selected_rows); + new_data->Append(append_state, chunk); + } + + // override the existing collection and row_ids + inlined_data.data = std::move(new_data); + inlined_data.row_ids = std::move(new_row_ids); +} + +static void RemoveFieldStats(map &column_stats, const DuckLakeFieldId &field_id) { + column_stats.erase(field_id.GetFieldIndex()); + for (auto &child_id : field_id.Children()) { + RemoveFieldStats(column_stats, *child_id); + } +} + +void LocalTableChanges::AddColumnToLocalInlinedData(ClientContext &context, TableIndex table_id, + const LogicalType &new_column_type, FieldIndex new_field_index, + const Value &default_value) { + lock_guard guard(lock); + auto entry = changes.find(table_id); + if (entry == changes.end()) { + throw InternalException("AddColumnToLocalInlinedData called but no transaction-local data exists"); + } + auto &table_changes = entry->second; + if (!table_changes.new_inlined_data) { + throw InternalException("AddColumnToLocalInlinedData called but no inlined data exists"); + } + + auto &existing = *table_changes.new_inlined_data->data; + + // New types: existing + new column + auto new_types = existing.Types(); + new_types.push_back(new_column_type); + + auto new_data = make_uniq(context, new_types); + + ColumnDataAppendState append_state; + new_data->InitializeAppend(append_state); + + bool has_default = !default_value.IsNull(); + + for (auto &chunk : existing.Chunks()) { + DataChunk new_chunk; + new_chunk.Initialize(context, new_types); + + // Copy existing columns + for (idx_t col_idx = 0; col_idx < chunk.ColumnCount(); col_idx++) { + new_chunk.data[col_idx].Reference(chunk.data[col_idx]); + } + + // New column: use default value or NULL + auto &new_col_vector = new_chunk.data[chunk.ColumnCount()]; + if (has_default) { + new_col_vector.Reference(default_value); + } else { + new_col_vector.SetVectorType(VectorType::CONSTANT_VECTOR); + ConstantVector::SetNull(new_col_vector, true); + } + + new_chunk.SetCardinality(chunk.size()); + new_data->Append(append_state, new_chunk); + } + + // Add stats for new column + idx_t total_rows = existing.Count(); + DuckLakeColumnStats new_col_stats(new_column_type); + new_col_stats.num_values = total_rows; + new_col_stats.has_num_values = true; + if (has_default) { + new_col_stats.null_count = 0; + new_col_stats.has_null_count = true; + new_col_stats.any_valid = true; + } else { + new_col_stats.null_count = total_rows; + new_col_stats.has_null_count = true; + new_col_stats.any_valid = false; + } + + table_changes.new_inlined_data->column_stats.emplace(new_field_index, std::move(new_col_stats)); + table_changes.new_inlined_data->data = std::move(new_data); +} + +void LocalTableChanges::RemoveColumnFromLocalInlinedData(ClientContext &context, TableIndex table_id, + LogicalIndex removed_column_index, + const DuckLakeFieldId &field_id) { + lock_guard guard(lock); + auto entry = changes.find(table_id); + if (entry == changes.end()) { + throw InternalException("RemoveColumnFromLocalInlinedData called but no transaction-local data exists"); + } + auto &table_changes = entry->second; + if (!table_changes.new_inlined_data) { + throw InternalException("RemoveColumnFromLocalInlinedData called but no inlined data exists"); + } + + auto &existing = *table_changes.new_inlined_data->data; + + // New types: existing minus the removed column + vector new_types; + for (idx_t col_idx = 0; col_idx < existing.Types().size(); col_idx++) { + if (col_idx == removed_column_index.index) { + continue; + } + new_types.push_back(existing.Types()[col_idx]); + } + + auto new_data = make_uniq(context, new_types); + + ColumnDataAppendState append_state; + new_data->InitializeAppend(append_state); + + for (auto &chunk : existing.Chunks()) { + DataChunk new_chunk; + new_chunk.Initialize(context, new_types); + + idx_t new_col_idx = 0; + for (idx_t col_idx = 0; col_idx < chunk.ColumnCount(); col_idx++) { + if (col_idx == removed_column_index.index) { + continue; + } + new_chunk.data[new_col_idx].Reference(chunk.data[col_idx]); + new_col_idx++; + } + + new_chunk.SetCardinality(chunk.size()); + new_data->Append(append_state, new_chunk); + } + + // Remove stats for the dropped field and all its children + RemoveFieldStats(table_changes.new_inlined_data->column_stats, field_id); + + table_changes.new_inlined_data->data = std::move(new_data); +} + +optional_ptr LocalTableChanges::GetInlinedDeletes(TableIndex table_id, + const string &table_name) const { + lock_guard guard(lock); + auto entry = changes.find(table_id); + if (entry == changes.end()) { + return nullptr; + } + auto &table_changes = entry->second; + auto delete_entry = table_changes.new_inlined_data_deletes.find(table_name); + if (delete_entry == table_changes.new_inlined_data_deletes.end()) { + return nullptr; + } + return delete_entry->second.get(); +} + +void LocalTableChanges::AddNewInlinedFileDeletes(TableIndex table_id, idx_t file_id, set new_deletes) { + if (new_deletes.empty()) { + return; + } + lock_guard guard(lock); + auto &table_changes = changes[table_id]; + if (!table_changes.new_inlined_file_deletes) { + table_changes.new_inlined_file_deletes = make_uniq(); + } + auto &file_deletes = table_changes.new_inlined_file_deletes->file_deletes[file_id]; + for (auto &row_id : new_deletes) { + file_deletes.insert(row_id); + } +} + +void LocalTableChanges::AddCompaction(TableIndex table_id, DuckLakeCompactionEntry entry) { + lock_guard guard(lock); + auto &table_changes = changes[table_id]; + table_changes.compactions.push_back(std::move(entry)); +} + +bool LocalTableChanges::HasLocalDeletes(TableIndex table_id) const { + lock_guard guard(lock); + auto entry = changes.find(table_id); + if (entry == changes.end()) { + return false; + } + return !entry->second.new_delete_files.empty(); +} + +bool LocalTableChanges::HasAnyLocalChanges(TableIndex table_id) const { + lock_guard guard(lock); + auto entry = changes.find(table_id); + if (entry != changes.end() && !entry->second.IsEmpty()) { + return true; + } + return false; +} + +void LocalTableChanges::GetLocalDeleteForFile(TableIndex table_id, const string &path, DuckLakeFileData &result) const { + lock_guard guard(lock); + auto entry = changes.find(table_id); + if (entry == changes.end()) { + return; + } + auto &table_changes = entry->second; + auto file_entry = table_changes.new_delete_files.find(path); + if (file_entry == table_changes.new_delete_files.end() || file_entry->second.empty()) { + return; + } + auto &delete_file = file_entry->second.back(); + result.path = delete_file.file_name; + result.file_size_bytes = delete_file.file_size_bytes; + result.footer_size = delete_file.footer_size; + result.encryption_key = delete_file.encryption_key; +} + +bool LocalTableChanges::HasLocalInlinedFileDeletes(TableIndex table_id) const { + lock_guard guard(lock); + auto entry = changes.find(table_id); + if (entry == changes.end()) { + return false; + } + auto &table_changes = entry->second; + if (!table_changes.new_inlined_file_deletes) { + return false; + } + return !table_changes.new_inlined_file_deletes->file_deletes.empty(); +} + +void LocalTableChanges::GetLocalInlinedFileDeletesForFile(TableIndex table_id, idx_t file_id, + set &result) const { + lock_guard guard(lock); + auto entry = changes.find(table_id); + if (entry == changes.end()) { + return; + } + auto &table_changes = entry->second; + if (!table_changes.new_inlined_file_deletes) { + return; + } + auto file_entry = table_changes.new_inlined_file_deletes->file_deletes.find(file_id); + if (file_entry == table_changes.new_inlined_file_deletes->file_deletes.end()) { + return; + } + // Merge the inlined deletes into the result set + for (auto &row_id : file_entry->second) { + result.insert(row_id); + } +} + +void LocalTableChanges::TransactionLocalDelete(ClientContext &context, TableIndex table_id, + const string &data_file_path, DuckLakeDeleteFile delete_file) { + lock_guard guard(lock); + auto entry = changes.find(table_id); + if (entry == changes.end()) { + throw InternalException( + "Transaction local delete called for table which does not have transaction local insertions"); + } + auto &table_changes = entry->second; + for (auto &file : table_changes.new_data_files) { + if (file.file_name == data_file_path) { + if (!file.delete_files.empty()) { + auto &fs = FileSystem::GetFileSystem(context); + vector files_to_delete; + files_to_delete.reserve(file.delete_files.size()); + for (auto &old_file : file.delete_files) { + files_to_delete.push_back(old_file.file_name); + } + fs.RemoveFiles(files_to_delete); + file.delete_files.clear(); + } + file.delete_files.push_back(std::move(delete_file)); + return; + } + } + throw InternalException("Failed to find matching transaction-local file for written delete file"); +} + +void LocalTableChanges::CleanupFiles(ClientContext &context, TableIndex table_id) { + lock_guard guard(lock); + auto table_entry = changes.find(table_id); + if (table_entry != changes.end()) { + auto &table_changes = table_entry->second; + auto &fs = FileSystem::GetFileSystem(context); + for (auto &file : table_changes.new_data_files) { + fs.RemoveFile(file.file_name); + } + changes.erase(table_entry); + } +} + +void LocalTableChanges::AddDeletesToMap(ClientContext &context, vector new_deletes, + unordered_map> &table_delete_map) { + for (auto &file : new_deletes) { + auto &data_file_path = file.data_file_path; + if (data_file_path.empty()) { + throw InternalException("Data file path needs to be set in delete"); + } + if (file.source == DeleteFileSource::FLUSH) { + // If we have a snapshot, this is a flushed delete file, we add it to the rooster + table_delete_map[data_file_path].push_back(std::move(file)); + } else { + // If not is a regular deletion + auto existing_entry = table_delete_map.find(data_file_path); + if (existing_entry != table_delete_map.end() && !existing_entry->second.empty()) { + // If a file already exists we remove it + auto &fs = FileSystem::GetFileSystem(context); + for (auto &old_file : existing_entry->second) { + fs.RemoveFile(old_file.file_name); + } + existing_entry->second.clear(); + } + // We add the new file in + table_delete_map[data_file_path].push_back(std::move(file)); + } + } +} + +void LocalTableChanges::AddDeletes(ClientContext &context, TableIndex table_id, vector files) { + if (files.empty()) { + return; + } + lock_guard guard(lock); + auto &table_changes = changes[table_id]; + auto &table_delete_map = table_changes.new_delete_files; + LocalTableChanges::AddDeletesToMap(context, std::move(files), table_delete_map); +} + +LocalTableChangeIterationHelper::LocalTableChangeIterationHelper( + mutex &local_changes_lock, const map &changes_p) + : lock(local_changes_lock), changes(changes_p) { +} + +LocalTableChangeIterationHelper::LocalTableChangeIteratorEntry::LocalTableChangeIteratorEntry() { +} + +TableIndex LocalTableChangeIterationHelper::LocalTableChangeIteratorEntry::GetTableIndex() const { + return table_id; +} + +const LocalTableDataChanges &LocalTableChangeIterationHelper::LocalTableChangeIteratorEntry::GetTableChanges() const { + return *changes; +} + +LocalTableChangeIterationHelper::LocalTableChangeIterator::LocalTableChangeIterator( + map::const_iterator it_p, + map::const_iterator end_it_p) + : it(std::move(it_p)), end_it(std::move(end_it_p)) { + if (it != end_it) { + entry.table_id = it->first; + entry.changes = it->second; + } +} + +LocalTableChangeIterationHelper::LocalTableChangeIterator & +LocalTableChangeIterationHelper::LocalTableChangeIterator::operator++() { + it++; + if (it != end_it) { + entry.table_id = it->first; + entry.changes = it->second; + } + return *this; +} + +bool LocalTableChangeIterationHelper::LocalTableChangeIterator::operator!=( + const LocalTableChangeIterator &other) const { + return it != other.it; +} + +const LocalTableChangeIterationHelper::LocalTableChangeIteratorEntry & +LocalTableChangeIterationHelper::LocalTableChangeIterator::operator*() const { + return entry; +} + +LocalTableChangeIterationHelper LocalTableChanges::Changes() const { + return LocalTableChangeIterationHelper(lock, changes); +} + DuckLakeTransaction::DuckLakeTransaction(DuckLakeCatalog &ducklake_catalog, TransactionManager &manager, ClientContext &context) : Transaction(manager, context), ducklake_catalog(ducklake_catalog), db(*context.db), @@ -68,8 +669,7 @@ void DuckLakeTransaction::Commit() { connection->Commit(); } connection.reset(); - - table_data_changes.clear(); + local_changes.Clear(); } void DuckLakeTransaction::Rollback() { @@ -79,8 +679,7 @@ void DuckLakeTransaction::Rollback() { connection.reset(); } CleanupFiles(); - - table_data_changes.clear(); + local_changes.Clear(); } Connection &DuckLakeTransaction::GetConnection() { @@ -105,14 +704,14 @@ Connection &DuckLakeTransaction::GetConnection() { return *connection; } -bool DuckLakeTransaction::SchemaChangesMade() { +bool DuckLakeTransaction::SchemaChangesMade() const { return !new_tables.empty() || !dropped_tables.empty() || new_schemas || !dropped_schemas.empty() || !dropped_views.empty() || !new_macros.empty() || !dropped_scalar_macros.empty() || !dropped_table_macros.empty(); } -bool DuckLakeTransaction::ChangesMade() { - return SchemaChangesMade() || !table_data_changes.empty() || !dropped_files.empty() || +bool DuckLakeTransaction::ChangesMade() const { + return SchemaChangesMade() || local_changes.HasChanges() || !dropped_files.empty() || !new_name_maps.name_maps.empty(); } @@ -223,7 +822,7 @@ void GetTransactionViewChanges(reference view_entry, TransactionCh } } -TransactionChangeInformation DuckLakeTransaction::GetTransactionChanges() { +TransactionChangeInformation DuckLakeTransaction::GetTransactionChanges() const { TransactionChangeInformation changes; for (auto &dropped_table_idx : dropped_tables) { changes.dropped_tables.insert(dropped_table_idx); @@ -281,13 +880,13 @@ TransactionChangeInformation DuckLakeTransaction::GetTransactionChanges() { } } changes.tables_deleted_from = tables_deleted_from; - for (auto &entry : table_data_changes) { - auto table_id = entry.first; + for (auto &entry : local_changes.Changes()) { + auto table_id = entry.GetTableIndex(); if (table_id.IsTransactionLocal()) { // don't report transaction-local tables yet - these will get added later on continue; } - auto &table_changes = entry.second; + auto &table_changes = entry.GetTableChanges(); AddTableChanges(table_id, table_changes, changes); } return changes; @@ -300,6 +899,9 @@ struct DuckLakeCommitState { DuckLakeSnapshot &commit_snapshot; map committed_schemas; map committed_tables; + map committed_partition_ids; + map committed_mapping_indexes; + map> local_delete_files; void RemapIdentifier(SchemaIndex &schema_id) const { auto entry = committed_schemas.find(schema_id); @@ -313,6 +915,21 @@ struct DuckLakeCommitState { table_id = entry->second; } } + void RemapPartitionId(optional_idx &partition_id) const { + if (!partition_id.IsValid()) { + return; + } + auto entry = committed_partition_ids.find(partition_id.GetIndex()); + if (entry != committed_partition_ids.end()) { + partition_id = entry->second; + } + } + void RemapMappingIndex(MappingIndex &table_id) const { + auto entry = committed_mapping_indexes.find(table_id); + if (entry != committed_mapping_indexes.end()) { + table_id = entry->second; + } + } SchemaIndex GetSchemaId(DuckLakeSchemaEntry &schema) const { auto schema_id = schema.GetSchemaId(); @@ -336,7 +953,7 @@ struct DuckLakeCommitState { }; void DuckLakeTransaction::AddTableChanges(TableIndex table_id, const LocalTableDataChanges &table_changes, - TransactionChangeInformation &changes) { + TransactionChangeInformation &changes) const { bool inserted_data = false; bool flushed_inline_data = false; for (auto &file : table_changes.new_data_files) { @@ -391,14 +1008,14 @@ void AddChangeInfo(DuckLakeCommitState &commit_state, SnapshotChangeInfo &change } string DuckLakeTransaction::WriteSnapshotChanges(DuckLakeCommitState &commit_state, - TransactionChangeInformation &changes) { + TransactionChangeInformation &changes) const { SnapshotChangeInfo change_info; // re-add all inserted tables - transaction-local table identifiers should have been converted at this stage changes.tables_deleted_from = tables_deleted_from; - for (auto &entry : table_data_changes) { - auto table_id = commit_state.GetTableId(entry.first); - auto &table_changes = entry.second; + for (auto &entry : local_changes.Changes()) { + auto table_id = commit_state.GetTableId(entry.GetTableIndex()); + auto &table_changes = entry.GetTableChanges(); AddTableChanges(table_id, table_changes, changes); } for (auto &entry : changes.dropped_schemas) { @@ -488,26 +1105,7 @@ string DuckLakeTransaction::WriteSnapshotChanges(DuckLakeCommitState &commit_sta void DuckLakeTransaction::CleanupFiles() { // remove any files that were written - auto context_ref = context.lock(); - auto &fs = FileSystem::GetFileSystem(db); - for (auto &entry : table_data_changes) { - auto &table_changes = entry.second; - for (auto &file : table_changes.new_data_files) { - if (file.created_by_ducklake) { - fs.TryRemoveFile(file.file_name); - } - for (auto &del_file : file.delete_files) { - fs.TryRemoveFile(del_file.file_name); - } - } - for (auto &file : table_changes.new_delete_files) { - for (auto &delete_files : file.second) { - fs.TryRemoveFile(delete_files.file_name); - } - } - table_changes.new_data_files.clear(); - table_changes.new_delete_files.clear(); - } + local_changes.CleanupFiles(db); } template @@ -574,7 +1172,7 @@ void ConflictCheck(const case_insensitive_map_t> & void DuckLakeTransaction::CheckForConflicts(const TransactionChangeInformation &changes, const SnapshotChangeInformation &other_changes, - DuckLakeSnapshot transaction_snapshot) { + DuckLakeSnapshot transaction_snapshot) const { // check if we are dropping the same table as another transaction for (auto &dropped_idx : changes.dropped_tables) { ConflictCheck(dropped_idx, other_changes.dropped_tables, "drop table", "dropped it already"); @@ -658,8 +1256,8 @@ void DuckLakeTransaction::CheckForConflicts(const TransactionChangeInformation & if (check_for_matches) { // If we have deletes on the tables, check for files being deleted const auto deleted_files = metadata_manager->GetFilesDeletedOrDroppedAfterSnapshot(transaction_snapshot); - for (auto &entry : table_data_changes) { - auto &table_changes = entry.second; + for (auto &entry : local_changes.Changes()) { + auto &table_changes = entry.GetTableChanges(); for (auto &file_entry : table_changes.new_delete_files) { for (auto &file : file_entry.second) { ConflictCheck(file.data_file_id, deleted_files.deleted_from_files, "delete from file", @@ -780,16 +1378,7 @@ DuckLakePartitionInfo DuckLakeTransaction::GetNewPartitionKey(DuckLakeCommitStat } partition_key.fields.push_back(std::move(partition_field)); } - - // if we wrote any data with this partition id - rewrite it to the latest partition id - for (auto &entry : table_data_changes) { - auto &table_changes = entry.second; - for (auto &file : table_changes.new_data_files) { - if (file.partition_id.IsValid() && file.partition_id.GetIndex() == local_partition_id) { - file.partition_id = partition_id; - } - } - } + commit_state.committed_partition_ids[local_partition_id] = partition_id; return partition_key; } @@ -1104,7 +1693,7 @@ void DuckLakeTransaction::GetNewMacroInfo(DuckLakeCommitState &commit_state, ref new_macro_info.macro_id = MacroIndex(commit_state.commit_snapshot.next_catalog_id++); new_macro_info.macro_name = macro_entry.name; - new_macro_info.schema_id = ducklake_schema.GetSchemaId(); + new_macro_info.schema_id = commit_state.GetSchemaId(ducklake_schema); // Let's do the implementations for (const auto &impl : macro_entry.macros) { DuckLakeMacroImplementation macro_impl; @@ -1317,8 +1906,9 @@ DuckLakeColumnStatsInfo DuckLakeColumnStatsInfo::FromColumnStats(FieldIndex fiel return column_stats; } -DuckLakeFileInfo DuckLakeTransaction::GetNewDataFile(DuckLakeDataFile &file, DuckLakeSnapshot &commit_snapshot, +DuckLakeFileInfo DuckLakeTransaction::GetNewDataFile(const DuckLakeDataFile &file, DuckLakeCommitState &commit_state, TableIndex table_id, optional_idx row_id_start) { + auto &commit_snapshot = commit_state.commit_snapshot; DuckLakeFileInfo data_file; data_file.id = DataFileIndex(commit_snapshot.next_file_id++); data_file.table_id = table_id; @@ -1333,6 +1923,8 @@ DuckLakeFileInfo DuckLakeTransaction::GetNewDataFile(DuckLakeDataFile &file, Duc data_file.begin_snapshot = file.begin_snapshot; data_file.max_partial_file_snapshot = file.max_partial_file_snapshot; data_file.column_stats = file.column_stats; + commit_state.RemapPartitionId(data_file.partition_id); + commit_state.RemapMappingIndex(data_file.mapping_id); for (auto &partition_entry : file.partition_values) { DuckLakeFilePartitionInfo partition_info; partition_info.partition_column_idx = partition_entry.partition_column_idx; @@ -1357,12 +1949,12 @@ NewDataInfo DuckLakeTransaction::GetNewDataFiles(string &batch_query, DuckLakeCo auto &schema = ducklake_catalog.GetSchemaForSnapshot(*this, GetSnapshot()); dl_stats = ducklake_catalog.ConstructStatsMap(*stats, schema); } - for (auto &entry : table_data_changes) { - auto table_id = commit_state.GetTableId(entry.first); + for (auto &entry : local_changes.Changes()) { + auto table_id = commit_state.GetTableId(entry.GetTableIndex()); if (table_id.IsTransactionLocal()) { throw InternalException("Cannot commit transaction local files - these should have been cleaned up before"); } - auto &table_changes = entry.second; + auto &table_changes = entry.GetTableChanges(); if (table_changes.new_data_files.empty() && !table_changes.new_inlined_data) { // no new data - skip this entry continue; @@ -1390,7 +1982,7 @@ NewDataInfo DuckLakeTransaction::GetNewDataFiles(string &batch_query, DuckLakeCo // row_id_start auto row_id_start = file.flush_row_id_start.IsValid() ? file.flush_row_id_start.GetIndex() : new_stats.next_row_id; - auto data_file = GetNewDataFile(file, commit_state.commit_snapshot, table_id, row_id_start); + auto data_file = GetNewDataFile(file, commit_state, table_id, row_id_start); for (auto &del_file : file.delete_files) { // this transaction-local file already has deletes - write them out DuckLakeDeleteFile delete_file = del_file; @@ -1410,8 +2002,8 @@ NewDataInfo DuckLakeTransaction::GetNewDataFiles(string &batch_query, DuckLakeCo } result.new_files.push_back(std::move(data_file)); } - // write any deletes that were made on top of these transaction-local files - AddDeletes(table_id, std::move(delete_files)); + // add any delete files that were made on top of these transaction-local files + commit_state.local_delete_files[table_id] = std::move(delete_files); if (table_changes.new_inlined_data) { auto &inlined_data = *table_changes.new_inlined_data; @@ -1429,8 +2021,17 @@ NewDataInfo DuckLakeTransaction::GetNewDataFiles(string &batch_query, DuckLakeCo // update global stats new_stats.record_count += record_count; - new_stats.next_row_id += record_count; - + if (!inlined_data.HasPreservedRowIds()) { + // regular insert, we advance next_row_id + new_stats.next_row_id += record_count; + } else { + // mixed insert and updates, we only advance inserted row_ids + for (auto &rid : inlined_data.row_ids) { + if (DuckLakeConstants::IsTransactionLocalRowId(rid)) { + new_stats.next_row_id++; + } + } + } // add the file to the to-be-written inlined data list new_inlined_data.data = table_changes.new_inlined_data.get(); result.new_inlined_data.push_back(new_inlined_data); @@ -1446,34 +2047,55 @@ NewDataInfo DuckLakeTransaction::GetNewDataFiles(string &batch_query, DuckLakeCo return result; } +DuckLakeDeleteFileInfo DuckLakeTransaction::GetNewDeleteFile(TableIndex table_id, + const DuckLakeCommitState &commit_state, + const DuckLakeDeleteFile &file) const { + DuckLakeDeleteFileInfo delete_file; + delete_file.id = DataFileIndex(commit_state.commit_snapshot.next_file_id++); + delete_file.table_id = table_id; + delete_file.data_file_id = file.data_file_id; + delete_file.path = file.file_name; + delete_file.delete_count = file.delete_count; + delete_file.file_size_bytes = file.file_size_bytes; + delete_file.footer_size = file.footer_size; + delete_file.encryption_key = file.encryption_key; + delete_file.begin_snapshot = file.begin_snapshot; + delete_file.max_snapshot = file.max_snapshot; + return delete_file; +} + vector DuckLakeTransaction::GetNewDeleteFiles(const DuckLakeCommitState &commit_state, vector &overwritten_delete_files) const { vector result; - for (auto &entry : table_data_changes) { - auto table_id = commit_state.GetTableId(entry.first); - auto &table_changes = entry.second; + // handle delete files made to existing files + for (auto &entry : local_changes.Changes()) { + auto table_id = commit_state.GetTableId(entry.GetTableIndex()); + auto &table_changes = entry.GetTableChanges(); for (auto &file_entry : table_changes.new_delete_files) { for (auto &file : file_entry.second) { if (file.overwritten_delete_file.delete_file_id.IsValid()) { // track the old delete file for deletion from metadata and disk overwritten_delete_files.push_back(file.overwritten_delete_file); } - DuckLakeDeleteFileInfo delete_file; - delete_file.id = DataFileIndex(commit_state.commit_snapshot.next_file_id++); - delete_file.table_id = table_id; - delete_file.data_file_id = file.data_file_id; - delete_file.path = file.file_name; - delete_file.delete_count = file.delete_count; - delete_file.file_size_bytes = file.file_size_bytes; - delete_file.footer_size = file.footer_size; - delete_file.encryption_key = file.encryption_key; - delete_file.begin_snapshot = file.begin_snapshot; - delete_file.max_snapshot = file.max_snapshot; + auto delete_file = GetNewDeleteFile(table_id, commit_state, file); result.push_back(std::move(delete_file)); } } } + // handle any delete files that were added to data files that are ALSO added in this transaction + for (auto &entry : commit_state.local_delete_files) { + auto table_id = commit_state.GetTableId(entry.first); + for (auto &file : entry.second) { + if (file.overwritten_delete_file.delete_file_id.IsValid()) { + throw InternalException("Local delete files should not overwrite files"); + // track the old delete file for deletion from metadata and disk + overwritten_delete_files.push_back(file.overwritten_delete_file); + } + auto delete_file = GetNewDeleteFile(table_id, commit_state, file); + result.push_back(std::move(delete_file)); + } + } return result; } @@ -1501,7 +2123,7 @@ void ConvertNameMapColumn(const DuckLakeNameMapEntry &name_map_entry, MappingInd NewNameMapInfo DuckLakeTransaction::GetNewNameMaps(DuckLakeCommitState &commit_state) { NewNameMapInfo result; - map remap_mapping_index; + auto &committed_mapping_indexes = commit_state.committed_mapping_indexes; for (auto &entry : new_name_maps.name_maps) { // generate a new mapping id auto local_map_id = entry.first; @@ -1523,29 +2145,17 @@ NewNameMapInfo DuckLakeTransaction::GetNewNameMaps(DuckLakeCommitState &commit_s } result.new_column_mappings.push_back(std::move(map_info)); - remap_mapping_index[local_map_id] = new_map_id; - } - // iterate over the data files to point them towards any new mapping ids - for (auto &entry : table_data_changes) { - auto &table_changes = entry.second; - for (auto &data_file : table_changes.new_data_files) { - if (!data_file.mapping_id.IsValid()) { - continue; - } - auto entry = remap_mapping_index.find(data_file.mapping_id); - if (entry != remap_mapping_index.end()) { - data_file.mapping_id = entry->second; - } - } + committed_mapping_indexes[local_map_id] = new_map_id; } return result; } -vector DuckLakeTransaction::GetNewInlinedDeletes(DuckLakeCommitState &commit_state) { +vector +DuckLakeTransaction::GetNewInlinedDeletes(DuckLakeCommitState &commit_state) const { vector result; - for (auto &entry : table_data_changes) { - auto table_id = commit_state.GetTableId(entry.first); - auto &table_changes = entry.second; + for (auto &entry : local_changes.Changes()) { + auto table_id = commit_state.GetTableId(entry.GetTableIndex()); + auto &table_changes = entry.GetTableChanges(); for (auto &delete_entry : table_changes.new_inlined_data_deletes) { DuckLakeDeletedInlinedDataInfo info; info.table_id = table_id; @@ -1640,9 +2250,11 @@ string DuckLakeTransaction::CommitChanges(DuckLakeCommitState &commit_state, } // write new data / data files - if (!table_data_changes.empty()) { + bool has_table_data_changes = local_changes.HasChanges(); + if (has_table_data_changes) { auto result = GetNewDataFiles(batch_queries, commit_state, stats); - batch_queries += metadata_manager->WriteNewDataFiles(commit_snapshot, result.new_files, new_tables_result, new_schemas_result); + batch_queries += metadata_manager->WriteNewDataFiles(commit_snapshot, result.new_files, new_tables_result, + new_schemas_result); batch_queries += metadata_manager->WriteNewInlinedData(commit_snapshot, result.new_inlined_data, new_tables_result, new_inlined_data_tables_result); } @@ -1656,7 +2268,7 @@ string DuckLakeTransaction::CommitChanges(DuckLakeCommitState &commit_state, batch_queries += metadata_manager->DropDataFiles(dropped_indexes); } - if (!table_data_changes.empty()) { + if (has_table_data_changes) { // write new delete files vector overwritten_delete_files; auto file_list = GetNewDeleteFiles(commit_state, overwritten_delete_files); @@ -1673,15 +2285,15 @@ string DuckLakeTransaction::CommitChanges(DuckLakeCommitState &commit_state, // write compactions auto compaction_merge_adjacent_changes = - GetCompactionChanges(commit_snapshot, CompactionType::MERGE_ADJACENT_TABLES); + GetCompactionChanges(commit_state, CompactionType::MERGE_ADJACENT_TABLES); batch_queries += metadata_manager->WriteCompactions(compaction_merge_adjacent_changes.compacted_files, CompactionType::MERGE_ADJACENT_TABLES); - batch_queries += metadata_manager->WriteNewDataFiles(commit_snapshot, compaction_merge_adjacent_changes.new_files, - new_tables_result, new_schemas_result); + batch_queries += metadata_manager->WriteNewDataFiles( + commit_snapshot, compaction_merge_adjacent_changes.new_files, new_tables_result, new_schemas_result); - auto compaction_rewrite_delete_changes = GetCompactionChanges(commit_snapshot, CompactionType::REWRITE_DELETES); - batch_queries += metadata_manager->WriteNewDataFiles(commit_snapshot, compaction_rewrite_delete_changes.new_files, - new_tables_result, new_schemas_result); + auto compaction_rewrite_delete_changes = GetCompactionChanges(commit_state, CompactionType::REWRITE_DELETES); + batch_queries += metadata_manager->WriteNewDataFiles( + commit_snapshot, compaction_rewrite_delete_changes.new_files, new_tables_result, new_schemas_result); batch_queries += metadata_manager->WriteCompactions(compaction_rewrite_delete_changes.compacted_files, CompactionType::REWRITE_DELETES); } @@ -1705,43 +2317,53 @@ string DuckLakeTransaction::CommitChanges(DuckLakeCommitState &commit_state, return batch_queries; } -CompactionInformation DuckLakeTransaction::GetCompactionChanges(DuckLakeSnapshot &commit_snapshot, +CompactionInformation DuckLakeTransaction::GetCompactionChanges(DuckLakeCommitState &commit_state, CompactionType type) { + auto &commit_snapshot = commit_state.commit_snapshot; CompactionInformation result; - for (auto &entry : table_data_changes) { - auto table_id = entry.first; - auto &table_changes = entry.second; + for (auto &entry : local_changes.Changes()) { + auto table_id = entry.GetTableIndex(); + auto &table_changes = entry.GetTableChanges(); for (auto &compaction : table_changes.compactions) { if (type != compaction.type) { continue; } - auto new_file = GetNewDataFile(compaction.written_file, commit_snapshot, table_id, compaction.row_id_start); - switch (type) { - case CompactionType::REWRITE_DELETES: - new_file.begin_snapshot = commit_snapshot.snapshot_id; - break; - case CompactionType::MERGE_ADJACENT_TABLES: { - // For MERGE_ADJACENT_TABLES, track the max partial snapshot across all source files - optional_idx merged_max_partial_snapshot; - idx_t first_begin_snapshot = compaction.source_files[0].file.begin_snapshot; - for (auto &compacted_file : compaction.source_files) { - idx_t file_max_snapshot = compacted_file.max_partial_file_snapshot.IsValid() - ? compacted_file.max_partial_file_snapshot.GetIndex() - : compacted_file.file.begin_snapshot; - if (!merged_max_partial_snapshot.IsValid() || - file_max_snapshot > merged_max_partial_snapshot.GetIndex()) { - merged_max_partial_snapshot = file_max_snapshot; + bool has_new_file = !compaction.written_file.file_name.empty(); + DuckLakeFileInfo new_file; + + if (!has_new_file) { + if (type != CompactionType::REWRITE_DELETES) { + throw InternalException("Compaction error - expected output file for non-rewrite compaction"); + } + } else { + new_file = GetNewDataFile(compaction.written_file, commit_state, table_id, compaction.row_id_start); + switch (type) { + case CompactionType::REWRITE_DELETES: + new_file.begin_snapshot = commit_snapshot.snapshot_id; + break; + case CompactionType::MERGE_ADJACENT_TABLES: { + // For MERGE_ADJACENT_TABLES, track the max partial snapshot across all source files + optional_idx merged_max_partial_snapshot; + idx_t first_begin_snapshot = compaction.source_files[0].file.begin_snapshot; + for (auto &compacted_file : compaction.source_files) { + idx_t file_max_snapshot = compacted_file.max_partial_file_snapshot.IsValid() + ? compacted_file.max_partial_file_snapshot.GetIndex() + : compacted_file.file.begin_snapshot; + if (!merged_max_partial_snapshot.IsValid() || + file_max_snapshot > merged_max_partial_snapshot.GetIndex()) { + merged_max_partial_snapshot = file_max_snapshot; + } + } + // Use the first source file's begin_snapshot for proper time travel support + new_file.begin_snapshot = first_begin_snapshot; + if (compaction.source_files.size() > 1) { + new_file.max_partial_file_snapshot = merged_max_partial_snapshot; } + break; } - // Use the first source file's begin_snapshot for proper time travel support - new_file.begin_snapshot = first_begin_snapshot; - if (compaction.source_files.size() > 1) { - new_file.max_partial_file_snapshot = merged_max_partial_snapshot; + default: + throw InternalException("DuckLakeTransaction::GetCompactionChanges Compaction type is invalid"); } - break; - } - default: - throw InternalException("DuckLakeTransaction::GetCompactionChanges Compaction type is invalid"); } idx_t row_id_limit = 0; @@ -1753,22 +2375,30 @@ CompactionInformation DuckLakeTransaction::GetCompactionChanges(DuckLakeSnapshot DuckLakeCompactedFileInfo file_info; file_info.path = compacted_file.file.data.path; file_info.source_id = compacted_file.file.id; - file_info.new_id = new_file.id; + if (has_new_file) { + file_info.new_id = new_file.id; + } if (!compacted_file.delete_files.empty()) { file_info.delete_file_path = compacted_file.delete_files.back().data.path; file_info.delete_file_id = compacted_file.delete_files.back().delete_file_id; file_info.start_snapshot = compacted_file.file.begin_snapshot; - file_info.table_index = entry.first; + file_info.table_index = entry.GetTableIndex(); file_info.delete_file_start_snapshot = commit_snapshot.snapshot_id; file_info.delete_file_end_snapshot = compacted_file.delete_files.back().end_snapshot; } - if (row_id_limit > new_file.row_count) { + if (has_new_file && row_id_limit > new_file.row_count) { throw InternalException("Compaction error - row id limit is larger than the row count of the file"); } result.compacted_files.push_back(std::move(file_info)); } - result.new_files.push_back(std::move(new_file)); + if (!has_new_file && row_id_limit != 0) { + throw InternalException( + "Compaction error - rewrite compaction without output file must fully delete source files"); + } + if (has_new_file) { + result.new_files.push_back(std::move(new_file)); + } } } return result; @@ -1915,432 +2545,138 @@ void DuckLakeTransaction::DeleteSnapshots(const vector &sn metadata_manager.DeleteSnapshots(snapshots); } -void DuckLakeTransaction::DeleteInlinedData(const DuckLakeInlinedTableInfo &inlined_table) { - auto &metadata_manager = GetMetadataManager(); - metadata_manager.DeleteInlinedData(inlined_table); -} - -unique_ptr DuckLakeTransaction::Query(string query) { - auto &connection = GetConnection(); - DuckLakeMetadataManager::FillCatalogArgs(query, ducklake_catalog); - return connection.Query(query); -} - -string DuckLakeTransaction::GetDefaultSchemaName() { - auto &metadata_context = *connection->context; - auto &db_manager = DatabaseManager::Get(metadata_context); - auto metadb = db_manager.GetDatabase(metadata_context, ducklake_catalog.MetadataDatabaseName()); - return metadb->GetCatalog().GetDefaultSchema(); -} - -DuckLakeSnapshot DuckLakeTransaction::GetSnapshot() { - auto catalog_snapshot = ducklake_catalog.CatalogSnapshot(); - if (catalog_snapshot) { - // the catalog was opened at a specific snapshot - load that snapshot - return GetSnapshot(catalog_snapshot); - } - lock_guard guard(snapshot_lock); - if (!snapshot) { - // no snapshot loaded yet for this transaction - load it - snapshot = metadata_manager->GetSnapshot(); - } - return *snapshot; -} - -DuckLakeSnapshot DuckLakeTransaction::GetSnapshot(optional_ptr at_clause, SnapshotBound bound) { - if (!at_clause) { - // no AT-clause - get the latest snapshot - return GetSnapshot(); - } - // construct a struct value from the AT clause in the form of {"unit": value} (e.g. {"version": 2} - // this is used as a caching key for the snapshot - child_list_t values; - values.push_back(make_pair(at_clause->Unit(), at_clause->GetValue())); - auto snapshot_value = Value::STRUCT(std::move(values)); - - lock_guard guard(snapshot_lock); - auto entry = snapshot_cache.find(snapshot_value); - if (entry != snapshot_cache.end()) { - // we already found this snapshot - return it - return entry->second; - } - // find the snapshot and cache it - auto result_snapshot = *metadata_manager->GetSnapshot(*at_clause, bound); - snapshot_cache.insert(make_pair(std::move(snapshot_value), result_snapshot)); - return result_snapshot; -} - -idx_t DuckLakeTransaction::GetLocalCatalogId() { - return local_catalog_id++; -} - -bool DuckLakeTransaction::HasTransactionLocalInserts(TableIndex table_id) const { - auto entry = table_data_changes.find(table_id); - if (entry == table_data_changes.end()) { - return false; - } - auto &table_changes = entry->second; - return !table_changes.new_data_files.empty() || table_changes.new_inlined_data; -} - -bool DuckLakeTransaction::HasTransactionInlinedData(TableIndex table_id) const { - auto entry = table_data_changes.find(table_id); - if (entry == table_data_changes.end()) { - return false; - } - auto &table_changes = entry->second; - return table_changes.new_inlined_data != nullptr; -} - -vector DuckLakeTransaction::GetTransactionLocalFiles(TableIndex table_id) { - auto entry = table_data_changes.find(table_id); - if (entry == table_data_changes.end()) { - return vector(); - } - return entry->second.new_data_files; -} - -shared_ptr DuckLakeTransaction::GetTransactionLocalInlinedData(TableIndex table_id) { - auto entry = table_data_changes.find(table_id); - if (entry == table_data_changes.end()) { - return nullptr; - } - auto &table_changes = entry->second; - if (!table_changes.new_inlined_data) { - return nullptr; - } - auto &local_changes = *table_changes.new_inlined_data; - auto context_ref = context.lock(); - auto result = make_shared_ptr(); - result->data = make_uniq(*context_ref, local_changes.data->Types()); - for (auto &chunk : local_changes.data->Chunks()) { - result->data->Append(chunk); - } - return result; -} - -void DuckLakeTransaction::DropTransactionLocalFile(TableIndex table_id, const string &path) { - auto entry = table_data_changes.find(table_id); - if (entry == table_data_changes.end()) { - throw InternalException( - "DropTransactionLocalFile called for a table for which no transaction-local files exist"); - } - auto &table_changes = entry->second; - auto &table_files = table_changes.new_data_files; - auto context_ref = context.lock(); - auto &fs = FileSystem::GetFileSystem(*context_ref); - for (idx_t i = 0; i < table_files.size(); i++) { - auto &file = table_files[i]; - if (file.file_name == path) { - for (auto &del_file : file.delete_files) { - fs.RemoveFile(del_file.file_name); - } - file.delete_files.clear(); - // found the file - delete it from the table list and from disk - table_files.erase_at(i); - fs.RemoveFile(path); - if (table_changes.IsEmpty()) { - // no more files remaining - table_data_changes.erase(entry); - } - return; - } - } - throw InternalException("Failed to find matching transaction-local file for DropTransactionLocalFile"); -} - -void DuckLakeTransaction::AppendFiles(TableIndex table_id, vector files) { - if (files.empty()) { - return; - } - lock_guard guard(table_data_changes_lock); - auto &table_changes = table_data_changes[table_id]; - if (table_changes.new_data_files.empty()) { - // If empty, just move the entire vector - table_changes.new_data_files = std::move(files); - } else { - // Reserve to avoid reallocations during insertion - table_changes.new_data_files.reserve(table_changes.new_data_files.size() + files.size()); - // Use move_iterator for efficient batch move - table_changes.new_data_files.insert( - table_changes.new_data_files.end(), - std::make_move_iterator(files.begin()), - std::make_move_iterator(files.end())); - } -} - -void DuckLakeTransaction::AppendInlinedData(TableIndex table_id, unique_ptr new_data) { - lock_guard guard(table_data_changes_lock); - auto &table_changes = table_data_changes[table_id]; - if (table_changes.new_inlined_data) { - // already exists - append - auto &existing_data = *table_changes.new_inlined_data; - auto &existing_types = existing_data.data->Types(); - auto &new_types = new_data->data->Types(); - // check if types changed (e.g. due to ALTER COLUMN TYPE) - if (existing_types != new_types) { - // if types differ we gotta add a cast. - auto context_ref = context.lock(); - auto casted_data = make_uniq(*context_ref, new_types); - ColumnDataAppendState append_state; - casted_data->InitializeAppend(append_state); - for (auto &chunk : existing_data.data->Chunks()) { - DataChunk casted_chunk; - casted_chunk.Initialize(*context_ref, new_types); - for (idx_t col_idx = 0; col_idx < chunk.ColumnCount(); col_idx++) { - if (existing_types[col_idx] != new_types[col_idx]) { - VectorOperations::Cast(*context_ref, chunk.data[col_idx], casted_chunk.data[col_idx], - chunk.size()); - } else { - casted_chunk.data[col_idx].Reference(chunk.data[col_idx]); - } - } - casted_chunk.SetCardinality(chunk.size()); - casted_data->Append(append_state, casted_chunk); - } - existing_data.data = std::move(casted_data); - } - ColumnDataAppendState append_state; - existing_data.data->InitializeAppend(append_state); - for (auto &chunk : new_data->data->Chunks()) { - existing_data.data->Append(chunk); - } - for (auto &entry : new_data->column_stats) { - auto stats_entry = existing_data.column_stats.find(entry.first); - if (stats_entry == existing_data.column_stats.end()) { - throw InternalException("Missing stats when merging inlined data"); - } - stats_entry->second.MergeStats(entry.second); - } - } else { - // does not exist yet - set it - table_changes.new_inlined_data = std::move(new_data); - } -} - -void DuckLakeTransaction::AddNewInlinedDeletes(TableIndex table_id, const string &table_name, set new_deletes) { - if (new_deletes.empty()) { - return; - } - lock_guard guard(table_data_changes_lock); - auto &table_changes = table_data_changes[table_id]; - auto &table_deletes = table_changes.new_inlined_data_deletes; - auto entry = table_deletes.find(table_name); - if (entry != table_deletes.end()) { - // merge deletes - auto &existing_rows = entry->second->rows; - for (auto &row_idx : new_deletes) { - existing_rows.insert(row_idx); - } - } else { - auto new_data = make_uniq(); - new_data->rows = std::move(new_deletes); - table_deletes.emplace(table_name, std::move(new_data)); - } -} - -void DuckLakeTransaction::DeleteFromLocalInlinedData(TableIndex table_id, set new_deletes) { - lock_guard guard(table_data_changes_lock); - auto entry = table_data_changes.find(table_id); - if (entry == table_data_changes.end()) { - throw InternalException("DeleteFromLocalInlinedData called but no transaction-local data exists for table"); - } - auto &table_changes = entry->second; - auto &existing = *table_changes.new_inlined_data->data; - // construct a new collection from the existing data minus the deletes - auto context_ref = context.lock(); - auto new_data = make_uniq(*context_ref, existing.Types()); - - idx_t base_row_id = 0; - ColumnDataAppendState append_state; - new_data->InitializeAppend(append_state); - for (auto &chunk : existing.Chunks()) { - // slice out non-deleted rows - SelectionVector sel(chunk.size()); - idx_t selected_rows = 0; - - for (idx_t r = 0; r < chunk.size(); r++) { - auto row_id = base_row_id + r; - if (new_deletes.find(row_id) != new_deletes.end()) { - // deleted - skip - continue; - } - sel.set_index(selected_rows++, r); - } - base_row_id += chunk.size(); - if (selected_rows == 0) { - continue; - } - chunk.Slice(sel, selected_rows); - new_data->Append(append_state, chunk); - } - - // override the existing collection - table_changes.new_inlined_data->data = std::move(new_data); -} - -void DuckLakeTransaction::AddColumnToLocalInlinedData(TableIndex table_id, const LogicalType &new_column_type, - FieldIndex new_field_index, const Value &default_value) { - lock_guard guard(table_data_changes_lock); - auto entry = table_data_changes.find(table_id); - if (entry == table_data_changes.end()) { - throw InternalException("AddColumnToLocalInlinedData called but no transaction-local data exists"); - } - auto &table_changes = entry->second; - if (!table_changes.new_inlined_data) { - throw InternalException("AddColumnToLocalInlinedData called but no inlined data exists"); - } - - auto &existing = *table_changes.new_inlined_data->data; - - // New types: existing + new column - auto new_types = existing.Types(); - new_types.push_back(new_column_type); - - auto context_ref = context.lock(); - auto new_data = make_uniq(*context_ref, new_types); - - ColumnDataAppendState append_state; - new_data->InitializeAppend(append_state); - - bool has_default = !default_value.IsNull(); - - for (auto &chunk : existing.Chunks()) { - DataChunk new_chunk; - new_chunk.Initialize(*context_ref, new_types); +void DuckLakeTransaction::DeleteInlinedData(const DuckLakeInlinedTableInfo &inlined_table) { + auto &metadata_manager = GetMetadataManager(); + metadata_manager.DeleteInlinedData(inlined_table); +} - // Copy existing columns - for (idx_t col_idx = 0; col_idx < chunk.ColumnCount(); col_idx++) { - new_chunk.data[col_idx].Reference(chunk.data[col_idx]); - } +unique_ptr DuckLakeTransaction::Query(string query) { + auto &connection = GetConnection(); + DuckLakeMetadataManager::FillCatalogArgs(query, ducklake_catalog); + return connection.Query(query); +} - // New column: use default value or NULL - auto &new_col_vector = new_chunk.data[chunk.ColumnCount()]; - if (has_default) { - new_col_vector.Reference(default_value); - } else { - new_col_vector.SetVectorType(VectorType::CONSTANT_VECTOR); - ConstantVector::SetNull(new_col_vector, true); - } +string DuckLakeTransaction::GetDefaultSchemaName() { + auto &metadata_context = *connection->context; + auto &db_manager = DatabaseManager::Get(metadata_context); + auto metadb = db_manager.GetDatabase(metadata_context, ducklake_catalog.MetadataDatabaseName()); + return metadb->GetCatalog().GetDefaultSchema(); +} - new_chunk.SetCardinality(chunk.size()); - new_data->Append(append_state, new_chunk); +DuckLakeSnapshot DuckLakeTransaction::GetSnapshot() { + auto catalog_snapshot = ducklake_catalog.CatalogSnapshot(); + if (catalog_snapshot) { + // the catalog was opened at a specific snapshot - load that snapshot + return GetSnapshot(catalog_snapshot); + } + lock_guard guard(snapshot_lock); + if (!snapshot) { + // no snapshot loaded yet for this transaction - load it + snapshot = metadata_manager->GetSnapshot(); } + return *snapshot; +} - // Add stats for new column - idx_t total_rows = existing.Count(); - DuckLakeColumnStats new_col_stats(new_column_type); - new_col_stats.num_values = total_rows; - new_col_stats.has_num_values = true; - if (has_default) { - new_col_stats.null_count = 0; - new_col_stats.has_null_count = true; - new_col_stats.any_valid = true; - } else { - new_col_stats.null_count = total_rows; - new_col_stats.has_null_count = true; - new_col_stats.any_valid = false; +DuckLakeSnapshot DuckLakeTransaction::GetSnapshot(optional_ptr at_clause, SnapshotBound bound) { + if (!at_clause) { + // no AT-clause - get the latest snapshot + return GetSnapshot(); } + // construct a struct value from the AT clause in the form of {"unit": value} (e.g. {"version": 2} + // this is used as a caching key for the snapshot + child_list_t values; + values.push_back(make_pair(at_clause->Unit(), at_clause->GetValue())); + auto snapshot_value = Value::STRUCT(std::move(values)); - table_changes.new_inlined_data->column_stats.emplace(new_field_index, std::move(new_col_stats)); - table_changes.new_inlined_data->data = std::move(new_data); + lock_guard guard(snapshot_lock); + auto entry = snapshot_cache.find(snapshot_value); + if (entry != snapshot_cache.end()) { + // we already found this snapshot - return it + return entry->second; + } + // find the snapshot and cache it + auto result_snapshot = *metadata_manager->GetSnapshot(*at_clause, bound); + snapshot_cache.insert(make_pair(std::move(snapshot_value), result_snapshot)); + return result_snapshot; } -static void RemoveFieldStats(map &column_stats, const DuckLakeFieldId &field_id) { - column_stats.erase(field_id.GetFieldIndex()); - for (auto &child_id : field_id.Children()) { - RemoveFieldStats(column_stats, *child_id); - } +idx_t DuckLakeTransaction::GetLocalCatalogId() { + return local_catalog_id++; } -void DuckLakeTransaction::RemoveColumnFromLocalInlinedData(TableIndex table_id, LogicalIndex removed_column_index, - const DuckLakeFieldId &field_id) { - lock_guard guard(table_data_changes_lock); - auto entry = table_data_changes.find(table_id); - if (entry == table_data_changes.end()) { - throw InternalException("RemoveColumnFromLocalInlinedData called but no transaction-local data exists"); - } - auto &table_changes = entry->second; - if (!table_changes.new_inlined_data) { - throw InternalException("RemoveColumnFromLocalInlinedData called but no inlined data exists"); - } +bool DuckLakeTransaction::HasTransactionLocalInserts(TableIndex table_id) const { + return local_changes.HasTransactionLocalInserts(table_id); +} - auto &existing = *table_changes.new_inlined_data->data; +bool DuckLakeTransaction::HasTransactionInlinedData(TableIndex table_id) const { + return local_changes.HasTransactionInlinedData(table_id); +} - // New types: existing minus the removed column - vector new_types; - for (idx_t col_idx = 0; col_idx < existing.Types().size(); col_idx++) { - if (col_idx == removed_column_index.index) { - continue; - } - new_types.push_back(existing.Types()[col_idx]); - } +vector DuckLakeTransaction::GetTransactionLocalFiles(TableIndex table_id) const { + return local_changes.GetTransactionLocalFiles(table_id); +} +shared_ptr DuckLakeTransaction::GetTransactionLocalInlinedData(TableIndex table_id) const { auto context_ref = context.lock(); - auto new_data = make_uniq(*context_ref, new_types); + return local_changes.GetTransactionLocalInlinedData(*context_ref, table_id); +} - ColumnDataAppendState append_state; - new_data->InitializeAppend(append_state); +void DuckLakeTransaction::DropTransactionLocalFile(TableIndex table_id, const string &path) { + auto context_ref = context.lock(); + local_changes.DropTransactionLocalFile(*context_ref, table_id, path); +} - for (auto &chunk : existing.Chunks()) { - DataChunk new_chunk; - new_chunk.Initialize(*context_ref, new_types); +void DuckLakeTransaction::AppendFiles(TableIndex table_id, vector files) { + if (files.empty()) { + return; + } + local_changes.AppendFiles(table_id, std::move(files)); +} - idx_t new_col_idx = 0; - for (idx_t col_idx = 0; col_idx < chunk.ColumnCount(); col_idx++) { - if (col_idx == removed_column_index.index) { - continue; - } - new_chunk.data[new_col_idx].Reference(chunk.data[col_idx]); - new_col_idx++; - } +void DuckLakeTransaction::AppendInlinedData(TableIndex table_id, unique_ptr new_data) { + auto context_ref = context.lock(); + local_changes.AppendInlinedData(*context_ref, table_id, std::move(new_data)); +} - new_chunk.SetCardinality(chunk.size()); - new_data->Append(append_state, new_chunk); +void DuckLakeTransaction::AddNewInlinedDeletes(TableIndex table_id, const string &table_name, set new_deletes) { + if (new_deletes.empty()) { + return; } + local_changes.AddNewInlinedDeletes(table_id, table_name, std::move(new_deletes)); +} - // Remove stats for the dropped field and all its children - RemoveFieldStats(table_changes.new_inlined_data->column_stats, field_id); +void DuckLakeTransaction::DeleteFromLocalInlinedData(TableIndex table_id, set new_deletes) { + auto context_ref = context.lock(); + local_changes.DeleteFromLocalInlinedData(*context_ref, table_id, std::move(new_deletes)); +} - table_changes.new_inlined_data->data = std::move(new_data); +void DuckLakeTransaction::AddColumnToLocalInlinedData(TableIndex table_id, const LogicalType &new_column_type, + FieldIndex new_field_index, const Value &default_value) { + auto context_ref = context.lock(); + local_changes.AddColumnToLocalInlinedData(*context_ref, table_id, new_column_type, new_field_index, default_value); +} + +void DuckLakeTransaction::RemoveColumnFromLocalInlinedData(TableIndex table_id, LogicalIndex removed_column_index, + const DuckLakeFieldId &field_id) { + auto context_ref = context.lock(); + local_changes.RemoveColumnFromLocalInlinedData(*context_ref, table_id, removed_column_index, field_id); } optional_ptr DuckLakeTransaction::GetInlinedDeletes(TableIndex table_id, - const string &table_name) { - lock_guard guard(table_data_changes_lock); - auto entry = table_data_changes.find(table_id); - if (entry == table_data_changes.end()) { - return nullptr; - } - auto &table_changes = entry->second; - auto delete_entry = table_changes.new_inlined_data_deletes.find(table_name); - if (delete_entry == table_changes.new_inlined_data_deletes.end()) { - return nullptr; - } - return delete_entry->second.get(); + const string &table_name) const { + return local_changes.GetInlinedDeletes(table_id, table_name); } void DuckLakeTransaction::AddNewInlinedFileDeletes(TableIndex table_id, idx_t file_id, set new_deletes) { - if (new_deletes.empty()) { - return; - } - lock_guard guard(table_data_changes_lock); - auto &table_changes = table_data_changes[table_id]; - if (!table_changes.new_inlined_file_deletes) { - table_changes.new_inlined_file_deletes = make_uniq(); - } - auto &file_deletes = table_changes.new_inlined_file_deletes->file_deletes[file_id]; - for (auto &row_id : new_deletes) { - file_deletes.insert(row_id); - } + local_changes.AddNewInlinedFileDeletes(table_id, file_id, std::move(new_deletes)); } vector DuckLakeTransaction::GetNewInlinedFileDeletes(DuckLakeCommitState &commit_state) { vector result; - for (auto &entry : table_data_changes) { - auto table_id = commit_state.GetTableId(entry.first); - auto &table_changes = entry.second; + for (auto &entry : local_changes.Changes()) { + auto table_id = commit_state.GetTableId(entry.GetTableIndex()); + auto &table_changes = entry.GetTableChanges(); if (!table_changes.new_inlined_file_deletes) { continue; } @@ -2357,136 +2693,43 @@ DuckLakeTransaction::GetNewInlinedFileDeletes(DuckLakeCommitState &commit_state) } void DuckLakeTransaction::AddDeletes(TableIndex table_id, vector files) { - if (files.empty()) { - return; - } - lock_guard guard(table_data_changes_lock); - auto &table_changes = table_data_changes[table_id]; - auto &table_delete_map = table_changes.new_delete_files; - for (auto &file : files) { - auto &data_file_path = file.data_file_path; - if (data_file_path.empty()) { - throw InternalException("Data file path needs to be set in delete"); - } - if (file.source == DeleteFileSource::FLUSH) { - // If we have a snapshot, this is a flushed delete file, we add it to the rooster - table_delete_map[data_file_path].push_back(std::move(file)); - } else { - // If not is a regular deletion - auto existing_entry = table_delete_map.find(data_file_path); - if (existing_entry != table_delete_map.end() && !existing_entry->second.empty()) { - // If a file already exists we remove it - auto context_ref = context.lock(); - auto &fs = FileSystem::GetFileSystem(*context_ref); - for (auto &old_file : existing_entry->second) { - fs.RemoveFile(old_file.file_name); - } - existing_entry->second.clear(); - } - // We add the new file in - table_delete_map[data_file_path].push_back(std::move(file)); - } - } + auto context_ref = context.lock(); + local_changes.AddDeletes(*context_ref, table_id, std::move(files)); } void DuckLakeTransaction::AddCompaction(TableIndex table_id, DuckLakeCompactionEntry entry) { - lock_guard guard(table_data_changes_lock); - auto &table_changes = table_data_changes[table_id]; - table_changes.compactions.push_back(std::move(entry)); + local_changes.AddCompaction(table_id, std::move(entry)); } -bool DuckLakeTransaction::HasLocalDeletes(TableIndex table_id) { - auto entry = table_data_changes.find(table_id); - if (entry == table_data_changes.end()) { - return false; - } - return !entry->second.new_delete_files.empty(); +bool DuckLakeTransaction::HasLocalDeletes(TableIndex table_id) const { + return local_changes.HasLocalDeletes(table_id); } -bool DuckLakeTransaction::HasAnyLocalChanges(TableIndex table_id) { - auto entry = table_data_changes.find(table_id); - if (entry != table_data_changes.end() && !entry->second.IsEmpty()) { +bool DuckLakeTransaction::HasAnyLocalChanges(TableIndex table_id) const { + if (local_changes.HasAnyLocalChanges(table_id)) { return true; } return tables_deleted_from.find(table_id) != tables_deleted_from.end(); } -void DuckLakeTransaction::GetLocalDeleteForFile(TableIndex table_id, const string &path, DuckLakeFileData &result) { - auto entry = table_data_changes.find(table_id); - if (entry == table_data_changes.end()) { - return; - } - auto &table_changes = entry->second; - auto file_entry = table_changes.new_delete_files.find(path); - if (file_entry == table_changes.new_delete_files.end() || file_entry->second.empty()) { - return; - } - auto &delete_file = file_entry->second.back(); - result.path = delete_file.file_name; - result.file_size_bytes = delete_file.file_size_bytes; - result.footer_size = delete_file.footer_size; - result.encryption_key = delete_file.encryption_key; +void DuckLakeTransaction::GetLocalDeleteForFile(TableIndex table_id, const string &path, + DuckLakeFileData &result) const { + local_changes.GetLocalDeleteForFile(table_id, path, result); } -bool DuckLakeTransaction::HasLocalInlinedFileDeletes(TableIndex table_id) { - lock_guard guard(table_data_changes_lock); - auto entry = table_data_changes.find(table_id); - if (entry == table_data_changes.end()) { - return false; - } - auto &table_changes = entry->second; - if (!table_changes.new_inlined_file_deletes) { - return false; - } - return !table_changes.new_inlined_file_deletes->file_deletes.empty(); +bool DuckLakeTransaction::HasLocalInlinedFileDeletes(TableIndex table_id) const { + return local_changes.HasLocalInlinedFileDeletes(table_id); } -void DuckLakeTransaction::GetLocalInlinedFileDeletesForFile(TableIndex table_id, idx_t file_id, set &result) { - lock_guard guard(table_data_changes_lock); - auto entry = table_data_changes.find(table_id); - if (entry == table_data_changes.end()) { - return; - } - auto &table_changes = entry->second; - if (!table_changes.new_inlined_file_deletes) { - return; - } - auto file_entry = table_changes.new_inlined_file_deletes->file_deletes.find(file_id); - if (file_entry == table_changes.new_inlined_file_deletes->file_deletes.end()) { - return; - } - // Merge the inlined deletes into the result set - for (auto &row_id : file_entry->second) { - result.insert(row_id); - } +void DuckLakeTransaction::GetLocalInlinedFileDeletesForFile(TableIndex table_id, idx_t file_id, + set &result) const { + local_changes.GetLocalInlinedFileDeletesForFile(table_id, file_id, result); } void DuckLakeTransaction::TransactionLocalDelete(TableIndex table_id, const string &data_file_path, DuckLakeDeleteFile delete_file) { - auto entry = table_data_changes.find(table_id); - if (entry == table_data_changes.end()) { - throw InternalException( - "Transaction local delete called for table which does not have transaction local insertions"); - } - auto &table_changes = entry->second; - for (auto &file : table_changes.new_data_files) { - if (file.file_name == data_file_path) { - if (!file.delete_files.empty()) { - auto context_ref = context.lock(); - auto &fs = FileSystem::GetFileSystem(*context_ref); - vector files_to_delete; - files_to_delete.reserve(file.delete_files.size()); - for (auto &old_file : file.delete_files) { - files_to_delete.push_back(old_file.file_name); - } - fs.RemoveFiles(files_to_delete); - file.delete_files.clear(); - } - file.delete_files.push_back(std::move(delete_file)); - return; - } - } - throw InternalException("Failed to find matching transaction-local file for written delete file"); + auto context_ref = context.lock(); + local_changes.TransactionLocalDelete(*context_ref, table_id, data_file_path, std::move(delete_file)); } DuckLakeTransaction &DuckLakeTransaction::Get(ClientContext &context, Catalog &catalog) { @@ -2527,16 +2770,8 @@ void DuckLakeTransaction::DropTable(DuckLakeTableEntry &table) { auto table_id = table.GetTableId(); schema_entry->second->DropEntry(table.name); // if we have written any files for this table - clean them up - auto table_entry = table_data_changes.find(table_id); - if (table_entry != table_data_changes.end()) { - auto &table_changes = table_entry->second; - auto context_ref = context.lock(); - auto &fs = FileSystem::GetFileSystem(*context_ref); - for (auto &file : table_changes.new_data_files) { - fs.RemoveFile(file.file_name); - } - table_data_changes.erase(table_entry); - } + auto context_ref = context.lock(); + local_changes.CleanupFiles(*context_ref, table_id); new_tables.erase(schema_entry); } else { auto table_id = table.GetTableId(); diff --git a/src/storage/ducklake_transaction_changes.cpp b/src/storage/ducklake_transaction_changes.cpp index 5a0a61c524d..5efc1f62500 100644 --- a/src/storage/ducklake_transaction_changes.cpp +++ b/src/storage/ducklake_transaction_changes.cpp @@ -147,7 +147,7 @@ SnapshotChangeInformation SnapshotChangeInformation::ParseChangesMade(const stri } case ChangeType::CREATED_TABLE_MACRO: { auto catalog_value = DuckLakeUtil::ParseCatalogEntry(entry.change_value); - result.created_scalar_macros[catalog_value.schema].insert( + result.created_table_macros[catalog_value.schema].insert( make_pair(std::move(catalog_value.name), "table_macro")); break; } diff --git a/src/storage/ducklake_update.cpp b/src/storage/ducklake_update.cpp index 0a60c5dbfda..cd4ff531224 100644 --- a/src/storage/ducklake_update.cpp +++ b/src/storage/ducklake_update.cpp @@ -8,8 +8,12 @@ #include "duckdb/planner/expression.hpp" #include "duckdb/planner/expression/bound_reference_expression.hpp" #include "storage/ducklake_delete.hpp" +#include "storage/ducklake_inline_data.hpp" #include "storage/ducklake_table_entry.hpp" #include "storage/ducklake_catalog.hpp" +#include "storage/ducklake_schema_entry.hpp" +#include "storage/ducklake_transaction.hpp" +#include "common/ducklake_util.hpp" #include "duckdb/planner/operator/logical_update.hpp" #include "duckdb/parallel/thread_context.hpp" #include "duckdb/parser/expression/cast_expression.hpp" @@ -37,11 +41,10 @@ struct FileRowIdHash { }; DuckLakeUpdate::DuckLakeUpdate(PhysicalPlan &physical_plan, DuckLakeTableEntry &table, vector columns_p, - PhysicalOperator &child, PhysicalOperator ©_op, PhysicalOperator &delete_op, - PhysicalOperator &insert_op, vector> &expressions) - : PhysicalOperator(physical_plan, PhysicalOperatorType::EXTENSION, {LogicalType::BIGINT}, 1), table(table), - columns(std::move(columns_p)), copy_op(copy_op), delete_op(delete_op), insert_op(insert_op), - expressions(std::move(expressions)) { + PhysicalOperator &child, PhysicalOperator &delete_op, + vector> &expressions) + : PhysicalOperator(physical_plan, PhysicalOperatorType::EXTENSION, {}, 1), table(table), + columns(std::move(columns_p)), delete_op(delete_op), expressions(std::move(expressions)) { children.push_back(child); row_id_index = columns.size(); } @@ -49,7 +52,7 @@ DuckLakeUpdate::DuckLakeUpdate(PhysicalPlan &physical_plan, DuckLakeTableEntry & //===--------------------------------------------------------------------===// // States //===--------------------------------------------------------------------===// -class DuckLakeUpdateGlobalState : public GlobalSinkState { +class DuckLakeUpdateGlobalState : public GlobalOperatorState { public: DuckLakeUpdateGlobalState() : total_updated_count(0) { } @@ -61,9 +64,8 @@ class DuckLakeUpdateGlobalState : public GlobalSinkState { unordered_set seen_rows; }; -class DuckLakeUpdateLocalState : public LocalSinkState { +class DuckLakeUpdateLocalState : public OperatorState { public: - unique_ptr copy_local_state; unique_ptr delete_local_state; unique_ptr expression_executor; //! Chunk where the updated expressions are executed. @@ -73,16 +75,15 @@ class DuckLakeUpdateLocalState : public LocalSinkState { idx_t updated_count = 0; }; -unique_ptr DuckLakeUpdate::GetGlobalSinkState(ClientContext &context) const { +unique_ptr DuckLakeUpdate::GetGlobalOperatorState(ClientContext &context) const { auto result = make_uniq(); - copy_op.sink_state = copy_op.GetGlobalSinkState(context); + // init delete_op sink state delete_op.sink_state = delete_op.GetGlobalSinkState(context); return std::move(result); } -unique_ptr DuckLakeUpdate::GetLocalSinkState(ExecutionContext &context) const { +unique_ptr DuckLakeUpdate::GetOperatorState(ExecutionContext &context) const { auto result = make_uniq(); - result->copy_local_state = copy_op.GetLocalSinkState(context); result->delete_local_state = delete_op.GetLocalSinkState(context); vector delete_types; @@ -96,47 +97,37 @@ unique_ptr DuckLakeUpdate::GetLocalSinkState(ExecutionContext &c expression_types.push_back(expr->return_type); } - for (auto &type : expression_types) { - if (DuckLakeTypes::RequiresCast(type)) { - type = DuckLakeTypes::GetCastedType(type); - } - } result->update_expression_chunk.Initialize(context.client, expression_types); - // updates also write the row id to the file, so the final version needs the row_id placed - // right after the physical columns (before computed partition columns). - vector insert_types = expression_types; - auto physical_column_count = columns.size(); - D_ASSERT(physical_column_count <= insert_types.size()); - insert_types.insert(insert_types.begin() + physical_column_count, LogicalType::BIGINT); - result->insert_chunk.Initialize(context.client, insert_types); + result->insert_chunk.Initialize(context.client, types); result->delete_chunk.Initialize(context.client, delete_types); return std::move(result); } //===--------------------------------------------------------------------===// -// Sink +// Execute //===--------------------------------------------------------------------===// -SinkResultType DuckLakeUpdate::Sink(ExecutionContext &context, DataChunk &chunk, OperatorSinkInput &input) const { - auto &gstate = input.global_state.Cast(); - auto &lstate = input.local_state.Cast(); +OperatorResultType DuckLakeUpdate::Execute(ExecutionContext &context, DataChunk &input, DataChunk &chunk, + GlobalOperatorState &gstate_p, OperatorState &state_p) const { + auto &gstate = gstate_p.Cast(); + auto &lstate = state_p.Cast(); // filter duplicate row IDs using deletion info (last 3 columns) - idx_t delete_idx_start = chunk.ColumnCount() - DELETION_INFO_SIZE; - auto &file_index_vec = chunk.data[delete_idx_start + 1]; - auto &row_number_vec = chunk.data[delete_idx_start + 2]; + idx_t delete_idx_start = input.ColumnCount() - DELETION_INFO_SIZE; + auto &file_index_vec = input.data[delete_idx_start + 1]; + auto &row_number_vec = input.data[delete_idx_start + 2]; UnifiedVectorFormat file_index_data, row_number_data; - file_index_vec.ToUnifiedFormat(chunk.size(), file_index_data); - row_number_vec.ToUnifiedFormat(chunk.size(), row_number_data); + file_index_vec.ToUnifiedFormat(input.size(), file_index_data); + row_number_vec.ToUnifiedFormat(input.size(), row_number_data); auto file_indices = UnifiedVectorFormat::GetData(file_index_data); auto row_numbers = UnifiedVectorFormat::GetData(row_number_data); - SelectionVector sel(chunk.size()); + SelectionVector sel(input.size()); idx_t sel_count = 0; { lock_guard guard(gstate.seen_rows_lock); - for (idx_t i = 0; i < chunk.size(); i++) { + for (idx_t i = 0; i < input.size(); i++) { auto file_idx = file_index_data.sel->get_index(i); auto row_idx = row_number_data.sel->get_index(i); FileRowId key {file_indices[file_idx], row_numbers[row_idx]}; @@ -148,148 +139,68 @@ SinkResultType DuckLakeUpdate::Sink(ExecutionContext &context, DataChunk &chunk, if (sel_count == 0) { // all rows were duplicates - return SinkResultType::NEED_MORE_INPUT; + return OperatorResultType::NEED_MORE_INPUT; } // slice to non-duplicate rows only - chunk.Slice(sel, sel_count); + input.Slice(sel, sel_count); - // push the to-be-inserted data into the copy + // evaluate update expressions auto &update_expression_chunk = lstate.update_expression_chunk; auto &insert_chunk = lstate.insert_chunk; - update_expression_chunk.SetCardinality(chunk.size()); - insert_chunk.SetCardinality(chunk.size()); - lstate.expression_executor->Execute(chunk, update_expression_chunk); + update_expression_chunk.SetCardinality(input.size()); + insert_chunk.SetCardinality(input.size()); + lstate.expression_executor->Execute(input, update_expression_chunk); const idx_t physical_column_count = columns.size(); - const idx_t expression_column_count = update_expression_chunk.ColumnCount(); - D_ASSERT(expression_column_count >= physical_column_count); - const idx_t partition_column_count = expression_column_count - physical_column_count; - const idx_t insert_column_count = insert_chunk.ColumnCount(); - D_ASSERT(insert_column_count >= expression_column_count); - // virtual columns (PlanUpdate sets WRITE_ROW_ID, so there is exactly one: the row id) must sit between the physical - // and partition columns - const idx_t virtual_column_count = insert_column_count - expression_column_count; - D_ASSERT(virtual_column_count == 1); - - // copy the physical columns directly + + // build output, physical columns + row_id for (idx_t i = 0; i < physical_column_count; i++) { insert_chunk.data[i].Reference(update_expression_chunk.data[i]); } - // reference the row id right after the physical columns - insert_chunk.data[physical_column_count].Reference(chunk.data[row_id_index]); - // place computed partition columns after the virtual columns - for (idx_t part_idx = 0; part_idx < partition_column_count; part_idx++) { - insert_chunk.data[physical_column_count + virtual_column_count + part_idx].Reference( - update_expression_chunk.data[physical_column_count + part_idx]); - } + // we place row_id right after physical columns + insert_chunk.data[physical_column_count].Reference(input.data[row_id_index]); - OperatorSinkInput copy_input {*copy_op.sink_state, *lstate.copy_local_state, input.interrupt_state}; - copy_op.Sink(context, insert_chunk, copy_input); + chunk.Reference(insert_chunk); - // push the rowids into the delete auto &delete_chunk = lstate.delete_chunk; - delete_chunk.SetCardinality(chunk.size()); + delete_chunk.SetCardinality(input.size()); for (idx_t i = 0; i < DELETION_INFO_SIZE; i++) { - delete_chunk.data[i].Reference(chunk.data[delete_idx_start + i]); + delete_chunk.data[i].Reference(input.data[delete_idx_start + i]); } - OperatorSinkInput delete_input {*delete_op.sink_state, *lstate.delete_local_state, input.interrupt_state}; + InterruptState interrupt_state; + OperatorSinkInput delete_input {*delete_op.sink_state, *lstate.delete_local_state, interrupt_state}; delete_op.Sink(context, delete_chunk, delete_input); - lstate.updated_count += chunk.size(); - return SinkResultType::NEED_MORE_INPUT; + lstate.updated_count += input.size(); + return OperatorResultType::NEED_MORE_INPUT; } -//===--------------------------------------------------------------------===// -// Combine -//===--------------------------------------------------------------------===// -SinkCombineResultType DuckLakeUpdate::Combine(ExecutionContext &context, OperatorSinkCombineInput &input) const { - auto &global_state = input.global_state.Cast(); - auto &local_state = input.local_state.Cast(); - OperatorSinkCombineInput copy_combine_input {*copy_op.sink_state, *local_state.copy_local_state, - input.interrupt_state}; - auto result = copy_op.Combine(context, copy_combine_input); - if (result != SinkCombineResultType::FINISHED) { - throw InternalException("DuckLakeUpdate::Combine does not support async child operators"); - } - OperatorSinkCombineInput del_combine_input {*delete_op.sink_state, *local_state.delete_local_state, - input.interrupt_state}; - result = delete_op.Combine(context, del_combine_input); +OperatorFinalizeResultType DuckLakeUpdate::FinalExecute(ExecutionContext &context, DataChunk &chunk, + GlobalOperatorState &gstate_p, OperatorState &state_p) const { + auto &gstate = gstate_p.Cast(); + auto &lstate = state_p.Cast(); + + InterruptState interrupt_state; + OperatorSinkCombineInput del_combine_input {*delete_op.sink_state, *lstate.delete_local_state, interrupt_state}; + auto result = delete_op.Combine(context, del_combine_input); if (result != SinkCombineResultType::FINISHED) { - throw InternalException("DuckLakeUpdate::Combine does not support async child operators"); + throw InternalException("DuckLakeUpdate::FinalExecute does not support async child operators"); } - global_state.total_updated_count += local_state.updated_count; - return SinkCombineResultType::FINISHED; + gstate.total_updated_count += lstate.updated_count; + return OperatorFinalizeResultType::FINISHED; } -//===--------------------------------------------------------------------===// -// Finalize -//===--------------------------------------------------------------------===// -SinkFinalizeType DuckLakeUpdate::Finalize(Pipeline &pipeline, Event &event, ClientContext &context, - OperatorSinkFinalizeInput &input) const { - OperatorSinkFinalizeInput copy_finalize_input {*copy_op.sink_state, input.interrupt_state}; - auto result = copy_op.Finalize(pipeline, event, context, copy_finalize_input); - if (result != SinkFinalizeType::READY) { - throw InternalException("DuckLakeUpdate::Finalize does not support async child operators"); - } +OperatorFinalResultType DuckLakeUpdate::OperatorFinalize(Pipeline &pipeline, Event &event, ClientContext &context, + OperatorFinalizeInput &input) const { OperatorSinkFinalizeInput del_finalize_input {*delete_op.sink_state, input.interrupt_state}; - result = delete_op.Finalize(pipeline, event, context, del_finalize_input); + auto result = delete_op.Finalize(pipeline, event, context, del_finalize_input); if (result != SinkFinalizeType::READY) { - throw InternalException("DuckLakeUpdate::Finalize does not support async child operators"); + throw InternalException("DuckLakeUpdate::OperatorFinalize does not support async child operators"); } - - // scan the copy operator and sink into the insert operator - ThreadContext thread_context(context); - ExecutionContext execution_context(context, thread_context, nullptr); - auto global_source = copy_op.GetGlobalSourceState(context); - auto local_source = copy_op.GetLocalSourceState(execution_context, *global_source); - - DataChunk copy_source_chunk; - copy_source_chunk.Initialize(context, copy_op.types); - - auto global_sink = insert_op.GetGlobalSinkState(context); - auto local_sink = insert_op.GetLocalSinkState(execution_context); - - OperatorSourceInput source_input {*global_source, *local_source, input.interrupt_state}; - OperatorSinkInput sink_input {*global_sink, *local_sink, input.interrupt_state}; - while (true) { - auto source_result = copy_op.GetData(execution_context, copy_source_chunk, source_input); - if (copy_source_chunk.size() == 0) { - break; - } - if (source_result == SourceResultType::BLOCKED) { - throw InternalException("DuckLakeUpdate::Finalize does not support async child operators"); - } - - auto sink_result = insert_op.Sink(execution_context, copy_source_chunk, sink_input); - if (sink_result == SinkResultType::BLOCKED) { - throw InternalException("DuckLakeUpdate::Finalize does not support async child operators"); - } - if (source_result == SourceResultType::FINISHED) { - break; - } - } - - OperatorSinkFinalizeInput insert_finalize_input {*global_sink, input.interrupt_state}; - result = insert_op.Finalize(pipeline, event, context, insert_finalize_input); - if (result != SinkFinalizeType::READY) { - throw InternalException("DuckLakeUpdate::Finalize does not support async child operators"); - } - return SinkFinalizeType::READY; -} - -//===--------------------------------------------------------------------===// -// GetData -//===--------------------------------------------------------------------===// -SourceResultType DuckLakeUpdate::GetDataInternal(ExecutionContext &context, DataChunk &chunk, - OperatorSourceInput &input) const { - auto &global_state = sink_state->Cast(); - auto value = Value::BIGINT(NumericCast(global_state.total_updated_count.load())); - chunk.SetCardinality(1); - chunk.SetValue(0, 0, value); - return SourceResultType::FINISHED; + return OperatorFinalResultType::FINISHED; } //===--------------------------------------------------------------------===// @@ -305,103 +216,74 @@ InsertionOrderPreservingMap DuckLakeUpdate::ParamsToString() const { return result; } -static unique_ptr GetFunction(ClientContext &context, unique_ptr column_reference, - const string &function_name) { - vector> children; - children.emplace_back(std::move(column_reference)); - ErrorData error; - FunctionBinder binder(context); - auto function = binder.BindScalarFunction(DEFAULT_SCHEMA, function_name, std::move(children), error, false); - if (!function) { - error.Throw(); - } - return function; -} - -static unique_ptr GetPartitionExpressionForUpdate(ClientContext &context, - unique_ptr column_reference, - const DuckLakePartitionField &field) { - switch (field.transform.type) { - case DuckLakeTransformType::IDENTITY: - return std::move(column_reference); - case DuckLakeTransformType::YEAR: - return GetFunction(context, std::move(column_reference), "year"); - case DuckLakeTransformType::MONTH: - return GetFunction(context, std::move(column_reference), "month"); - case DuckLakeTransformType::DAY: - return GetFunction(context, std::move(column_reference), "day"); - case DuckLakeTransformType::HOUR: - return GetFunction(context, std::move(column_reference), "hour"); - default: - throw NotImplementedException("Unsupported partition transform type in GetPartitionExpressionForUpdate"); - } -} - -PhysicalOperator &DuckLakeCatalog::PlanUpdate(ClientContext &context, PhysicalPlanGenerator &planner, LogicalUpdate &op, - PhysicalOperator &child_plan) { - if (op.return_chunk) { - throw BinderException("RETURNING clause not yet supported for updates of a DuckLake table"); - } +DuckLakeUpdate &DuckLakeUpdate::PlanUpdateOperator(ClientContext &context, PhysicalPlanGenerator &planner, + LogicalUpdate &op, PhysicalOperator &child_plan, + DuckLakeCopyInput ©_input) { for (auto &expr : op.expressions) { if (expr->type == ExpressionType::VALUE_DEFAULT) { throw BinderException("SET DEFAULT is not yet supported for updates of a DuckLake table"); } } auto &table = op.table.Cast(); - // FIXME: we should take the inlining limit into account here and write new updates to the inline data tables if - // possible updates are executed as a delete + insert - generate the two nodes (delete and insert) plan the copy for - // the insert - DuckLakeCopyInput copy_input(context, table); - copy_input.virtual_columns = InsertVirtualColumns::WRITE_ROW_ID; - auto ©_op = DuckLakeInsert::PlanCopyForInsert(context, planner, copy_input, nullptr); - // plan the delete + vector row_id_indexes; for (idx_t i = 0; i < DuckLakeUpdate::DELETION_INFO_SIZE; i++) { row_id_indexes.push_back(i); } auto &delete_op = DuckLakeDelete::PlanDelete(context, planner, table, child_plan, std::move(row_id_indexes), copy_input.encryption_key, false); - // plan the actual insert - auto &insert_op = DuckLakeInsert::PlanInsert(context, planner, table, copy_input.encryption_key); + // build update expressions (physical columns only, no partition cols, no casts) vector> expressions; unordered_map expression_map; - for (idx_t i = 0; i < op.columns.size(); i++) { expression_map[op.columns[i].index] = i; } for (idx_t i = 0; i < op.columns.size(); i++) { expressions.push_back(op.expressions[expression_map[i]]->Copy()); } - if (copy_input.partition_data) { - bool all_identity = true; - for (auto &field : copy_input.partition_data->fields) { - if (field.transform.type != DuckLakeTransformType::IDENTITY) { - all_identity = false; - break; - } - } - if (!all_identity) { - for (auto &field : copy_input.partition_data->fields) { - optional_idx col_idx; - DuckLakeInsert::GetTopLevelColumn(copy_input, field.field_id, col_idx); - D_ASSERT(col_idx.IsValid()); - auto &child_expression = expressions[col_idx.GetIndex()]->Cast(); - auto column_reference = - make_uniq(child_expression.return_type, child_expression.index); - expressions.push_back(GetPartitionExpressionForUpdate(context, std::move(column_reference), field)); - } - } + + auto &update_op = + planner.Make(table, op.columns, child_plan, delete_op, expressions).Cast(); + + // set output types we use physical column types + BIGINT row_id + vector update_output_types; + for (auto &expr : update_op.expressions) { + update_output_types.push_back(expr->return_type); } + update_output_types.push_back(LogicalType::BIGINT); + update_op.types = std::move(update_output_types); + return update_op; +} - for (auto &expr : expressions) { - if (DuckLakeTypes::RequiresCast(expr->return_type)) { - auto target_type = DuckLakeTypes::GetCastedType(expr->return_type); - expr = BoundCastExpression::AddCastToType(context, std::move(expr), std::move(target_type)); - } +PhysicalOperator &DuckLakeCatalog::PlanUpdate(ClientContext &context, PhysicalPlanGenerator &planner, LogicalUpdate &op, + PhysicalOperator &child_plan) { + if (op.return_chunk) { + throw BinderException("RETURNING clause not yet supported for updates of a DuckLake table"); } + auto &table = op.table.Cast(); + + DuckLakeCopyInput copy_input(context, table); + copy_input.virtual_columns = InsertVirtualColumns::WRITE_ROW_ID; + auto &update_op = DuckLakeUpdate::PlanUpdateOperator(context, planner, op, child_plan, copy_input); - return planner.Make(table, op.columns, child_plan, copy_op, delete_op, insert_op, expressions); + // follow the insert path for inlining + optional_ptr plan = &update_op; + optional_ptr inline_data; + + idx_t data_inlining_row_limit = GetInliningLimit(context, table, plan->types); + if (data_inlining_row_limit > 0) { + plan = planner.Make(*plan, data_inlining_row_limit); + inline_data = plan->Cast(); + } + + auto &physical_copy = DuckLakeInsert::PlanCopyForInsert(context, planner, copy_input, plan); + auto &insert_op = DuckLakeInsert::PlanInsert(context, planner, table, std::move(copy_input.encryption_key)); + if (inline_data) { + inline_data->insert = insert_op.Cast(); + } + insert_op.children.push_back(physical_copy); + return insert_op; } void DuckLakeTableEntry::BindUpdateConstraints(Binder &binder, LogicalGet &get, LogicalProjection &proj, diff --git a/src/storage/ducklake_view_entry.cpp b/src/storage/ducklake_view_entry.cpp index abca0bbedc2..6a09edd7905 100644 --- a/src/storage/ducklake_view_entry.cpp +++ b/src/storage/ducklake_view_entry.cpp @@ -104,4 +104,13 @@ string DuckLakeViewEntry::GetQuerySQL() { return query_sql; } +void DuckLakeViewEntry::BindView(ClientContext &context, BindViewAction action) { + try { + ViewCatalogEntry::BindView(context, action); + } catch (...) { + // If binding fails, we reset the view + UpdateBinding({}, {}); + } +} + } // namespace duckdb diff --git a/test/configs/minio.json b/test/configs/minio.json index 11fb6eac24d..4fcd0ca3713 100644 --- a/test/configs/minio.json +++ b/test/configs/minio.json @@ -29,8 +29,9 @@ "test/sql/migration/v01_partitioned.test", "test/sql/delete/delete_ignore_extra_columns.test", "test/sql/add_files/add_file_with_three_level_list.test", - "test/sql/migration/test_compaction.test" + "test/sql/migration/test_compaction.test", + "test/sql/remove_orphans/metadata_in_data_path.test" ] } ] -} \ No newline at end of file +} diff --git a/test/configs/postgres.json b/test/configs/postgres.json index 8eb787eb051..e778e0b3220 100644 --- a/test/configs/postgres.json +++ b/test/configs/postgres.json @@ -24,7 +24,8 @@ "test/sql/general/default_path.test", "test/sql/autoloading/autoload_data_path.test", "test/sql/general/metadata_parameters.test", - "test/sql/metadata/ducklake_settings.test" + "test/sql/metadata/ducklake_settings.test", + "test/sql/remove_orphans/metadata_in_data_path.test" ] }, { @@ -60,4 +61,4 @@ ] } ] -} \ No newline at end of file +} diff --git a/test/configs/sqlite.json b/test/configs/sqlite.json index e22a35aa9ee..af0d2acc4ad 100644 --- a/test/configs/sqlite.json +++ b/test/configs/sqlite.json @@ -109,6 +109,12 @@ "paths": [ "test/sql/deletion_inlining/test_deletion_inlining_concurrent.test" ] + }, + { + "reason": "FIXME: old sqlite extension has issues with this query, can be removed after v1.5.1", + "paths": [ + "test/sql/metadata/ducklake_ui_catalog_query.test" + ] } ] } \ No newline at end of file diff --git a/test/sql/alter/test_insert_select_alter_column_persistence.test b/test/sql/alter/test_insert_select_alter_column_persistence.test new file mode 100644 index 00000000000..a69f7840516 --- /dev/null +++ b/test/sql/alter/test_insert_select_alter_column_persistence.test @@ -0,0 +1,56 @@ +# name: test/sql/alter/test_insert_select_alter_column_persistence.test +# description: test that INSERT ... SELECT on ALTER-added column is persisted across DETACH/ATTACH +# group: [alter] + +require ducklake + +require parquet + +statement ok +ATTACH 'ducklake:__TEST_DIR__/alter_insert_select.db' AS dl (DATA_PATH '__TEST_DIR__/alter_insert_select') + +statement ok +USE dl + +statement ok +CREATE TABLE t (id INT, name VARCHAR) + +statement ok +INSERT INTO t VALUES (1, 'a') + +statement ok +ALTER TABLE t ADD COLUMN ts TIMESTAMPTZ + +statement ok +CREATE TABLE src (id INT, name VARCHAR) + +statement ok +INSERT INTO src VALUES (2, 'b') + +statement ok +INSERT INTO t (id, name, ts) SELECT id, name, '2026-01-01'::TIMESTAMPTZ AS ts FROM src + +query IIT +SELECT * FROM t ORDER BY id +---- +1 a NULL +2 b 2026-01-01 00:00:00+00 + +# Detach and reattach +statement ok +USE memory + +statement ok +DETACH dl + +statement ok +ATTACH 'ducklake:__TEST_DIR__/alter_insert_select.db' AS dl (DATA_PATH '__TEST_DIR__/alter_insert_select') + +statement ok +USE dl + +query IIT +SELECT * FROM t ORDER BY id +---- +1 a NULL +2 b 2026-01-01 00:00:00+00 diff --git a/test/sql/compaction/compaction_per_thread_output.test b/test/sql/compaction/compaction_per_thread_output.test new file mode 100644 index 00000000000..d853d04df2d --- /dev/null +++ b/test/sql/compaction/compaction_per_thread_output.test @@ -0,0 +1,65 @@ +# name: test/sql/compaction/compaction_per_thread_output.test +# description: test compaction with per_thread_output enabled +# group: [compaction] + +require ducklake + +require parquet + +test-env DUCKLAKE_CONNECTION __TEST_DIR__/{UUID}.db + +test-env DATA_PATH __TEST_DIR__ + +statement ok +ATTACH 'ducklake:${DUCKLAKE_CONNECTION}' AS ducklake (DATA_PATH '${DATA_PATH}/ducklake_compaction_per_thread_output') + +statement ok +CALL ducklake.set_option('per_thread_output', 'true'); + +statement ok +CREATE TABLE ducklake.test(i INTEGER); + +statement ok +INSERT INTO ducklake.test VALUES (1); + +statement ok +CALL ducklake_flush_inlined_data('ducklake') + +statement ok +INSERT INTO ducklake.test VALUES (2); + +statement ok +CALL ducklake_flush_inlined_data('ducklake') + +statement ok +INSERT INTO ducklake.test VALUES (3); + +statement ok +CALL ducklake_flush_inlined_data('ducklake') + +# merge_adjacent_files should work with per_thread_output enabled +statement ok +CALL ducklake_merge_adjacent_files('ducklake'); + +# verify data integrity after merge +query I +SELECT * FROM ducklake.test ORDER BY ALL +---- +1 +2 +3 + +# delete from the merged file to create a delete file +statement ok +DELETE FROM ducklake.test WHERE i = 3; + +# rewrite_data_files should work with per_thread_output enabled +statement ok +CALL ducklake_rewrite_data_files('ducklake', delete_threshold => 0); + +# verify data integrity after rewrite +query I +SELECT * FROM ducklake.test ORDER BY ALL +---- +1 +2 diff --git a/test/sql/compaction/merge_adjacent_max_files_partitioned.test b/test/sql/compaction/merge_adjacent_max_files_partitioned.test new file mode 100644 index 00000000000..3b472a9e312 --- /dev/null +++ b/test/sql/compaction/merge_adjacent_max_files_partitioned.test @@ -0,0 +1,39 @@ +# name: test/sql/compaction/merge_adjacent_max_files_partitioned.test +# description: test that max_compacted_files respects global limit for partitioned tables +# group: [compaction] + +require ducklake + +require parquet + +test-env DUCKLAKE_CONNECTION __TEST_DIR__/{UUID}.db + +test-env DATA_PATH __TEST_DIR__ + +statement ok +ATTACH 'ducklake:${DUCKLAKE_CONNECTION}' AS ducklake (DATA_PATH '${DATA_PATH}/merge_adjacent_max_files_partitioned/', DATA_INLINING_ROW_LIMIT 0) + +statement ok +USE ducklake; + +statement ok +CREATE TABLE t (v INTEGER, p INTEGER); + +statement ok +ALTER TABLE t SET PARTITIONED BY (p); + +statement ok +INSERT INTO t VALUES (1, 0), (2, 1); + +statement ok +INSERT INTO t VALUES (3, 0), (4, 1); + +query I +SELECT count(*) FROM ducklake_merge_adjacent_files('ducklake', 't', max_compacted_files => 1) +---- +1 + +query I +SELECT count(*) FROM t +---- +4 diff --git a/test/sql/compaction/rewrite_deletes_full_file_delete_after_flush.test b/test/sql/compaction/rewrite_deletes_full_file_delete_after_flush.test new file mode 100644 index 00000000000..105fb8217bd --- /dev/null +++ b/test/sql/compaction/rewrite_deletes_full_file_delete_after_flush.test @@ -0,0 +1,103 @@ +# name: test/sql/compaction/rewrite_deletes_full_file_delete_after_flush.test +# description: rewrite_deletes should handle a fully deleted file created by ducklake_flush_inlined_data +# group: [compaction] + +require ducklake + +require parquet + +test-env DUCKLAKE_CONNECTION __TEST_DIR__/{UUID}.db + +test-env DATA_PATH __TEST_DIR__ + + +statement ok +ATTACH 'ducklake:${DUCKLAKE_CONNECTION}' AS ducklake (DATA_PATH '${DATA_PATH}/rewrite_deletes_full_file_delete_after_flush', DATA_INLINING_ROW_LIMIT 100, METADATA_CATALOG 'ducklake_meta') + +statement ok +USE ducklake; + +statement ok +CREATE TABLE t (id INTEGER); + +statement ok +INSERT INTO t SELECT i FROM range(20) tbl(i); + +query I +SELECT COUNT(*) FROM ducklake_meta.ducklake_data_file WHERE end_snapshot IS NULL; +---- +0 + +statement ok +SET VARIABLE v_after_insert = (SELECT max(snapshot_id) FROM ducklake_snapshots('ducklake')); + +query I +SELECT COUNT(*) FROM t AT (VERSION => getvariable('v_after_insert')) +---- +20 + +statement ok +DELETE FROM t WHERE id < 20; + +statement ok +SET VARIABLE v_after_delete = (SELECT max(snapshot_id) FROM ducklake_snapshots('ducklake')); + +query I +SELECT COUNT(*) FROM t; +---- +0 + +query I +SELECT COUNT(*) FROM t AT (VERSION => getvariable('v_after_insert')) +---- +20 + +query I +SELECT COUNT(*) FROM t AT (VERSION => getvariable('v_after_delete')) +---- +0 + +query I +SELECT COUNT(*) FROM ducklake_flush_inlined_data('ducklake'); +---- +1 + +query I +SELECT COUNT(*) FROM t AT (VERSION => getvariable('v_after_insert')) +---- +20 + +query I +SELECT COUNT(*) FROM t AT (VERSION => getvariable('v_after_delete')) +---- +0 + +query II +SELECT COUNT(*), COALESCE(SUM(record_count), 0) FROM ducklake_meta.ducklake_data_file WHERE end_snapshot IS NULL; +---- +1 20 + +query II +SELECT COUNT(*), COALESCE(SUM(delete_count), 0) FROM ducklake_meta.ducklake_delete_file WHERE end_snapshot IS NULL; +---- +1 20 + +query II +SELECT COUNT(*), SUM(files_created) FROM ducklake_rewrite_data_files('ducklake', 't', delete_threshold => 0); +---- +1 0 + +query I +SELECT COUNT(*) FROM t AT (VERSION => getvariable('v_after_insert')) +---- +20 + +query I +SELECT COUNT(*) FROM t AT (VERSION => getvariable('v_after_delete')) +---- +0 + +query I +SELECT COUNT(*) FROM t; +---- +0 diff --git a/test/sql/data_inlining/data_inlining_interleaved_update.test b/test/sql/data_inlining/data_inlining_interleaved_update.test index 9592c87d92b..b85b8d07313 100644 --- a/test/sql/data_inlining/data_inlining_interleaved_update.test +++ b/test/sql/data_inlining/data_inlining_interleaved_update.test @@ -55,13 +55,14 @@ SELECT * FROM ducklake.test ORDER BY id 4 d # verify rowids - the updated row (id=1) should preserve its original rowid (0) +# INSERT rows get sequential row_ids from next_row_id (2), UPDATE rows keep preserved row_ids query III SELECT rowid, * FROM ducklake.test ORDER BY id ---- 0 1 aa 1 2 b -3 3 c -4 4 d +2 3 c +3 4 d # verify table_changes shows the update correctly (not as separate delete+insert) query IIIII @@ -69,5 +70,31 @@ FROM ducklake.table_changes('test', 3, 3) ORDER BY ALL ---- 3 0 update_postimage 1 aa 3 0 update_preimage 1 a -3 3 insert 3 c -3 4 insert 4 d +3 2 insert 3 c +3 3 insert 4 d + +# UPDATE -> INSERT in one transaction (update-only existing data, then insert) +# this tests the case where existing_data has only preserved row_ids (from the update) +# and the new insert data needs TRANSACTION_LOCAL_ROW_ID_START-based ids +statement ok +BEGIN + +query I +UPDATE ducklake.test SET val='bb' WHERE id=2 +---- +1 + +statement ok +INSERT INTO ducklake.test VALUES (5, 'e') + +statement ok +COMMIT + +query III +SELECT rowid, * FROM ducklake.test ORDER BY id +---- +0 1 aa +1 2 bb +2 3 c +3 4 d +4 5 e diff --git a/test/sql/data_inlining/data_inlining_update_inline_verification.test b/test/sql/data_inlining/data_inlining_update_inline_verification.test new file mode 100644 index 00000000000..9b714fc8536 --- /dev/null +++ b/test/sql/data_inlining/data_inlining_update_inline_verification.test @@ -0,0 +1,106 @@ +# name: test/sql/data_inlining/data_inlining_update_inline_verification.test +# description: verify that a small update on file-backed data gets inlined (insert + delete tables, no new file) +# group: [data_inlining] + +require ducklake + +require parquet + +test-env DUCKLAKE_CONNECTION __TEST_DIR__/{UUID}.db + +test-env DATA_PATH __TEST_DIR__ + +statement ok +ATTACH 'ducklake:${DUCKLAKE_CONNECTION}' AS ducklake (DATA_PATH '${DATA_PATH}/ducklake_update_inline_verify_files', METADATA_CATALOG 'ducklake_meta', DATA_INLINING_ROW_LIMIT 10) + +statement ok +CREATE TABLE ducklake.test AS SELECT i, 'val_' || i::VARCHAR AS j FROM range(20) t(i) + +query I +SELECT COUNT(*) = 1 cnt FROM GLOB('${DATA_PATH}/ducklake_update_inline_verify_files/**') +---- +true + +query I +UPDATE ducklake.test SET j='updated' WHERE i=5 +---- +1 + +query I +SELECT COUNT(*) = 1 FROM GLOB('${DATA_PATH}/ducklake_update_inline_verify_files/**') +---- +true + +# verify the data is correct +query II +SELECT * FROM ducklake.test WHERE i=5 +---- +5 updated + +# the inlined insert table should have the new row from the update +query II +SELECT i, j FROM ducklake_meta.ducklake_inlined_data_1_1 +---- +5 updated + +# the inlined delete table should track the old row that was removed from the file +query I +SELECT COUNT(*) FROM ducklake_meta.ducklake_inlined_delete_1 +---- +1 + +# verify rowids are preserved +query III +SELECT rowid, * FROM ducklake.test WHERE i=5 +---- +5 5 updated + +# do another update +query I +UPDATE ducklake.test SET j='changed' WHERE i=10 +---- +1 + +# still no new files +query I +SELECT COUNT(*) = 1FROM GLOB('${DATA_PATH}/ducklake_update_inline_verify_files/**') +---- +true + +# inlined insert table now has 2 rows +query II +SELECT i, j FROM ducklake_meta.ducklake_inlined_data_1_1 ORDER BY i +---- +5 updated +10 changed + +# inlined delete table now has 2 entries +query I +SELECT COUNT(*) FROM ducklake_meta.ducklake_inlined_delete_1 +---- +2 + +# verify all data is correct +query II +SELECT * FROM ducklake.test ORDER BY i +---- +0 val_0 +1 val_1 +2 val_2 +3 val_3 +4 val_4 +5 updated +6 val_6 +7 val_7 +8 val_8 +9 val_9 +10 changed +11 val_11 +12 val_12 +13 val_13 +14 val_14 +15 val_15 +16 val_16 +17 val_17 +18 val_18 +19 val_19 \ No newline at end of file diff --git a/test/sql/issues/issue_865_update_wrong_result.test b/test/sql/issues/issue_865_update_wrong_result.test new file mode 100644 index 00000000000..589b4600dac --- /dev/null +++ b/test/sql/issues/issue_865_update_wrong_result.test @@ -0,0 +1,60 @@ +# name: test/sql/issues/issue_865_update_wrong_result.test +# description: UPDATE on ducklake table produces duplicate rows when a data file has both a committed delete file and committed inlined file deletions +# group: [issues] + +require ducklake + +require parquet + +test-env DUCKLAKE_CONNECTION __TEST_DIR__/{UUID}.db + +test-env DATA_PATH __TEST_DIR__ + +statement ok +ATTACH 'ducklake:${DUCKLAKE_CONNECTION}' AS ducklake (DATA_PATH '${DATA_PATH}/issue_865', DATA_INLINING_ROW_LIMIT 10) + +# Create table with enough rows to NOT be inlined +statement ok +CREATE TABLE ducklake.test AS SELECT i AS id, 'original' AS val FROM range(100) t(i) + +# Create delete file +statement ok +DELETE FROM ducklake.test WHERE id >= 80 + +query I +SELECT count(*) FROM ducklake.test +---- +80 + +# Inline deletes +statement ok +DELETE FROM ducklake.test WHERE id >= 75 + +query I +SELECT count(*) FROM ducklake.test +---- +75 + +# If we update more than the inline value, inlined deletion merge will corrupt de delete data +query I +UPDATE ducklake.test SET val = 'updated' WHERE id < 20 +---- +20 + +# After update: should still have 75 rows +query I +SELECT count(*) FROM ducklake.test +---- +75 + +# The 20 updated rows should exist exactly once with 'updated' +query I +SELECT count(*) FROM ducklake.test WHERE val = 'updated' +---- +20 + +# No duplicate ids +query I +SELECT count(*) FROM (SELECT id, count(*) cnt FROM ducklake.test GROUP BY id HAVING count(*) > 1) +---- +0 diff --git a/test/sql/macros/test_macro_window_function.test b/test/sql/macros/test_macro_window_function.test new file mode 100644 index 00000000000..f40f27dc165 --- /dev/null +++ b/test/sql/macros/test_macro_window_function.test @@ -0,0 +1,45 @@ +# name: test/sql/macros/test_macro_window_function.test +# description: test table macros with window functions +# group: [macros] + +require ducklake + +require parquet + +test-env DUCKLAKE_CONNECTION __TEST_DIR__/{UUID}.db + +test-env DATA_PATH __TEST_DIR__ + +statement ok +ATTACH 'ducklake:${DUCKLAKE_CONNECTION}' AS ducklake (DATA_PATH '${DATA_PATH}/test_macro_window'); + +statement ok +USE ducklake; + +statement ok +CREATE TABLE tbl (id INT, val REAL); + +statement ok +INSERT INTO tbl VALUES (1, 1.0), (2, 2.0), (3, 3.0), (4, 10.0); + +statement ok +CREATE MACRO macro_no_partition() AS TABLE (SELECT avg(val) OVER () AS avg_val FROM tbl); + +query I +FROM macro_no_partition(); +---- +4.0 +4.0 +4.0 +4.0 + +statement ok +CREATE MACRO macro_partition() AS TABLE (SELECT avg(val) OVER (PARTITION BY id) AS avg_val FROM tbl); + +query I +FROM macro_partition(); +---- +1.0 +2.0 +3.0 +10.0 diff --git a/test/sql/macros/test_snapshots_after_macro.test b/test/sql/macros/test_snapshots_after_macro.test new file mode 100644 index 00000000000..6e32219b001 --- /dev/null +++ b/test/sql/macros/test_snapshots_after_macro.test @@ -0,0 +1,36 @@ +# name: test/sql/macros/test_snapshots_after_macro.test +# description: test that snapshots() works after creating a macro +# group: [macros] + +require ducklake + +require parquet + +test-env DUCKLAKE_CONNECTION __TEST_DIR__/{UUID}.db + +test-env DATA_PATH __TEST_DIR__ + +statement ok +ATTACH 'ducklake:${DUCKLAKE_CONNECTION}' AS mylake (DATA_PATH '${DATA_PATH}/test_snapshots_after_macro') + +statement ok +use mylake + +statement ok +CREATE TABLE tbl (id INT, val VARCHAR) + +statement ok +INSERT INTO tbl VALUES (1, 'a'), (2, 'b') + +query I +SELECT count(*) FROM mylake.snapshots() +---- +3 + +statement ok +CREATE MACRO testmacro() AS TABLE (SELECT * FROM tbl) + +query I +SELECT count(*) FROM mylake.snapshots() +---- +4 diff --git a/test/sql/merge/merge_partition_update.test b/test/sql/merge/merge_partition_update.test index bc6f7476d5b..fd7929c13b0 100644 --- a/test/sql/merge/merge_partition_update.test +++ b/test/sql/merge/merge_partition_update.test @@ -39,6 +39,10 @@ MERGE INTO my_timeseries ON my_timeseries.ts = timeseries_updates.ts WHEN MATCHED THEN UPDATE; +# flush inlined data from the MERGE update +statement ok +CALL ducklake_flush_inlined_data('ducklake') + query I SELECT COUNT(*) FROM GLOB('${DATA_PATH}/merge_partition_update/**/year=2025/*') ---- @@ -51,6 +55,10 @@ UPDATE my_timeseries SET x = 43::DOUBLE PRECISION WHERE ts = '2025-09-17'::TIMESTAMP; +# flush inlined data from the UPDATE +statement ok +CALL ducklake_flush_inlined_data('ducklake') + query I SELECT COUNT(*) FROM GLOB('${DATA_PATH}/merge_partition_update/**/year=2025/*') ---- diff --git a/test/sql/metadata/ducklake_ui_catalog_query.test b/test/sql/metadata/ducklake_ui_catalog_query.test new file mode 100644 index 00000000000..25646f0e231 --- /dev/null +++ b/test/sql/metadata/ducklake_ui_catalog_query.test @@ -0,0 +1,52 @@ +# name: test/sql/metadata/ducklake_ui_catalog_query.test +# description: Reproducer for UI metadata query against ducklake catalog +# group: [metadata] + +require ducklake + +require parquet + +test-env DUCKLAKE_CONNECTION __TEST_DIR__/{UUID}.db + +test-env DATA_PATH __TEST_DIR__ + +statement ok +SET threads=8 + +statement ok +ATTACH 'ducklake:${DUCKLAKE_CONNECTION}' AS lake (DATA_PATH '${DATA_PATH}/ducklake_ui_catalog_query_files') + +statement ok +CREATE TABLE lake.sample(i INTEGER, note VARCHAR); + +query I +SELECT COUNT(*)::INTEGER +FROM ( + SELECT + schemata.schema_name AS schema_name, + tables.table_name AS table_name, + tables.table_type AS table_type, + tables.table_comment AS table_comment, + table_metadata.estimated_size AS table_estimated_size, + columns.ordinal_position AS column_position, + columns.column_name AS column_name, + columns.data_type AS column_data_type, + columns.column_comment AS column_comment + FROM system.information_schema.schemata AS schemata + LEFT JOIN system.information_schema.tables AS tables + ON tables.table_catalog = schemata.catalog_name + AND tables.table_schema = schemata.schema_name + LEFT JOIN duckdb_tables() AS table_metadata + ON table_metadata.table_name = tables.table_name + AND table_metadata.schema_name = tables.table_schema + AND table_metadata.database_name = tables.table_catalog + LEFT JOIN system.information_schema.columns AS columns + ON columns.table_catalog = tables.table_catalog + AND columns.table_schema = tables.table_schema + AND columns.table_name = tables.table_name + WHERE schemata.catalog_name = 'lake' + AND schemata.schema_name NOT IN ('information_schema', 'pg_catalog') + ORDER BY schemata.schema_name, tables.table_name, columns.ordinal_position +) q; +---- +2 diff --git a/test/sql/remove_orphans/metadata_in_data_path.test b/test/sql/remove_orphans/metadata_in_data_path.test new file mode 100644 index 00000000000..28633265ec2 --- /dev/null +++ b/test/sql/remove_orphans/metadata_in_data_path.test @@ -0,0 +1,53 @@ +# name: test/sql/remove_orphans/metadata_in_data_path.test +# description: ducklake_delete_orphaned_files should not delete the metadata database when it resides in the data path +# group: [remove_orphans] + +require ducklake + +require parquet + +# Place the metadata DB directly inside the data path directory +test-env DUCKLAKE_CONNECTION __TEST_DIR__/{UUID}.db + +test-env DATA_PATH __TEST_DIR__ + +statement ok +ATTACH 'ducklake:${DUCKLAKE_CONNECTION}' AS ducklake (DATA_PATH '${DATA_PATH}', METADATA_CATALOG 'ducklake_metadata') + +statement ok +USE ducklake + +statement ok +CREATE TABLE test (a integer) + +statement ok +INSERT INTO test VALUES (1); + +statement ok +CALL ducklake_flush_inlined_data('ducklake') + +# The metadata DB and the parquet data file must exist before the call +query I +SELECT count(*) FROM GLOB('${DATA_PATH}/*.db') +---- +1 + +query I +SELECT count(*) FROM GLOB('${DATA_PATH}/**/*.parquet') +---- +1 + +statement ok +CALL ducklake_delete_orphaned_files('ducklake', cleanup_all => true); + +# Verify the metadata DB file was NOT deleted +query I +SELECT count(*) FROM GLOB('${DATA_PATH}/*.db') +---- +1 + +# Verify the parquet data file was NOT deleted +query I +SELECT count(*) FROM GLOB('${DATA_PATH}/**/*.parquet') +---- +1 diff --git a/test/sql/remove_orphans/orphan_after_expire.test b/test/sql/remove_orphans/orphan_after_expire.test new file mode 100644 index 00000000000..29e3a4f1d65 --- /dev/null +++ b/test/sql/remove_orphans/orphan_after_expire.test @@ -0,0 +1,73 @@ +# name: test/sql/remove_orphans/orphan_after_expire.test +# description: Test that active data files are NOT reported as orphaned after snapshot expiration +# group: [remove_orphans] + +require ducklake + +require parquet + +require icu + +test-env DUCKLAKE_CONNECTION __TEST_DIR__/{UUID}.db + +test-env DATA_PATH __TEST_DIR__ + +statement ok +ATTACH 'ducklake:${DUCKLAKE_CONNECTION}' AS lake (DATA_PATH '${DATA_PATH}/orphan_after_expire//', METADATA_CATALOG 'ducklake_meta') + +statement ok +USE lake + +statement ok +CREATE TABLE lake.t1 (id INTEGER, val VARCHAR) + +statement ok +INSERT INTO lake.t1 VALUES (1, 'Hello'), (2, 'World') + +statement ok +CALL ducklake_flush_inlined_data('lake') + +statement ok +CREATE TABLE lake.t2 (id INTEGER, val VARCHAR) + +statement ok +INSERT INTO lake.t2 VALUES (1, 'Foo'), (2, 'Bar') + +statement ok +CALL ducklake_flush_inlined_data('lake') + +statement ok +UPDATE lake.t1 SET val = 'DuckLake' WHERE id = 2 + +statement ok +ALTER TABLE lake.t1 ADD name VARCHAR + +statement ok +INSERT INTO lake.t1 VALUES (3, 'Test', 'Test') + +statement ok +CALL ducklake_flush_inlined_data('lake') + +statement ok +CALL ducklake_merge_adjacent_files('lake') + +statement ok +CALL ducklake_rewrite_data_files('lake') + +statement ok +CALL ducklake_expire_snapshots('lake', older_than => now() - INTERVAL '1 second') + +query I +SELECT COUNT(*) > 0 FROM lake.t1 +---- +true + +query I +SELECT COUNT(*) > 0 FROM lake.t2 +---- +true + +query I +SELECT count(*) FROM ducklake_delete_orphaned_files('lake', dry_run => true, cleanup_all => true) +---- +0 diff --git a/test/sql/rewrite_data_files/test_rewrite_large_file_with_deletes.test b/test/sql/rewrite_data_files/test_rewrite_large_file_with_deletes.test new file mode 100644 index 00000000000..38698b65037 --- /dev/null +++ b/test/sql/rewrite_data_files/test_rewrite_large_file_with_deletes.test @@ -0,0 +1,89 @@ +# name: test/sql/rewrite_data_files/test_rewrite_large_file_with_deletes.test +# description: Test that rewrite_data_files does not skip large files with delete files +# group: [rewrite_data_files] + +require ducklake + +require parquet + +test-env DUCKLAKE_CONNECTION __TEST_DIR__/{UUID}.db + +test-env DATA_PATH __TEST_DIR__ + + +statement ok +ATTACH 'ducklake:${DUCKLAKE_CONNECTION}' AS ducklake (DATA_PATH '${DATA_PATH}/rewrite_large_deletes', METADATA_CATALOG 'ducklake_metadata') + +statement ok +USE ducklake + +# Set a very small target_file_size so that the data file exceeds it +statement ok +CALL ducklake.set_option('target_file_size', '1KB'); + +statement ok +CREATE TABLE test (key INTEGER, value VARCHAR); + +statement ok +INSERT INTO test SELECT i, concat('thisisastring_', i) FROM range(1000) t(i) + +statement ok +CALL ducklake_flush_inlined_data('ducklake') + +# Verify we have 1 data file, no delete files +query I +SELECT count(*) FROM ducklake_metadata.ducklake_data_file WHERE end_snapshot IS NULL +---- +1 + +query I +SELECT count(*) FROM ducklake_metadata.ducklake_delete_file WHERE end_snapshot IS NULL +---- +0 + +# Delete some rows to create a delete file +statement ok +DELETE FROM test WHERE key < 500 + +statement ok +CALL ducklake_flush_inlined_data('ducklake') + +# Verify we now have a delete file +query I +SELECT count(*) FROM ducklake_metadata.ducklake_delete_file WHERE end_snapshot IS NULL +---- +1 + +# The data file should be larger than target_file_size (1KB) +# Without the fix, rewrite_data_files would skip this file +query III +SELECT table_name, files_processed, files_created FROM ducklake_rewrite_data_files('ducklake', delete_threshold => 0) ORDER BY ALL +---- +test 1 1 + +# After rewrite: delete files should be gone, data file should be rewritten +query I +SELECT count(*) FROM ducklake_metadata.ducklake_delete_file WHERE end_snapshot IS NULL +---- +0 + +query I +SELECT count(*) FROM ducklake_metadata.ducklake_data_file WHERE end_snapshot IS NULL +---- +1 + +# Verify data correctness +query I +SELECT count(*) FROM test +---- +500 + +query I +SELECT min(key) FROM test +---- +500 + +query I +SELECT max(key) FROM test +---- +999 diff --git a/test/sql/settings/disabled_filesystems.test b/test/sql/settings/disabled_filesystems.test new file mode 100644 index 00000000000..4b88f852d94 --- /dev/null +++ b/test/sql/settings/disabled_filesystems.test @@ -0,0 +1,19 @@ +# name: test/sql/settings/disabled_filesystems.test +# description: StorePath/LoadPath breaks when LocalFileSystem is disabled +# group: [settings] + +require ducklake + +require httpfs + +statement ok +ATTACH 'ducklake::memory:' AS ducklake (DATA_PATH 's3://any-bucket/data/') + +statement ok +USE ducklake + +statement ok +SET disabled_filesystems = 'LocalFileSystem' + +statement ok +CREATE TABLE test (id INTEGER) diff --git a/test/sql/transaction/update_null_column.test b/test/sql/transaction/update_null_column.test new file mode 100644 index 00000000000..15ee48c69c1 --- /dev/null +++ b/test/sql/transaction/update_null_column.test @@ -0,0 +1,47 @@ +# name: test/sql/transaction/update_null_column.test +# description: test that boolean stats pruning works correctly after update/insert +# group: [transaction] + +require ducklake + +require parquet + +test-env DUCKLAKE_CONNECTION __TEST_DIR__/{UUID}.db + +test-env DATA_PATH __TEST_DIR__ + +statement ok +ATTACH 'ducklake:${DUCKLAKE_CONNECTION}' AS ducklake (DATA_PATH '${DATA_PATH}/update_null_column/', DATA_INLINING_ROW_LIMIT 0); + +statement ok +USE ducklake; + +statement ok +CREATE TABLE test (active BOOLEAN); + +statement ok +INSERT INTO test (active) VALUES (false); + +query I +SELECT COUNT(*) FROM test WHERE active = false; +---- +1 + +statement ok +CREATE TABLE t (id BIGINT, tag VARCHAR); + +statement ok +INSERT INTO t (id) VALUES (1); + +statement ok +UPDATE t SET tag = 'new'; + +query IT +SELECT * FROM t; +---- +1 new + +query IT +SELECT * FROM t WHERE tag = 'new'; +---- +1 new diff --git a/test/sql/view/dangling_view_columns.test b/test/sql/view/dangling_view_columns.test new file mode 100644 index 00000000000..b0ec42c4f5c --- /dev/null +++ b/test/sql/view/dangling_view_columns.test @@ -0,0 +1,102 @@ +# name: test/sql/view/dangling_view_columns.test +# description: Test that dangling views don't error on inspection +# group: [view] + +require ducklake + +require parquet + +test-env DUCKLAKE_CONNECTION __TEST_DIR__/{UUID}.db + +test-env DATA_PATH __TEST_DIR__ + +statement ok +ATTACH 'ducklake:${DUCKLAKE_CONNECTION}' AS ducklake (DATA_PATH '${DATA_PATH}/dangling_view_files'); + +statement ok +USE ducklake + +statement ok +CREATE SCHEMA IF NOT EXISTS main + +statement ok +CREATE TABLE main.t1 (id INTEGER, label VARCHAR) + +statement ok +CREATE VIEW main.v1 AS SELECT 1 AS id, 'x' AS label + +statement ok +CREATE VIEW main.v2 AS SELECT id, label FROM main.v1 + +statement ok +CREATE VIEW main.v3 AS SELECT 100 AS score, 'hello' AS greeting + +statement ok +DROP VIEW main.v1 + +query I +SELECT view_name FROM duckdb_views() WHERE database_name = 'ducklake' AND schema_name = 'main' ORDER BY view_name +---- +v2 +v3 + +query III +SELECT table_name, column_name, data_type FROM information_schema.columns WHERE table_schema = 'main' AND table_name IN ('t1', 'v2', 'v3') ORDER BY table_name, ordinal_position +---- +t1 id INTEGER +t1 label VARCHAR +v3 score INTEGER +v3 greeting VARCHAR + +query III +SELECT table_name, column_name, data_type FROM duckdb_columns() WHERE database_name = 'ducklake' AND schema_name = 'main' AND table_name IN ('t1', 'v2', 'v3') ORDER BY table_name, column_index +---- +t1 id INTEGER +t1 label VARCHAR +v3 score INTEGER +v3 greeting VARCHAR + +statement error +SELECT * FROM main.v2 +---- +Table with name v1 does not exist + +statement error +SELECT * FROM main.v2 +---- +Table with name v1 does not exist + +statement ok +CREATE VIEW main.v1 AS SELECT 42 AS id, 'y' AS label + +query II +SELECT * FROM main.v2 +---- +42 y + +statement ok +DROP VIEW main.v1 + +query III +SELECT table_name, column_name, data_type FROM information_schema.columns WHERE table_schema = 'main' AND table_name IN ('t1', 'v2', 'v3') ORDER BY table_name, ordinal_position +---- +t1 id INTEGER +t1 label VARCHAR +v3 score INTEGER +v3 greeting VARCHAR + +query III +SELECT table_name, column_name, data_type FROM duckdb_columns() WHERE database_name = 'ducklake' AND schema_name = 'main' AND table_name IN ('t1', 'v2', 'v3') ORDER BY table_name, column_index +---- +t1 id INTEGER +t1 label VARCHAR +v3 score INTEGER +v3 greeting VARCHAR + +statement ok +CREATE VIEW main.v1 AS SELECT 1 AS a, 'x' AS b + +statement error +SELECT * FROM main.v2 +---- +Referenced column "id" not found in FROM clause