diff --git a/benchmarks/duckdb-bench/src/lib.rs b/benchmarks/duckdb-bench/src/lib.rs index fed9f82b004..69e4da63853 100644 --- a/benchmarks/duckdb-bench/src/lib.rs +++ b/benchmarks/duckdb-bench/src/lib.rs @@ -16,7 +16,6 @@ use vortex_bench::Format; use vortex_bench::IdempotentPath; use vortex_bench::generate_duckdb_registration_sql; use vortex_bench::runner::BenchmarkQueryResult; -use vortex_duckdb::duckdb::Config; use vortex_duckdb::duckdb::Connection; use vortex_duckdb::duckdb::Database; use vortex_duckdb::duckdb::QueryResult; @@ -75,30 +74,21 @@ impl DuckClient { path: Option, threads: Option, ) -> Result<(Database, Connection)> { - let mut config = Config::new().vortex_expect("failed to create duckdb config"); - - // Set DuckDB thread count if specified - if let Some(thread_count) = threads { - config.set("threads", &format!("{}", thread_count))?; - } - let db = match path { - Some(path) => Database::open_with_config(path, config), - None => Database::open_in_memory_with_config(config), + Some(path) => Database::open(path), + None => Database::open_in_memory(), }?; let connection = db.connect()?; vortex_duckdb::initialize(&db)?; - // Enable Parquet metadata cache for all benchmark runs. - // + if let Some(thread_count) = threads { + connection.query(&format!("SET threads = {thread_count}"))?; + } + // `parquet_metadata_cache` is an extension-specific option that's // only available after the Parquet extension is loaded. The Parquet // extension is loaded after the connection is established. - // - // Passing the option to `open_with_config` before leads to - // "Invalid Input Error: The following options were not recognized: - // parquet_metadata_cache" when running DuckDB in debug mode. connection.query("SET parquet_metadata_cache = true")?; Ok((db, connection)) diff --git a/vortex-duckdb/build.rs b/vortex-duckdb/build.rs index b51d529ebe0..949ab9545c6 100644 --- a/vortex-duckdb/build.rs +++ b/vortex-duckdb/build.rs @@ -27,30 +27,19 @@ const DEFAULT_DUCKDB_VERSION: &str = "1.5.3"; const BUILD_ARTIFACTS: [&str; 3] = ["libduckdb.dylib", "libduckdb.so", "libduckdb_static.a"]; -const SOURCE_FILES: [&str; 18] = [ - "cpp/client_context.cpp", - "cpp/config.cpp", +const SOURCE_FILES: [&str; 7] = [ + "cpp/vortex_duckdb.cpp", "cpp/copy_function.cpp", - "cpp/data.cpp", - "cpp/data_chunk.cpp", - "cpp/error.cpp", "cpp/expr.cpp", - "cpp/file_system.cpp", - "cpp/logical_type.cpp", "cpp/optimizer.cpp", - "cpp/replacement_scan.cpp", - "cpp/reusable_dict.cpp", - "cpp/scalar_function.cpp", "cpp/table_filter.cpp", "cpp/table_function.cpp", - "cpp/value.cpp", "cpp/vector.cpp", - "cpp/vector_buffer.cpp", ]; // Duckdb C API function we use. // This lowers codegen'd src/cpp.rs by four times. -const DUCKDB_C_API_FUNCTIONS: [&str; 134] = [ +const DUCKDB_C_API_FUNCTIONS: [&str; 133] = [ "duckdb_array_type_array_size", "duckdb_array_type_child_type", "duckdb_array_vector_get_child", @@ -184,7 +173,16 @@ const DUCKDB_C_API_FUNCTIONS: [&str; 134] = [ "duckdb_vector_reference_value", "duckdb_vector_reference_vector", "duckdb_vector_size", - "duckdb_vector_to_string", +]; + +const DUCKDB_C_API_HEADERS: [&str; 7] = [ + "cpp/include/vortex_duckdb.h", + "cpp/include/expr.h", + "cpp/include/table_filter.h", + "cpp/include/vector.h", + "cpp/include/copy_function.h", + "cpp/include/table_function.h", + "cpp/include/optimizer.h", ]; const DOWNLOAD_MAX_RETRIES: i32 = 3; @@ -462,7 +460,7 @@ fn try_build_duckdb( /// Generate rust functions with bindgen from C sources. fn bindgen_c2rust(crate_dir: &Path, duckdb_include_dir: &Path) { let mut builder = bindgen::Builder::default() - .header("cpp/include/duckdb_vx.h") + .headers(DUCKDB_C_API_HEADERS) .override_abi(Abi::CUnwind, ".*") .raw_line("#![allow(dead_code)]") .raw_line("#![allow(non_camel_case_types)]") diff --git a/vortex-duckdb/cpp/client_context.cpp b/vortex-duckdb/cpp/client_context.cpp deleted file mode 100644 index 996842fb0f3..00000000000 --- a/vortex-duckdb/cpp/client_context.cpp +++ /dev/null @@ -1,35 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#include -#include - -extern "C" duckdb_client_context duckdb_vx_connection_get_client_context(duckdb_connection conn) { - try { - auto connection = reinterpret_cast(conn); - return reinterpret_cast(connection->context.get()); - } catch (...) { - return nullptr; - } -} - -extern "C" duckdb_value duckdb_client_context_try_get_current_setting(duckdb_client_context context, - const char *key) { - if (!context || !key) { - return nullptr; - } - - try { - auto client_context = reinterpret_cast(context); - duckdb::Value result; - auto lookup_result = client_context->TryGetCurrentSetting(key, result); - - if (lookup_result) { - return reinterpret_cast(new duckdb::Value(result)); - } - - return nullptr; - } catch (...) { - return nullptr; - } -} diff --git a/vortex-duckdb/cpp/config.cpp b/vortex-duckdb/cpp/config.cpp deleted file mode 100644 index 1e09bc7c930..00000000000 --- a/vortex-duckdb/cpp/config.cpp +++ /dev/null @@ -1,153 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#include "include/duckdb_vx/config.h" - -#include "duckdb.hpp" -#include "duckdb/main/capi/capi_internal.hpp" -#include "duckdb/main/config.hpp" - -#include -#include -#include -#include -#include - -using namespace duckdb; - -extern "C" { - -duckdb_config duckdb_vx_database_get_config(duckdb_database database) { - if (!database) { - return nullptr; - } - - auto wrapper = reinterpret_cast(database); - auto &config = DBConfig::GetConfig(*wrapper->database->instance); - return reinterpret_cast(&config); -} - -duckdb_state duckdb_vx_get_config_value(duckdb_config config, const char *key, duckdb_value *out_value) { - if (!config || !key || !out_value) { - return DuckDBError; - } - - try { - // Cast the config to DuckDB's internal config type - auto *db_config = reinterpret_cast(config); - - if (!db_config) { - return DuckDBError; - } - - std::string key_str(key); - - // First check set_variable_defaults (the primary location for config values) - if (db_config->options.set_variable_defaults.contains(key_str)) { - *out_value = - reinterpret_cast(new Value(db_config->options.set_variable_defaults[key_str])); - return DuckDBSuccess; - } - - // Then check user_options - if (db_config->options.user_options.contains(key_str)) { - *out_value = reinterpret_cast(new Value(db_config->options.user_options[key_str])); - return DuckDBSuccess; - } - - // Key not found in any map - return DuckDBError; - - } catch (...) { - return DuckDBError; - } -} - -int duckdb_vx_config_has_key(duckdb_config config, const char *key) { - if (!config || !key) { - return 0; - } - - try { - auto *db_config = reinterpret_cast(config); - if (!db_config) { - return 0; - } - - std::string key_str(key); - - // Check if the key exists in set_variable_defaults (primary location) - if (db_config->options.set_variable_defaults.contains(key_str)) { - return 1; - } - - // Check if the key exists in user_options - if (db_config->options.user_options.contains(key_str)) { - return 1; - } - - return 0; - - } catch (...) { - return 0; - } -} - -char *duckdb_vx_value_to_string(duckdb_value value) { - if (!value) { - return nullptr; - } - - try { - // Cast the value to DuckDB's internal Value type - auto *ddb_value = reinterpret_cast(value); - - if (!ddb_value) { - return nullptr; - } - - // Use the ToString method to get the string representation - std::string str_value = ddb_value->ToString(); - - size_t str_len = str_value.length() + 1; - char *result = static_cast(duckdb_malloc(str_len)); - if (!result) { - return nullptr; - } - - // Copy the string and null terminate - std::memcpy(result, str_value.c_str(), str_len); - return result; - - } catch (...) { - return nullptr; - } -} - -duckdb_state duckdb_vx_add_extension_option(duckdb_config config, - const char *name, - const char *description, - duckdb_logical_type logical_type, - duckdb_value default_value) { - if (!name || !description || !logical_type || !default_value) { - return DuckDBError; - } - - try { - auto *db_config = reinterpret_cast(config); - if (!db_config) { - return DuckDBError; - } - - auto *type = reinterpret_cast(logical_type); - auto *value = reinterpret_cast(default_value); - - db_config->AddExtensionOption(string(name), string(description), *type, *value); - - return DuckDBSuccess; - } catch (...) { - return DuckDBError; - } -} - -} // extern "C" diff --git a/vortex-duckdb/cpp/copy_function.cpp b/vortex-duckdb/cpp/copy_function.cpp index 89d07aca462..4740b92bbc3 100644 --- a/vortex-duckdb/cpp/copy_function.cpp +++ b/vortex-duckdb/cpp/copy_function.cpp @@ -1,8 +1,9 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#include "duckdb_vx/data.hpp" -#include "duckdb_vx/error.hpp" -#include "duckdb_vx/table_function.h" +#include "data.hpp" +#include "error.hpp" +#include "vortex_duckdb.h" +#include "table_function.h" #include "vortex.h" #include "duckdb/function/copy_function.hpp" #include "duckdb/main/capi/capi_internal.hpp" @@ -11,8 +12,6 @@ #include "duckdb/parser/parsed_data/create_copy_function_info.hpp" using namespace duckdb; -using vortex::CData; -using vortex::IntoErrString; struct CopyBindData final : TableFunctionData { CopyBindData(unique_ptr ffi_data) : ffi_data(std::move(ffi_data)) { diff --git a/vortex-duckdb/cpp/data.cpp b/vortex-duckdb/cpp/data.cpp deleted file mode 100644 index 5d7e37b7667..00000000000 --- a/vortex-duckdb/cpp/data.cpp +++ /dev/null @@ -1,29 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#include "duckdb.h" - -#include "duckdb_vx/data.hpp" - -namespace vortex { - -CData::CData(void *data_ptr, duckdb_delete_callback_t callback) : data(data_ptr), delete_callback(callback) { -} - -CData::~CData() { - if (data && delete_callback) { - delete_callback(data); - } - data = nullptr; - delete_callback = nullptr; -} - -void *CData::DataPtr() const { - return data; -} - -extern "C" duckdb_vx_data duckdb_vx_data_create(void *data, duckdb_delete_callback_t delete_callback) { - return reinterpret_cast(new CData(data, delete_callback)); -} - -} // namespace vortex diff --git a/vortex-duckdb/cpp/data_chunk.cpp b/vortex-duckdb/cpp/data_chunk.cpp deleted file mode 100644 index b9d4d1ab13e..00000000000 --- a/vortex-duckdb/cpp/data_chunk.cpp +++ /dev/null @@ -1,31 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#include "duckdb_vx/data_chunk.h" -#include "duckdb/common/types/data_chunk.hpp" - -const char *duckdb_data_chunk_to_string(duckdb_data_chunk chunk, duckdb_vx_error *err) { - try { - auto dchunk = reinterpret_cast(chunk); - auto str = dchunk->ToString(); - auto result = static_cast(duckdb_malloc(str.size() + 1)); - memcpy(result, str.c_str(), str.size() + 1); - *err = nullptr; - return result; - } catch (std::runtime_error &e) { - auto s = e.what(); - *err = duckdb_vx_error_create(s, strlen(s)); - return nullptr; - } -} - -void duckdb_data_chunk_verify(duckdb_data_chunk chunk, duckdb_vx_error *err) { - try { - auto dchunk = reinterpret_cast(chunk); - dchunk->Verify(); - *err = nullptr; - } catch (std::runtime_error &e) { - auto s = e.what(); - *err = duckdb_vx_error_create(s, strlen(s)); - } -} diff --git a/vortex-duckdb/cpp/error.cpp b/vortex-duckdb/cpp/error.cpp deleted file mode 100644 index 62fe8b6250b..00000000000 --- a/vortex-duckdb/cpp/error.cpp +++ /dev/null @@ -1,40 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#include -#include "duckdb_vx/error.h" -#include "duckdb.h" -#include "duckdb/common/assert.hpp" - -extern "C" duckdb_vx_error duckdb_vx_error_create(const char *message, size_t message_length) { - return reinterpret_cast(new std::string(message, message_length)); -} - -extern "C" const char *duckdb_vx_error_value(duckdb_vx_error err) { - auto str = reinterpret_cast(err); - return str->c_str(); -} - -extern "C" void duckdb_vx_error_free(duckdb_vx_error err) { - auto str = reinterpret_cast(err); - delete str; -} - -namespace vortex { - -std::string IntoErrString(duckdb_vx_error error) { - if (!error) { - return {}; - } - std::string *const error_str = reinterpret_cast(error); - std::string out = std::move(*error_str); - duckdb_vx_error_free(error); - return out; -} - -duckdb_state SetError(duckdb_vx_error *error_out, std::string_view message) { - D_ASSERT(error_out != nullptr && "SetError called with null error_out"); - *error_out = duckdb_vx_error_create(message.data(), message.size()); - return DuckDBError; -} -} // namespace vortex diff --git a/vortex-duckdb/cpp/expr.cpp b/vortex-duckdb/cpp/expr.cpp index b15ce8e02cd..6470a9d338d 100644 --- a/vortex-duckdb/cpp/expr.cpp +++ b/vortex-duckdb/cpp/expr.cpp @@ -1,7 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#include "duckdb_vx/expr.h" +#include "expr.h" +#include "duckdb/function/scalar_function.hpp" #include "duckdb/planner/expression/bound_between_expression.hpp" #include "duckdb/planner/expression/bound_columnref_expression.hpp" #include "duckdb/planner/expression/bound_comparison_expression.hpp" @@ -12,6 +13,14 @@ using namespace duckdb; +extern "C" const char *duckdb_vx_sfunc_name(duckdb_vx_sfunc ffi_func) { + if (!ffi_func) { + return nullptr; + } + auto func = reinterpret_cast(ffi_func); + return func->name.c_str(); +} + extern "C" const char *duckdb_vx_expr_to_string(duckdb_vx_expr ffi_expr) { if (!ffi_expr) { return nullptr; diff --git a/vortex-duckdb/cpp/file_system.cpp b/vortex-duckdb/cpp/file_system.cpp deleted file mode 100644 index 6852d4d559b..00000000000 --- a/vortex-duckdb/cpp/file_system.cpp +++ /dev/null @@ -1,165 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#include "duckdb_vx.h" -#include "duckdb_vx/error.hpp" - -#include -#include -#include -#include - -#include - -using namespace duckdb; -using vortex::SetError; - -extern "C" duckdb_vx_file_handle -duckdb_vx_fs_open(duckdb_client_context ctx, const char *path, duckdb_vx_error *error_out) { - if (!ctx || !path) { - SetError(error_out, "Invalid filesystem open arguments"); - return nullptr; - } - - auto *client_context = reinterpret_cast(ctx); - - try { - auto &fs = FileSystem::GetFileSystem(*client_context); - auto handle = fs.OpenFile(path, FileFlags::FILE_FLAGS_READ | FileFlags::FILE_FLAGS_PARALLEL_ACCESS); - return reinterpret_cast(handle.release()); - } catch (const std::exception &e) { - SetError(error_out, e.what()); - return nullptr; - } -} - -extern "C" duckdb_vx_file_handle -duckdb_vx_fs_create(duckdb_client_context ctx, const char *path, duckdb_vx_error *error_out) { - if (!ctx || !path) { - SetError(error_out, "Invalid filesystem create arguments"); - return nullptr; - } - - constexpr auto flags = FileFlags::FILE_FLAGS_WRITE | FileFlags::FILE_FLAGS_FILE_CREATE_NEW | - FileFlags::FILE_FLAGS_PARALLEL_ACCESS; - auto *client_context = reinterpret_cast(ctx); - - try { - auto &fs = FileSystem::GetFileSystem(*client_context); - auto handle = fs.OpenFile(path, flags); - return reinterpret_cast(handle.release()); - } catch (const std::exception &e) { - SetError(error_out, e.what()); - return nullptr; - } -} - -extern "C" void duckdb_vx_fs_close(duckdb_vx_file_handle *handle) { - if (handle && *handle) { - delete reinterpret_cast(std::exchange(*handle, nullptr)); - } -} - -extern "C" duckdb_state -duckdb_vx_fs_get_size(duckdb_vx_file_handle handle, idx_t *size_out, duckdb_vx_error *error_out) { - if (!handle || !size_out) { - return SetError(error_out, "Invalid arguments to fs_get_size"); - } - - try { - *size_out = reinterpret_cast(handle)->GetFileSize(); - } catch (const std::exception &e) { - return SetError(error_out, e.what()); - } - return DuckDBSuccess; -} - -extern "C" duckdb_state duckdb_vx_fs_read(duckdb_vx_file_handle handle, - idx_t offset, - idx_t len, - uint8_t *buffer, - idx_t *out_len, - duckdb_vx_error *error_out) { - if (!handle || !buffer || !out_len) { - return SetError(error_out, "Invalid arguments to fs_read"); - } - - try { - reinterpret_cast(handle)->Read(buffer, len, offset); - *out_len = len; - } catch (const std::exception &e) { - return SetError(error_out, e.what()); - } - return DuckDBSuccess; -} - -extern "C" duckdb_state duckdb_vx_fs_write(duckdb_vx_file_handle handle, - idx_t offset, - idx_t len, - uint8_t *buffer, - idx_t *out_len, - duckdb_vx_error *error_out) { - if (!handle || !buffer || !out_len) { - return SetError(error_out, "Invalid arguments to fs_write"); - } - - try { - reinterpret_cast(handle)->Write(QueryContext(), buffer, len, offset); - *out_len = len; - } catch (const std::exception &e) { - return SetError(error_out, e.what()); - } - return DuckDBSuccess; -} - -extern "C" duckdb_state duckdb_vx_fs_list_files(duckdb_client_context ctx, - const char *directory, - duckdb_vx_list_files_callback callback, - void *user_data, - duckdb_vx_error *error_out) { - if (!ctx || !directory || !callback) { - return SetError(error_out, "Invalid arguments to fs_list_files"); - } - - auto fn = [&](const string &name, bool is_dir) { - callback(name.c_str(), is_dir, user_data); - }; - auto *client_context = reinterpret_cast(ctx); - - try { - FileSystem::GetFileSystem(*client_context).ListFiles(directory, fn); - return DuckDBSuccess; - } catch (const std::exception &e) { - return SetError(error_out, e.what()); - } - return DuckDBSuccess; -} - -extern "C" duckdb_state duckdb_vx_fs_sync(duckdb_vx_file_handle handle, duckdb_vx_error *error_out) { - if (!handle) { - return SetError(error_out, "Invalid arguments to fs_sync"); - } - - try { - reinterpret_cast(handle)->Sync(); - } catch (const std::exception &e) { - return SetError(error_out, e.what()); - } - return DuckDBSuccess; -} - -extern "C" duckdb_state -duckdb_vx_fs_remove(duckdb_client_context ctx, const char *path, duckdb_vx_error *error_out) { - if (!ctx || !path) { - return SetError(error_out, "Invalid arguments to fs_remove"); - } - - auto *client_context = reinterpret_cast(ctx); - - try { - FileSystem::GetFileSystem(*client_context).RemoveFile(path); - } catch (const std::exception &e) { - return SetError(error_out, e.what()); - } - return DuckDBSuccess; -} diff --git a/vortex-duckdb/cpp/include/copy_function.h b/vortex-duckdb/cpp/include/copy_function.h new file mode 100644 index 00000000000..e7e9bc827f6 --- /dev/null +++ b/vortex-duckdb/cpp/include/copy_function.h @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#pragma once +#include "duckdb.h" + +#ifdef __cplusplus +extern "C" { +#endif + +duckdb_state duckdb_vx_register_copy_function(duckdb_database ffi_db); + +#ifdef __cplusplus +} +#endif diff --git a/vortex-duckdb/cpp/include/duckdb_vx/data.hpp b/vortex-duckdb/cpp/include/data.hpp similarity index 67% rename from vortex-duckdb/cpp/include/duckdb_vx/data.hpp rename to vortex-duckdb/cpp/include/data.hpp index 185e33719e3..ade51784e70 100644 --- a/vortex-duckdb/cpp/include/duckdb_vx/data.hpp +++ b/vortex-duckdb/cpp/include/data.hpp @@ -3,27 +3,18 @@ #pragma once -#include "duckdb_vx/data.h" - -namespace vortex { +#include "duckdb.h" +// Owns a pointer with a DuckDB delete callback, freeing it on destruction. class CData final { public: CData(void *data_ptr, duckdb_delete_callback_t callback); - - // Disable copy constructor to prevent accidental copies. CData(const CData &) = delete; - - // Disable assignment operator to prevent accidental assignments. CData &operator=(const CData &) = delete; - ~CData(); - void *DataPtr() const; private: void *data = nullptr; duckdb_delete_callback_t delete_callback = nullptr; }; - -} // namespace vortex diff --git a/vortex-duckdb/cpp/include/duckdb_vx.h b/vortex-duckdb/cpp/include/duckdb_vx.h deleted file mode 100644 index 176b40a415a..00000000000 --- a/vortex-duckdb/cpp/include/duckdb_vx.h +++ /dev/null @@ -1,23 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#pragma once - -#include "duckdb_vx/client_context.h" -#include "duckdb_vx/optimizer.h" -#include "duckdb_vx/config.h" -#include "duckdb_vx/copy_function.h" -#include "duckdb_vx/data.h" -#include "duckdb_vx/data_chunk.h" -#include "duckdb_vx/error.h" -#include "duckdb_vx/expr.h" -#include "duckdb_vx/file_system.h" -#include "duckdb_vx/logical_type.h" -#include "duckdb_vx/reusable_dict.h" -#include "duckdb_vx/replacement_scan.h" -#include "duckdb_vx/scalar_function.h" -#include "duckdb_vx/table_filter.h" -#include "duckdb_vx/table_function.h" -#include "duckdb_vx/value.h" -#include "duckdb_vx/vector.h" -#include "duckdb_vx/vector_buffer.h" diff --git a/vortex-duckdb/cpp/include/duckdb_vx/client_context.h b/vortex-duckdb/cpp/include/duckdb_vx/client_context.h deleted file mode 100644 index 1683afa2ec1..00000000000 --- a/vortex-duckdb/cpp/include/duckdb_vx/client_context.h +++ /dev/null @@ -1,22 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#pragma once -#include "duckdb.h" - -#ifdef __cplusplus /* If compiled as C++, use C ABI */ -extern "C" { -#endif - -// Get the client context from a DuckDB connection. -// This reference is valid as long as the connection is valid. -duckdb_client_context duckdb_vx_connection_get_client_context(duckdb_connection conn); - -// Try to get the current value of a configuration setting. -// Returns a duckdb_value if the setting exists, or NULL if it doesn't. -// The caller is responsible for freeing the returned value using duckdb_destroy_value. -duckdb_value duckdb_client_context_try_get_current_setting(duckdb_client_context context, const char *key); - -#ifdef __cplusplus /* End C ABI */ -} -#endif diff --git a/vortex-duckdb/cpp/include/duckdb_vx/config.h b/vortex-duckdb/cpp/include/duckdb_vx/config.h deleted file mode 100644 index 86e0135652e..00000000000 --- a/vortex-duckdb/cpp/include/duckdb_vx/config.h +++ /dev/null @@ -1,59 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#pragma once - -#include "duckdb.h" - -#ifdef __cplusplus -extern "C" { -#endif - -/// Get the DuckDB configuration object for a given database instance. -/// -/// @param database The DuckDB database instance -/// @return A DuckDB configuration object that can be used to query or modify settings for the database -duckdb_config duckdb_vx_database_get_config(duckdb_database database); - -/// Get a configuration value from a DuckDB config object by key name. -/// Returns a DuckDB value containing the config value, or INVALID if the key doesn't exist. -/// The returned value must be freed with duckdb_destroy_value. -/// -/// @param config The DuckDB configuration object -/// @param key The configuration key to retrieve -/// @param out_value Pointer to store the resulting DuckDB value -/// @return DuckDBSuccess on success, DuckDBError if the key doesn't exist or on error -duckdb_state duckdb_vx_get_config_value(duckdb_config config, const char *key, duckdb_value *out_value); - -/// Check if a configuration key exists in the given config object. -/// -/// @param config The DuckDB configuration object -/// @param key The configuration key to check -/// @return 1 if the key exists, 0 if it doesn't exist or on error -int duckdb_vx_config_has_key(duckdb_config config, const char *key); - -/// Convert a DuckDB value to a string representation. -/// The returned string must be freed with duckdb_free. -/// -/// @param value The DuckDB value to convert -/// @return A newly allocated string containing the value's string representation, or NULL on error -char *duckdb_vx_value_to_string(duckdb_value value); - -/// Add an extension-specific configuration option to DuckDB. -/// This allows extensions to register custom SET variables. -/// -/// @param config The DuckDB config instance -/// @param name The name of the configuration option -/// @param description A description of what the option does -/// @param logical_type The DuckDB logical type for the option -/// @param default_value The default value for the option -/// @return DuckDBSuccess on success, DuckDBError on error -duckdb_state duckdb_vx_add_extension_option(duckdb_config config, - const char *name, - const char *description, - duckdb_logical_type logical_type, - duckdb_value default_value); - -#ifdef __cplusplus -} -#endif \ No newline at end of file diff --git a/vortex-duckdb/cpp/include/duckdb_vx/copy_function.h b/vortex-duckdb/cpp/include/duckdb_vx/copy_function.h deleted file mode 100644 index 73b33a3d818..00000000000 --- a/vortex-duckdb/cpp/include/duckdb_vx/copy_function.h +++ /dev/null @@ -1,20 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors -#pragma once -#include "duckdb.h" - -// TODO(joe): expose via c api -// typedef enum copy_function_execution_mode_ { -// REGULAR_COPY_TO_FILE = 1, -// PARALLEL_COPY_TO_FILE, -// BATCH_COPY_TO_FILE -// } copy_function_execution_mode; -// -// TODO(joe): expose via c api -// copy_function_execution_mode (*execution_mode)(bool preserve_insertion_order, bool - -#ifdef __cplusplus -extern "C" duckdb_state duckdb_vx_register_copy_function(duckdb_database ffi_db); -#else -duckdb_state duckdb_vx_register_copy_function(duckdb_database ffi_db); -#endif diff --git a/vortex-duckdb/cpp/include/duckdb_vx/data.h b/vortex-duckdb/cpp/include/duckdb_vx/data.h deleted file mode 100644 index 88d45d82a45..00000000000 --- a/vortex-duckdb/cpp/include/duckdb_vx/data.h +++ /dev/null @@ -1,20 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#pragma once - -#include "duckdb.h" - -#ifdef __cplusplus /* If compiled as C++, use C ABI */ -extern "C" { -#endif - -// Opaque data object with a deletion callback. -typedef struct duckdb_vx_data_ *duckdb_vx_data; - -// Create an opaque data object with a delete callback. -duckdb_vx_data duckdb_vx_data_create(void *data_ptr, duckdb_delete_callback_t delete_callback); - -#ifdef __cplusplus /* End C ABI */ -} -#endif diff --git a/vortex-duckdb/cpp/include/duckdb_vx/data_chunk.h b/vortex-duckdb/cpp/include/duckdb_vx/data_chunk.h deleted file mode 100644 index c2d1395ba33..00000000000 --- a/vortex-duckdb/cpp/include/duckdb_vx/data_chunk.h +++ /dev/null @@ -1,20 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#pragma once - -#include "duckdb.h" - -#include "error.h" - -#ifdef __cplusplus /* If compiled as C++, use C ABI */ -extern "C" { -#endif - -const char *duckdb_data_chunk_to_string(duckdb_data_chunk chunk, duckdb_vx_error *err); - -void duckdb_data_chunk_verify(duckdb_data_chunk chunk, duckdb_vx_error *err); - -#ifdef __cplusplus /* End C ABI */ -} -#endif diff --git a/vortex-duckdb/cpp/include/duckdb_vx/error.h b/vortex-duckdb/cpp/include/duckdb_vx/error.h deleted file mode 100644 index d2a0b53b4ef..00000000000 --- a/vortex-duckdb/cpp/include/duckdb_vx/error.h +++ /dev/null @@ -1,24 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#pragma once - -#include - -#ifdef __cplusplus /* If compiled as C++, use C ABI */ -extern "C" { -#endif - -typedef struct duckdb_vx_error_ *duckdb_vx_error; - -//! Create a DuckDB vortex error. -duckdb_vx_error duckdb_vx_error_create(const char *message, size_t message_length); - -// Borrows the message owned by the err type. -const char *duckdb_vx_error_value(duckdb_vx_error err); - -void duckdb_vx_error_free(duckdb_vx_error err); - -#ifdef __cplusplus /* End C ABI */ -} -#endif diff --git a/vortex-duckdb/cpp/include/duckdb_vx/file_system.h b/vortex-duckdb/cpp/include/duckdb_vx/file_system.h deleted file mode 100644 index fe1f224bb00..00000000000 --- a/vortex-duckdb/cpp/include/duckdb_vx/file_system.h +++ /dev/null @@ -1,79 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#pragma once - -#include "duckdb.h" - -#include "duckdb_vx/client_context.h" -#include "duckdb_vx/error.h" - -#ifdef __cplusplus /* If compiled as C++, use C ABI */ -extern "C" { -#endif - -typedef struct duckdb_vx_file_handle_ *duckdb_vx_file_handle; - -// Open a file using DuckDB's filesystem (supports httpfs, s3, etc.). -duckdb_vx_file_handle -duckdb_vx_fs_open(duckdb_client_context ctx, const char *path, duckdb_vx_error *error_out); - -// Close a previously opened file handle. -void duckdb_vx_fs_close(duckdb_vx_file_handle *handle); - -// Get the size of an opened file. -duckdb_state duckdb_vx_fs_get_size(duckdb_vx_file_handle handle, idx_t *size_out, duckdb_vx_error *error_out); - -// Read up to len bytes at the given offset into buffer. Returns bytes read via out_len. -// TODO(myrrc) Here we use duckdb's positional read which returns nothing, -// and thus we 1. don't know whether this read succeeded in the function itself, -// only when out-of-bounds exception is thrown -// 2. Always have out_len=len. -// Maybe the issue is in past-the-end reads which propagate an Error -// in rust code, but here we throw an exception -duckdb_state duckdb_vx_fs_read(duckdb_vx_file_handle handle, - idx_t offset, - idx_t len, - uint8_t *buffer, - idx_t *out_len, - duckdb_vx_error *error_out); - -/// Callback invoked for each entry returned by `duckdb_vx_fs_list_files`. -/// -/// @param name The entry's path, full for remote files, relative for local files -/// @param is_dir Whether the entry is a directory. -/// @param user_data Opaque pointer forwarded from the caller. -typedef void (*duckdb_vx_list_files_callback)(const char *name, bool is_dir, void *user_data); - -/// Non-recursively list entries in a directory using DuckDB's filesystem. -/// -/// Invokes `callback` once for each entry (file or subdirectory) found directly -/// inside `directory`. The caller is responsible for recursing into subdirectories -/// if a recursive listing is desired. -duckdb_state duckdb_vx_fs_list_files(duckdb_client_context ctx, - const char *directory, - duckdb_vx_list_files_callback callback, - void *user_data, - duckdb_vx_error *error_out); - -// Create/truncate a file for writing using DuckDB's filesystem. -duckdb_vx_file_handle -duckdb_vx_fs_create(duckdb_client_context ctx, const char *path, duckdb_vx_error *error_out); - -// Write len bytes at the given offset from buffer. -duckdb_state duckdb_vx_fs_write(duckdb_vx_file_handle handle, - idx_t offset, - idx_t len, - uint8_t *buffer, - idx_t *out_len, - duckdb_vx_error *error_out); - -// Flush pending writes to storage. -duckdb_state duckdb_vx_fs_sync(duckdb_vx_file_handle handle, duckdb_vx_error *error_out); - -// Delete a file using DuckDB's filesystem. -duckdb_state duckdb_vx_fs_remove(duckdb_client_context ctx, const char *path, duckdb_vx_error *error_out); - -#ifdef __cplusplus /* End C ABI */ -} -#endif diff --git a/vortex-duckdb/cpp/include/duckdb_vx/function.h b/vortex-duckdb/cpp/include/duckdb_vx/function.h deleted file mode 100644 index b669fed54af..00000000000 --- a/vortex-duckdb/cpp/include/duckdb_vx/function.h +++ /dev/null @@ -1,15 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#pragma once - -#ifdef __cplusplus /* If compiled as C++, use C ABI */ -extern "C" { -#endif - -/// Wrapper around FunctionInfo from . -typedef struct duckdb_vx_func_info_ *duckdb_vx_func_info; - -#ifdef __cplusplus /* End C ABI */ -} -#endif diff --git a/vortex-duckdb/cpp/include/duckdb_vx/logical_type.h b/vortex-duckdb/cpp/include/duckdb_vx/logical_type.h deleted file mode 100644 index a4b89c87550..00000000000 --- a/vortex-duckdb/cpp/include/duckdb_vx/logical_type.h +++ /dev/null @@ -1,21 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#pragma once - -#include "duckdb.h" - -#ifdef __cplusplus /* If compiled as C++, use C ABI */ -extern "C" { -#endif - -char *duckdb_vx_logical_type_stringify(duckdb_logical_type ty); -duckdb_logical_type duckdb_vx_logical_type_copy(duckdb_logical_type ty); - -/// Creates a GEOMETRY logical type with the given CRS (Coordinate Reference System). -/// `crs` must be a NUL-terminated UTF-8 string. Pass an empty string for no CRS. -duckdb_logical_type duckdb_vx_create_geometry(const char *crs); - -#ifdef __cplusplus /* End C ABI */ -} -#endif diff --git a/vortex-duckdb/cpp/include/duckdb_vx/replacement_scan.h b/vortex-duckdb/cpp/include/duckdb_vx/replacement_scan.h deleted file mode 100644 index 00db251a5d4..00000000000 --- a/vortex-duckdb/cpp/include/duckdb_vx/replacement_scan.h +++ /dev/null @@ -1,14 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#pragma once - -#ifdef __cplusplus /* If compiled as C++, use C ABI */ -extern "C" { -#endif - -duckdb_state duckdb_vx_register_scan_replacement(duckdb_database duckdb_database); - -#ifdef __cplusplus /* End C ABI */ -} -#endif diff --git a/vortex-duckdb/cpp/include/duckdb_vx/reusable_dict.h b/vortex-duckdb/cpp/include/duckdb_vx/reusable_dict.h deleted file mode 100644 index 15a1bdf15b6..00000000000 --- a/vortex-duckdb/cpp/include/duckdb_vx/reusable_dict.h +++ /dev/null @@ -1,36 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#pragma once - -#include "duckdb.h" - -#include "duckdb_vx/error.h" - -#ifdef __cplusplus /* If compiled as C++, use C ABI */ -extern "C" { -#endif - -typedef struct duckdb_vx_reusable_dict_ *duckdb_vx_reusable_dict; - -/// Creates a new reusable dictionary from a logical type and size. -/// The returned dictionary can be used with duckdb_vx_vector_dictionary_reusable. -duckdb_vx_reusable_dict duckdb_vx_reusable_dict_create(duckdb_logical_type logical_type, idx_t size); - -/// Destroys the reusable dictionary. -void duckdb_vx_reusable_dict_destroy(duckdb_vx_reusable_dict *dict); - -/// Clones the reusable dictionary. -duckdb_vx_reusable_dict duckdb_vx_reusable_dict_clone(duckdb_vx_reusable_dict dict); - -/// Get the internal vector of the reusable dictionary. -void duckdb_vx_reusable_dict_set_vector(duckdb_vx_reusable_dict reusable, duckdb_vector *out_vector); - -/// Creates a dictionary vector using a reusable dictionary and a selection vector. -void duckdb_vx_vector_dictionary_reusable(duckdb_vector vector, - duckdb_vx_reusable_dict reusable, - duckdb_selection_vector sel_vec); - -#ifdef __cplusplus /* End C ABI */ -} -#endif diff --git a/vortex-duckdb/cpp/include/duckdb_vx/scalar_function.h b/vortex-duckdb/cpp/include/duckdb_vx/scalar_function.h deleted file mode 100644 index fb20812a961..00000000000 --- a/vortex-duckdb/cpp/include/duckdb_vx/scalar_function.h +++ /dev/null @@ -1,20 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#pragma once - -#include "duckdb.h" - -#ifdef __cplusplus /* If compiled as C++, use C ABI */ -extern "C" { -#endif - -typedef struct duckdb_vx_sfunc_ *duckdb_vx_sfunc; - -const char *duckdb_vx_sfunc_name(duckdb_vx_sfunc ffi_func); - -duckdb_logical_type duckdb_vx_sfunc_return_type(duckdb_vx_sfunc ffi_func); - -#ifdef __cplusplus /* End C ABI */ -} -#endif diff --git a/vortex-duckdb/cpp/include/duckdb_vx/value.h b/vortex-duckdb/cpp/include/duckdb_vx/value.h deleted file mode 100644 index cc02d78e128..00000000000 --- a/vortex-duckdb/cpp/include/duckdb_vx/value.h +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#pragma once - -#include "duckdb.h" - -#ifdef __cplusplus /* If compiled as C++, use C ABI */ -extern "C" { -#endif - -// Create a null value with a reference to a logical type. -duckdb_value duckdb_vx_value_create_null(duckdb_logical_type ty); - -/// Creates a GEOMETRY value containing the given WKB bytes and CRS. -/// -/// `wkb` points to `len` bytes of well-known-binary geometry data; the bytes are not validated. -/// `crs` must be a NUL-terminated UTF-8 string; pass NULL or an empty string for no CRS. -duckdb_value duckdb_vx_value_create_geometry(const uint8_t *wkb, idx_t len, const char *crs); - -/// Extracts the raw WKB bytes from a GEOMETRY value as a duckdb_blob. -/// -/// This bypasses the GEOMETRY -> BLOB default cast (which would require the spatial extension to -/// be loaded). The returned `data` pointer must be freed with `duckdb_free`. Returns `{nullptr, 0}` -/// if `value` is null or not a GEOMETRY value. -duckdb_blob duckdb_vx_value_get_geometry(duckdb_value value); - -#ifdef __cplusplus /* End C ABI */ -} -#endif diff --git a/vortex-duckdb/cpp/include/duckdb_vx/vector_buffer.h b/vortex-duckdb/cpp/include/duckdb_vx/vector_buffer.h deleted file mode 100644 index 1f45f1dc991..00000000000 --- a/vortex-duckdb/cpp/include/duckdb_vx/vector_buffer.h +++ /dev/null @@ -1,26 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#pragma once - -#include "duckdb.h" - -#include "duckdb_vx/data.h" -#include "duckdb_vx/error.h" - -#ifdef __cplusplus /* If compiled as C++, use C ABI */ -extern "C" { -#endif - -typedef struct duckdb_vx_vector_buffer_ *duckdb_vx_vector_buffer; - -// Create a external vector buffer from an existing data buffer, used to allow DuckDB to keep a reference to -// the buffer. -duckdb_vx_vector_buffer duckdb_vx_vector_buffer_create(duckdb_vx_data buffer); - -// Destroy the vector buffer. -void duckdb_vx_vector_buffer_destroy(duckdb_vx_vector_buffer *buffer); - -#ifdef __cplusplus /* End C ABI */ -} -#endif diff --git a/vortex-duckdb/cpp/include/duckdb_vx/vector_buffer.hpp b/vortex-duckdb/cpp/include/duckdb_vx/vector_buffer.hpp deleted file mode 100644 index 0ce107cec3f..00000000000 --- a/vortex-duckdb/cpp/include/duckdb_vx/vector_buffer.hpp +++ /dev/null @@ -1,24 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#pragma once - -#include "duckdb/common/vector.hpp" -#include "duckdb/common/types/vector.hpp" - -#include "duckdb_vx/data.hpp" - -namespace vortex { - -// This is a wrapper around an externally managed buffer, which can be assigned to a Vector and -// freed once the vector is done with the buffer. -class ExternalVectorBuffer : public duckdb::VectorBuffer { -public: - explicit ExternalVectorBuffer(duckdb::unique_ptr data) : data(std::move(data)) { - } - -private: - duckdb::unique_ptr data; -}; - -} // namespace vortex \ No newline at end of file diff --git a/vortex-duckdb/cpp/include/duckdb_vx/error.hpp b/vortex-duckdb/cpp/include/error.hpp similarity index 70% rename from vortex-duckdb/cpp/include/duckdb_vx/error.hpp rename to vortex-duckdb/cpp/include/error.hpp index 66602683f5e..27b5c96b6a9 100644 --- a/vortex-duckdb/cpp/include/duckdb_vx/error.hpp +++ b/vortex-duckdb/cpp/include/error.hpp @@ -3,14 +3,9 @@ #pragma once -#include #include -#include "duckdb.h" +#include "vortex_duckdb.h" -#include "duckdb_vx/error.h" - -namespace vortex { std::string IntoErrString(duckdb_vx_error error); duckdb_state SetError(duckdb_vx_error *error_out, std::string_view message); -} // namespace vortex diff --git a/vortex-duckdb/cpp/include/duckdb_vx/expr.h b/vortex-duckdb/cpp/include/expr.h similarity index 98% rename from vortex-duckdb/cpp/include/duckdb_vx/expr.h rename to vortex-duckdb/cpp/include/expr.h index b1344341403..457a944e5d5 100644 --- a/vortex-duckdb/cpp/include/duckdb_vx/expr.h +++ b/vortex-duckdb/cpp/include/expr.h @@ -3,15 +3,19 @@ #pragma once -#include "scalar_function.h" +#include "duckdb.h" #ifdef __cplusplus /* If compiled as C++, use C ABI */ extern "C" { #endif +typedef struct duckdb_vx_sfunc_ *duckdb_vx_sfunc; + +const char *duckdb_vx_sfunc_name(duckdb_vx_sfunc ffi_func); + typedef struct duckdb_vx_expr_ *duckdb_vx_expr; -/// Return the string representation of the expression. Must be freed with `duckdb_vx_free`. +/// Return the string representation of the expression. Must be freed with `duckdb_free`. const char *duckdb_vx_expr_to_string(duckdb_vx_expr expr); void duckdb_vx_destroy_expr(duckdb_vx_expr *expr); diff --git a/vortex-duckdb/cpp/include/optimizer.h b/vortex-duckdb/cpp/include/optimizer.h new file mode 100644 index 00000000000..82f1ba47975 --- /dev/null +++ b/vortex-duckdb/cpp/include/optimizer.h @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#pragma once +#include "duckdb.h" + +#ifdef __cplusplus +extern "C" { +#endif + +duckdb_state duckdb_vx_optimizer_extension_register(duckdb_database ffi_db); + +#ifdef __cplusplus +} +#endif diff --git a/vortex-duckdb/cpp/include/duckdb_vx/optimizer.h b/vortex-duckdb/cpp/include/optimizer.hpp similarity index 96% rename from vortex-duckdb/cpp/include/duckdb_vx/optimizer.h rename to vortex-duckdb/cpp/include/optimizer.hpp index 051c725715b..27ec19d917f 100644 --- a/vortex-duckdb/cpp/include/duckdb_vx/optimizer.h +++ b/vortex-duckdb/cpp/include/optimizer.hpp @@ -1,19 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#pragma once -#include "duckdb.h" - -#ifdef __cplusplus -extern "C" { -#endif - -duckdb_state duckdb_vx_optimizer_extension_register(duckdb_database ffi_db); -#ifdef __cplusplus -} -#endif +#pragma once -#ifdef __cplusplus #include "duckdb/optimizer/optimizer_extension.hpp" #include "duckdb/planner/expression/bound_columnref_expression.hpp" #include "duckdb/planner/expression/bound_function_expression.hpp" @@ -137,5 +126,3 @@ struct GetBinding { * GET nor part of PROJECTION wrapping a GET. */ std::optional Resolve(ColumnBinding binding, Analyses &analyses, const Projections &projections); - -#endif diff --git a/vortex-duckdb/cpp/include/duckdb_vx/table_filter.h b/vortex-duckdb/cpp/include/table_filter.h similarity index 99% rename from vortex-duckdb/cpp/include/duckdb_vx/table_filter.h rename to vortex-duckdb/cpp/include/table_filter.h index 10c08617ab0..a559f269796 100644 --- a/vortex-duckdb/cpp/include/duckdb_vx/table_filter.h +++ b/vortex-duckdb/cpp/include/table_filter.h @@ -4,8 +4,7 @@ #pragma once #include "duckdb.h" - -#include "duckdb_vx/expr.h" +#include "expr.h" #ifdef __cplusplus /* If compiled as C++, use C ABI */ extern "C" { diff --git a/vortex-duckdb/cpp/include/duckdb_vx/table_function.h b/vortex-duckdb/cpp/include/table_function.h similarity index 78% rename from vortex-duckdb/cpp/include/duckdb_vx/table_function.h rename to vortex-duckdb/cpp/include/table_function.h index 550e1cf3635..806af620773 100644 --- a/vortex-duckdb/cpp/include/duckdb_vx/table_function.h +++ b/vortex-duckdb/cpp/include/table_function.h @@ -2,34 +2,10 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors #pragma once -#include "duckdb_vx/data.h" -#include "duckdb_vx/error.h" -#include "duckdb_vx/expr.h" +#include "duckdb.h" #include "table_filter.h" #include -#ifdef __cplusplus -static_assert(sizeof(idx_t) == 8); - -#include "duckdb/main/capi/capi_internal.hpp" - -duckdb::unique_ptr -duckdb_vx_table_function_bind(duckdb::ClientContext &context, - duckdb::TableFunctionBindInput &input, - duckdb::vector &return_types, - duckdb::vector &names); - -struct TableFunctionProjectionExpressionInput { - const duckdb::LogicalGet &get; - const duckdb::Expression &expression; - idx_t projection_idx; -}; - -// true if we can push down the expression, false otherwise -bool projection_expression_pushdown(duckdb::ClientContext &context, - const TableFunctionProjectionExpressionInput &input); -#endif - #ifdef __cplusplus extern "C" { #endif diff --git a/vortex-duckdb/cpp/include/table_function.hpp b/vortex-duckdb/cpp/include/table_function.hpp new file mode 100644 index 00000000000..8cdb813f4a7 --- /dev/null +++ b/vortex-duckdb/cpp/include/table_function.hpp @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#pragma once + +#include "duckdb.h" +#include "duckdb/function/function.hpp" +#include "duckdb/function/table_function.hpp" + +static_assert(sizeof(idx_t) == 8); + +// We need this exposed to compare function addresses in optimizer.cpp +duckdb::unique_ptr +duckdb_vx_table_function_bind(duckdb::ClientContext &context, + duckdb::TableFunctionBindInput &input, + duckdb::vector &return_types, + duckdb::vector &names); + +struct TableFunctionProjectionExpressionInput { + const duckdb::LogicalGet &get; + const duckdb::Expression &expression; + idx_t projection_idx; +}; + +// true if we can push down the expression, false otherwise +bool projection_expression_pushdown(duckdb::ClientContext &context, + const TableFunctionProjectionExpressionInput &input); diff --git a/vortex-duckdb/cpp/include/duckdb_vx/vector.h b/vortex-duckdb/cpp/include/vector.h similarity index 79% rename from vortex-duckdb/cpp/include/duckdb_vx/vector.h rename to vortex-duckdb/cpp/include/vector.h index 41c95bd2c7f..24d01748666 100644 --- a/vortex-duckdb/cpp/include/duckdb_vx/vector.h +++ b/vortex-duckdb/cpp/include/vector.h @@ -1,29 +1,15 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors - #pragma once +#include "vortex_duckdb.h" -#include "duckdb.h" - -#include "duckdb_vx/data.h" -#include "duckdb_vx/error.h" -#include "duckdb_vx/vector_buffer.h" - -#ifdef __cplusplus /* If compiled as C++, use C ABI */ +#ifdef __cplusplus extern "C" { #endif // Create a vector that slices another vector between a pair of offsets [offset, end) duckdb_vector duckdb_vx_vector_slice(duckdb_vector ffi_vector, idx_t offset, idx_t end); -/// Slice the vector to a new dictionary vector, using the current vector's values and -/// the provided selection vector. -/// -/// A dictionary slice holds a strong reference to all memory it uses. -void duckdb_vx_vector_slice_to_dictionary(duckdb_vector ffi_vector, - duckdb_selection_vector selection_vector, - idx_t selection_vector_length); - /// Creates a dictionary vector for a given values vector and selection vector. /// /// A dictionary holds a strong reference to all memory it uses. @@ -38,6 +24,27 @@ void duckdb_vx_vector_dictionary(duckdb_vector ffi_vector, void duckdb_vx_set_dictionary_vector_length(duckdb_vector dict, unsigned int len); +// Reset vector's validity mask to nullptr, making all vector's elements valid. +// vector must not be a DictionaryVector or a SequenceVector +void duckdb_vx_vector_set_all_valid(duckdb_vector ffi_vector); + +// Set the data pointer for the vector. This is the start of the values array in the vector. +void duckdb_vx_vector_set_data_ptr(duckdb_vector ffi_vector, void *ptr); + +// Converts a duckdb flat vector into a Sequence vector. +void duckdb_vx_sequence_vector(duckdb_vector c_vector, int64_t start, int64_t step, idx_t capacity); + +void duckdb_vector_flatten(duckdb_vector vector, unsigned long len); + +duckdb_value duckdb_vx_vector_get_value(duckdb_vector ffi_vector, idx_t index); + +typedef struct duckdb_vx_vector_buffer_ *duckdb_vx_vector_buffer; + +// Create a external vector buffer from an existing data buffer +duckdb_vx_vector_buffer duckdb_vx_vector_buffer_create(duckdb_vx_data buffer); + +void duckdb_vx_vector_buffer_destroy(duckdb_vx_vector_buffer *buffer); + // Add the buffer to the string vector (basically, keep it alive as long as the vector). void duckdb_vx_string_vector_add_vector_data_buffer(duckdb_vector ffi_vector, duckdb_vx_vector_buffer buffer); @@ -46,13 +53,6 @@ void duckdb_vx_string_vector_add_vector_data_buffer(duckdb_vector ffi_vector, du // valid. void duckdb_vx_vector_set_vector_data_buffer(duckdb_vector ffi_vector, duckdb_vx_vector_buffer buffer); -// Reset vector's validity mask to nullptr, making all vector's elements valid. -// vector must not be a DictionaryVector or a SequenceVector -void duckdb_vx_vector_set_all_valid(duckdb_vector ffi_vector); - -// Set the data pointer for the vector. This is the start of the values array in the vector. -void duckdb_vx_vector_set_data_ptr(duckdb_vector ffi_vector, void *ptr); - // Set the validity pointer for the vector to external data, and store the buffer in auxiliary // to keep it alive. The validity pointer is derived from data_ptr at the given u64 offset. // The buffer is attached purely as a keep-alive. This enables zero-copy export of validity masks. @@ -62,15 +62,6 @@ void duckdb_vx_vector_set_validity_data(duckdb_vector ffi_vector, duckdb_vx_vector_buffer buffer, void *data_ptr); -// Converts a duckdb flat vector into a Sequence vector. -void duckdb_vx_sequence_vector(duckdb_vector c_vector, int64_t start, int64_t step, idx_t capacity); - -void duckdb_vector_flatten(duckdb_vector vector, unsigned long len); - -const char *duckdb_vector_to_string(duckdb_vector vector, unsigned long len, duckdb_vx_error *err); - -duckdb_value duckdb_vx_vector_get_value(duckdb_vector ffi_vector, idx_t index); - -#ifdef __cplusplus /* End C ABI */ +#ifdef __cplusplus } #endif diff --git a/vortex-duckdb/cpp/include/vector.hpp b/vortex-duckdb/cpp/include/vector.hpp new file mode 100644 index 00000000000..f59d51ef2e5 --- /dev/null +++ b/vortex-duckdb/cpp/include/vector.hpp @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#pragma once + +#include "data.hpp" +#include "duckdb/common/types/vector_buffer.hpp" + +// A DuckDB vector buffer that keeps externally-owned data alive for the +// lifetime of the vector. +class ExternalVectorBuffer final : public duckdb::VectorBuffer { + duckdb::unique_ptr data; + +public: + explicit inline ExternalVectorBuffer(duckdb::unique_ptr data) : data(std::move(data)) { + } +}; diff --git a/vortex-duckdb/cpp/include/vortex_duckdb.h b/vortex-duckdb/cpp/include/vortex_duckdb.h new file mode 100644 index 00000000000..3e2a5e958dc --- /dev/null +++ b/vortex-duckdb/cpp/include/vortex_duckdb.h @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#pragma once + +#include "duckdb.h" +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct duckdb_vx_error_ *duckdb_vx_error; +typedef struct duckdb_vx_data_ *duckdb_vx_data; +typedef struct duckdb_vx_reusable_dict_ *duckdb_vx_reusable_dict; + +//! Create a DuckDB vortex error. +duckdb_vx_error duckdb_vx_error_create(const char *message, size_t message_length); + +// Borrows the message owned by the err type. +const char *duckdb_vx_error_value(duckdb_vx_error err); + +void duckdb_vx_error_free(duckdb_vx_error err); + +// Create an opaque data object with a delete callback. +duckdb_vx_data duckdb_vx_data_create(void *data_ptr, duckdb_delete_callback_t delete_callback); + +/// Convert a DuckDB value to a string representation. +/// The returned string must be freed with duckdb_free. +char *duckdb_vx_value_to_string(duckdb_value value); + +const char *duckdb_data_chunk_to_string(duckdb_data_chunk chunk, duckdb_vx_error *err); + +void duckdb_data_chunk_verify(duckdb_data_chunk chunk, duckdb_vx_error *err); + +char *duckdb_vx_logical_type_stringify(duckdb_logical_type ty); +duckdb_logical_type duckdb_vx_logical_type_copy(duckdb_logical_type ty); + +/// Creates a GEOMETRY logical type with the given CRS (Coordinate Reference System). +/// `crs` must be a NUL-terminated UTF-8 string. Pass an empty string for no CRS. +duckdb_logical_type duckdb_vx_create_geometry(const char *crs); + +duckdb_state duckdb_vx_register_scan_replacement(duckdb_database duckdb_database); + +/// Creates a new reusable dictionary from a logical type and size. +duckdb_vx_reusable_dict duckdb_vx_reusable_dict_create(duckdb_logical_type logical_type, idx_t size); + +/// Destroys the reusable dictionary. +void duckdb_vx_reusable_dict_destroy(duckdb_vx_reusable_dict *dict); + +/// Clones the reusable dictionary. +duckdb_vx_reusable_dict duckdb_vx_reusable_dict_clone(duckdb_vx_reusable_dict dict); + +/// Get the internal vector of the reusable dictionary. +void duckdb_vx_reusable_dict_set_vector(duckdb_vx_reusable_dict reusable, duckdb_vector *out_vector); + +/// Creates a dictionary vector using a reusable dictionary and a selection vector. +void duckdb_vx_vector_dictionary_reusable(duckdb_vector vector, + duckdb_vx_reusable_dict reusable, + duckdb_selection_vector sel_vec); + +// Create a null value with a reference to a logical type. +duckdb_value duckdb_vx_value_create_null(duckdb_logical_type ty); + +/// Creates a GEOMETRY value containing the given WKB bytes and CRS. +duckdb_value duckdb_vx_value_create_geometry(const uint8_t *wkb, idx_t len, const char *crs); + +/// Extracts the raw WKB bytes from a GEOMETRY value as a duckdb_blob. +/// The returned `data` pointer must be freed with `duckdb_free`. Returns `{nullptr, 0}` +/// if `value` is null or not a GEOMETRY value. +duckdb_blob duckdb_vx_value_get_geometry(duckdb_value value); + +#ifdef __cplusplus +} +#endif diff --git a/vortex-duckdb/cpp/logical_type.cpp b/vortex-duckdb/cpp/logical_type.cpp deleted file mode 100644 index d21896ddc4d..00000000000 --- a/vortex-duckdb/cpp/logical_type.cpp +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#include "duckdb_vx/logical_type.h" -#include "duckdb/common/types.hpp" -#include -#include - -duckdb_logical_type duckdb_vx_logical_type_copy(duckdb_logical_type ty) { - D_ASSERT(ty); - auto *src = reinterpret_cast(ty); - auto copy = duckdb::make_uniq(*src); - return reinterpret_cast(copy.release()); -} - -char *duckdb_vx_logical_type_stringify(duckdb_logical_type c_type) { - auto type = reinterpret_cast(c_type); - auto str = type->ToString(); - auto result = static_cast(duckdb_malloc(str.size() + 1)); - memcpy(result, str.c_str(), str.size() + 1); - return result; -} - -duckdb_logical_type duckdb_vx_create_geometry(const char *crs) { - D_ASSERT(crs); - auto geom = - (*crs == '\0') ? duckdb::LogicalType::GEOMETRY() : duckdb::LogicalType::GEOMETRY(std::string(crs)); - auto copy = duckdb::make_uniq(std::move(geom)); - return reinterpret_cast(copy.release()); -} diff --git a/vortex-duckdb/cpp/optimizer.cpp b/vortex-duckdb/cpp/optimizer.cpp index ef647fe7dbb..645c621b201 100644 --- a/vortex-duckdb/cpp/optimizer.cpp +++ b/vortex-duckdb/cpp/optimizer.cpp @@ -1,10 +1,11 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +#include "optimizer.hpp" +#include "table_function.hpp" #include "duckdb/catalog/catalog.hpp" +#include "duckdb/main/config.hpp" +#include "duckdb/main/capi/capi_internal.hpp" #include "duckdb/planner/operator/logical_projection.hpp" -#include "duckdb_vx/optimizer.h" -#include "duckdb_vx/table_function.h" -#include "vortex.h" #include /** diff --git a/vortex-duckdb/cpp/replacement_scan.cpp b/vortex-duckdb/cpp/replacement_scan.cpp deleted file mode 100644 index 1dc82ea3c3d..00000000000 --- a/vortex-duckdb/cpp/replacement_scan.cpp +++ /dev/null @@ -1,52 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#include "duckdb/main/capi/capi_internal.hpp" -#include "duckdb/parser/expression/constant_expression.hpp" -#include "duckdb/parser/expression/function_expression.hpp" -#include "duckdb/parser/tableref/table_function_ref.hpp" - -#include "duckdb_vx.h" - -namespace vortex { - -static duckdb::unique_ptr -VortexScanReplacement(duckdb::ClientContext &context, - duckdb::ReplacementScanInput &input, - duckdb::optional_ptr /*data*/) { - auto table_name = duckdb::ReplacementScan::GetFullPath(input); - if (!duckdb::ReplacementScan::CanReplace(table_name, {"vortex"})) { - return nullptr; - } - auto table_function = duckdb::make_uniq(); - - duckdb::vector> children(1); - children[0] = duckdb::make_uniq(duckdb::Value(table_name)); - table_function->function = - duckdb::make_uniq("read_vortex", std::move(children)); - - if (!duckdb::FileSystem::HasGlob(table_name)) { - auto &fs = duckdb::FileSystem::GetFileSystem(context); - table_function->alias = fs.ExtractBaseName(table_name); - } - - return table_function; -} - -} // namespace vortex - -extern "C" duckdb_state duckdb_vx_register_scan_replacement(duckdb_database duckdb_database) { - if (!duckdb_database) { - return DuckDBError; - } - - auto wrapper = reinterpret_cast(duckdb_database); - if (!wrapper) { - return DuckDBError; - } - - auto &config = duckdb::DBConfig::GetConfig(*wrapper->database->instance); - config.replacement_scans.emplace_back(vortex::VortexScanReplacement); - - return DuckDBSuccess; -} diff --git a/vortex-duckdb/cpp/reusable_dict.cpp b/vortex-duckdb/cpp/reusable_dict.cpp deleted file mode 100644 index d72854eaa6b..00000000000 --- a/vortex-duckdb/cpp/reusable_dict.cpp +++ /dev/null @@ -1,50 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#include "duckdb/common/types/vector.hpp" -#include "duckdb_vx.h" - -using namespace duckdb; - -// buffer_ptr is shared_ptr, two pointers long, but duckdb_vx_reusable_dict is -// one pointer long, so we need a wrapper. -using Buffer = buffer_ptr; -struct ReusableDict { - Buffer buffer; - ReusableDict(Buffer buffer) : buffer(std::move(buffer)) { - } -}; - -extern "C" duckdb_vx_reusable_dict duckdb_vx_reusable_dict_create(duckdb_logical_type ffi_type, idx_t size) { - const LogicalType &type = *reinterpret_cast(ffi_type); - auto buffer = DictionaryVector::CreateReusableDictionary(type, size); - auto ptr = std::make_unique(std::move(buffer)); - return reinterpret_cast(ptr.release()); -} - -extern "C" void duckdb_vx_reusable_dict_destroy(duckdb_vx_reusable_dict *dict) { - if (dict && *dict) { - delete reinterpret_cast(*dict); - } -} - -extern "C" duckdb_vx_reusable_dict duckdb_vx_reusable_dict_clone(duckdb_vx_reusable_dict dict) { - ReusableDict *wrapper = reinterpret_cast(dict); - auto ptr = std::make_unique(wrapper->buffer); - return reinterpret_cast(ptr.release()); -} - -extern "C" void duckdb_vx_reusable_dict_set_vector(duckdb_vx_reusable_dict reusable, - duckdb_vector *out_vector) { - auto *wrapper = reinterpret_cast(reusable); - *out_vector = reinterpret_cast(&wrapper->buffer->data); -} - -extern "C" void duckdb_vx_vector_dictionary_reusable(duckdb_vector ffi_vector, - duckdb_vx_reusable_dict reusable, - duckdb_selection_vector ffi_sel_vec) { - auto vector = reinterpret_cast(ffi_vector); - auto *wrapper = reinterpret_cast(reusable); - auto sel_vec = reinterpret_cast(ffi_sel_vec); - vector->Dictionary(wrapper->buffer, *sel_vec); -} diff --git a/vortex-duckdb/cpp/scalar_function.cpp b/vortex-duckdb/cpp/scalar_function.cpp deleted file mode 100644 index 5c692e1b090..00000000000 --- a/vortex-duckdb/cpp/scalar_function.cpp +++ /dev/null @@ -1,24 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#include "duckdb/function/scalar_function.hpp" - -#include "duckdb_vx.h" - -using namespace duckdb; - -extern "C" const char *duckdb_vx_sfunc_name(duckdb_vx_sfunc ffi_func) { - if (!ffi_func) { - return nullptr; - } - auto func = reinterpret_cast(ffi_func); - return func->name.c_str(); -} - -extern "C" duckdb_logical_type duckdb_vx_sfunc_return_type(duckdb_vx_sfunc ffi_func) { - if (!ffi_func) { - return nullptr; - } - auto func = reinterpret_cast(ffi_func); - return reinterpret_cast(&func->return_type); -} diff --git a/vortex-duckdb/cpp/table_filter.cpp b/vortex-duckdb/cpp/table_filter.cpp index bc52bfe42c8..27630cc7c5b 100644 --- a/vortex-duckdb/cpp/table_filter.cpp +++ b/vortex-duckdb/cpp/table_filter.cpp @@ -1,7 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#include "duckdb_vx.h" +#include "table_filter.h" +#include #include "duckdb/planner/table_filter.hpp" #include "duckdb/planner/filter/conjunction_filter.hpp" @@ -75,12 +76,14 @@ extern "C" void duckdb_vx_table_filter_get_conjunction_and(duckdb_vx_table_filte } // Wrapper to hold the shared pointer for dynamic filter data. +namespace { struct DynamicFilterDataWrapper { shared_ptr data; explicit DynamicFilterDataWrapper(shared_ptr d) : data(std::move(d)) { } }; +} // namespace extern "C" void duckdb_vx_table_filter_get_dynamic(duckdb_vx_table_filter ffi_filter, duckdb_vx_table_filter_dynamic *out) { @@ -92,7 +95,7 @@ extern "C" void duckdb_vx_table_filter_get_dynamic(duckdb_vx_table_filter ffi_fi // Hold the lock while accessing the filter data. std::lock_guard lock(filter.filter_data->lock); - auto data_wrapper = duckdb::make_uniq(filter.filter_data); + auto data_wrapper = make_uniq(filter.filter_data); out->data = reinterpret_cast(data_wrapper.release()); out->comparison_type = static_cast(filter.filter_data->filter->comparison_type); } diff --git a/vortex-duckdb/cpp/table_function.cpp b/vortex-duckdb/cpp/table_function.cpp index f18557a2d11..e037b2de6d1 100644 --- a/vortex-duckdb/cpp/table_function.cpp +++ b/vortex-duckdb/cpp/table_function.cpp @@ -1,10 +1,11 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#include "duckdb_vx/data.hpp" -#include "duckdb_vx/error.hpp" -#include "duckdb_vx/table_function.h" -#include "duckdb_vx/expr.h" +#include "data.hpp" +#include "error.hpp" +#include "table_function.hpp" +#include "vortex_duckdb.h" +#include "table_function.h" #include "vortex.h" #include "duckdb.h" @@ -19,8 +20,6 @@ using namespace std::string_literals; using namespace duckdb; -using vortex::CData; -using vortex::IntoErrString; constexpr column_t COLUMN_IDENTIFIER_FILE_INDEX = MultiFileReader::COLUMN_IDENTIFIER_FILE_INDEX; constexpr column_t COLUMN_IDENTIFIER_FILE_ROW_NUMBER = MultiFileReader::COLUMN_IDENTIFIER_FILE_ROW_NUMBER; diff --git a/vortex-duckdb/cpp/value.cpp b/vortex-duckdb/cpp/value.cpp deleted file mode 100644 index 9a3154a43a0..00000000000 --- a/vortex-duckdb/cpp/value.cpp +++ /dev/null @@ -1,40 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#include "duckdb/common/types/geometry_crs.hpp" -#include "duckdb/common/types/value.hpp" - -#include "duckdb_vx.h" - -extern "C" duckdb_value duckdb_vx_value_create_null(duckdb_logical_type ty) { - const auto logical_type = reinterpret_cast(ty); - auto value = duckdb::make_uniq(*logical_type); - return reinterpret_cast(value.release()); -} - -extern "C" duckdb_value duckdb_vx_value_create_geometry(const uint8_t *wkb, idx_t len, const char *crs) { - const auto bytes = reinterpret_cast(wkb); - auto value = - (crs == nullptr || *crs == '\0') - ? duckdb::Value::GEOMETRY(bytes, len) - : duckdb::Value::GEOMETRY(bytes, len, duckdb::CoordinateReferenceSystem(std::string(crs))); - auto owned = duckdb::make_uniq(std::move(value)); - return reinterpret_cast(owned.release()); -} - -extern "C" duckdb_blob duckdb_vx_value_get_geometry(duckdb_value value) { - if (value == nullptr) { - return {nullptr, 0}; - } - const auto val = reinterpret_cast(value); - if (val->type().id() != duckdb::LogicalTypeId::GEOMETRY) { - return {nullptr, 0}; - } - const auto &str = duckdb::StringValue::Get(*val); - const auto size = str.size(); - auto buf = reinterpret_cast(duckdb_malloc(size)); - if (size > 0) { - memcpy(buf, str.c_str(), size); - } - return {buf, size}; -} diff --git a/vortex-duckdb/cpp/vector.cpp b/vortex-duckdb/cpp/vector.cpp index 5ede3e4fd4b..b6666ff6dcb 100644 --- a/vortex-duckdb/cpp/vector.cpp +++ b/vortex-duckdb/cpp/vector.cpp @@ -1,13 +1,12 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -#include "include/duckdb_vx/vector.h" -#include "duckdb/common/vector.hpp" +#include "vector.h" +#include "vector.hpp" + #include "duckdb/common/types/value.hpp" #include "duckdb/common/types/vector.hpp" -#include "duckdb_vx/vector_buffer.hpp" - using namespace duckdb; extern "C" duckdb_vector duckdb_vx_vector_slice(duckdb_vector ffi_vector, idx_t offset, idx_t end) { @@ -16,14 +15,6 @@ extern "C" duckdb_vector duckdb_vx_vector_slice(duckdb_vector ffi_vector, idx_t return reinterpret_cast(sliced); } -extern "C" void duckdb_vx_vector_slice_to_dictionary(duckdb_vector ffi_vector, - duckdb_selection_vector ffi_sel_vec, - idx_t selection_vector_length) { - auto vector = reinterpret_cast(ffi_vector); - auto sel_vec = reinterpret_cast(ffi_sel_vec); - vector->Slice(*sel_vec, selection_vector_length); -} - extern "C" void duckdb_vx_vector_dictionary(duckdb_vector ffi_vector, duckdb_vector ffi_dict, idx_t dictionary_size, @@ -46,10 +37,10 @@ duckdb_vx_sequence_vector(duckdb_vector c_vector, int64_t start, int64_t step, i vector->Sequence(start, step, capacity); } -namespace vortex { - // This is a complete hack to access the data buffer and pointer of a vector. -class DataVector : public Vector { +// Duckdb passes us Vectors and not VortexVectors. This only works because +// VortexVector doesn't add any members. +class VortexVector final : public Vector { public: inline void SetDataBuffer(buffer_ptr new_buffer) { buffer = std::move(new_buffer); @@ -63,9 +54,10 @@ class DataVector : public Vector { return validity; }; }; +static_assert(sizeof(VortexVector) == sizeof(Vector)); // Same hack for ValidityMask: access protected fields via inheritance. -class ExternalValidityMask : public ValidityMask { +class ExternalValidityMask final : public ValidityMask { public: inline void SetExternal(idx_t u64_offset, idx_t cap, buffer_ptr keeper) { validity_data = std::move(keeper); @@ -74,27 +66,26 @@ class ExternalValidityMask : public ValidityMask { capacity = cap; }; }; - -} // namespace vortex +static_assert(sizeof(ExternalValidityMask) == sizeof(ValidityMask)); extern "C" void duckdb_vx_string_vector_add_vector_data_buffer(duckdb_vector ffi_vector, duckdb_vx_vector_buffer buffer) { auto vector = reinterpret_cast(ffi_vector); - auto data = reinterpret_cast *>(buffer); + auto data = reinterpret_cast *>(buffer); StringVector::AddBuffer(*vector, *data); } extern "C" void duckdb_vx_vector_set_vector_data_buffer(duckdb_vector ffi_vector, duckdb_vx_vector_buffer buffer) { auto vector = reinterpret_cast(ffi_vector); - auto dvector = reinterpret_cast(vector); - auto data = reinterpret_cast *>(buffer); + auto dvector = reinterpret_cast(vector); + auto data = reinterpret_cast *>(buffer); dvector->SetDataBuffer(*data); } extern "C" void duckdb_vx_vector_set_data_ptr(duckdb_vector ffi_vector, void *ptr) { auto vector = reinterpret_cast(ffi_vector); - auto dvector = reinterpret_cast(vector); + auto dvector = reinterpret_cast(vector); dvector->SetDataPtr((data_ptr_t)ptr); } @@ -103,16 +94,16 @@ extern "C" void duckdb_vx_vector_set_validity_data(duckdb_vector ffi_vector, idx_t capacity, duckdb_vx_vector_buffer buffer, void *data_ptr) { - auto dvector = reinterpret_cast(ffi_vector); + auto dvector = reinterpret_cast(ffi_vector); auto &validity = dvector->GetValidity(); // ExternalValidityMask adds no members, so this downcast only exposes // access to ValidityMask's protected fields. - auto ext_validity = static_cast(&validity); + auto ext_validity = static_cast(&validity); // Use the shared_ptr aliasing constructor: the control block ref-counts the // ExternalVectorBuffer (preventing the Rust buffer from being freed), // while the stored pointer points to the explicit data_ptr. - auto ext_buf = reinterpret_cast *>(buffer); + auto ext_buf = reinterpret_cast *>(buffer); auto keeper = shared_ptr>( *ext_buf, reinterpret_cast *>(data_ptr)); @@ -132,21 +123,6 @@ void duckdb_vector_flatten(duckdb_vector vector, unsigned long len) { dvector->Flatten(len); } -const char *duckdb_vector_to_string(duckdb_vector vector, unsigned long len, duckdb_vx_error *err) { - try { - auto dvector = reinterpret_cast(vector); - auto str = dvector->ToString(len); - auto result = static_cast(duckdb_malloc(str.size() + 1)); - memcpy(result, str.c_str(), str.size() + 1); - *err = nullptr; - return result; - } catch (std::runtime_error &e) { - auto s = e.what(); - *err = duckdb_vx_error_create(s, strlen(s)); - return nullptr; - } -} - void duckdb_vx_vector_set_all_valid(duckdb_vector ffi_vector) { using enum VectorType; Vector &vector = *reinterpret_cast(ffi_vector); @@ -163,3 +139,18 @@ void duckdb_vx_vector_set_all_valid(duckdb_vector ffi_vector) { __builtin_unreachable(); } } + +extern "C" duckdb_vx_vector_buffer duckdb_vx_vector_buffer_create(duckdb_vx_data buffer) { + auto data = reinterpret_cast(buffer); + auto *shared_buffer = + new shared_ptr(make_shared_ptr(unique_ptr(data))); + return reinterpret_cast(shared_buffer); +} + +extern "C" void duckdb_vx_vector_buffer_destroy(duckdb_vx_vector_buffer *buffer) { + if (buffer != nullptr && *buffer != nullptr) { + auto shared_buffer = reinterpret_cast *>(*buffer); + delete shared_buffer; + *buffer = nullptr; + } +} diff --git a/vortex-duckdb/cpp/vector_buffer.cpp b/vortex-duckdb/cpp/vector_buffer.cpp deleted file mode 100644 index 908db60b467..00000000000 --- a/vortex-duckdb/cpp/vector_buffer.cpp +++ /dev/null @@ -1,26 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -#include "duckdb/common/vector.hpp" -#include "duckdb/common/types/vector.hpp" - -#include "duckdb_vx.h" -#include "duckdb_vx/data.hpp" -#include "duckdb_vx/vector_buffer.hpp" - -using namespace duckdb; - -extern "C" duckdb_vx_vector_buffer duckdb_vx_vector_buffer_create(duckdb_vx_data buffer) { - auto data = reinterpret_cast(buffer); - auto *shared_buffer = new duckdb::shared_ptr( - duckdb::make_shared_ptr(unique_ptr(data))); - return reinterpret_cast(shared_buffer); -} - -extern "C" void duckdb_vx_vector_buffer_destroy(duckdb_vx_vector_buffer *buffer) { - if (buffer != nullptr && *buffer != nullptr) { - auto shared_buffer = reinterpret_cast *>(*buffer); - delete shared_buffer; - *buffer = nullptr; - } -} diff --git a/vortex-duckdb/cpp/vortex_duckdb.cpp b/vortex-duckdb/cpp/vortex_duckdb.cpp new file mode 100644 index 00000000000..1f34cb8c2d2 --- /dev/null +++ b/vortex-duckdb/cpp/vortex_duckdb.cpp @@ -0,0 +1,265 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +#include "data.hpp" +#include "error.hpp" +#include "vortex_duckdb.h" + +#include "duckdb/common/assert.hpp" +#include "duckdb/common/types.hpp" +#include "duckdb/common/types/data_chunk.hpp" +#include "duckdb/common/types/geometry_crs.hpp" +#include "duckdb/common/types/value.hpp" +#include "duckdb/common/types/vector.hpp" +#include "duckdb/main/capi/capi_internal.hpp" +#include "duckdb/main/client_context.hpp" +#include "duckdb/main/config.hpp" +#include "duckdb/main/connection.hpp" +#include "duckdb/parser/expression/constant_expression.hpp" +#include "duckdb/parser/expression/function_expression.hpp" +#include "duckdb/parser/tableref/table_function_ref.hpp" + +#include +#include + +using namespace duckdb; + +extern "C" char *duckdb_vx_value_to_string(duckdb_value value) { + if (!value) { + return nullptr; + } + + try { + // Cast the value to DuckDB's internal Value type + auto *ddb_value = reinterpret_cast(value); + + if (!ddb_value) { + return nullptr; + } + + // Use the ToString method to get the string representation + std::string str_value = ddb_value->ToString(); + + size_t str_len = str_value.length() + 1; + char *result = static_cast(duckdb_malloc(str_len)); + if (!result) { + return nullptr; + } + + // Copy the string and null terminate + std::memcpy(result, str_value.c_str(), str_len); + return result; + + } catch (...) { + return nullptr; + } +} + +CData::CData(void *data_ptr, duckdb_delete_callback_t callback) : data(data_ptr), delete_callback(callback) { +} + +CData::~CData() { + if (data && delete_callback) { + delete_callback(data); + } + data = nullptr; + delete_callback = nullptr; +} + +void *CData::DataPtr() const { + return data; +} + +extern "C" duckdb_vx_data duckdb_vx_data_create(void *data, duckdb_delete_callback_t delete_callback) { + return reinterpret_cast(new CData(data, delete_callback)); +} + +extern "C" const char *duckdb_data_chunk_to_string(duckdb_data_chunk chunk, duckdb_vx_error *err) { + try { + auto dchunk = reinterpret_cast(chunk); + auto str = dchunk->ToString(); + auto result = static_cast(duckdb_malloc(str.size() + 1)); + memcpy(result, str.c_str(), str.size() + 1); + *err = nullptr; + return result; + } catch (std::runtime_error &e) { + auto s = e.what(); + *err = duckdb_vx_error_create(s, strlen(s)); + return nullptr; + } +} + +extern "C" void duckdb_data_chunk_verify(duckdb_data_chunk chunk, duckdb_vx_error *err) { + try { + auto dchunk = reinterpret_cast(chunk); + dchunk->Verify(); + *err = nullptr; + } catch (std::runtime_error &e) { + auto s = e.what(); + *err = duckdb_vx_error_create(s, strlen(s)); + } +} + +extern "C" duckdb_vx_error duckdb_vx_error_create(const char *message, size_t message_length) { + return reinterpret_cast(new std::string(message, message_length)); +} + +extern "C" const char *duckdb_vx_error_value(duckdb_vx_error err) { + auto str = reinterpret_cast(err); + return str->c_str(); +} + +extern "C" void duckdb_vx_error_free(duckdb_vx_error err) { + auto str = reinterpret_cast(err); + delete str; +} + +std::string IntoErrString(duckdb_vx_error error) { + if (!error) { + return {}; + } + std::string *const error_str = reinterpret_cast(error); + std::string out = std::move(*error_str); + duckdb_vx_error_free(error); + return out; +} + +duckdb_state SetError(duckdb_vx_error *error_out, std::string_view message) { + D_ASSERT(error_out != nullptr && "SetError called with null error_out"); + *error_out = duckdb_vx_error_create(message.data(), message.size()); + return DuckDBError; +} + +extern "C" duckdb_logical_type duckdb_vx_logical_type_copy(duckdb_logical_type ty) { + D_ASSERT(ty); + auto *src = reinterpret_cast(ty); + auto copy = make_uniq(*src); + return reinterpret_cast(copy.release()); +} + +extern "C" char *duckdb_vx_logical_type_stringify(duckdb_logical_type c_type) { + auto type = reinterpret_cast(c_type); + auto str = type->ToString(); + auto result = static_cast(duckdb_malloc(str.size() + 1)); + memcpy(result, str.c_str(), str.size() + 1); + return result; +} + +extern "C" duckdb_logical_type duckdb_vx_create_geometry(const char *crs) { + D_ASSERT(crs); + auto geom = (*crs == '\0') ? LogicalType::GEOMETRY() : LogicalType::GEOMETRY(std::string(crs)); + auto copy = make_uniq(std::move(geom)); + return reinterpret_cast(copy.release()); +} + +static unique_ptr VortexScanReplacement(ClientContext &context, + ReplacementScanInput &input, + optional_ptr) { + auto table_name = ReplacementScan::GetFullPath(input); + if (!ReplacementScan::CanReplace(table_name, {"vortex"})) { + return nullptr; + } + auto table_function = make_uniq(); + + vector> children(1); + children[0] = make_uniq(Value(table_name)); + table_function->function = make_uniq("read_vortex", std::move(children)); + + if (!FileSystem::HasGlob(table_name)) { + auto &fs = FileSystem::GetFileSystem(context); + table_function->alias = fs.ExtractBaseName(table_name); + } + + return table_function; +} + +extern "C" duckdb_state duckdb_vx_register_scan_replacement(duckdb_database duckdb_database) { + if (!duckdb_database) { + return DuckDBError; + } + + auto wrapper = reinterpret_cast(duckdb_database); + if (!wrapper) { + return DuckDBError; + } + + auto &config = DBConfig::GetConfig(*wrapper->database->instance); + config.replacement_scans.emplace_back(VortexScanReplacement); + + return DuckDBSuccess; +} + +// buffer_ptr is shared_ptr, two pointers long, but duckdb_vx_reusable_dict is +// one pointer long, so we need a wrapper. +using Buffer = buffer_ptr; +struct ReusableDict { + Buffer buffer; + ReusableDict(Buffer buffer) : buffer(std::move(buffer)) { + } +}; + +extern "C" duckdb_vx_reusable_dict duckdb_vx_reusable_dict_create(duckdb_logical_type ffi_type, idx_t size) { + const LogicalType &type = *reinterpret_cast(ffi_type); + auto buffer = DictionaryVector::CreateReusableDictionary(type, size); + auto ptr = std::make_unique(std::move(buffer)); + return reinterpret_cast(ptr.release()); +} + +extern "C" void duckdb_vx_reusable_dict_destroy(duckdb_vx_reusable_dict *dict) { + if (dict && *dict) { + delete reinterpret_cast(*dict); + } +} + +extern "C" duckdb_vx_reusable_dict duckdb_vx_reusable_dict_clone(duckdb_vx_reusable_dict dict) { + ReusableDict *wrapper = reinterpret_cast(dict); + auto ptr = std::make_unique(wrapper->buffer); + return reinterpret_cast(ptr.release()); +} + +extern "C" void duckdb_vx_reusable_dict_set_vector(duckdb_vx_reusable_dict reusable, + duckdb_vector *out_vector) { + auto *wrapper = reinterpret_cast(reusable); + *out_vector = reinterpret_cast(&wrapper->buffer->data); +} + +extern "C" void duckdb_vx_vector_dictionary_reusable(duckdb_vector ffi_vector, + duckdb_vx_reusable_dict reusable, + duckdb_selection_vector ffi_sel_vec) { + auto vector = reinterpret_cast(ffi_vector); + auto *wrapper = reinterpret_cast(reusable); + auto sel_vec = reinterpret_cast(ffi_sel_vec); + vector->Dictionary(wrapper->buffer, *sel_vec); +} + +extern "C" duckdb_value duckdb_vx_value_create_null(duckdb_logical_type ty) { + const auto logical_type = reinterpret_cast(ty); + auto value = make_uniq(*logical_type); + return reinterpret_cast(value.release()); +} + +extern "C" duckdb_value duckdb_vx_value_create_geometry(const uint8_t *wkb, idx_t len, const char *crs) { + const auto bytes = reinterpret_cast(wkb); + auto value = (crs == nullptr || *crs == '\0') + ? Value::GEOMETRY(bytes, len) + : Value::GEOMETRY(bytes, len, CoordinateReferenceSystem(std::string(crs))); + auto owned = make_uniq(std::move(value)); + return reinterpret_cast(owned.release()); +} + +extern "C" duckdb_blob duckdb_vx_value_get_geometry(duckdb_value value) { + if (value == nullptr) { + return {nullptr, 0}; + } + const auto val = reinterpret_cast(value); + if (val->type().id() != LogicalTypeId::GEOMETRY) { + return {nullptr, 0}; + } + const auto &str = StringValue::Get(*val); + const auto size = str.size(); + auto buf = reinterpret_cast(duckdb_malloc(size)); + if (size > 0) { + memcpy(buf, str.c_str(), size); + } + return {buf, size}; +} diff --git a/vortex-duckdb/src/duckdb/client_context.rs b/vortex-duckdb/src/duckdb/client_context.rs deleted file mode 100644 index 4cb646b82c5..00000000000 --- a/vortex-duckdb/src/duckdb/client_context.rs +++ /dev/null @@ -1,48 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use std::ffi::CStr; - -use crate::cpp; -use crate::duckdb::Value; -use crate::lifetime_wrapper; - -lifetime_wrapper!( - /// A DuckDB client context wrapper. - ClientContext, - cpp::duckdb_client_context, - cpp::duckdb_destroy_client_context -); - -// SAFETY: ClientContext carries an opaque pointer. It is safe to send/share across threads -// under the same guarantees: the underlying DuckDB context is valid for the connection -// lifetime and DuckDB synchronizes internal state. -unsafe impl Send for ClientContextRef {} -unsafe impl Sync for ClientContextRef {} - -impl ClientContextRef { - /// Erases the lifetime of this reference, returning a `&'static ClientContextRef`. - /// - /// # Safety - /// - /// The caller must ensure that the underlying `ClientContext` outlives all uses of the - /// returned reference. In practice, the `ClientContext` is owned by the `Connection` - /// and lives as long as the connection, so this is safe as long as the connection is kept alive. - pub unsafe fn erase_lifetime(&self) -> &'static Self { - unsafe { &*(self as *const Self) } - } - - /// Try to get the current value of a configuration setting. - /// Returns None if the setting doesn't exist. - pub fn try_get_current_setting(&self, key: &CStr) -> Option { - unsafe { - let value_ptr = - cpp::duckdb_client_context_try_get_current_setting(self.as_ptr(), key.as_ptr()); - if value_ptr.is_null() { - None - } else { - Some(Value::own(value_ptr)) - } - } - } -} diff --git a/vortex-duckdb/src/duckdb/config.rs b/vortex-duckdb/src/duckdb/config.rs deleted file mode 100644 index 710306b8b7a..00000000000 --- a/vortex-duckdb/src/duckdb/config.rs +++ /dev/null @@ -1,296 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: Copyright the Vortex contributors - -use std::ffi::CStr; -use std::ffi::CString; -use std::ffi::c_void; -use std::os::raw::c_char; -use std::ptr; - -use vortex::error::VortexResult; -use vortex::error::vortex_err; - -use crate::cpp; -use crate::duckdb::DatabaseRef; -use crate::duckdb::Value; -use crate::duckdb_try; -use crate::lifetime_wrapper; - -lifetime_wrapper!( - /// A DuckDB configuration instance. - Config, - cpp::duckdb_config, - cpp::duckdb_destroy_config -); - -impl Config { - /// Creates a new DuckDB configuration. - pub fn new() -> VortexResult { - let mut ptr: cpp::duckdb_config = ptr::null_mut(); - duckdb_try!( - unsafe { cpp::duckdb_create_config(&raw mut ptr) }, - "Failed to create DuckDB config" - ); - - Ok(unsafe { Self::own(ptr) }) - } -} - -impl ConfigRef { - /// Sets a key-value configuration parameter. - pub fn set(&mut self, key: &str, value: &str) -> VortexResult<()> { - let key_cstr = - CString::new(key).map_err(|_| vortex_err!("Invalid key: contains null bytes"))?; - let value_cstr = - CString::new(value).map_err(|_| vortex_err!("Invalid value: contains null bytes"))?; - - duckdb_try!( - unsafe { - cpp::duckdb_set_config(self.as_ptr(), key_cstr.as_ptr(), value_cstr.as_ptr()) - }, - "Failed to set config parameter '{}' to '{}'", - key, - value - ); - - Ok(()) - } - - /// Gets the value of a configuration parameter that was previously set. - /// Returns None if the parameter was never set on this Config instance. - pub fn get(&self, key: &str) -> Option { - let key_cstr = CString::new(key).ok()?; - - let mut value: cpp::duckdb_value = ptr::null_mut(); - let result = unsafe { - cpp::duckdb_vx_get_config_value(self.as_ptr(), key_cstr.as_ptr(), &raw mut value) - }; - - (result == cpp::duckdb_state::DuckDBSuccess && !value.is_null()) - .then(|| unsafe { Value::own(value) }) - } - - pub fn get_str(&self, key: &str) -> Option { - self.get(key).and_then(|value| { - let c_str = unsafe { cpp::duckdb_vx_value_to_string(value.as_ptr()) }; - - if !c_str.is_null() { - let rust_str = unsafe { CStr::from_ptr(c_str).to_string_lossy().into_owned() }; - - // Free the C string allocated by our function - unsafe { cpp::duckdb_free(c_str as *mut c_void) }; - - return Some(rust_str); - } - None - }) - } - - /// Checks if a configuration key has been set on this config instance. - pub fn has_key(&self, key: &str) -> bool { - let Ok(key_cstr) = CString::new(key) else { - return false; - }; - - let result = unsafe { cpp::duckdb_vx_config_has_key(self.as_ptr(), key_cstr.as_ptr()) }; - result == 1 - } - - /// Returns the number of configuration parameters available in DuckDB. - pub fn count() -> usize { - unsafe { cpp::duckdb_config_count() } - } - - /// Gets information about a configuration option by index. - /// Returns (name, description) if the index is valid, None otherwise. - pub fn get_config_flag(index: usize) -> VortexResult> { - let mut name_ptr: *const c_char = ptr::null(); - let mut desc_ptr: *const c_char = ptr::null(); - - let result = - unsafe { cpp::duckdb_get_config_flag(index, &raw mut name_ptr, &raw mut desc_ptr) }; - - if result != cpp::duckdb_state::DuckDBSuccess { - return Ok(None); - } - - let name = unsafe { CStr::from_ptr(name_ptr).to_string_lossy().into_owned() }; - let description = unsafe { CStr::from_ptr(desc_ptr).to_string_lossy().into_owned() }; - - Ok(Some((name, description))) - } - - /// Returns a list of all available configuration options. - pub fn list_available_options() -> VortexResult> { - let count = Self::count(); - let mut options = Vec::with_capacity(count); - - for i in 0..count { - if let Some((name, desc)) = Self::get_config_flag(i)? { - options.push((name, desc)); - } - } - - Ok(options) - } - - /// Add a new extension option. - pub fn add_extension_options( - &self, - name: &str, - description: &str, - logical_type: LogicalType, - default_value: Value, - ) -> VortexResult<()> { - let name_cstr = - CString::new(name).map_err(|_| vortex_err!("Invalid name: contains null bytes"))?; - let desc_cstr = CString::new(description) - .map_err(|_| vortex_err!("Invalid description: contains null bytes"))?; - - duckdb_try!(unsafe { - cpp::duckdb_vx_add_extension_option( - self.as_ptr(), - name_cstr.as_ptr(), - desc_cstr.as_ptr(), - logical_type.as_ptr(), - default_value.as_ptr(), - ) - }); - Ok(()) - } -} - -use crate::duckdb::LogicalType; - -impl DatabaseRef { - pub fn config(&self) -> &ConfigRef { - unsafe { Config::borrow(cpp::duckdb_vx_database_get_config(self.as_ptr())) } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::duckdb::Database; - - #[test] - fn test_config_creation() { - let config = Config::new(); - assert!(config.is_ok()); - } - - #[test] - fn test_config_set_and_get() { - let mut config = Config::new().unwrap(); - - // Set some values - assert!(config.set("memory_limit", "1GB").is_ok()); - assert!(config.set("threads", "2").is_ok()); - - // Verify they can be retrieved - assert_eq!(config.get_str("memory_limit"), Some("1GB".to_string())); - assert_eq!(config.get_str("threads"), Some("2".to_string())); - - // Non-existent key should return None - assert_eq!(config.get_str("nonexistent_key"), None); - } - - #[test] - fn test_config_get_all() { - let mut config = Config::new().unwrap(); - - assert!(config.set("memory_limit", "512MB").is_ok()); - assert!(config.set("threads", "4").is_ok()); - assert!(config.set("max_memory", "1GB").is_ok()); - - assert_eq!(config.get_str("memory_limit"), Some("512MB".to_string())); - assert_eq!(config.get_str("threads"), Some("4".to_string())); - assert_eq!(config.get_str("max_memory"), Some("1GB".to_string())); - } - - #[test] - fn test_config_update_value() { - let mut config = Config::new().unwrap(); - - // Set initial value - assert!(config.set("threads", "2").is_ok()); - assert_eq!(config.get_str("threads"), Some("2".to_string())); - - // Update the value - assert!(config.set("threads", "8").is_ok()); - assert_eq!(config.get_str("threads"), Some("8".to_string())); - } - - #[test] - fn test_config_persistence_through_database() { - // Create config with specific settings - let mut config = Config::new().unwrap(); - config.set("memory_limit", "256MB").unwrap(); - config.set("threads", "1").unwrap(); - - // Verify values are stored before using - assert_eq!(config.get_str("memory_limit"), Some("256MB".to_string())); - assert_eq!(config.get_str("threads"), Some("1".to_string())); - - // Use config to create database (this consumes the config) - let db = Database::open_in_memory_with_config(config); - assert!(db.is_ok()); - - // Verify database was created successfully - let conn = db.unwrap().connect(); - assert!(conn.is_ok()); - } - - #[test] - fn test_config_invalid_key() { - let mut config = Config::new().unwrap(); - let result = config.set("key\0with\0nulls", "value"); - assert!(result.is_err()); - } - - #[test] - fn test_config_invalid_value() { - let mut config = Config::new().unwrap(); - let result = config.set("key", "value\0with\0nulls"); - assert!(result.is_err()); - } - - #[test] - fn test_config_count() { - let count = ConfigRef::count(); - assert!(count > 0, "DuckDB should have configuration options"); - } - - #[test] - fn test_config_list_available_options() { - let options = ConfigRef::list_available_options(); - assert!(options.is_ok()); - - let options = options.unwrap(); - assert!(!options.is_empty(), "Should have available config options"); - - // Check that we got valid option names and descriptions - for (name, desc) in options.iter().take(5) { - assert!(!name.is_empty(), "Option name should not be empty"); - assert!(!desc.is_empty(), "Option description should not be empty"); - } - - println!("First few DuckDB config options:"); - for (name, desc) in options.iter().take(3) { - println!(" {}: {}", name, desc); - } - } - - #[test] - fn test_config_get_flag() { - // Test getting the first config option - let first_option = ConfigRef::get_config_flag(0); - assert!(first_option.is_ok()); - assert!(first_option.unwrap().is_some()); - - // Test getting an invalid index - let invalid_option = ConfigRef::get_config_flag(999999); - assert!(invalid_option.is_ok()); - assert!(invalid_option.unwrap().is_none()); - } -} diff --git a/vortex-duckdb/src/duckdb/connection.rs b/vortex-duckdb/src/duckdb/connection.rs index 05734c5e6a4..87a6c2df56e 100644 --- a/vortex-duckdb/src/duckdb/connection.rs +++ b/vortex-duckdb/src/duckdb/connection.rs @@ -5,12 +5,9 @@ use std::ffi::CStr; use std::ptr; use vortex::error::VortexResult; -use vortex::error::vortex_bail; use vortex::error::vortex_err; use crate::cpp; -use crate::duckdb::ClientContext; -use crate::duckdb::ClientContextRef; use crate::duckdb::DatabaseRef; use crate::duckdb::QueryResult; use crate::duckdb_try; @@ -60,20 +57,6 @@ impl ConnectionRef { Ok(unsafe { QueryResult::new(result) }) } - - /// Get the client context for this connection. - pub fn client_context(&self) -> VortexResult<&ClientContextRef> { - unsafe { - let client_context = cpp::duckdb_vx_connection_get_client_context(self.as_ptr()); - if client_context.is_null() { - vortex_bail!( - "Failed to get client context: connection={:p}", - self.as_ptr() - ) - } - Ok(ClientContext::borrow(client_context)) - } - } } #[cfg(test)] diff --git a/vortex-duckdb/src/duckdb/database.rs b/vortex-duckdb/src/duckdb/database.rs index 6ad9623c4e8..ab86503b291 100644 --- a/vortex-duckdb/src/duckdb/database.rs +++ b/vortex-duckdb/src/duckdb/database.rs @@ -2,17 +2,14 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use std::ffi::CString; -use std::os::raw::c_char; use std::path::Path; use std::ptr; use cpp::duckdb_database; use vortex::error::VortexResult; -use vortex::error::vortex_bail; use vortex::error::vortex_err; use crate::cpp; -use crate::duckdb::Config; use crate::duckdb::connection::Connection; use crate::duckdb_try; use crate::lifetime_wrapper; @@ -54,80 +51,6 @@ impl Database { ); Ok(unsafe { Self::own(ptr) }) } - - /// Opens a DuckDB database from a file path with custom configuration. - /// - /// Creates a new file in case the path does not exist. - pub fn open_with_config>(path: P, config: Config) -> VortexResult { - let path_str = path - .as_ref() - .to_str() - .ok_or_else(|| vortex_err!("Invalid path: path contains non-UTF8 characters"))?; - let path_cstr = CString::new(path_str) - .map_err(|_| vortex_err!("Invalid path: path contains null bytes"))?; - - let mut ptr: duckdb_database = ptr::null_mut(); - let mut error: *mut c_char = ptr::null_mut(); - - // duckdb_open_ext borrows the config (copies it internally), so we pass as_ptr() - // and let the Config drop naturally at the end of this function. - let result = unsafe { - cpp::duckdb_open_ext( - path_cstr.as_ptr(), - &raw mut ptr, - config.as_ptr(), - &raw mut error, - ) - }; - - if result != cpp::duckdb_state::DuckDBSuccess { - if !error.is_null() { - let error_msg = unsafe { - std::ffi::CStr::from_ptr(error) - .to_string_lossy() - .to_string() - }; - unsafe { cpp::duckdb_free(error.cast()) }; - vortex_bail!( - "Failed to open DuckDB database at path '{}': {}", - path_str, - error_msg - ); - } else { - vortex_bail!("Failed to open DuckDB database at path: {}", path_str); - } - } - - Ok(unsafe { Self::own(ptr) }) - } - - /// Opens an in-memory DuckDB database with custom configuration. - pub fn open_in_memory_with_config(config: Config) -> VortexResult { - let mut ptr: duckdb_database = ptr::null_mut(); - let mut error: *mut c_char = ptr::null_mut(); - - // duckdb_open_ext borrows the config (copies it internally), so we pass as_ptr() - // and let the Config drop naturally at the end of this function. - let result = unsafe { - cpp::duckdb_open_ext(ptr::null(), &raw mut ptr, config.as_ptr(), &raw mut error) - }; - - if result != cpp::duckdb_state::DuckDBSuccess { - if !error.is_null() { - let error_msg = unsafe { - std::ffi::CStr::from_ptr(error) - .to_string_lossy() - .to_string() - }; - unsafe { cpp::duckdb_free(error.cast()) }; - vortex_bail!("Failed to open in-memory DuckDB database: {}", error_msg); - } else { - vortex_bail!("Failed to open in-memory DuckDB database"); - } - } - - Ok(unsafe { Self::own(ptr) }) - } } impl DatabaseRef { @@ -168,35 +91,3 @@ impl DatabaseRef { Ok(()) } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_database_with_config() { - let mut config = Config::new().unwrap(); - config.set("memory_limit", "512MB").unwrap(); - config.set("threads", "1").unwrap(); - - let db = Database::open_in_memory_with_config(config); - assert!(db.is_ok()); - - let conn = db.unwrap().connect(); - assert!(conn.is_ok()); - } - - #[test] - fn test_file_database_with_config() { - let mut config = Config::new().unwrap(); - config.set("memory_limit", "256MB").unwrap(); - - let db = Database::open_with_config("test_config_unit.db", config); - assert!(db.is_ok()); - - let conn = db.unwrap().connect(); - assert!(conn.is_ok()); - - std::fs::remove_file("test_config_unit.db").ok(); - } -} diff --git a/vortex-duckdb/src/duckdb/ddb_string.rs b/vortex-duckdb/src/duckdb/ddb_string.rs index 6fbe12a8c4f..ee233f09dc9 100644 --- a/vortex-duckdb/src/duckdb/ddb_string.rs +++ b/vortex-duckdb/src/duckdb/ddb_string.rs @@ -6,8 +6,6 @@ use std::fmt::Display; use std::fmt::Formatter; use vortex::dtype::FieldName; -use vortex::error::VortexExpect; -use vortex::error::vortex_err; use crate::cpp::duckdb_free; use crate::lifetime_wrapper; @@ -19,21 +17,6 @@ lifetime_wrapper!( |ptr: &mut *mut std::ffi::c_char| unsafe { duckdb_free((*ptr).cast()) } ); -impl DDBString { - /// Creates an owned DDBString from a C string pointer, validating it is UTF-8. - /// - /// # Safety - /// - /// The pointer must be a valid, non-null, null-terminated C string allocated by DuckDB. - pub unsafe fn from_c_str(ptr: *mut std::ffi::c_char) -> Self { - unsafe { CStr::from_ptr(ptr) } - .to_str() - .map_err(|e| vortex_err!("Failed to convert C string to str: {e}")) - .vortex_expect("DuckDB string should be valid UTF-8"); - unsafe { Self::own(ptr) } - } -} - impl Display for DDBString { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.as_ref()) diff --git a/vortex-duckdb/src/duckdb/mod.rs b/vortex-duckdb/src/duckdb/mod.rs index c393df8dff7..27ea994b1e7 100644 --- a/vortex-duckdb/src/duckdb/mod.rs +++ b/vortex-duckdb/src/duckdb/mod.rs @@ -2,8 +2,6 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors mod bind_input; -mod client_context; -mod config; mod connection; mod data; mod data_chunk; @@ -27,8 +25,6 @@ use std::ffi::c_void; use std::ptr; pub use bind_input::*; -pub use client_context::*; -pub use config::*; pub use connection::*; pub use data::*; pub use data_chunk::*; diff --git a/vortex-duckdb/src/duckdb/query_result.rs b/vortex-duckdb/src/duckdb/query_result.rs index 02f24c8097d..c32c1debf0d 100644 --- a/vortex-duckdb/src/duckdb/query_result.rs +++ b/vortex-duckdb/src/duckdb/query_result.rs @@ -1,12 +1,9 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -use std::ffi::CStr; - use vortex::error::VortexExpect; +#[cfg(test)] use vortex::error::VortexResult; -use vortex::error::vortex_bail; -use vortex::error::vortex_err; use crate::cpp; use crate::cpp::DUCKDB_TYPE; @@ -58,8 +55,14 @@ impl QueryResultRef { } /// Get the name of a column by index. + #[cfg(test)] pub fn column_name(&self, col_idx: usize) -> VortexResult<&str> { unsafe { + use std::ffi::CStr; + + use vortex::error::vortex_bail; + use vortex::error::vortex_err; + let name_ptr = cpp::duckdb_column_name(self.as_ptr(), col_idx as u64); if name_ptr.is_null() { vortex_bail!("Invalid column index: {}", col_idx); diff --git a/vortex-duckdb/src/duckdb/scalar_function.rs b/vortex-duckdb/src/duckdb/scalar_function.rs index f400779278d..175b0e70d5c 100644 --- a/vortex-duckdb/src/duckdb/scalar_function.rs +++ b/vortex-duckdb/src/duckdb/scalar_function.rs @@ -5,8 +5,6 @@ use vortex::error::VortexExpect; use vortex::error::vortex_err; use crate::cpp; -use crate::duckdb::LogicalType; -use crate::duckdb::LogicalTypeRef; use crate::lifetime_wrapper; lifetime_wrapper!(ScalarFunction, cpp::duckdb_vx_sfunc, |_| {}); @@ -21,8 +19,4 @@ impl ScalarFunctionRef { .vortex_expect("scalar function name should be valid UTF-8") } } - - pub fn return_type(&self) -> &LogicalTypeRef { - unsafe { LogicalType::borrow(cpp::duckdb_vx_sfunc_return_type(self.as_ptr())) } - } } diff --git a/vortex-duckdb/src/duckdb/vector.rs b/vortex-duckdb/src/duckdb/vector.rs index 697ce52da50..5ac3ad01aa9 100644 --- a/vortex-duckdb/src/duckdb/vector.rs +++ b/vortex-duckdb/src/duckdb/vector.rs @@ -4,9 +4,7 @@ use std::ffi::CStr; use std::ffi::c_void; use std::ops::Range; -use std::ptr; -use bitvec::macros::internal::funty::Fundamental; use bitvec::slice::BitSlice; use bitvec::view::BitView; use vortex::array::dtype::Nullability; @@ -18,7 +16,6 @@ use vortex::error::vortex_bail; use vortex::mask::Mask; use crate::cpp; -use crate::cpp::duckdb_vx_error; use crate::cpp::idx_t; use crate::duckdb::LogicalType; use crate::duckdb::LogicalTypeRef; @@ -282,20 +279,6 @@ impl VectorRef { } } - pub fn try_to_string(&self, len: u64) -> VortexResult { - let mut err: duckdb_vx_error = ptr::null_mut(); - let debug = - unsafe { cpp::duckdb_vector_to_string(self.as_ptr(), len.as_u64(), &raw mut err) }; - if !err.is_null() { - vortex_bail!("{}", unsafe { - CStr::from_ptr(cpp::duckdb_vx_error_value(err)).to_string_lossy() - }) - } - let string = unsafe { CStr::from_ptr(debug).to_string_lossy() }.to_string(); - unsafe { cpp::duckdb_free(debug.cast_mut().cast()) }; - Ok(string) - } - pub fn list_vector_reserve(&self, required_capacity: u64) -> VortexResult<()> { let state = unsafe { cpp::duckdb_list_vector_reserve(self.as_ptr(), required_capacity) }; match state { diff --git a/vortex-duckdb/src/lib.rs b/vortex-duckdb/src/lib.rs index f70d8f0847b..55039b88737 100644 --- a/vortex-duckdb/src/lib.rs +++ b/vortex-duckdb/src/lib.rs @@ -18,8 +18,6 @@ use vortex::session::VortexSession; use crate::duckdb::Database; use crate::duckdb::DatabaseRef; -use crate::duckdb::LogicalType; -use crate::duckdb::Value; mod column_statistics; mod convert; @@ -67,12 +65,6 @@ fn init_tracing() { /// Note: This also registers extension options. If you want to register options /// separately (e.g., before creating connections), call `register_extension_options` first. pub fn initialize(db: &DatabaseRef) -> VortexResult<()> { - db.config().add_extension_options( - "vortex_filesystem", - "Whether to use Vortex's filesystem ('vortex') or DuckDB's filesystems ('duckdb').", - LogicalType::varchar(), - Value::from("vortex"), - )?; db.register_table_functions()?; db.register_optimizer_extension()?; db.register_copy_function()