diff --git a/lang/cpp/include/vortex/array.hpp b/lang/cpp/include/vortex/array.hpp index 43b986f4b42..47652dd14fa 100644 --- a/lang/cpp/include/vortex/array.hpp +++ b/lang/cpp/include/vortex/array.hpp @@ -6,6 +6,7 @@ #include "vortex/dtype.hpp" #include "vortex/error.hpp" #include "vortex/expression.hpp" +#include "vortex/scalar.hpp" #include "vortex/session.hpp" #include @@ -211,6 +212,8 @@ class Array { // Bulk view over Binary values. BytesView bytes(const Session &session) const; + Scalar scalar_at(const Session &session, size_t index) const; + private: friend struct detail::Access; friend class StringView; diff --git a/lang/cpp/include/vortex/scalar.hpp b/lang/cpp/include/vortex/scalar.hpp index f454d918df3..f9ead083d75 100644 --- a/lang/cpp/include/vortex/scalar.hpp +++ b/lang/cpp/include/vortex/scalar.hpp @@ -26,20 +26,93 @@ class Scalar { bool is_null() const; DataType dtype() const; + /** + * Read scalar's value. + * + * Supported types are bool, primitives, std::string_view, and BinaryView. + * + * std::string_view and BinaryView returned borrow from scalar and stay + * valid while scalar is valid. + * + * Throws if scalar type does not match T or value is Null. + */ + template + T get() const; + + /** + * Read a decimal scalar's unscaled value. + * int8/16/32/64_t are supported. + * Throws if scalar is not a decimal, is Null, or value does not fit in T. + */ + template + T get_decimal() const; + private: friend struct detail::Access; - explicit Scalar(vx_scalar *owned) : handle_(owned) { + explicit Scalar(const vx_scalar *owned) : handle_(owned) { } - vx_scalar *release() && { + const vx_scalar *release() && { return handle_.release(); } struct Deleter { - void operator()(vx_scalar *ptr) const noexcept; + void operator()(const vx_scalar *ptr) const noexcept; }; - std::unique_ptr handle_; + std::unique_ptr handle_; }; +template +T Scalar::get() const { + const vx_scalar *h = handle_.get(); + if constexpr (std::is_same_v) { + return vx_scalar_get_bool(h); + } else if constexpr (std::is_same_v) { + vx_view v = vx_scalar_get_utf8(h); + return std::string_view(v.ptr, v.len); + } else if constexpr (std::is_same_v) { + vx_view v = vx_scalar_get_binary(h); + return BinaryView(reinterpret_cast(v.ptr), v.len); + } else if constexpr (std::is_same_v) { + return vx_scalar_get_u8(h); + } else if constexpr (std::is_same_v) { + return vx_scalar_get_u16(h); + } else if constexpr (std::is_same_v) { + return vx_scalar_get_u32(h); + } else if constexpr (std::is_same_v) { + return vx_scalar_get_u64(h); + } else if constexpr (std::is_same_v) { + return vx_scalar_get_i8(h); + } else if constexpr (std::is_same_v) { + return vx_scalar_get_i16(h); + } else if constexpr (std::is_same_v) { + return vx_scalar_get_i32(h); + } else if constexpr (std::is_same_v) { + return vx_scalar_get_i64(h); + } else if constexpr (std::is_same_v) { + return vx_scalar_get_f32(h); + } else if constexpr (std::is_same_v) { + return vx_scalar_get_f64(h); + } else { + static_assert(false, "f16 scalar get is not supported"); + } +} + +template +T Scalar::get_decimal() const { + const vx_scalar *h = handle_.get(); + if constexpr (std::is_same_v) { + return vx_scalar_get_decimal_i8(h); + } else if constexpr (std::is_same_v) { + return vx_scalar_get_decimal_i16(h); + } else if constexpr (std::is_same_v) { + return vx_scalar_get_decimal_i32(h); + } else if constexpr (std::is_same_v) { + return vx_scalar_get_decimal_i64(h); + } else { + static_assert(false, "unsupported decimal scalar get type"); + } +} + namespace detail { vx_scalar *make_bool(bool value, bool nullable); vx_scalar *make_primitive(vx_ptype ptype, const void *value, bool nullable); diff --git a/lang/cpp/src/array.cpp b/lang/cpp/src/array.cpp index b4adf69fe5c..5e46c953bba 100644 --- a/lang/cpp/src/array.cpp +++ b/lang/cpp/src/array.cpp @@ -5,6 +5,8 @@ #include "vortex/common.hpp" #include "vortex/dtype.hpp" #include "vortex/error.hpp" +#include "vortex/scalar.hpp" +#include "vortex/session.hpp" #include @@ -324,7 +326,19 @@ BytesView Array::bytes(const Session &session) const { return BytesView(std::move(canonical), std::move(validity), len); } +Scalar Array::scalar_at(const Session &session, size_t index) const { + vx_error *error = nullptr; + const vx_scalar *scalar = vx_array_get_scalar(Access::c_ptr(session), handle_.get(), index, &error); + throw_on_error(error); + return Access::adopt(scalar); +} + bool PrimitiveView::value(size_t i) const { + if (i >= size_) { + throw VortexException("index " + std::to_string(i) + " out of bounds for view of size " + + std::to_string(size_), + ErrorCode::OutOfBounds); + } return vx_array_get_bool(Access::c_ptr(canonical_), i); } diff --git a/lang/cpp/src/scalar.cpp b/lang/cpp/src/scalar.cpp index 2bef2699bba..f0427aec647 100644 --- a/lang/cpp/src/scalar.cpp +++ b/lang/cpp/src/scalar.cpp @@ -17,7 +17,7 @@ namespace vortex { using detail::Access; using detail::throw_on_error; -void Scalar::Deleter::operator()(vx_scalar *ptr) const noexcept { +void Scalar::Deleter::operator()(const vx_scalar *ptr) const noexcept { vx_scalar_free(ptr); } diff --git a/lang/cpp/tests/expression.cpp b/lang/cpp/tests/expression.cpp index 1a55364df02..8145ab6b6d4 100644 --- a/lang/cpp/tests/expression.cpp +++ b/lang/cpp/tests/expression.cpp @@ -85,6 +85,7 @@ TEST_CASE("Operator overloading", "[expr]") { REQUIRE(bits.value(1)); REQUIRE(bits.value(2)); REQUIRE_FALSE(bits.value(3)); + REQUIRE_THROWS_AS(bits.value(data.size()), VortexException); } TEST_CASE("Apply error", "[expr]") { diff --git a/lang/cpp/tests/scalar.cpp b/lang/cpp/tests/scalar.cpp index 27ff73f656a..178130d0f24 100644 --- a/lang/cpp/tests/scalar.cpp +++ b/lang/cpp/tests/scalar.cpp @@ -10,6 +10,8 @@ using namespace std::string_view_literals; namespace { using enum vortex::PType; +using scalar::decimal; +using scalar::of; TEST_CASE("Boolean scalar", "[scalar]") { Scalar s = scalar::of(true); @@ -29,7 +31,7 @@ TEST_CASE("Integer scalars", "[scalar]") { } TEST_CASE("Float scalars", "[scalar]") { - REQUIRE(scalar::of(1.5F).dtype().primitive_type() == F32); + REQUIRE(scalar::of(1.5f).dtype().primitive_type() == F32); REQUIRE(scalar::of(1.5).dtype().primitive_type() == F64); REQUIRE(scalar::of(float16_t {0x3C00}).dtype().primitive_type() == F16); } @@ -87,6 +89,32 @@ TEST_CASE("Decimal scalars", "[scalar]") { REQUIRE(d64.dtype().decimal_scale() == 3); } +TEST_CASE("Primitive getters", "[scalar]") { + REQUIRE(of(1ULL << 40).get() == (1ULL << 40)); + REQUIRE(of(-7).get() == -7); + REQUIRE(of(2.5).get() == 2.5); + REQUIRE(of(true).get()); +} + +TEST_CASE("String getters", "[scalar]") { + Scalar s = of("hello"sv); + REQUIRE(s.get() == "hello"sv); + + const std::byte bytes[] = {std::byte {1}, std::byte {2}, std::byte {0}, std::byte {4}}; + Scalar b = of(std::span {bytes}); + BinaryView view = b.get(); + REQUIRE(view.size() == 4); + REQUIRE(view[0] == std::byte {1}); + REQUIRE(view[3] == std::byte {4}); +} + +TEST_CASE("Decimal getters", "[scalar]") { + REQUIRE(decimal(56, 5, 2).get_decimal() == 56); + REQUIRE(decimal(1234, 5, 2).get_decimal() == 1234); + REQUIRE(decimal(5678, 6, 2).get_decimal() == 5678); + REQUIRE(decimal(99999, 12, 3).get_decimal() == 99999); +} + TEST_CASE("Copy scalar", "[scalar]") { Scalar a = scalar::of(42); Scalar b = a; diff --git a/vortex-ffi/cinclude/vortex.h b/vortex-ffi/cinclude/vortex.h index 5476c6e494d..c2892698d82 100644 --- a/vortex-ffi/cinclude/vortex.h +++ b/vortex-ffi/cinclude/vortex.h @@ -429,7 +429,10 @@ typedef struct vx_partition vx_partition; /** * A vx_scalar is a single value with an associated vx_dtype. + * * Scalar value may be Null is vx_dtype is nullable. + * One example where you can get a Null scalar is vx_array_get_scalar + * where the element at some index is invalid/null. */ typedef struct vx_scalar vx_scalar; @@ -645,7 +648,10 @@ const vx_array *vx_array_slice(const vx_array *array, size_t start, size_t stop, * validity array. Sets error if index is out of bounds or underlying validity * array is corrupted. */ -bool vx_array_element_is_invalid(const vx_array *array, size_t index, vx_error **error); +bool vx_array_element_is_invalid(const vx_session *session, + const vx_array *array, + size_t index, + vx_error **error); /** * Check how many items in the array are invalid (null). @@ -708,50 +714,6 @@ const vx_array *vx_array_new_primitive(vx_ptype ptype, const vx_array * vx_array_from_arrow(FFI_ArrowArray *array, FFI_ArrowSchema *schema, bool nullable, vx_error **error_out); -uint8_t vx_array_get_u8(const vx_array *array, size_t index); - -uint8_t vx_array_get_storage_u8(const vx_array *array, size_t index); - -uint16_t vx_array_get_u16(const vx_array *array, size_t index); - -uint16_t vx_array_get_storage_u16(const vx_array *array, size_t index); - -uint32_t vx_array_get_u32(const vx_array *array, size_t index); - -uint32_t vx_array_get_storage_u32(const vx_array *array, size_t index); - -uint64_t vx_array_get_u64(const vx_array *array, size_t index); - -uint64_t vx_array_get_storage_u64(const vx_array *array, size_t index); - -int8_t vx_array_get_i8(const vx_array *array, size_t index); - -int8_t vx_array_get_storage_i8(const vx_array *array, size_t index); - -int16_t vx_array_get_i16(const vx_array *array, size_t index); - -int16_t vx_array_get_storage_i16(const vx_array *array, size_t index); - -int32_t vx_array_get_i32(const vx_array *array, size_t index); - -int32_t vx_array_get_storage_i32(const vx_array *array, size_t index); - -int64_t vx_array_get_i64(const vx_array *array, size_t index); - -int64_t vx_array_get_storage_i64(const vx_array *array, size_t index); - -uint16_t vx_array_get_f16(const vx_array *array, size_t index); - -uint16_t vx_array_get_storage_f16(const vx_array *array, size_t index); - -float vx_array_get_f32(const vx_array *array, size_t index); - -float vx_array_get_storage_f32(const vx_array *array, size_t index); - -double vx_array_get_f64(const vx_array *array, size_t index); - -double vx_array_get_storage_f64(const vx_array *array, size_t index); - /** * Return UTF-8 string at "index" in a canonical Utf8 array. * @@ -783,6 +745,19 @@ vx_view vx_array_binary_at(const vx_array *array, size_t index, vx_error **error */ bool vx_array_get_bool(const vx_array *array, size_t index); +/** + * Get array's element at position "index". + * + * If element at index is invalid, returns a Null vx_scalar. + * + * This is an expensive operation. If you need bulk access, use + * vx_array_data_ptr_primitive or vx_data_ptr_bool. + * + * Errors if "index" is out of bounds. + */ +const vx_scalar * +vx_array_get_scalar(const vx_session *session, const vx_array *array, size_t index, vx_error **error_out); + /** * Decode array into its canonical form. * @@ -1174,7 +1149,7 @@ vx_scalar *vx_scalar_clone(const vx_scalar *scalar); const vx_dtype *vx_scalar_dtype(const vx_scalar *scalar); /** - * Return whether scalar is a typed null value. + * Return whether scalar is a typed Null value. */ bool vx_scalar_is_null(const vx_scalar *scalar); @@ -1184,81 +1159,175 @@ bool vx_scalar_is_null(const vx_scalar *scalar); vx_scalar *vx_scalar_new_bool(bool value, bool is_nullable); /** - * Create an unsigned 8-bit integer scalar. + * Return the boolean value stored in the scalar. + * + * Panics if the scalar is not a Bool scalar, or is null. + */ +bool vx_scalar_get_bool(const vx_scalar *scalar); + +/** + * Create a u8 scalar. */ vx_scalar *vx_scalar_new_u8(uint8_t value, bool is_nullable); /** - * Create an unsigned 16-bit integer scalar. + * Return u8 value stored in scalar. + * + * Panics if scalar is not a primitive scalar of this type or is null. + */ +uint8_t vx_scalar_get_u8(const vx_scalar *scalar); + +/** + * Create a u16 scalar. */ vx_scalar *vx_scalar_new_u16(uint16_t value, bool is_nullable); /** - * Create an unsigned 32-bit integer scalar. + * Return u16 value stored in scalar. + * + * Panics if scalar is not a primitive scalar of this type or is null. + */ +uint16_t vx_scalar_get_u16(const vx_scalar *scalar); + +/** + * Create a u32 scalar. */ vx_scalar *vx_scalar_new_u32(uint32_t value, bool is_nullable); /** - * Create an unsigned 64-bit integer scalar. + * Return u32 value stored in scalar. + * + * Panics if scalar is not a primitive scalar of this type or is null. + */ +uint32_t vx_scalar_get_u32(const vx_scalar *scalar); + +/** + * Create a u64 scalar. */ vx_scalar *vx_scalar_new_u64(uint64_t value, bool is_nullable); /** - * Create a signed 8-bit integer scalar. + * Return u64 value stored in scalar. + * + * Panics if scalar is not a primitive scalar of this type or is null. + */ +uint64_t vx_scalar_get_u64(const vx_scalar *scalar); + +/** + * Create a i8 scalar. */ vx_scalar *vx_scalar_new_i8(int8_t value, bool is_nullable); /** - * Create a signed 16-bit integer scalar. + * Return i8 value stored in scalar. + * + * Panics if scalar is not a primitive scalar of this type or is null. + */ +int8_t vx_scalar_get_i8(const vx_scalar *scalar); + +/** + * Create a i16 scalar. */ vx_scalar *vx_scalar_new_i16(int16_t value, bool is_nullable); /** - * Create a signed 32-bit integer scalar. + * Return i16 value stored in scalar. + * + * Panics if scalar is not a primitive scalar of this type or is null. + */ +int16_t vx_scalar_get_i16(const vx_scalar *scalar); + +/** + * Create a i32 scalar. */ vx_scalar *vx_scalar_new_i32(int32_t value, bool is_nullable); /** - * Create a signed 64-bit integer scalar. + * Return i32 value stored in scalar. + * + * Panics if scalar is not a primitive scalar of this type or is null. + */ +int32_t vx_scalar_get_i32(const vx_scalar *scalar); + +/** + * Create a i64 scalar. */ vx_scalar *vx_scalar_new_i64(int64_t value, bool is_nullable); /** - * Create a 32-bit floating point scalar. + * Return i64 value stored in scalar. + * + * Panics if scalar is not a primitive scalar of this type or is null. + */ +int64_t vx_scalar_get_i64(const vx_scalar *scalar); + +/** + * Create a f32 scalar. */ vx_scalar *vx_scalar_new_f32(float value, bool is_nullable); /** - * Create a 64-bit floating point scalar. + * Return f32 value stored in scalar. + * + * Panics if scalar is not a primitive scalar of this type or is null. + */ +float vx_scalar_get_f32(const vx_scalar *scalar); + +/** + * Create a f64 scalar. */ vx_scalar *vx_scalar_new_f64(double value, bool is_nullable); /** - * Create a 16-bit floating point scalar. + * Return f64 value stored in scalar. * - * The value is read from raw half-precision bits because C has no portable - * half-precision floating point ABI. + * Panics if scalar is not a primitive scalar of this type or is null. + */ +double vx_scalar_get_f64(const vx_scalar *scalar); + +/** + * Create a 16-bit floating point scalar. + * The value is read from raw uint16_t. */ vx_scalar *vx_scalar_new_f16_bits(uint16_t bits, bool is_nullable); /** * Create a UTF-8 scalar. * - * The string bytes are copied into the scalar. Invalid UTF-8 returns NULL and - * writes the error output. + * "value" bytes are copied into scalar. + * Errors on invalid UTF-8. */ vx_scalar *vx_scalar_new_utf8(vx_view value, bool is_nullable, vx_error **err); /** * Create a binary scalar. * - * The byte range is copied into the scalar. NULL "ptr" is allowed only when - * len == 0. + * Byte range is copied into the scalar. + * + * NULL "ptr" is allowed only when len == 0. * * Returns NULL and sets "err" on error. */ vx_scalar *vx_scalar_new_binary(const uint8_t *ptr, size_t len, bool is_nullable, vx_error **err); +/** + * Return UTF-8 string stored in scalar. + * + * Returned view borrows the scalar and is valid as long as "scalar" is valid. + * + * Panics if scalar is not a Utf8 scalar, or is null. + */ +vx_view vx_scalar_get_utf8(const vx_scalar *scalar); + +/** + * Return binary bytes stored in the scalar. + * + * Returned view borrows scalar and is valid as long as "scalar" is valid. + * + * Panics if scalar is not a Binary scalar, or is null. + */ +vx_view vx_scalar_get_binary(const vx_scalar *scalar); + /** * Create a typed null scalar. * @@ -1270,9 +1339,7 @@ vx_scalar *vx_scalar_new_binary(const uint8_t *ptr, size_t len, bool is_nullable vx_scalar *vx_scalar_new_null(const vx_dtype *dtype, vx_error **err); /** - * Create a decimal scalar. - * - * The unscaled value is provided as a signed 8-bit integer. + * Create a decimal scalar from a signed i8 unscaled value. * * Returns NULL and sets "err" on error. */ @@ -1280,9 +1347,15 @@ vx_scalar * vx_scalar_new_decimal_i8(int8_t value, uint8_t precision, int8_t scale, bool is_nullable, vx_error **err); /** - * Create a decimal scalar. + * Return the unscaled i8 value of a decimal scalar. * - * The unscaled value is provided as a signed 16-bit integer. + * Panics if the scalar is not a decimal scalar, is null, or the + * unscaled value does not fit in i8. + */ +int8_t vx_scalar_get_decimal_i8(const vx_scalar *scalar); + +/** + * Create a decimal scalar from a signed i16 unscaled value. * * Returns NULL and sets "err" on error. */ @@ -1290,9 +1363,15 @@ vx_scalar * vx_scalar_new_decimal_i16(int16_t value, uint8_t precision, int8_t scale, bool is_nullable, vx_error **err); /** - * Create a decimal scalar. + * Return the unscaled i16 value of a decimal scalar. * - * The unscaled value is provided as a signed 32-bit integer. + * Panics if the scalar is not a decimal scalar, is null, or the + * unscaled value does not fit in i16. + */ +int16_t vx_scalar_get_decimal_i16(const vx_scalar *scalar); + +/** + * Create a decimal scalar from a signed i32 unscaled value. * * Returns NULL and sets "err" on error. */ @@ -1300,15 +1379,29 @@ vx_scalar * vx_scalar_new_decimal_i32(int32_t value, uint8_t precision, int8_t scale, bool is_nullable, vx_error **err); /** - * Create a decimal scalar. + * Return the unscaled i32 value of a decimal scalar. * - * The unscaled value is provided as a signed 64-bit integer. + * Panics if the scalar is not a decimal scalar, is null, or the + * unscaled value does not fit in i32. + */ +int32_t vx_scalar_get_decimal_i32(const vx_scalar *scalar); + +/** + * Create a decimal scalar from a signed i64 unscaled value. * * Returns NULL and sets "err" on error. */ vx_scalar * vx_scalar_new_decimal_i64(int64_t value, uint8_t precision, int8_t scale, bool is_nullable, vx_error **err); +/** + * Return the unscaled i64 value of a decimal scalar. + * + * Panics if the scalar is not a decimal scalar, is null, or the + * unscaled value does not fit in i64. + */ +int64_t vx_scalar_get_decimal_i64(const vx_scalar *scalar); + /** * Create a decimal scalar. * diff --git a/vortex-ffi/src/array.rs b/vortex-ffi/src/array.rs index ae2a2e3d93a..a7fce6fd27d 100644 --- a/vortex-ffi/src/array.rs +++ b/vortex-ffi/src/array.rs @@ -10,7 +10,6 @@ use arrow_array::array::make_array; use arrow_array::ffi::FFI_ArrowArray; use arrow_array::ffi::FFI_ArrowSchema; use arrow_array::ffi::from_ffi; -use paste::paste; use vortex::array::ArrayRef; use vortex::array::Canonical; use vortex::array::IntoArray; @@ -45,11 +44,11 @@ use crate::error::vx_error; use crate::error::write_error; use crate::expression::vx_expression; use crate::ptype::vx_ptype; +use crate::scalar::vx_scalar; use crate::session::vx_session; use crate::session::vx_session_ref; -use crate::string::vx_view; +use crate::vx_view; -// ArrayRef has an Arc inside box_wrapper!( /// Arrays are reference-counted handles to owned memory buffers that hold /// scalars. These buffers can be held in a number of physical encodings to @@ -264,13 +263,14 @@ pub unsafe extern "C-unwind" fn vx_array_slice( #[unsafe(no_mangle)] #[allow(clippy::disallowed_methods)] pub unsafe extern "C-unwind" fn vx_array_element_is_invalid( + session: *const vx_session, array: *const vx_array, index: usize, error: *mut *mut vx_error, ) -> bool { try_or_default(error, || { - vortex_ensure!(!array.is_null()); - vx_array::as_ref(array).is_invalid(index, &mut legacy_session().create_execution_ctx()) + let session = unsafe { vx_session_ref(session) }?; + vx_array::as_ref(array).is_invalid(index, &mut session.create_execution_ctx()) }) } @@ -421,54 +421,6 @@ pub unsafe extern "C-unwind" fn vx_array_from_arrow( }) } -macro_rules! ffiarray_get_ptype { - ($ptype:ident) => { - paste! { - #[unsafe(no_mangle)] - pub unsafe extern "C-unwind" fn [](array: *const vx_array, index: usize) -> $ptype { - let array = vx_array::as_ref(array); - // TODO(joe): propagate this error up instead of expecting - #[allow(clippy::disallowed_methods)] - let value = array - .execute_scalar(index, &mut legacy_session().create_execution_ctx()) - .vortex_expect("scalar_at failed"); - // TODO(joe): propagate this error up instead of expecting - value.as_primitive() - .as_::<$ptype>() - .vortex_expect("null value") - } - - #[unsafe(no_mangle)] - pub unsafe extern "C-unwind" fn [](array: *const vx_array, index: usize) -> $ptype { - let array = vx_array::as_ref(array); - // TODO(joe): propagate this error up instead of expecting - #[allow(clippy::disallowed_methods)] - let value = array - .execute_scalar(index, &mut legacy_session().create_execution_ctx()) - .vortex_expect("scalar_at failed"); - // TODO(joe): propagate this error up instead of expecting - value.as_extension() - .to_storage_scalar() - .as_primitive() - .as_::<$ptype>() - .vortex_expect("null value") - } - } - }; -} - -ffiarray_get_ptype!(u8); -ffiarray_get_ptype!(u16); -ffiarray_get_ptype!(u32); -ffiarray_get_ptype!(u64); -ffiarray_get_ptype!(i8); -ffiarray_get_ptype!(i16); -ffiarray_get_ptype!(i32); -ffiarray_get_ptype!(i64); -ffiarray_get_ptype!(f16); -ffiarray_get_ptype!(f32); -ffiarray_get_ptype!(f64); - /// SAFETY: "array" must be null or a valid "vx_array" unsafe fn varbinview_at( array: *const vx_array, @@ -550,6 +502,29 @@ pub unsafe extern "C-unwind" fn vx_array_get_bool(array: *const vx_array, index: bits.value(index) } +/// Get array's element at position "index". +/// +/// If element at index is invalid, returns a Null vx_scalar. +/// +/// This is an expensive operation. If you need bulk access, use +/// vx_array_data_ptr_primitive or vx_data_ptr_bool. +/// +/// Errors if "index" is out of bounds. +#[unsafe(no_mangle)] +pub unsafe extern "C-unwind" fn vx_array_get_scalar( + session: *const vx_session, + array: *const vx_array, + index: usize, + error_out: *mut *mut vx_error, +) -> *const vx_scalar { + try_or_default(error_out, || { + let session = vx_session::as_ref(session); + let array = vx_array::as_ref(array); + let scalar = array.execute_scalar(index, &mut session.create_execution_ctx())?; + Ok(vx_scalar::new(scalar)) + }) +} + /// Decode array into its canonical form. /// /// On error returns NULL and "sets error_out". @@ -560,8 +535,8 @@ pub unsafe extern "C-unwind" fn vx_array_canonicalize( error_out: *mut *mut vx_error, ) -> *const vx_array { try_or_default(error_out, || { - let session = unsafe { vx_session_ref(session) }?; - let array = unsafe { vx_array_ref(array) }?; + let session = vx_session::as_ref(session); + let array = vx_array::as_ref(array); let mut ctx = session.create_execution_ctx(); let canonical = array.clone().execute::(&mut ctx)?; Ok(vx_array::new(canonical.into_array())) @@ -578,7 +553,7 @@ pub unsafe extern "C-unwind" fn vx_array_data_ptr_primitive( error_out: *mut *mut vx_error, ) -> *const c_void { try_or(error_out, ptr::null(), || { - let array = unsafe { vx_array_ref(array) }?; + let array = vx_array::as_ref(array); let primitive = array.as_opt::().ok_or_else(|| { vortex_err!( "vx_array_data_ptr_primitive requires a canonical Primitive array, got {}", @@ -607,7 +582,7 @@ pub unsafe extern "C-unwind" fn vx_array_data_ptr_bool( error_out: *mut *mut vx_error, ) -> *const c_void { try_or(error_out, ptr::null(), || { - let array = unsafe { vx_array_ref(array) }?; + let array = vx_array::as_ref(array); vortex_ensure!(!bit_offset_out.is_null(), "null bit_offset_out"); let bool_array = array.as_opt::().ok_or_else(|| { vortex_err!( @@ -655,8 +630,6 @@ mod tests { use vortex::array::arrays::bool::BoolArrayExt; use vortex::array::validity::Validity; use vortex::buffer::buffer; - #[cfg(not(miri))] - use vortex::dtype::half::f16; use vortex::expr::eq; use vortex::expr::lit; use vortex::expr::root; @@ -667,16 +640,36 @@ mod tests { use crate::dtype::vx_dtype_variant; use crate::error::vx_error_free; use crate::expression::vx_expression_free; + use crate::scalar::*; use crate::session::vx_session_free; use crate::session::vx_session_new; use crate::tests::assert_error; use crate::tests::assert_no_error; + unsafe fn get_i32(session: *const vx_session, array: *const vx_array, index: usize) -> i32 { + let mut error = ptr::null_mut(); + let scalar = unsafe { vx_array_get_scalar(session, array, index, &raw mut error) }; + assert_no_error(error); + let value = unsafe { vx_scalar_get_i32(scalar) }; + unsafe { vx_scalar_free(scalar.cast_mut()) }; + value + } + + unsafe fn get_u8(session: *const vx_session, array: *const vx_array, index: usize) -> u8 { + let mut error = ptr::null_mut(); + let scalar = unsafe { vx_array_get_scalar(session, array, index, &raw mut error) }; + assert_no_error(error); + let value = unsafe { vx_scalar_get_u8(scalar) }; + unsafe { vx_scalar_free(scalar.cast_mut()) }; + value + } + #[test] // TODO(joe): enable once this is fixed https://github.com/Amanieu/parking_lot/issues/477 #[cfg_attr(miri, ignore)] fn test_simple() { unsafe { + let session = vx_session_new(); let primitive = PrimitiveArray::new(buffer![1i32, 2i32, 3i32], Validity::NonNullable); let ffi_array = vx_array::new(primitive.into_array()); @@ -688,12 +681,26 @@ mod tests { vx_dtype_variant::DTYPE_PRIMITIVE ); - assert_eq!(vx_array_get_i32(ffi_array, 0), 1); - assert_eq!(vx_array_get_i32(ffi_array, 1), 2); - assert_eq!(vx_array_get_i32(ffi_array, 2), 3); + let mut error = ptr::null_mut(); + + let scalar = vx_array_get_scalar(session, ffi_array, 0, &raw mut error); + assert_no_error(error); + assert_eq!(vx_scalar_get_i32(scalar), 1); + vx_scalar_free(scalar); + + let scalar = vx_array_get_scalar(session, ffi_array, 1, &raw mut error); + assert_no_error(error); + assert_eq!(vx_scalar_get_i32(scalar), 2); + vx_scalar_free(scalar); + + let scalar = vx_array_get_scalar(session, ffi_array, 2, &raw mut error); + assert_no_error(error); + assert_eq!(vx_scalar_get_i32(scalar), 3); + vx_scalar_free(scalar); vx_dtype_free(array_dtype); vx_array_free(ffi_array); + vx_session_free(session); } } @@ -715,6 +722,7 @@ mod tests { #[cfg_attr(miri, ignore)] fn test_slice() { unsafe { + let session = vx_session_new(); let primitive = PrimitiveArray::new(buffer![1i32, 2i32, 3i32, 4i32, 5i32], Validity::NonNullable); let ffi_array = vx_array::new(primitive.into_array()); @@ -723,12 +731,12 @@ mod tests { let sliced = vx_array_slice(ffi_array, 1, 4, &raw mut error); assert_no_error(error); assert_eq!(vx_array_len(sliced), 3); - assert_eq!(vx_array_get_i32(sliced, 0), 2); - assert_eq!(vx_array_get_i32(sliced, 1), 3); - assert_eq!(vx_array_get_i32(sliced, 2), 4); + assert_eq!(get_i32(session, sliced, 0), 2); + assert_eq!(get_i32(session, sliced, 2), 4); vx_array_free(sliced); vx_array_free(ffi_array); + vx_session_free(session); } } @@ -737,6 +745,7 @@ mod tests { #[cfg_attr(miri, ignore)] fn test_null_operations() { unsafe { + let session = vx_session_new(); let primitive = PrimitiveArray::new( buffer![1i32, 2i32, 3i32], Validity::from_iter([true, false, true]), @@ -744,11 +753,26 @@ mod tests { let ffi_array = vx_array::new(primitive.into_array()); let mut error = ptr::null_mut(); - assert!(!vx_array_element_is_invalid(ffi_array, 0, &raw mut error)); + assert!(!vx_array_element_is_invalid( + session, + ffi_array, + 0, + &raw mut error + )); assert_no_error(error); - assert!(vx_array_element_is_invalid(ffi_array, 1, &raw mut error)); + assert!(vx_array_element_is_invalid( + session, + ffi_array, + 1, + &raw mut error + )); assert_no_error(error); - assert!(!vx_array_element_is_invalid(ffi_array, 2, &raw mut error)); + assert!(!vx_array_element_is_invalid( + session, + ffi_array, + 2, + &raw mut error + )); assert_no_error(error); let null_count = vx_array_invalid_count(ffi_array, &raw mut error); @@ -756,6 +780,7 @@ mod tests { assert_eq!(null_count, 1); vx_array_free(ffi_array); + vx_session_free(session); } } @@ -764,6 +789,7 @@ mod tests { #[cfg_attr(miri, ignore)] fn test_get_field() { unsafe { + let session = vx_session_new(); let names = VarBinViewArray::from_iter_str(["Alice", "Bob", "Charlie"]); let ages = PrimitiveArray::new(buffer![30u8, 25u8, 35u8], Validity::NonNullable); let struct_array = StructArray::try_new( @@ -783,9 +809,9 @@ mod tests { let field1 = vx_array_get_field(ffi_array, 1, &raw mut error); assert_no_error(error); assert_eq!(vx_array_len(field1), 3); - assert_eq!(vx_array_get_u8(field1, 0), 30); - assert_eq!(vx_array_get_u8(field1, 1), 25); - assert_eq!(vx_array_get_u8(field1, 2), 35); + assert_eq!(get_u8(session, field1, 0), 30); + assert_eq!(get_u8(session, field1, 1), 25); + assert_eq!(get_u8(session, field1, 2), 35); // Test out of bounds let field_oob = vx_array_get_field(ffi_array, 2, &raw mut error); @@ -796,95 +822,9 @@ mod tests { vx_array_free(field0); vx_array_free(field1); vx_array_free(ffi_array); + vx_session_free(session); } } - - #[test] - // TODO(joe): enable once this is fixed https://github.com/Amanieu/parking_lot/issues/477 - #[cfg_attr(miri, ignore)] - fn test_primitive_getters() { - unsafe { - // Test a representative sample of primitive types - // The macro generates identical code for all types, so exhaustive testing is redundant - - // Test signed integer with edge cases - let mut error = ptr::null_mut(); - let validity = vx_validity { - r#type: vx_validity_type::VX_VALIDITY_NON_NULLABLE, - array: ptr::null(), - }; - - let i32_array = [i32::MAX, i32::MIN, 0]; - let ffi_i32 = vx_array_new_primitive( - vx_ptype::PTYPE_I32, - i32_array.as_ptr() as *const c_void, - i32_array.len(), - &raw const validity, - &raw mut error, - ); - assert_no_error(error); - assert!(!ffi_i32.is_null()); - - assert!(vx_array_is_primitive(ffi_i32, vx_ptype::PTYPE_I32)); - assert_eq!(vx_array_get_i32(ffi_i32, 0), i32::MAX); - assert_eq!(vx_array_get_i32(ffi_i32, 1), i32::MIN); - assert_eq!(vx_array_get_i32(ffi_i32, 2), 0); - vx_array_free(ffi_i32); - - // Test unsigned integer - let u64_array = [u64::MAX, 0u64, 42u64]; - let ffi_u64 = vx_array_new_primitive( - vx_ptype::PTYPE_U64, - u64_array.as_ptr() as *const c_void, - u64_array.len(), - &raw const validity, - &raw mut error, - ); - assert_no_error(error); - assert!(!ffi_u64.is_null()); - assert!(vx_array_is_primitive(ffi_u64, vx_ptype::PTYPE_U64)); - assert_eq!(vx_array_get_u64(ffi_u64, 0), u64::MAX); - assert_eq!(vx_array_get_u64(ffi_u64, 1), 0); - assert_eq!(vx_array_get_u64(ffi_u64, 2), 42); - vx_array_free(ffi_u64); - - // Test floating point including special values - let f64_array = [f64::NEG_INFINITY, 0.0f64, f64::NAN]; - let ffi_f64 = vx_array_new_primitive( - vx_ptype::PTYPE_F64, - f64_array.as_ptr() as *const c_void, - f64_array.len(), - &raw const validity, - &raw mut error, - ); - assert_no_error(error); - assert!(!ffi_f64.is_null()); - assert!(vx_array_is_primitive(ffi_f64, vx_ptype::PTYPE_F64)); - assert_eq!(vx_array_get_f64(ffi_f64, 0), f64::NEG_INFINITY); - assert_eq!(vx_array_get_f64(ffi_f64, 1), 0.0); - assert!(vx_array_get_f64(ffi_f64, 2).is_nan()); - vx_array_free(ffi_f64); - - // Test f16 (special half-precision type) - skip in Miri due to inline assembly - #[cfg(not(miri))] - { - let f16_array = [f16::from_f32(1.0), f16::from_f32(-0.5)]; - let ffi_f16 = vx_array_new_primitive( - vx_ptype::PTYPE_F16, - f16_array.as_ptr() as *const c_void, - f16_array.len(), - &raw const validity, - &raw mut error, - ); - assert_no_error(error); - assert!(!ffi_f16.is_null()); - assert_eq!(vx_array_get_f16(ffi_f16, 0), f16::from_f32(1.0)); - assert_eq!(vx_array_get_f16(ffi_f16, 1), f16::from_f32(-0.5)); - vx_array_free(ffi_f16); - } - } - } - #[test] // TODO(joe): enable once this is fixed https://github.com/Amanieu/parking_lot/issues/477 #[cfg_attr(miri, ignore)] @@ -1049,6 +989,8 @@ mod tests { assert!(!vx.is_null()); unsafe { + let session = vx_session_new(); + assert!(vx_array_has_dtype(vx, vx_dtype_variant::DTYPE_STRUCT)); assert_eq!(vx_array_len(vx), 3); assert!(!vx_array_is_nullable(vx)); @@ -1056,31 +998,19 @@ mod tests { let a = vx_array_get_field(vx, 0, &raw mut error); assert_no_error(error); assert!(vx_array_is_primitive(a, vx_ptype::PTYPE_I32)); - assert_eq!(vx_array_get_i32(a, 0), 1); - assert_eq!(vx_array_get_i32(a, 2), 3); + assert_eq!(get_i32(session, a, 0), 1); + assert_eq!(get_i32(session, a, 2), 3); vx_array_free(a); let b = vx_array_get_field(vx, 1, &raw mut error); assert_no_error(error); assert!(vx_array_has_dtype(b, vx_dtype_variant::DTYPE_UTF8)); - assert!(vx_array_element_is_invalid(b, 1, &raw mut error)); + assert!(vx_array_element_is_invalid(session, b, 1, &raw mut error)); assert_no_error(error); vx_array_free(b); vx_array_free(vx); - } - } - - #[test] - #[cfg_attr(miri, ignore)] - fn test_get_bool() { - let bools = BoolArray::from_iter([true, false, true]); - unsafe { - let array = vx_array::new(bools.into_array()); - assert!(vx_array_get_bool(array, 0)); - assert!(!vx_array_get_bool(array, 1)); - assert!(vx_array_get_bool(array, 2)); - vx_array_free(array); + vx_session_free(session); } } @@ -1218,4 +1148,17 @@ mod tests { vx_array_free(array); } } + + #[test] + #[cfg_attr(miri, ignore)] + fn test_get_bool() { + let bools = BoolArray::from_iter([true, false, true]); + unsafe { + let array = vx_array::new(bools.into_array()); + assert!(vx_array_get_bool(array, 0)); + assert!(!vx_array_get_bool(array, 1)); + assert!(vx_array_get_bool(array, 2)); + vx_array_free(array); + } + } } diff --git a/vortex-ffi/src/scalar.rs b/vortex-ffi/src/scalar.rs index 792834a7028..492659bc8ec 100644 --- a/vortex-ffi/src/scalar.rs +++ b/vortex-ffi/src/scalar.rs @@ -1,18 +1,17 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors -//! FFI interface for working with Vortex scalar values. - use std::ptr; use std::slice; -use std::str; use std::sync::Arc; +use paste::paste; use vortex::dtype::DType; use vortex::dtype::DecimalDType; use vortex::dtype::Nullability; use vortex::dtype::half::f16; use vortex::dtype::i256; +use vortex::error::VortexExpect; use vortex::error::VortexResult; use vortex::error::vortex_bail; use vortex::error::vortex_ensure; @@ -28,7 +27,10 @@ use crate::string::vx_view; box_wrapper!( /// A vx_scalar is a single value with an associated vx_dtype. + /// /// Scalar value may be Null is vx_dtype is nullable. + /// One example where you can get a Null scalar is vx_array_get_scalar + /// where the element at some index is invalid/null. Scalar, vx_scalar ); @@ -45,7 +47,7 @@ pub unsafe extern "C-unwind" fn vx_scalar_dtype(scalar: *const vx_scalar) -> *co vx_dtype::new(vx_scalar::as_ref(scalar).dtype().clone()) } -/// Return whether scalar is a typed null value. +/// Return whether scalar is a typed Null value. #[unsafe(no_mangle)] pub unsafe extern "C-unwind" fn vx_scalar_is_null(scalar: *const vx_scalar) -> bool { vx_scalar::as_ref(scalar).is_null() @@ -60,70 +62,58 @@ pub unsafe extern "C-unwind" fn vx_scalar_new_bool( vx_scalar::new(Scalar::bool(value, Nullability::from(is_nullable))) } -/// Create an unsigned 8-bit integer scalar. -#[unsafe(no_mangle)] -pub unsafe extern "C-unwind" fn vx_scalar_new_u8(value: u8, is_nullable: bool) -> *mut vx_scalar { - vx_scalar::new(Scalar::primitive(value, Nullability::from(is_nullable))) -} - -/// Create an unsigned 16-bit integer scalar. -#[unsafe(no_mangle)] -pub unsafe extern "C-unwind" fn vx_scalar_new_u16(value: u16, is_nullable: bool) -> *mut vx_scalar { - vx_scalar::new(Scalar::primitive(value, Nullability::from(is_nullable))) -} - -/// Create an unsigned 32-bit integer scalar. -#[unsafe(no_mangle)] -pub unsafe extern "C-unwind" fn vx_scalar_new_u32(value: u32, is_nullable: bool) -> *mut vx_scalar { - vx_scalar::new(Scalar::primitive(value, Nullability::from(is_nullable))) -} - -/// Create an unsigned 64-bit integer scalar. -#[unsafe(no_mangle)] -pub unsafe extern "C-unwind" fn vx_scalar_new_u64(value: u64, is_nullable: bool) -> *mut vx_scalar { - vx_scalar::new(Scalar::primitive(value, Nullability::from(is_nullable))) -} - -/// Create a signed 8-bit integer scalar. -#[unsafe(no_mangle)] -pub unsafe extern "C-unwind" fn vx_scalar_new_i8(value: i8, is_nullable: bool) -> *mut vx_scalar { - vx_scalar::new(Scalar::primitive(value, Nullability::from(is_nullable))) -} - -/// Create a signed 16-bit integer scalar. -#[unsafe(no_mangle)] -pub unsafe extern "C-unwind" fn vx_scalar_new_i16(value: i16, is_nullable: bool) -> *mut vx_scalar { - vx_scalar::new(Scalar::primitive(value, Nullability::from(is_nullable))) -} - -/// Create a signed 32-bit integer scalar. -#[unsafe(no_mangle)] -pub unsafe extern "C-unwind" fn vx_scalar_new_i32(value: i32, is_nullable: bool) -> *mut vx_scalar { - vx_scalar::new(Scalar::primitive(value, Nullability::from(is_nullable))) -} - -/// Create a signed 64-bit integer scalar. +/// Return the boolean value stored in the scalar. +/// +/// Panics if the scalar is not a Bool scalar, or is null. #[unsafe(no_mangle)] -pub unsafe extern "C-unwind" fn vx_scalar_new_i64(value: i64, is_nullable: bool) -> *mut vx_scalar { - vx_scalar::new(Scalar::primitive(value, Nullability::from(is_nullable))) -} +pub unsafe extern "C-unwind" fn vx_scalar_get_bool(scalar: *const vx_scalar) -> bool { + vx_scalar::as_ref(scalar) + .as_bool() + .value() + .vortex_expect("scalar is null or not a bool") +} + +macro_rules! scalar_primitive { + ($ptype:ident) => { + paste! { + #[doc = concat!(" Create a ", stringify!($ptype), " scalar.")] + #[unsafe(no_mangle)] + pub unsafe extern "C-unwind" fn []( + value: $ptype, + is_nullable: bool, + ) -> *mut vx_scalar { + vx_scalar::new(Scalar::primitive(value, Nullability::from(is_nullable))) + } -/// Create a 32-bit floating point scalar. -#[unsafe(no_mangle)] -pub unsafe extern "C-unwind" fn vx_scalar_new_f32(value: f32, is_nullable: bool) -> *mut vx_scalar { - vx_scalar::new(Scalar::primitive(value, Nullability::from(is_nullable))) + #[doc = concat!(" Return ", stringify!($ptype), " value stored in scalar.")] + /// + /// Panics if scalar is not a primitive scalar of this type or is null. + #[unsafe(no_mangle)] + pub unsafe extern "C-unwind" fn []( + scalar: *const vx_scalar, + ) -> $ptype { + vx_scalar::as_ref(scalar) + .as_primitive() + .typed_value::<$ptype>() + .vortex_expect(concat!("scalar is null or not a ", stringify!($ptype))) + } + } + }; } -/// Create a 64-bit floating point scalar. -#[unsafe(no_mangle)] -pub unsafe extern "C-unwind" fn vx_scalar_new_f64(value: f64, is_nullable: bool) -> *mut vx_scalar { - vx_scalar::new(Scalar::primitive(value, Nullability::from(is_nullable))) -} +scalar_primitive!(u8); +scalar_primitive!(u16); +scalar_primitive!(u32); +scalar_primitive!(u64); +scalar_primitive!(i8); +scalar_primitive!(i16); +scalar_primitive!(i32); +scalar_primitive!(i64); +scalar_primitive!(f32); +scalar_primitive!(f64); /// Create a 16-bit floating point scalar. -/// -/// The value is read from raw half-precision bits because C has no portable -/// half-precision floating point ABI. +/// The value is read from raw uint16_t. #[unsafe(no_mangle)] pub unsafe extern "C-unwind" fn vx_scalar_new_f16_bits( bits: u16, @@ -137,8 +127,8 @@ pub unsafe extern "C-unwind" fn vx_scalar_new_f16_bits( /// Create a UTF-8 scalar. /// -/// The string bytes are copied into the scalar. Invalid UTF-8 returns NULL and -/// writes the error output. +/// "value" bytes are copied into scalar. +/// Errors on invalid UTF-8. #[unsafe(no_mangle)] pub unsafe extern "C-unwind" fn vx_scalar_new_utf8( value: vx_view, @@ -156,8 +146,9 @@ pub unsafe extern "C-unwind" fn vx_scalar_new_utf8( /// Create a binary scalar. /// -/// The byte range is copied into the scalar. NULL "ptr" is allowed only when -/// len == 0. +/// Byte range is copied into the scalar. +/// +/// NULL "ptr" is allowed only when len == 0. /// /// Returns NULL and sets "err" on error. #[unsafe(no_mangle)] @@ -176,96 +167,102 @@ pub unsafe extern "C-unwind" fn vx_scalar_new_binary( }) } -/// Create a typed null scalar. +/// Return UTF-8 string stored in scalar. /// -/// Returned scalar uses a nullable copy of that logical type, regardless of -/// the input type's top-level nullability. +/// Returned view borrows the scalar and is valid as long as "scalar" is valid. /// -/// Returns NULL and sets "err" on error. +/// Panics if scalar is not a Utf8 scalar, or is null. #[unsafe(no_mangle)] -pub unsafe extern "C-unwind" fn vx_scalar_new_null( - dtype: *const vx_dtype, - err: *mut *mut vx_error, -) -> *mut vx_scalar { - try_or(err, ptr::null_mut(), || { - Ok(vx_scalar::new(Scalar::null( - vx_dtype::as_ref(dtype).as_nullable(), - ))) - }) +pub unsafe extern "C-unwind" fn vx_scalar_get_utf8(scalar: *const vx_scalar) -> vx_view { + let value = vx_scalar::as_ref(scalar) + .as_utf8() + .value() + .vortex_expect("scalar is null or not a utf8"); + vx_view::from_str(value.as_str()) } -/// Create a decimal scalar. +/// Return binary bytes stored in the scalar. /// -/// The unscaled value is provided as a signed 8-bit integer. +/// Returned view borrows scalar and is valid as long as "scalar" is valid. /// -/// Returns NULL and sets "err" on error. +/// Panics if scalar is not a Binary scalar, or is null. #[unsafe(no_mangle)] -pub unsafe extern "C-unwind" fn vx_scalar_new_decimal_i8( - value: i8, - precision: u8, - scale: i8, - is_nullable: bool, - err: *mut *mut vx_error, -) -> *mut vx_scalar { - try_or(err, ptr::null_mut(), || { - decimal_scalar_from_value(DecimalValue::I8(value), precision, scale, is_nullable) - }) +pub unsafe extern "C-unwind" fn vx_scalar_get_binary(scalar: *const vx_scalar) -> vx_view { + let value = vx_scalar::as_ref(scalar) + .as_binary() + .value() + .vortex_expect("scalar is null or not a binary"); + vx_view::from_bytes(value.as_slice()) } -/// Create a decimal scalar. +/// Create a typed null scalar. /// -/// The unscaled value is provided as a signed 16-bit integer. +/// Returned scalar uses a nullable copy of that logical type, regardless of +/// the input type's top-level nullability. /// /// Returns NULL and sets "err" on error. #[unsafe(no_mangle)] -pub unsafe extern "C-unwind" fn vx_scalar_new_decimal_i16( - value: i16, - precision: u8, - scale: i8, - is_nullable: bool, +pub unsafe extern "C-unwind" fn vx_scalar_new_null( + dtype: *const vx_dtype, err: *mut *mut vx_error, ) -> *mut vx_scalar { try_or(err, ptr::null_mut(), || { - decimal_scalar_from_value(DecimalValue::I16(value), precision, scale, is_nullable) + Ok(vx_scalar::new(Scalar::null( + vx_dtype::as_ref(dtype).as_nullable(), + ))) }) } -/// Create a decimal scalar. -/// -/// The unscaled value is provided as a signed 32-bit integer. -/// -/// Returns NULL and sets "err" on error. -#[unsafe(no_mangle)] -pub unsafe extern "C-unwind" fn vx_scalar_new_decimal_i32( - value: i32, - precision: u8, - scale: i8, - is_nullable: bool, - err: *mut *mut vx_error, -) -> *mut vx_scalar { - try_or(err, ptr::null_mut(), || { - decimal_scalar_from_value(DecimalValue::I32(value), precision, scale, is_nullable) - }) -} +macro_rules! scalar_decimal { + ($int:ident, $variant:ident) => { + paste! { + #[doc = concat!(" Create a decimal scalar from a signed ", stringify!($int), " unscaled value.")] + /// + /// Returns NULL and sets "err" on error. + #[unsafe(no_mangle)] + pub unsafe extern "C-unwind" fn []( + value: $int, + precision: u8, + scale: i8, + is_nullable: bool, + err: *mut *mut vx_error, + ) -> *mut vx_scalar { + try_or(err, ptr::null_mut(), || { + decimal_scalar_from_value( + DecimalValue::$variant(value), + precision, + scale, + is_nullable, + ) + }) + } -/// Create a decimal scalar. -/// -/// The unscaled value is provided as a signed 64-bit integer. -/// -/// Returns NULL and sets "err" on error. -#[unsafe(no_mangle)] -pub unsafe extern "C-unwind" fn vx_scalar_new_decimal_i64( - value: i64, - precision: u8, - scale: i8, - is_nullable: bool, - err: *mut *mut vx_error, -) -> *mut vx_scalar { - try_or(err, ptr::null_mut(), || { - decimal_scalar_from_value(DecimalValue::I64(value), precision, scale, is_nullable) - }) + #[doc = concat!(" Return the unscaled ", stringify!($int), " value of a decimal scalar.")] + /// + /// Panics if the scalar is not a decimal scalar, is null, or the + #[doc = concat!(" unscaled value does not fit in ", stringify!($int), ".")] + #[unsafe(no_mangle)] + pub unsafe extern "C-unwind" fn []( + scalar: *const vx_scalar, + ) -> $int { + vx_scalar::as_ref(scalar) + .as_decimal() + .decimal_value() + .and_then(|value| value.cast::<$int>()) + .vortex_expect(concat!( + "scalar is null or its decimal value does not fit in ", + stringify!($int) + )) + } + } + }; } +scalar_decimal!(i8, I8); +scalar_decimal!(i16, I16); +scalar_decimal!(i32, I32); +scalar_decimal!(i64, I64); + /// Create a decimal scalar. /// /// The unscaled value is read from a 16-byte little-endian signed integer @@ -439,6 +436,10 @@ mod tests { use std::ptr; use std::sync::Arc; + use vortex::array::IntoArray; + use vortex::array::arrays::PrimitiveArray; + use vortex::array::validity::Validity; + use vortex::buffer::buffer; use vortex::dtype::DType; use vortex::dtype::DecimalDType; use vortex::dtype::Nullability; @@ -448,40 +449,16 @@ mod tests { use vortex::scalar::DecimalValue; use vortex::scalar::Scalar; + use crate::array::*; use crate::dtype::vx_dtype; use crate::dtype::vx_dtype_free; use crate::dtype::vx_dtype_new_bool; use crate::dtype::vx_dtype_new_primitive; use crate::ptype::vx_ptype; - use crate::scalar::vx_scalar; - use crate::scalar::vx_scalar_clone; - use crate::scalar::vx_scalar_dtype; - use crate::scalar::vx_scalar_free; - use crate::scalar::vx_scalar_is_null; - use crate::scalar::vx_scalar_new_binary; - use crate::scalar::vx_scalar_new_bool; - use crate::scalar::vx_scalar_new_decimal_i8; - use crate::scalar::vx_scalar_new_decimal_i16; - use crate::scalar::vx_scalar_new_decimal_i32; - use crate::scalar::vx_scalar_new_decimal_i64; - use crate::scalar::vx_scalar_new_decimal_i128_le; - use crate::scalar::vx_scalar_new_decimal_i256_le; - use crate::scalar::vx_scalar_new_f16_bits; - use crate::scalar::vx_scalar_new_f32; - use crate::scalar::vx_scalar_new_f64; - use crate::scalar::vx_scalar_new_fixed_size_list; - use crate::scalar::vx_scalar_new_i8; - use crate::scalar::vx_scalar_new_i16; - use crate::scalar::vx_scalar_new_i32; - use crate::scalar::vx_scalar_new_i64; - use crate::scalar::vx_scalar_new_list; - use crate::scalar::vx_scalar_new_null; - use crate::scalar::vx_scalar_new_struct; - use crate::scalar::vx_scalar_new_u8; - use crate::scalar::vx_scalar_new_u16; - use crate::scalar::vx_scalar_new_u32; - use crate::scalar::vx_scalar_new_u64; - use crate::scalar::vx_scalar_new_utf8; + use crate::scalar::*; + use crate::session::vx_session; + use crate::session::vx_session_free; + use crate::session::vx_session_new; use crate::string::vx_view; use crate::tests::assert_error; use crate::tests::assert_no_error; @@ -653,7 +630,7 @@ mod tests { ); assert_no_error(error); - let i256_value = vortex::dtype::i256::from_i128(12345); + let i256_value = i256::from_i128(12345); assert_scalar( vx_scalar_new_decimal_i256_le( i256_value.to_le_bytes().as_ptr(), @@ -801,4 +778,111 @@ mod tests { vx_dtype_free(dtype); } } + + #[test] + // TODO(joe): enable once this is fixed https://github.com/Amanieu/parking_lot/issues/477 + #[cfg_attr(miri, ignore)] + fn test_array_scalar_getters() { + unsafe fn get_i32(session: *const vx_session, array: *const vx_array, index: usize) -> i32 { + let mut error = ptr::null_mut(); + let scalar = unsafe { vx_array_get_scalar(session, array, index, &raw mut error) }; + assert_no_error(error); + let value = unsafe { vx_scalar_get_i32(scalar) }; + unsafe { vx_scalar_free(scalar.cast_mut()) }; + value + } + + unsafe fn get_f64(session: *const vx_session, array: *const vx_array, index: usize) -> f64 { + let mut error = ptr::null_mut(); + let scalar = unsafe { vx_array_get_scalar(session, array, index, &raw mut error) }; + assert_no_error(error); + let value = unsafe { vx_scalar_get_f64(scalar) }; + unsafe { vx_scalar_free(scalar.cast_mut()) }; + value + } + + unsafe { + let session = vx_session_new(); + + let i32_array = + PrimitiveArray::new(buffer![i32::MAX, i32::MIN, 0], Validity::NonNullable) + .into_array(); + let ffi_i32 = vx_array::new(i32_array); + assert!(vx_array_is_primitive(ffi_i32, vx_ptype::PTYPE_I32)); + assert_eq!(get_i32(session, ffi_i32, 0), i32::MAX); + assert_eq!(get_i32(session, ffi_i32, 1), i32::MIN); + assert_eq!(get_i32(session, ffi_i32, 2), 0); + vx_array_free(ffi_i32); + + let f64_array = PrimitiveArray::new( + buffer![f64::NEG_INFINITY, 0.0f64, f64::NAN], + Validity::NonNullable, + ) + .into_array(); + let ffi_f64 = vx_array::new(f64_array); + assert_eq!(get_f64(session, ffi_f64, 0), f64::NEG_INFINITY); + assert_eq!(get_f64(session, ffi_f64, 1), 0.0); + assert!(get_f64(session, ffi_f64, 2).is_nan()); + vx_array_free(ffi_f64); + + vx_session_free(session); + } + } + + #[test] + fn test_scalar_primitive_getters() { + unsafe { + let s = vx_scalar_new_i32(-42, false); + assert_eq!(vx_scalar_get_i32(s), -42); + vx_scalar_free(s); + + let s = vx_scalar_new_u64(u64::MAX, true); + assert_eq!(vx_scalar_get_u64(s), u64::MAX); + vx_scalar_free(s); + + let s = vx_scalar_new_f64(1.5, false); + assert_eq!(vx_scalar_get_f64(s), 1.5); + vx_scalar_free(s); + + let s = vx_scalar_new_bool(true, false); + assert!(vx_scalar_get_bool(s)); + vx_scalar_free(s); + } + } + + #[test] + fn test_scalar_string_getters() { + unsafe { + let mut error = ptr::null_mut(); + + let value = "hello"; + let s = vx_scalar_new_utf8(vx_view::from_str(value), false, &raw mut error); + assert_no_error(error); + assert_eq!(vx_scalar_get_utf8(s).as_str().unwrap(), value); + vx_scalar_free(s); + + let bytes = b"\xde\xad\xbe\xef"; + let s = vx_scalar_new_binary(bytes.as_ptr(), bytes.len(), false, &raw mut error); + assert_no_error(error); + assert_eq!(vx_scalar_get_binary(s).as_bytes().unwrap(), bytes); + vx_scalar_free(s); + } + } + + #[test] + fn test_scalar_decimal_getters() { + unsafe { + let mut error = ptr::null_mut(); + + let s = vx_scalar_new_decimal_i32(1234, 5, 2, false, &raw mut error); + assert_no_error(error); + assert_eq!(vx_scalar_get_decimal_i32(s), 1234); + vx_scalar_free(s); + + let s = vx_scalar_new_decimal_i64(99999, 12, 3, false, &raw mut error); + assert_no_error(error); + assert_eq!(vx_scalar_get_decimal_i64(s), 99999); + vx_scalar_free(s); + } + } } diff --git a/vortex-ffi/test/array.cpp b/vortex-ffi/test/array.cpp index f8db2661b8d..3a90fe909c2 100644 --- a/vortex-ffi/test/array.cpp +++ b/vortex-ffi/test/array.cpp @@ -19,6 +19,11 @@ TEST_CASE("Null array creation", "[array]") { } TEST_CASE("Primitive array creation", "[array]") { + vx_session *session = vx_session_new(); + defer { + vx_session_free(session); + }; + std::vector buffer(20, 1); buffer[3] = 8; @@ -39,13 +44,13 @@ TEST_CASE("Primitive array creation", "[array]") { REQUIRE(vx_array_len(array) == buffer.size()); for (size_t i = 0; i < buffer.size(); ++i) { - REQUIRE(buffer[i] == vx_array_get_u8(array, i)); + REQUIRE(buffer[i] == array_get_u8(session, array, i)); } buffer = {}; for (size_t i = 0; i < 20; ++i) { - REQUIRE(vx_array_get_u8(array, i) == (i == 3 ? 8 : 1)); + REQUIRE(array_get_u8(session, array, i) == (i == 3 ? 8 : 1)); } vx_array_free(array); diff --git a/vortex-ffi/test/common.h b/vortex-ffi/test/common.h index 81974b65bb9..92852ef48f9 100644 --- a/vortex-ffi/test/common.h +++ b/vortex-ffi/test/common.h @@ -27,6 +27,24 @@ inline void require_no_error(vx_error *error, bool assert = true) { } } +inline uint8_t array_get_u8(vx_session *session, const vx_array *array, size_t index) { + vx_error *error = nullptr; + const vx_scalar *scalar = vx_array_get_scalar(session, array, index, &error); + require_no_error(error); + const uint8_t value = vx_scalar_get_u8(scalar); + vx_scalar_free(scalar); + return value; +} + +inline uint16_t array_get_u16(vx_session *session, const vx_array *array, size_t index) { + vx_error *error = nullptr; + const vx_scalar *scalar = vx_array_get_scalar(session, array, index, &error); + require_no_error(error); + const uint16_t value = vx_scalar_get_u16(scalar); + vx_scalar_free(scalar); + return value; +} + template struct Defer { Defer(F &&f) : f(std::move(f)) { diff --git a/vortex-ffi/test/scalar.cpp b/vortex-ffi/test/scalar.cpp new file mode 100644 index 00000000000..5edfec557f3 --- /dev/null +++ b/vortex-ffi/test/scalar.cpp @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors +#include +#include +#include +#include +#include +#include "common.h" + +using namespace std::string_view_literals; + +TEST_CASE("Primitive getters", "[scalar]") { + vx_scalar *u64 = vx_scalar_new_u64(UINT64_MAX, true); + defer { + vx_scalar_free(u64); + }; + REQUIRE(vx_scalar_get_u64(u64) == UINT64_MAX); + + vx_scalar *f64 = vx_scalar_new_f64(1.5, false); + defer { + vx_scalar_free(f64); + }; + REQUIRE(vx_scalar_get_f64(f64) == 1.5); +} + +TEST_CASE("String getters", "[scalar]") { + vx_error *error = nullptr; + + const std::string_view text = "hello"sv; + vx_scalar *utf8 = vx_scalar_new_utf8(vx_view {text.data(), text.size()}, false, &error); + require_no_error(error); + defer { + vx_scalar_free(utf8); + }; + REQUIRE(to_string_view(vx_scalar_get_utf8(utf8)) == text); + + const uint8_t bytes[] = {0xde, 0xad, 0xbe, 0xef}; + vx_scalar *binary = vx_scalar_new_binary(bytes, sizeof(bytes), false, &error); + require_no_error(error); + defer { + vx_scalar_free(binary); + }; + const vx_view view = vx_scalar_get_binary(binary); + REQUIRE(view.len == sizeof(bytes)); + REQUIRE(std::memcmp(view.ptr, bytes, sizeof(bytes)) == 0); +} + +TEST_CASE("Decimal getters", "[scalar]") { + vx_error *error = nullptr; + + vx_scalar *d32 = vx_scalar_new_decimal_i32(1234, 5, 2, false, &error); + require_no_error(error); + defer { + vx_scalar_free(d32); + }; + REQUIRE(vx_scalar_get_decimal_i32(d32) == 1234); + + vx_scalar *d64 = vx_scalar_new_decimal_i64(99999, 12, 3, false, &error); + require_no_error(error); + defer { + vx_scalar_free(d64); + }; + REQUIRE(vx_scalar_get_decimal_i64(d64) == 99999); +} diff --git a/vortex-ffi/test/scan.cpp b/vortex-ffi/test/scan.cpp index e1f629dfde0..ba7ea3aa366 100644 --- a/vortex-ffi/test/scan.cpp +++ b/vortex-ffi/test/scan.cpp @@ -302,7 +302,7 @@ TEST_CASE("Write file and read dtypes", "[datasource]") { REQUIRE(to_string_view(height_name) == "height"); } -void verify_age_field(const vx_array *age_field) { +void verify_age_field(vx_session *session, const vx_array *age_field) { REQUIRE(vx_array_has_dtype(age_field, DTYPE_PRIMITIVE)); const vx_dtype *dtype = vx_array_dtype(age_field); defer { @@ -311,11 +311,11 @@ void verify_age_field(const vx_array *age_field) { REQUIRE(vx_dtype_primitive_ptype(dtype) == PTYPE_U8); REQUIRE(vx_array_len(age_field) == SAMPLE_ROWS); for (size_t i = 0; i < SAMPLE_ROWS; ++i) { - REQUIRE(vx_array_get_u8(age_field, i) == i); + REQUIRE(array_get_u8(session, age_field, i) == i); } } -void verify_height_field(const vx_array *height_field) { +void verify_height_field(vx_session *session, const vx_array *height_field) { REQUIRE(vx_array_has_dtype(height_field, DTYPE_PRIMITIVE)); const vx_dtype *dtype = vx_array_dtype(height_field); defer { @@ -324,11 +324,11 @@ void verify_height_field(const vx_array *height_field) { REQUIRE(vx_dtype_primitive_ptype(dtype) == PTYPE_U16); REQUIRE(vx_array_len(height_field) == SAMPLE_ROWS); for (size_t i = 0; i < SAMPLE_ROWS; ++i) { - REQUIRE(vx_array_get_u16(height_field, i) > 0); + REQUIRE(array_get_u16(session, height_field, i) > 0); } } -void verify_sample_array(const vx_array *array) { +void verify_sample_array(vx_session *session, const vx_array *array) { REQUIRE(vx_array_len(array) == SAMPLE_ROWS); REQUIRE(vx_array_has_dtype(array, DTYPE_STRUCT)); @@ -363,12 +363,12 @@ void verify_sample_array(const vx_array *array) { const vx_array *age_field = vx_array_get_field(array, 0, &error); require_no_error(error); - verify_age_field(age_field); + verify_age_field(session, age_field); vx_array_free(age_field); const vx_array *height_field = vx_array_get_field(array, 1, &error); require_no_error(error); - verify_height_field(height_field); + verify_height_field(session, height_field); vx_array_free(height_field); REQUIRE(vx_array_get_field(array, 2, &error) == nullptr); @@ -413,7 +413,7 @@ TEST_CASE("Requesting scans", "[datasource]") { } } -void basic_scan(const vx_data_source *ds) { +void basic_scan(vx_session *session, const vx_data_source *ds) { vx_error *error = nullptr; vx_estimate estimate = {}; vx_scan *scan = vx_data_source_scan(ds, nullptr, &estimate, &error); @@ -450,7 +450,7 @@ void basic_scan(const vx_data_source *ds) { REQUIRE(vx_partition_next(partition, &error) == nullptr); require_no_error(error); - verify_sample_array(array); + verify_sample_array(session, array); } TEST_CASE("Basic scan", "[datasource]") { @@ -472,7 +472,7 @@ TEST_CASE("Basic scan", "[datasource]") { vx_data_source_free(ds); }; - basic_scan(ds); + basic_scan(session, ds); } TEST_CASE("Basic scan from memory", "[datasource]") { @@ -496,7 +496,7 @@ TEST_CASE("Basic scan from memory", "[datasource]") { vx_data_source_free(ds); }; - basic_scan(ds); + basic_scan(session, ds); } TEST_CASE("Multithreaded scan", "[datasource]") { @@ -590,15 +590,11 @@ TEST_CASE("Multithreaded scan", "[datasource]") { defer { vx_array_free(array); }; - verify_sample_array(array); + verify_sample_array(session, array); } } -const vx_array *scan_with_options(vx_scan_options &options) { - vx_session *session = vx_session_new(); - defer { - vx_session_free(session); - }; +const vx_array *scan_with_options(vx_session *session, vx_scan_options &options) { TempPath path = write_sample(session); vx_error *error = nullptr; @@ -636,29 +632,41 @@ const vx_array *scan_with_options(vx_scan_options &options) { } TEST_CASE("Project all fields", "[projection]") { + vx_session *session = vx_session_new(); + defer { + vx_session_free(session); + }; vx_scan_options opts = {}; - const vx_array *array = scan_with_options(opts); + const vx_array *array = scan_with_options(session, opts); defer { vx_array_free(array); }; - verify_sample_array(array); + verify_sample_array(session, array); } TEST_CASE("Project root", "[projection]") { + vx_session *session = vx_session_new(); + defer { + vx_session_free(session); + }; vx_expression *root = vx_expression_root(); defer { vx_expression_free(root); }; vx_scan_options opts = {}; opts.projection = root; - const vx_array *array = scan_with_options(opts); + const vx_array *array = scan_with_options(session, opts); defer { vx_array_free(array); }; - verify_sample_array(array); + verify_sample_array(session, array); } TEST_CASE("Project single field", "[projection]") { + vx_session *session = vx_session_new(); + defer { + vx_session_free(session); + }; vx_expression *root = vx_expression_root(); defer { vx_expression_free(root); @@ -673,11 +681,11 @@ TEST_CASE("Project single field", "[projection]") { { opts.projection = age_field; - const vx_array *array = scan_with_options(opts); + const vx_array *array = scan_with_options(session, opts); defer { vx_array_free(array); }; - verify_age_field(array); + verify_age_field(session, array); } vx_expression *height_field = vx_expression_get_item(vx_view_from_cstr("height"), root); @@ -688,15 +696,19 @@ TEST_CASE("Project single field", "[projection]") { { opts.projection = height_field; - const vx_array *array = scan_with_options(opts); + const vx_array *array = scan_with_options(session, opts); defer { vx_array_free(array); }; - verify_height_field(array); + verify_height_field(session, array); } } TEST_CASE("Filter with literal expression", "[filter]") { + vx_session *session = vx_session_new(); + defer { + vx_session_free(session); + }; vx_expression *root = vx_expression_root(); defer { vx_expression_free(root); @@ -731,7 +743,7 @@ TEST_CASE("Filter with literal expression", "[filter]") { vx_scan_options opts = {}; opts.filter = filter; - const vx_array *array = scan_with_options(opts); + const vx_array *array = scan_with_options(session, opts); defer { vx_array_free(array); }; @@ -747,11 +759,15 @@ TEST_CASE("Filter with literal expression", "[filter]") { }; for (size_t i = 0; i < vx_array_len(filtered_age); ++i) { - REQUIRE(vx_array_get_u8(filtered_age, i) == static_cast(threshold + i)); + REQUIRE(array_get_u8(session, filtered_age, i) == static_cast(threshold + i)); } } TEST_CASE("Project UTF-8 literal expression", "[projection]") { + vx_session *session = vx_session_new(); + defer { + vx_session_free(session); + }; constexpr auto value = "constant"sv; vx_error *scalar_error = nullptr; vx_scalar *literal_scalar = @@ -772,17 +788,13 @@ TEST_CASE("Project UTF-8 literal expression", "[projection]") { vx_scan_options opts = {}; opts.projection = literal_expr; - const vx_array *array = scan_with_options(opts); + const vx_array *array = scan_with_options(session, opts); defer { vx_array_free(array); }; REQUIRE(vx_array_len(array) == SAMPLE_ROWS); - vx_session *session = vx_session_new(); - defer { - vx_session_free(session); - }; vx_error *canon_error = nullptr; const vx_array *canonical = vx_array_canonicalize(session, array, &canon_error); require_no_error(canon_error);