ffi: create bool arrays

lang/cpp/examples/CMakeLists.txt

@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright the Vortex contributors
 
-foreach(name reader writer dtype scan scan_to_arrow)
+foreach(name reader writer validity dtype scan scan_to_arrow)
     add_executable(${name} ${name}.cpp)
     target_link_libraries(${name} PRIVATE vortex_cxx_shared nanoarrow_shared)
 endforeach()

lang/cpp/examples/reader.cpp

@@ -10,7 +10,6 @@ using namespace std::string_view_literals;
 using namespace vortex;
 using namespace expr;
 using namespace ops; // overloaded >= for Expressions
-namespace fs = std::filesystem;
 
 int main() {
     const Session session;

lang/cpp/examples/validity.cpp

@@ -0,0 +1,45 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+#include "vortex/dtype.hpp"
+#include <iostream>
+
+#include <vortex.h>
+#include <vortex/data_source.hpp>
+#include <vortex/writer.hpp>
+
+using namespace std::string_view_literals;
+using namespace vortex;
+using dtype::Nullable;
+
+std::vector<uint8_t> bitpack(const std::vector<bool> &validity) {
+    std::vector<uint8_t> out;
+
+    return out;
+}
+
+int main() {
+    const std::string_view name = "validity.vortex";
+
+    const Session session;
+    const DataType dtype = dtype::int64(Nullable);
+    Writer writer = Writer::open(session, name, dtype);
+    writer.push(array);
+    writer.finish();
+
+    const DataSource ds = DataSource::open(session, {name});
+    Scan scan = ds.scan();
+    for (Partition &partition : scan.partitions()) {
+        for (Array &array : partition.batches()) {
+            const Array age = array.field("age");
+            const PrimitiveView<uint8_t> age_view = age.values<uint8_t>(session);
+            const std::span<const uint8_t> age_values = age_view.values();
+            for (uint8_t value : age_values) {
+                std::cout << int(value) << " ";
+            }
+        }
+    }
+    std::cout << "\n";
+
+    return 0;
+}

lang/cpp/include/vortex/array.hpp

@@ -11,6 +11,7 @@
 #include <vortex.h>
 
 #include <cstddef>
+#include <cstdint>
 #include <initializer_list>
 #include <memory>
 #include <span>
@@ -30,6 +31,46 @@ class Array;
 class StringView;
 class BytesView;
 
+/**
+ * Non-owning view over bitpacked booleans.
+ *
+ * Bits are laid out LSB-first.
+ * "bit_offset" is in [0; 8) and lets a view start at a non-byte-aligned bit.
+ *
+ * Example:
+ * "view" holds 6 boolean elements starting at bit 2, first 5 set to "true",
+ * last is "false".
+ *
+ * // [6][5][4][3][2][1][0][offset bit][offset bit]
+ * uint8_t word = 0b01111100;
+ * BoolView view = {&word, 6, 2};
+ */
+struct BoolView {
+    const uint8_t* ptr = nullptr;
+
+    /*
+     * Number of elements (bits) in the view. This is not the number of uint8_t
+     * words in "ptr", use words() for that.
+     */
+    size_t elements = 0;
+
+    /*
+     * Bit offset of first element in "bytes".
+     * Example: if bit_offset is 2, first bit is at bytes[0] & (1 << 2).
+     */
+    size_t bit_offset = 0;
+
+    // Get bit at "index". Index is in [0; elements).
+    inline bool operator[](size_t index) const {
+        return vx_bool_view_nth(*this, index);
+    }
+
+    // Number of uint8_t words storing elements.
+    inline size_t words() const {
+        return vx_bool_view_words(*this);
+    }
+};
+
 /*
  * Validity type tells us whether there are null/invalid values in an Array.
  */
@@ -101,8 +142,7 @@ class ValidityBits {
     ValidityBits(const Session &session, const vx_array *canonical);
 
     const vx_array *owner_ = nullptr;
-    const uint8_t *bits_ = nullptr;
-    size_t bit_offset_ = 0;
+    BoolView view_;
     bool all_invalid_ = false;
 };
 } // namespace detail
@@ -131,6 +171,17 @@ class Array {
         return primitive_raw(detail::to_ptype<T>(), data.data(), data.size(), validity);
     }
 
+    /**
+     * A Bool array copied from bitpacked storage.
+     *
+     * Example:
+     *
+     * std::vector<uint8_t> bits(2, 1);
+     * BoolView view = {bits, 0};
+     * auto array = Array::bool_array(view);
+     */
+    static Array bool_array(const BoolView &view, const Validity &validity = ValidityType::NonNullable);
+
     /**
      * Import an Arrow array. Consumes both "array" and "schema", do not use
      * or release them afterwards. For a record batch pass nullable = false.
@@ -293,17 +344,17 @@ class PrimitiveView {
 };
 
 /**
- * Read-only view over a Bool array. As Bool values are bit-packed, there's no
- * span. Read individual values with value(i).
+ * Read-only view over a Bool array.
  */
 template <>
 class PrimitiveView<bool> {
 public:
     /*
-     * Get raw value from this view. Values at null/invalid positions are
-     * unspecified.
+     * Get bitpacked storage for this view's values. Values at null/invalid
+     * positions are unspecified
      */
-    bool value(size_t index) const;
+    BoolView values() const;
+
     bool is_null(size_t index) const {
         return validity_.is_null(index);
     }

lang/cpp/src/array.cpp

@@ -76,11 +76,10 @@ bool ValidityBits::is_null(size_t index) const {
     if (all_invalid_) {
         return true;
     }
-    if (bits_ == nullptr) {
+    if (owner_ == nullptr) {
         return false;
     }
-    const size_t bit = bit_offset_ + index;
-    return (bits_[bit / 8] >> (bit % 8) & 1) == 0;
+    return view_[index];
 }
 
 ValidityBits::ValidityBits(const Session &session, const vx_array *canonical) {
@@ -104,29 +103,27 @@ ValidityBits::ValidityBits(const Session &session, const vx_array *canonical) {
     vx_array_free(raw.array);
     throw_on_error(error);
 
-    bits_ = static_cast<const uint8_t *>(vx_array_data_ptr_bool(owner_, &bit_offset_, &error));
+    vx_bool_view view = vx_array_data_ptr_bool(owner_, &error);
     if (error != nullptr) {
         vx_array_free(owner_);
     }
     throw_on_error(error);
+    view_.bit_offset = view.bit_offset;
+    view_.bytes = {view.ptr, view.elements};
 }
 
 ValidityBits::ValidityBits(ValidityBits &&other) noexcept
-    : owner_(other.owner_), bits_(other.bits_), bit_offset_(other.bit_offset_),
-      all_invalid_(other.all_invalid_) {
+    : owner_(other.owner_), view_(std::move(other.view_)), all_invalid_(other.all_invalid_) {
     other.owner_ = nullptr;
-    other.bits_ = nullptr;
 }
 
 ValidityBits &ValidityBits::operator=(ValidityBits &&other) noexcept {
     if (this != &other) {
         vx_array_free(owner_);
         owner_ = other.owner_;
-        bits_ = other.bits_;
-        bit_offset_ = other.bit_offset_;
+        view_ = std::move(other.view_);
         all_invalid_ = other.all_invalid_;
         other.owner_ = nullptr;
-        other.bits_ = nullptr;
     }
     return *this;
 }
@@ -177,6 +174,22 @@ Array Array::primitive_raw(vx_ptype ptype, const void *data, size_t len, const V
     return Access::adopt<Array>(out);
 }
 
+Array Array::bool_array(const BoolView &view, const Validity &validity) {
+    std::optional<Array> keep_alive;
+    vx_validity raw {};
+    raw.type = static_cast<vx_validity_type>(validity.type());
+    if (validity.type() == ValidityType::FromArray) {
+        keep_alive = validity.array();
+        raw.array = Access::c_ptr(*keep_alive);
+    }
+
+    vx_error *error = nullptr;
+    vx_bool_view bool_view {view.bytes.data(), view.elements(), view.bit_offset};
+    const vx_array *out = vx_array_new_bool(&bool_view, &raw, &error);
+    throw_on_error(error);
+    return Access::adopt<Array>(out);
+}
+
 Array Array::from_arrow(ArrowArray *array, ArrowSchema *schema, bool nullable) {
     vx_error *error = nullptr;
     const vx_array *out = vx_array_from_arrow(array, schema, nullable, &error);
@@ -324,8 +337,11 @@ BytesView Array::bytes(const Session &session) const {
     return BytesView(std::move(canonical), std::move(validity), len);
 }
 
-bool PrimitiveView<bool>::value(size_t i) const {
-    return vx_array_get_bool(Access::c_ptr(canonical_), i);
+BoolView PrimitiveView<bool>::values() const {
+    vx_error *error = nullptr;
+    const vx_bool_view view = vx_array_data_ptr_bool(Access::c_ptr(canonical_), &error);
+    throw_on_error(error);
+    return {{view.ptr, vx_bool_view_len(view)}, view.bit_offset};
 }
 
 std::string_view StringView::operator[](size_t i) const {

lang/cpp/tests/array.cpp

@@ -170,6 +170,46 @@ TEST_CASE("Slice", "[array]") {
     REQUIRE_THROWS_AS(a.slice(2, 100), VortexException);
 }
 
+TEST_CASE("Bool array", "[array]") {
+    Session session;
+
+    constexpr size_t ELEMENTS = 9;
+    std::array<bool, ELEMENTS> data = {true, false, true, true, false, true, true, false, true};
+    std::vector<uint8_t> bitpacked(2);
+    for (size_t i = 0; i < ELEMENTS; ++i) {
+        if (data[i]) {
+            bitpacked[i / 8] |= 1 << (i % 8);
+        }
+    }
+
+    BoolView bool_view = {bitpacked, ELEMENTS};
+    Array array = Array::bool_array(bool_view);
+    REQUIRE(array.size() == data.size());
+    REQUIRE(array.has_dtype(DataTypeVariant::Bool));
+    REQUIRE_FALSE(array.nullable());
+
+    auto view = array.bools(session);
+    BoolView values = view.values();
+    for (size_t i = 0; i < values.elements(); ++i) {
+        REQUIRE(values[i] == data[i]);
+    }
+
+    array = array.slice(3, array.size());
+    view = array.bools(session);
+    values = view.values();
+
+    Array roundtrip = Array::bool_array(values);
+    REQUIRE(roundtrip.size() == view.size());
+    REQUIRE_FALSE(roundtrip.nullable());
+
+    auto roundtrip_view = roundtrip.bools(session);
+    BoolView roundtrip_values = roundtrip_view.values();
+    for (size_t i = 0; i < view.size(); ++i) {
+        REQUIRE(roundtrip_values[i] == values[i]);
+        REQUIRE(roundtrip_values[i] == data[i + 3]);
+    }
+}
+
 TEST_CASE("Error with a code", "[array]") {
     std::vector<int16_t> data = {0};
     Array a = Array::primitive<int16_t>(data);

lang/cpp/tests/expression.cpp

@@ -79,12 +79,13 @@ TEST_CASE("Operator overloading", "[expr]") {
     Array array = Array::primitive<uint32_t>(data);
 
     Array applied = array.apply(expr::root() == expr::lit<uint32_t>(2));
-    auto bits = applied.bools(session);
-    REQUIRE(bits.size() == data.size());
-    REQUIRE_FALSE(bits.value(0));
-    REQUIRE(bits.value(1));
-    REQUIRE(bits.value(2));
-    REQUIRE_FALSE(bits.value(3));
+    auto view = applied.bools(session);
+    REQUIRE(view.size() == data.size());
+    BoolView values = view.values();
+    REQUIRE_FALSE(values[0]);
+    REQUIRE(values[1]);
+    REQUIRE(values[2]);
+    REQUIRE_FALSE(values[3]);
 }
 
 TEST_CASE("Apply error", "[expr]") {

vortex-ffi/cbindgen.toml

@@ -63,6 +63,12 @@ typedef struct ArrowSchema FFI_ArrowSchema;
 typedef struct ArrowArray FFI_ArrowArray;
 typedef struct ArrowArrayStream FFI_ArrowArrayStream;
 #endif
+
+// Number of uint8_t words holding view's elements
+#define vx_bool_view_words(X) (((X).elements + (X).bit_offset + 7) / 8)
+// I'th element (bit) in view, LSB-first. Already accounts for bit_offset
+#define vx_bool_view_nth(X, I) \
+    ((((X).ptr[((I) + (X).bit_offset) / 8] & (1 << (((I) + (X).bit_offset) % 8))) != 0))
 """
 
 trailer = """