diff --git a/vortex-array/src/arrays/dict/vtable/mod.rs b/vortex-array/src/arrays/dict/vtable/mod.rs index fa8515dd986..eb742182a10 100644 --- a/vortex-array/src/arrays/dict/vtable/mod.rs +++ b/vortex-array/src/arrays/dict/vtable/mod.rs @@ -46,7 +46,6 @@ use crate::executor::ExecutionResult; use crate::require_child; use crate::scalar::Scalar; use crate::serde::ArrayChildren; -use crate::validity::Validity; mod kernel; mod operations; @@ -179,7 +178,7 @@ impl VTable for Dict { let array = require_child!(array, array.codes(), DictSlots::CODES => Primitive); - if matches!(array.codes().validity()?, Validity::AllInvalid) { + if array.codes().validity()?.definitely_all_null() { return Ok(ExecutionResult::done(ConstantArray::new( Scalar::null(array.dtype().as_nullable()), array.codes().len(), diff --git a/vortex-array/src/arrays/masked/vtable/mod.rs b/vortex-array/src/arrays/masked/vtable/mod.rs index 4409c8ff3c6..ed5ea67cd62 100644 --- a/vortex-array/src/arrays/masked/vtable/mod.rs +++ b/vortex-array/src/arrays/masked/vtable/mod.rs @@ -165,7 +165,7 @@ impl VTable for Masked { let validity = array.masked_validity(); // Fast path: all masked means result is all nulls. - if matches!(validity, Validity::AllInvalid) { + if validity.definitely_all_null() { return Ok(ExecutionResult::done( ConstantArray::new(Scalar::null(array.dtype().as_nullable()), array.len()) .into_array(), diff --git a/vortex-array/src/arrays/primitive/array/top_value.rs b/vortex-array/src/arrays/primitive/array/top_value.rs index d3ee5eb5a65..7ae45c89405 100644 --- a/vortex-array/src/arrays/primitive/array/top_value.rs +++ b/vortex-array/src/arrays/primitive/array/top_value.rs @@ -17,7 +17,6 @@ use crate::arrays::primitive::NativeValue; use crate::dtype::NativePType; use crate::match_each_native_ptype; use crate::scalar::PValue; -use crate::validity::Validity; impl PrimitiveArray { /// Compute most common present value of this array @@ -26,7 +25,7 @@ impl PrimitiveArray { return Ok(None); } - if matches!(self.validity()?, Validity::AllInvalid) { + if self.validity()?.definitely_all_null() { return Ok(None); } diff --git a/vortex-array/src/arrays/varbin/array.rs b/vortex-array/src/arrays/varbin/array.rs index 4d435471ce0..220fe948a73 100644 --- a/vortex-array/src/arrays/varbin/array.rs +++ b/vortex-array/src/arrays/varbin/array.rs @@ -255,7 +255,7 @@ impl VarBinData { } _ => None, }; - let all_invalid = matches!(validity, Validity::AllInvalid); + let all_invalid = validity.definitely_all_null(); match_each_integer_ptype!(primitive_offsets.dtype().as_ptype(), |O| { let offsets_slice = primitive_offsets.as_slice::(); diff --git a/vortex-array/src/scalar_fn/fns/fill_null/kernel.rs b/vortex-array/src/scalar_fn/fns/fill_null/kernel.rs index eea3dd6ef7b..147658ddfb2 100644 --- a/vortex-array/src/scalar_fn/fns/fill_null/kernel.rs +++ b/vortex-array/src/scalar_fn/fns/fill_null/kernel.rs @@ -79,7 +79,7 @@ pub(super) fn precondition( } // If all values are null, replace the entire array with the fill value. - if matches!(array.validity()?, Validity::AllInvalid) { + if array.validity()?.definitely_all_null() { return Ok(Some( ConstantArray::new(fill_value.clone(), array.len()).into_array(), )); diff --git a/vortex-array/src/scalar_fn/fns/list_contains/mod.rs b/vortex-array/src/scalar_fn/fns/list_contains/mod.rs index 978a1da1caf..e16991763ed 100644 --- a/vortex-array/src/scalar_fn/fns/list_contains/mod.rs +++ b/vortex-array/src/scalar_fn/fns/list_contains/mod.rs @@ -415,7 +415,7 @@ fn list_is_not_empty( ctx: &mut ExecutionCtx, ) -> VortexResult { // Short-circuit for all invalid. - if matches!(list_array.validity()?, Validity::AllInvalid) { + if list_array.validity()?.definitely_all_null() { return Ok(ConstantArray::new( Scalar::null(DType::Bool(Nullability::Nullable)), list_array.len(), diff --git a/vortex-array/src/validity.rs b/vortex-array/src/validity.rs index 8d36c07405e..c45c4cf3094 100644 --- a/vortex-array/src/validity.rs +++ b/vortex-array/src/validity.rs @@ -123,6 +123,19 @@ impl Validity { matches!(self, Self::NonNullable | Self::AllValid) } + /// Returns `true` if this validity is *definitely* all-null (every value is null), i.e. it + /// is [`Validity::AllInvalid`]. + /// + /// Returning `false` does not prove that any value is valid: a [`Validity::Array`] may still + /// resolve to all-null once executed. Callers must treat `false` as "unknown without + /// compute". For a definitive answer, execute the validity with [`Self::execute_mask`] and + /// check whether the resulting [`Mask`] is all-false (`Mask::all_false`). This is the + /// all-null counterpart to [`Self::definitely_no_nulls`]. + #[inline] + pub fn definitely_all_null(&self) -> bool { + matches!(self, Self::AllInvalid) + } + /// Returns whether this validity contains no null values, executing the validity array if /// necessary. /// diff --git a/vortex-cuda/src/dynamic_dispatch/mod.rs b/vortex-cuda/src/dynamic_dispatch/mod.rs index 186f2297db1..557ec7468f5 100644 --- a/vortex-cuda/src/dynamic_dispatch/mod.rs +++ b/vortex-cuda/src/dynamic_dispatch/mod.rs @@ -32,7 +32,6 @@ use vortex::array::buffer::BufferHandle; use vortex::array::buffer::DeviceBufferExt; use vortex::array::match_each_unsigned_integer_ptype; use vortex::array::scalar::Scalar; -use vortex::array::validity::Validity; use vortex::buffer::Alignment; use vortex::buffer::ByteBuffer; use vortex::buffer::ByteBufferMut; @@ -434,7 +433,7 @@ impl MaterializedPlan { let output_ptype = self.dispatch_plan.output_ptype(); // All values are null — no need to touch the GPU. - if matches!(self.validity, Validity::AllInvalid) { + if self.validity.definitely_all_null() { let dtype = DType::Primitive(output_ptype, Nullability::Nullable); return ConstantArray::new(Scalar::null(dtype), len) .into_array() diff --git a/vortex-cuda/src/kernel/encodings/date_time_parts.rs b/vortex-cuda/src/kernel/encodings/date_time_parts.rs index d8c655c2558..7c4f4580c30 100644 --- a/vortex-cuda/src/kernel/encodings/date_time_parts.rs +++ b/vortex-cuda/src/kernel/encodings/date_time_parts.rs @@ -72,7 +72,7 @@ impl CudaExecute for DateTimePartsExecutor { return Ok(Canonical::empty(array.dtype())); } - if matches!(validity, Validity::AllInvalid) { + if validity.definitely_all_null() { let storage_ptype = ext.storage_dtype().as_ptype(); return Ok(Canonical::Extension( TemporalArray::new_timestamp( diff --git a/vortex-cuda/src/kernel/encodings/fsst.rs b/vortex-cuda/src/kernel/encodings/fsst.rs index 5d3d66eaf04..20a18b4e538 100644 --- a/vortex-cuda/src/kernel/encodings/fsst.rs +++ b/vortex-cuda/src/kernel/encodings/fsst.rs @@ -23,7 +23,6 @@ use vortex::array::arrays::varbinview::build_views::build_views; use vortex::array::buffer::DeviceBuffer; use vortex::array::match_each_integer_ptype; use vortex::array::match_each_unsigned_integer_ptype; -use vortex::array::validity::Validity; use vortex::buffer::Alignment; use vortex::buffer::Buffer; use vortex::dtype::NativePType; @@ -62,7 +61,7 @@ impl CudaExecute for FSSTExecutor { let dtype = fsst.dtype().clone(); let validity = fsst.codes().validity()?; - if fsst.is_empty() || matches!(validity, Validity::AllInvalid) { + if fsst.is_empty() || validity.definitely_all_null() { let empty = unsafe { VarBinViewArray::new_unchecked( Buffer::::zeroed(fsst.len()), diff --git a/vortex-cuda/src/kernel/encodings/runend.rs b/vortex-cuda/src/kernel/encodings/runend.rs index fca435478d8..01fc57abb86 100644 --- a/vortex-cuda/src/kernel/encodings/runend.rs +++ b/vortex-cuda/src/kernel/encodings/runend.rs @@ -75,7 +75,7 @@ impl CudaExecute for RunEndExecutor { ))); } - if matches!(values.validity()?, Validity::AllInvalid) { + if values.validity()?.definitely_all_null() { return ConstantArray::new(Scalar::null(values.dtype().clone()), output_len) .into_array() .execute::(ctx.execution_ctx()); diff --git a/vortex-duckdb/src/exporter/bool.rs b/vortex-duckdb/src/exporter/bool.rs index 84fd17f0789..4d0a2b29883 100644 --- a/vortex-duckdb/src/exporter/bool.rs +++ b/vortex-duckdb/src/exporter/bool.rs @@ -4,7 +4,6 @@ use vortex::array::ExecutionCtx; use vortex::array::arrays::BoolArray; use vortex::array::arrays::bool::BoolArrayExt; -use vortex::array::validity::Validity; use vortex::buffer::BitBuffer; use vortex::error::VortexResult; use vortex::mask::Mask; @@ -26,7 +25,7 @@ pub(crate) fn new_exporter( let bits = array.to_bit_buffer(); let validity = array.validity()?; - if matches!(validity, Validity::AllInvalid) { + if validity.definitely_all_null() { return Ok(all_invalid::new_exporter()); } let validity = validity.to_array(len).execute::(ctx)?; diff --git a/vortex-duckdb/src/exporter/decimal.rs b/vortex-duckdb/src/exporter/decimal.rs index f6b8d9607e7..b0683190b3a 100644 --- a/vortex-duckdb/src/exporter/decimal.rs +++ b/vortex-duckdb/src/exporter/decimal.rs @@ -8,7 +8,6 @@ use vortex::array::ExecutionCtx; use vortex::array::arrays::DecimalArray; use vortex::array::arrays::decimal::DecimalDataParts; use vortex::array::match_each_decimal_value_type; -use vortex::array::validity::Validity; use vortex::buffer::Buffer; use vortex::dtype::BigCast; use vortex::dtype::DecimalDType; @@ -49,7 +48,7 @@ pub(crate) fn new_exporter( } = array.into_data_parts(); let dest_values_type = precision_to_duckdb_storage_size(&decimal_dtype)?; - if matches!(validity, Validity::AllInvalid) { + if validity.definitely_all_null() { return Ok(all_invalid::new_exporter()); } let validity = validity.to_array(len).execute::(ctx)?; diff --git a/vortex-duckdb/src/exporter/fixed_size_list.rs b/vortex-duckdb/src/exporter/fixed_size_list.rs index 11b6e29a605..8fdb91df336 100644 --- a/vortex-duckdb/src/exporter/fixed_size_list.rs +++ b/vortex-duckdb/src/exporter/fixed_size_list.rs @@ -11,7 +11,6 @@ use vortex::array::ExecutionCtx; use vortex::array::arrays::FixedSizeListArray; use vortex::array::arrays::fixed_size_list::FixedSizeListArrayExt; -use vortex::array::validity::Validity; use vortex::error::VortexResult; use vortex::mask::Mask; @@ -43,7 +42,7 @@ pub(crate) fn new_exporter( let elements = parts.elements; let validity = parts.validity; - if matches!(validity, Validity::AllInvalid) { + if validity.definitely_all_null() { return Ok(all_invalid::new_exporter()); } diff --git a/vortex-duckdb/src/exporter/list.rs b/vortex-duckdb/src/exporter/list.rs index a0eb65ffd64..8569150a607 100644 --- a/vortex-duckdb/src/exporter/list.rs +++ b/vortex-duckdb/src/exporter/list.rs @@ -11,7 +11,6 @@ use vortex::array::arrays::ListArray; use vortex::array::arrays::PrimitiveArray; use vortex::array::arrays::list::ListDataParts; use vortex::array::match_each_integer_ptype; -use vortex::array::validity::Validity; use vortex::dtype::IntegerPType; use vortex::error::VortexResult; use vortex::error::vortex_ensure; @@ -55,7 +54,7 @@ pub(crate) fn new_exporter( } = array.into_data_parts(); let num_elements = elements.len(); - if matches!(validity, Validity::AllInvalid) { + if validity.definitely_all_null() { return Ok(all_invalid::new_exporter()); } let validity = validity.to_array(array_len).execute::(ctx)?; diff --git a/vortex-duckdb/src/exporter/list_view.rs b/vortex-duckdb/src/exporter/list_view.rs index a4cb61895f3..baf27949e5d 100644 --- a/vortex-duckdb/src/exporter/list_view.rs +++ b/vortex-duckdb/src/exporter/list_view.rs @@ -16,7 +16,6 @@ use vortex::array::arrays::listview::ListViewArrayExt; use vortex::array::arrays::listview::ListViewDataParts; use vortex::array::arrays::listview::ListViewRebuildMode; use vortex::array::match_each_integer_ptype; -use vortex::array::validity::Validity; use vortex::dtype::IntegerPType; use vortex::error::VortexExpect; use vortex::error::VortexResult; @@ -92,7 +91,7 @@ pub(crate) fn new_exporter( // Cache an `elements` vector up front so that future exports can reference it. let num_elements = elements.len(); - if matches!(validity, Validity::AllInvalid) { + if validity.definitely_all_null() { return Ok(all_invalid::new_exporter()); } let validity = validity.to_array(len).execute::(ctx)?; diff --git a/vortex-duckdb/src/exporter/primitive.rs b/vortex-duckdb/src/exporter/primitive.rs index a0b08c80e4b..4bda3fb3f0b 100644 --- a/vortex-duckdb/src/exporter/primitive.rs +++ b/vortex-duckdb/src/exporter/primitive.rs @@ -6,7 +6,6 @@ use std::marker::PhantomData; use vortex::array::ExecutionCtx; use vortex::array::arrays::PrimitiveArray; use vortex::array::match_each_native_ptype; -use vortex::array::validity::Validity; use vortex::dtype::NativePType; use vortex::error::VortexResult; use vortex::mask::Mask; @@ -29,7 +28,7 @@ pub fn new_exporter( ctx: &mut ExecutionCtx, ) -> VortexResult> { let validity = array.validity()?; - if matches!(validity, Validity::AllInvalid) { + if validity.definitely_all_null() { return Ok(all_invalid::new_exporter()); }; let validity = validity.to_array(array.len()).execute::(ctx)?; diff --git a/vortex-duckdb/src/exporter/struct_.rs b/vortex-duckdb/src/exporter/struct_.rs index 76c07d672a3..e9aca03a3bb 100644 --- a/vortex-duckdb/src/exporter/struct_.rs +++ b/vortex-duckdb/src/exporter/struct_.rs @@ -8,7 +8,6 @@ use vortex::array::arrays::StructArray; use vortex::array::arrays::bool::BoolArrayExt; use vortex::array::arrays::struct_::StructDataParts; use vortex::array::builtins::ArrayBuiltins; -use vortex::array::validity::Validity; use vortex::error::VortexResult; use crate::duckdb::VectorRef; @@ -35,7 +34,7 @@ pub(crate) fn new_exporter( .. } = array.into_data_parts(); - if matches!(validity, Validity::AllInvalid) { + if validity.definitely_all_null() { return Ok(all_invalid::new_exporter()); }; let validity = validity.to_array(len).execute::(ctx)?; diff --git a/vortex-duckdb/src/exporter/varbinview.rs b/vortex-duckdb/src/exporter/varbinview.rs index 557795f45f3..3e00384cd93 100644 --- a/vortex-duckdb/src/exporter/varbinview.rs +++ b/vortex-duckdb/src/exporter/varbinview.rs @@ -9,7 +9,6 @@ use vortex::array::arrays::VarBinViewArray; use vortex::array::arrays::varbinview::BinaryView; use vortex::array::arrays::varbinview::Inlined; use vortex::array::arrays::varbinview::VarBinViewDataParts; -use vortex::array::validity::Validity; use vortex::buffer::Buffer; use vortex::buffer::ByteBuffer; use vortex::error::VortexResult; @@ -39,7 +38,7 @@ pub(crate) fn new_exporter( buffers, } = array.into_data_parts(); - if matches!(validity, Validity::AllInvalid) { + if validity.definitely_all_null() { return Ok(all_invalid::new_exporter()); } let validity = validity.to_array(len).execute::(ctx)?;