Prevent non empty unions and clarify null docs

vortex-array/src/dtype/dtype_impl.rs

@@ -445,7 +445,10 @@ impl DType {
         }
     }
 
-    /// Get the [`UnionVariants`] if `self` is a [`DType::Union`], otherwise `None`.
+    /// Get the [`UnionVariants`] if `self` is a [`DType::Union`], otherwise [`None`].
+    ///
+    /// This [`Option`] only represents whether the dtype is a union; Union nullability does not
+    /// affect the result.
     pub fn as_union_variants_opt(&self) -> Option<&UnionVariants> {
         if let Union(uv, _) = self {
             Some(uv)
@@ -454,7 +457,7 @@ impl DType {
         }
     }
 
-    /// Owned version of [Self::as_union_variants_opt].
+    /// Owned version of [`Self::as_union_variants_opt`].
     pub fn into_union_variants_opt(self) -> Option<UnionVariants> {
         if let Union(uv, _) = self {
             Some(uv)

vortex-array/src/dtype/union.rs

@@ -19,8 +19,9 @@ use crate::dtype::FieldNames;
 /// Type information for a union array.
 ///
 /// A `UnionVariants` describes the possible alternative types for each row of a union, along with a
-/// per-variant `u8` type tag. We use the term **variants** (rather than "fields") because a union
-/// is a sum type: each row chooses exactly one alternative.
+/// per-variant `u8` type tag. A union must contain at least one variant. We use the term
+/// **variants** (rather than "fields") because a union is a sum type: each row chooses exactly one
+/// alternative.
 ///
 /// By default, tag `i` selects the child at offset `i` (`type_ids = [0, 1, ..., N-1]`). Vortex uses
 /// unsigned tags and supports up to 256 variants.
@@ -164,22 +165,7 @@ impl UnionVariants {
     }
 }
 
-impl Default for UnionVariants {
-    fn default() -> Self {
-        Self::empty()
-    }
-}
-
 impl UnionVariants {
-    /// The variants of the empty union.
-    pub fn empty() -> Self {
-        Self(Arc::new(UnionVariantsInner::from_fields(
-            FieldNames::default(),
-            Arc::from([]),
-            Arc::from([]),
-        )))
-    }
-
     /// Validate that `names`, `dtypes`, and `type_ids` are mutually consistent.
     fn validate_shape(names: &FieldNames, n_dtypes: usize, type_ids: &[u8]) -> VortexResult<()> {
         vortex_ensure_eq!(
@@ -196,6 +182,10 @@ impl UnionVariants {
             names.len(),
             type_ids.len()
         );
+        vortex_ensure!(
+            !names.is_empty(),
+            "union must have at least one variant (for now)"
+        );
 
         vortex_ensure!(
             type_ids.iter().all_unique(),
@@ -221,8 +211,9 @@ impl UnionVariants {
     ///
     /// # Errors
     ///
-    /// Returns an error if names, dtypes, or type IDs do not all have the same length, or if there
-    /// are any duplicate names or type IDs, or if there are more than 256 variants.
+    /// Returns an error if the union has no variants, if names, dtypes, or type IDs do not all have
+    /// the same length, if there are any duplicate names or type IDs, or if there are more than 256
+    /// variants.
     pub fn try_new(names: FieldNames, dtypes: Vec<DType>, type_ids: Vec<u8>) -> VortexResult<Self> {
         Self::validate_shape(&names, dtypes.len(), &type_ids)?;
 
@@ -238,8 +229,8 @@ impl UnionVariants {
     ///
     /// # Errors
     ///
-    /// `names` and `dtypes` must have the same length, and `names.len()` cannot be more than
-    /// `u8::MAX as usize + 1` (256).
+    /// The union must have at least one variant, `names` and `dtypes` must have the same length, and
+    /// `names.len()` cannot be more than `u8::MAX as usize + 1` (256).
     pub fn new(names: FieldNames, dtypes: Vec<DType>) -> VortexResult<Self> {
         const MAX_VARIANTS: usize = u8::MAX as usize + 1;
         vortex_ensure!(
@@ -265,8 +256,9 @@ impl UnionVariants {
     ///
     /// # Errors
     ///
-    /// Returns an error if names, dtypes, or type IDs do not all have the same length, or if there
-    /// are any duplicate names or type IDs, or if there are more than 256 variants.
+    /// Returns an error if the union has no variants, if names, dtypes, or type IDs do not all have
+    /// the same length, if there are any duplicate names or type IDs, or if there are more than 256
+    /// variants.
     pub(crate) fn try_from_fields(
         names: FieldNames,
         dtypes: Vec<FieldDType>,
@@ -330,7 +322,7 @@ impl UnionVariants {
         )
     }
 
-    /// Get the [`DType`] of a variant by offset.
+    /// Get the [`DType`] of a variant by offset, or [`None`] if `index` is out of bounds.
     pub fn variant_by_index(&self, index: usize) -> Option<DType> {
         Some(
             self.0
@@ -600,11 +592,19 @@ mod tests {
     }
 
     #[test]
-    fn test_empty() {
-        let v = UnionVariants::empty();
-        assert!(v.is_empty());
-        assert_eq!(v.len(), 0);
-        assert_eq!(v.type_ids(), &[] as &[u8]);
-        assert!(v.is_consecutive());
+    fn test_empty_rejected() {
+        let error = UnionVariants::new(FieldNames::default(), vec![]).unwrap_err();
+        assert!(
+            error
+                .to_string()
+                .contains("union must have at least one variant")
+        );
+
+        let error = UnionVariants::try_new(FieldNames::default(), vec![], vec![]).unwrap_err();
+        assert!(
+            error
+                .to_string()
+                .contains("union must have at least one variant")
+        );
     }
 }

vortex-array/src/scalar/constructor.rs

@@ -198,7 +198,9 @@ impl Scalar {
     /// Creates a union scalar from a type ID and child scalar.
     ///
     /// The selected child's dtype is verified and then discarded. Its raw value is stored so an
-    /// inner null child remains distinct from a null at the outer union level.
+    /// inner null child remains distinct from a null at the outer union level. Passing a null child
+    /// creates a non-null union with a selected type ID; use [`Scalar::null`] with a nullable
+    /// [`DType::Union`] to create an outer null union with no selected type ID.
     ///
     /// # Errors
     ///

vortex-array/src/scalar/downcast.rs

@@ -168,7 +168,13 @@ impl Scalar {
             .vortex_expect("Failed to convert scalar to union")
     }
 
-    /// Returns a view of the scalar as a union scalar if it has a union type.
+    /// Returns a view of the scalar as a union scalar if it has a union dtype.
+    ///
+    /// [`None`] means the scalar's dtype is not [`DType::Union`](crate::dtype::DType); it does not
+    /// describe the scalar's nullness.
+    ///
+    /// An outer null union still returns `Some(UnionScalar)` and can be inspected with
+    /// [`UnionScalar::is_null`](crate::scalar::UnionScalar::is_null).
     pub fn as_union_opt(&self) -> Option<UnionScalar<'_>> {
         if !self.dtype().is_union() {
             return None;

vortex-array/src/scalar/proto.rs

@@ -116,7 +116,7 @@ impl From<&ScalarValue> for pb::ScalarValue {
             ScalarValue::Union(v) => pb::ScalarValue {
                 kind: Some(Kind::UnionValue(Box::new(PbUnionValue {
                     type_id: u32::from(v.type_id()),
-                    value: Some(Box::new(ScalarValue::to_proto(v.value()))),
+                    value: Some(Box::new(ScalarValue::to_proto(v.child_value()))),
                 }))),
             },
             ScalarValue::Variant(v) => pb::ScalarValue {

vortex-array/src/scalar/scalar_impl.rs

@@ -101,10 +101,16 @@ impl Scalar {
     /// See [`Scalar::zero_value`] for more details about "zero" values.
     ///
     /// For non-nullable nested types, this function recursively creates valid default children.
+    /// For a union specifically:
+    ///
+    /// - A nullable union defaults to an **outer null**. It has no selected variant or type ID.
+    /// - A non-nullable union selects its first variant. That selected child may itself default to
+    ///   an **inner null** when its dtype is nullable; the enclosing union remains non-null and
+    ///   retains the first variant's type ID.
     ///
     /// # Panics
     ///
-    /// Panics if `dtype` has no default value, such as a non-nullable union.
+    /// Panics if `dtype` has no default value.
     pub fn default_value(dtype: &DType) -> Self {
         Self::try_default_value(dtype)
             .unwrap_or_else(|| vortex_panic!("{dtype} has no default value"))
@@ -133,16 +139,19 @@ impl Scalar {
     ///   element [`DType`]
     /// - `Struct`: A struct where each field has a zero value, which is determined by the field
     ///   [`DType`]
-    /// - `Union`: Does not have a "zero" value
+    /// - `Union`: A non-null union selecting the first variant with a non-null child zero value
     /// - `Extension`: The zero value of the storage [`DType`]
-    ///
     /// # Panics
     ///
-    /// Panics if the dtype has no non-null zero value, such as `Null` or a union.
+    /// Panics if the dtype has no non-null zero value, such as `Null`, or if a nested dtype needed
+    /// to construct the zero value has no non-null zero value.
+    ///
+    /// Unlike [`Scalar::default_value`], this never uses either an outer or inner null for a union:
+    /// even a nullable union's zero value is non-null and contains a non-null selected child.
     pub fn zero_value(dtype: &DType) -> Self {
         let value = ScalarValue::zero_value(dtype);
 
-        // SAFETY: We assume that `zero_value` creates a valid `ScalarValue` for the `DType`.
+        // SAFETY: `zero_value` creates a valid `ScalarValue` for the `DType`.
         unsafe { Self::new_unchecked(dtype.clone(), Some(value)) }
     }
 
@@ -186,8 +195,12 @@ impl Scalar {
 
     /// Returns `true` if the [`Scalar`] has a non-null zero value.
     ///
-    /// Returns `None` if the scalar is null, otherwise returns `Some(true)` if the value is zero
-    /// and `Some(false)` otherwise.
+    /// Returns `None` if the scalar is null or its zero-ness is undefined. Otherwise, returns
+    /// `Some(true)` if the value is zero and `Some(false)` if it is not.
+    ///
+    /// A union that selects a variant other than the first returns `Some(false)`. A union selecting
+    /// its first variant delegates to that child's [`Scalar::is_zero`], so an inner null child
+    /// returns `None` and a non-null zero child returns `Some(true)`.
     pub fn is_zero(&self) -> Option<bool> {
         let value = self.value()?;
 
@@ -212,8 +225,16 @@ impl Scalar {
                 .as_struct()
                 .fields_iter()
                 .is_some_and(|mut fields| fields.all(|f| f.is_zero() == Some(true))),
-            // We do not define union to have any valid "zero" value.
-            DType::Union(..) => false,
+            // Only the first variant of unions can be zero. Its child determines whether the
+            // zero-ness is true, false, or undefined.
+            DType::Union(..) => {
+                let union = self.as_union();
+                if union.child_index() != Some(0) {
+                    false
+                } else {
+                    union.child().and_then(|child| child.is_zero())?
+                }
+            }
             DType::Variant(_) => self.as_variant().is_zero()?,
             DType::Extension(_) => self.as_extension().to_storage_scalar().is_zero()?,
         };
@@ -284,7 +305,7 @@ impl Scalar {
                 .unwrap_or_default(),
             DType::Union(..) => self
                 .as_union()
-                .value()
+                .child()
                 .map_or(0, |value| 1 + value.approx_nbytes()),
             DType::Variant(_) => self.as_variant().value().map_or(0, Scalar::approx_nbytes),
             DType::Extension(_) => self.as_extension().to_storage_scalar().approx_nbytes(),
@@ -403,7 +424,7 @@ fn partial_cmp_non_null_scalar_values(
             let child_index = variants.tag_to_child_index(lhs.type_id())?;
             let child_dtype = variants.variant_by_index(child_index)?;
 
-            partial_cmp_scalar_values(&child_dtype, lhs.value(), rhs.value())
+            partial_cmp_scalar_values(&child_dtype, lhs.child_value(), rhs.child_value())
         }
         // Variant values can have a different dtype in each row, so it doesn't make sense to
         // compare them.

vortex-array/src/scalar/scalar_value.rs

@@ -9,6 +9,7 @@ use std::fmt::Formatter;
 use itertools::Itertools;
 use vortex_buffer::BufferString;
 use vortex_buffer::ByteBuffer;
+use vortex_error::VortexExpect;
 use vortex_error::vortex_panic;
 
 use crate::dtype::DType;
@@ -51,7 +52,7 @@ impl ScalarValue {
     }
 
     /// Returns the non-null zero value for `dtype`, or [`None`] if no such value exists.
-    fn try_zero_value(dtype: &DType) -> Option<Self> {
+    pub(super) fn try_zero_value(dtype: &DType) -> Option<Self> {
         Some(match dtype {
             DType::Null => return None,
             DType::Bool(_) => Self::Bool(false),
@@ -73,7 +74,17 @@ impl ScalarValue {
                     .collect::<Option<Vec<_>>>()?;
                 Self::Tuple(field_values)
             }
-            DType::Union(..) => return None,
+            DType::Union(variants, _) => {
+                let child_dtype = variants
+                    .variant_by_index(0)
+                    .vortex_expect("union must have at least one variant");
+                let child_value = Self::try_zero_value(&child_dtype)?;
+
+                Self::Union(UnionValue::new(
+                    variants.child_index_to_tag(0),
+                    Some(child_value),
+                ))
+            }
             DType::Variant(_) => Self::Variant(Box::new(Scalar::null(DType::Null))),
             DType::Extension(ext_dtype) => {
                 // Since we have no way to define a "zero" extension value (since we have no idea
@@ -114,7 +125,14 @@ impl ScalarValue {
                     .collect::<Option<Vec<_>>>()?;
                 Self::Tuple(field_values)
             }
-            DType::Union(..) => return None,
+            DType::Union(variants, _) => {
+                let child_dtype = variants
+                    .variant_by_index(0)
+                    .vortex_expect("union must have at least one variant");
+                let child_value = Self::try_default_value(&child_dtype)?;
+
+                Self::Union(UnionValue::new(variants.child_index_to_tag(0), child_value))
+            }
             DType::Variant(_) => Self::Variant(Box::new(Scalar::null(DType::Null))),
             DType::Extension(ext_dtype) => {
                 // Since we have no way to define a "default" extension value (since we have no idea
@@ -174,7 +192,7 @@ impl Display for ScalarValue {
             }
             ScalarValue::Union(value) => {
                 write!(f, "union@{}(", value.type_id())?;
-                match value.value() {
+                match value.child_value() {
                     Some(value) => write!(f, "{value}"),
                     None => write!(f, "null"),
                 }?;

vortex-array/src/scalar/tests/casting.rs

@@ -420,15 +420,15 @@ mod tests {
         assert_eq!(
             unchanged_child
                 .as_union()
-                .value()
+                .child()
                 .map(|scalar| scalar.dtype().clone()),
             Some(DType::Primitive(PType::I32, Nullability::NonNullable))
         );
         assert_eq!(changed_child.dtype(), &target_dtype);
         assert_eq!(
             changed_child
                 .as_union()
-                .value()
+                .child()
                 .map(|scalar| scalar.dtype().clone()),
             Some(DType::Utf8(Nullability::Nullable))
         );
@@ -482,7 +482,7 @@ mod tests {
         let cast = scalar.cast(&target_dtype)?;
         let selected_child = cast
             .as_union()
-            .value()
+            .child()
             .vortex_expect("present union must have a selected child");
 
         assert_eq!(cast.dtype(), &target_dtype);
@@ -531,7 +531,7 @@ mod tests {
         assert_eq!(cast.dtype(), &target_dtype);
         assert_eq!(union.dtype(), &target_union_dtype);
         assert_eq!(
-            union.as_union().value(),
+            union.as_union().child(),
             Some(Scalar::primitive(42_i32, Nullability::Nullable))
         );
 
@@ -605,7 +605,7 @@ mod tests {
         assert_eq!(
             nullable
                 .as_union()
-                .value()
+                .child()
                 .map(|scalar| scalar.dtype().clone()),
             Some(DType::Primitive(PType::I32, Nullability::NonNullable))
         );