Parquet page size mid batch

arrow-arith/src/temporal.rs

@@ -24,7 +24,6 @@ use arrow_array::cast::AsArray;
 use cast::as_primitive_array;
 use chrono::{Datelike, TimeZone, Timelike, Utc};
 
-use arrow_array::ree_map;
 use arrow_array::temporal_conversions::{
     MICROSECONDS, MICROSECONDS_IN_DAY, MILLISECONDS, MILLISECONDS_IN_DAY, NANOSECONDS,
     NANOSECONDS_IN_DAY, SECONDS_IN_DAY, date32_to_datetime, date64_to_datetime,
@@ -63,6 +62,10 @@ pub enum DatePart {
     DayOfWeekSunday0,
     /// Day of the week, in range `0..=6`, where Monday is `0`
     DayOfWeekMonday0,
+    /// Day of the week, in range `1..=7`, where Sunday is `1`
+    DayOfWeekSunday1,
+    /// ISO day of the week, in range `1..=7`, where Monday is `1`
+    DayOfWeekMonday1,
     /// Day of year, in range `1..=366`
     DayOfYear,
     /// Hour of the day, in range `0..=23`
@@ -100,11 +103,6 @@ impl std::fmt::Display for DatePart {
 /// - `century`, `decade`, `millennium` — no matching [`DatePart`] variant.
 /// - `timezone`, `timezone_hour`, `timezone_minute` — not modeled by
 ///   [`DatePart`].
-/// - `isodow` — PostgreSQL's `isodow` returns `1..=7` (Mon=1), but the
-///   closest variant ([`DatePart::DayOfWeekMonday0`]) returns `0..=6`.
-///   Accepting it here would silently shift the value range; callers that
-///   need the PostgreSQL semantic should map the alias themselves and
-///   add `1` to the extracted value.
 impl FromStr for DatePart {
     type Err = ArrowError;
 
@@ -118,6 +116,9 @@ impl FromStr for DatePart {
             "isoweek" => Self::WeekISO,
             "d" | "day" | "days" => Self::Day,
             "dow" | "dayofweek" => Self::DayOfWeekSunday0,
+            "dow1" | "dayofweek1" => Self::DayOfWeekSunday1,
+            "isodow0" => Self::DayOfWeekMonday0,
+            "isodow" => Self::DayOfWeekMonday1,
             "doy" | "dayofyear" => Self::DayOfYear,
             "h" | "hr" | "hrs" | "hour" | "hours" => Self::Hour,
             "m" | "min" | "mins" | "minute" | "minutes" => Self::Minute,
@@ -158,6 +159,8 @@ where
         DatePart::Day => |d| d.day() as i32,
         DatePart::DayOfWeekSunday0 => |d| d.num_days_from_sunday(),
         DatePart::DayOfWeekMonday0 => |d| d.num_days_from_monday(),
+        DatePart::DayOfWeekSunday1 => |d| d.num_days_from_sunday() + 1,
+        DatePart::DayOfWeekMonday1 => |d| d.num_days_from_monday() + 1,
         DatePart::DayOfYear => |d| d.ordinal() as i32,
         DatePart::Hour => |d| d.hour() as i32,
         DatePart::Minute => |d| d.minute() as i32,
@@ -249,15 +252,12 @@ pub fn date_part(array: &dyn Array, part: DatePart) -> Result<ArrayRef, ArrowErr
             let new_array = array.with_values(values);
             Ok(new_array)
         }
-        DataType::RunEndEncoded(k, _) => match k.data_type() {
-            DataType::Int16 => ree_map!(array, Int16Type, |a| date_part(a, part)),
-            DataType::Int32 => ree_map!(array, Int32Type, |a| date_part(a, part)),
-            DataType::Int64 => ree_map!(array, Int64Type, |a| date_part(a, part)),
-            _ => Err(ArrowError::InvalidArgumentError(format!(
-                "Invalid run-end type: {:?}",
-                k.data_type()
-            ))),
-        },
+        DataType::RunEndEncoded(_, _) => {
+            let array = array.as_any_ree();
+            let values = date_part(array.values(), part)?;
+            let new_array = array.with_values(values);
+            Ok(new_array)
+        }
         t => return_compute_error_with!(format!("{part} does not support"), t),
     )
 }
@@ -500,6 +500,8 @@ impl ExtractDatePartExt for PrimitiveArray<IntervalYearMonthType> {
             | DatePart::Day
             | DatePart::DayOfWeekSunday0
             | DatePart::DayOfWeekMonday0
+            | DatePart::DayOfWeekSunday1
+            | DatePart::DayOfWeekMonday1
             | DatePart::DayOfYear
             | DatePart::Hour
             | DatePart::Minute
@@ -536,6 +538,8 @@ impl ExtractDatePartExt for PrimitiveArray<IntervalDayTimeType> {
             | DatePart::Month
             | DatePart::DayOfWeekSunday0
             | DatePart::DayOfWeekMonday0
+            | DatePart::DayOfWeekSunday1
+            | DatePart::DayOfWeekMonday1
             | DatePart::DayOfYear => {
                 return_compute_error_with!(format!("{part} does not support"), self.data_type())
             }
@@ -578,6 +582,8 @@ impl ExtractDatePartExt for PrimitiveArray<IntervalMonthDayNanoType> {
             | DatePart::YearISO
             | DatePart::DayOfWeekSunday0
             | DatePart::DayOfWeekMonday0
+            | DatePart::DayOfWeekSunday1
+            | DatePart::DayOfWeekMonday1
             | DatePart::DayOfYear => {
                 return_compute_error_with!(format!("{part} does not support"), self.data_type())
             }
@@ -610,6 +616,8 @@ impl ExtractDatePartExt for PrimitiveArray<DurationSecondType> {
             | DatePart::Month
             | DatePart::DayOfWeekSunday0
             | DatePart::DayOfWeekMonday0
+            | DatePart::DayOfWeekSunday1
+            | DatePart::DayOfWeekMonday1
             | DatePart::DayOfYear => {
                 return_compute_error_with!(format!("{part} does not support"), self.data_type())
             }
@@ -642,6 +650,8 @@ impl ExtractDatePartExt for PrimitiveArray<DurationMillisecondType> {
             | DatePart::Month
             | DatePart::DayOfWeekSunday0
             | DatePart::DayOfWeekMonday0
+            | DatePart::DayOfWeekSunday1
+            | DatePart::DayOfWeekMonday1
             | DatePart::DayOfYear => {
                 return_compute_error_with!(format!("{part} does not support"), self.data_type())
             }
@@ -674,6 +684,8 @@ impl ExtractDatePartExt for PrimitiveArray<DurationMicrosecondType> {
             | DatePart::Month
             | DatePart::DayOfWeekSunday0
             | DatePart::DayOfWeekMonday0
+            | DatePart::DayOfWeekSunday1
+            | DatePart::DayOfWeekMonday1
             | DatePart::DayOfYear => {
                 return_compute_error_with!(format!("{part} does not support"), self.data_type())
             }
@@ -706,6 +718,8 @@ impl ExtractDatePartExt for PrimitiveArray<DurationNanosecondType> {
             | DatePart::Month
             | DatePart::DayOfWeekSunday0
             | DatePart::DayOfWeekMonday0
+            | DatePart::DayOfWeekSunday1
+            | DatePart::DayOfWeekMonday1
             | DatePart::DayOfYear => {
                 return_compute_error_with!(format!("{part} does not support"), self.data_type())
             }
@@ -985,6 +999,46 @@ mod tests {
         assert_eq!(2, b.value(2));
     }
 
+    #[test]
+    fn test_temporal_array_date64_dayofweek1() {
+        //1514764800000 -> 2018-01-01 (Monday)
+        //1550636625000 -> 2019-02-20 (Wednesday)
+        //1483228800000 -> 2017-01-01 (Sunday)
+        let a: PrimitiveArray<Date64Type> = vec![
+            Some(1514764800000),
+            None,
+            Some(1550636625000),
+            Some(1483228800000),
+        ]
+        .into();
+
+        let b = date_part_primitive(&a, DatePart::DayOfWeekSunday1).unwrap();
+        assert_eq!(2, b.value(0));
+        assert!(!b.is_valid(1));
+        assert_eq!(4, b.value(2));
+        assert_eq!(1, b.value(3));
+    }
+
+    #[test]
+    fn test_temporal_array_date64_isodow() {
+        //1514764800000 -> 2018-01-01 (Monday)
+        //1550636625000 -> 2019-02-20 (Wednesday)
+        //1483228800000 -> 2017-01-01 (Sunday)
+        let a: PrimitiveArray<Date64Type> = vec![
+            Some(1514764800000),
+            None,
+            Some(1550636625000),
+            Some(1483228800000),
+        ]
+        .into();
+
+        let b = date_part_primitive(&a, DatePart::DayOfWeekMonday1).unwrap();
+        assert_eq!(1, b.value(0));
+        assert!(!b.is_valid(1));
+        assert_eq!(3, b.value(2));
+        assert_eq!(7, b.value(3));
+    }
+
     #[test]
     fn test_temporal_array_date64_weekday0() {
         //1483228800000 -> 2017-01-01 (Sunday)
@@ -1579,11 +1633,15 @@ mod tests {
             let invalid_parts = [
                 DatePart::Quarter,
                 DatePart::Year,
+                DatePart::YearISO,
                 DatePart::Month,
                 DatePart::Week,
+                DatePart::WeekISO,
                 DatePart::Day,
                 DatePart::DayOfWeekSunday0,
                 DatePart::DayOfWeekMonday0,
+                DatePart::DayOfWeekSunday1,
+                DatePart::DayOfWeekMonday1,
                 DatePart::DayOfYear,
             ];
 
@@ -1813,8 +1871,12 @@ mod tests {
         fn ensure_returns_error(array: &dyn Array) {
             let invalid_parts = [
                 DatePart::Quarter,
+                DatePart::YearISO,
+                DatePart::WeekISO,
                 DatePart::DayOfWeekSunday0,
                 DatePart::DayOfWeekMonday0,
+                DatePart::DayOfWeekSunday1,
+                DatePart::DayOfWeekMonday1,
                 DatePart::DayOfYear,
             ];
 
@@ -1956,10 +2018,14 @@ mod tests {
         fn ensure_returns_error(array: &dyn Array) {
             let invalid_parts = [
                 DatePart::Year,
+                DatePart::YearISO,
                 DatePart::Quarter,
                 DatePart::Month,
+                DatePart::WeekISO,
                 DatePart::DayOfWeekSunday0,
                 DatePart::DayOfWeekMonday0,
+                DatePart::DayOfWeekSunday1,
+                DatePart::DayOfWeekMonday1,
                 DatePart::DayOfYear,
             ];
 
@@ -2157,6 +2223,11 @@ mod tests {
             ("day", DatePart::Day),
             ("dow", DatePart::DayOfWeekSunday0),
             ("DayOfWeek", DatePart::DayOfWeekSunday0),
+            ("dow1", DatePart::DayOfWeekSunday1),
+            ("dayofweek1", DatePart::DayOfWeekSunday1),
+            ("DAYOFWEEK1", DatePart::DayOfWeekSunday1),
+            ("isodow", DatePart::DayOfWeekMonday1),
+            ("ISODOW", DatePart::DayOfWeekMonday1),
             ("doy", DatePart::DayOfYear),
             ("DayOfYear", DatePart::DayOfYear),
             ("h", DatePart::Hour),
@@ -2195,8 +2266,6 @@ mod tests {
     #[test]
     fn test_date_part_from_str_unknown() {
         // Names intentionally rejected — see the FromStr doc comment for why.
-        // `isodow` is here because mapping it to `DayOfWeekMonday0` would
-        // silently shift the value range vs. PostgreSQL's `isodow` (1..=7).
         let unknown = [
             "epoch",
             "century",
@@ -2205,7 +2274,6 @@ mod tests {
             "timezone",
             "timezone_hour",
             "timezone_minute",
-            "isodow",
             // Whitespace is not trimmed — pin this so the behavior doesn't
             // change silently. Callers should trim before parsing.
             " year ",

arrow-array/src/array/boolean_array.rs

@@ -19,7 +19,6 @@ use crate::array::print_long_array;
 use crate::builder::BooleanBuilder;
 use crate::iterator::BooleanIter;
 use crate::{Array, ArrayAccessor, ArrayRef, Scalar};
-use arrow_buffer::bit_chunk_iterator::UnalignedBitChunk;
 use arrow_buffer::{BooleanBuffer, Buffer, MutableBuffer, NullBuffer, bit_util};
 use arrow_data::{ArrayData, ArrayDataBuilder};
 use arrow_schema::DataType;
@@ -157,16 +156,6 @@ impl BooleanArray {
         &self.values
     }
 
-    /// Block size for chunked fold operations in [`Self::has_true`] and [`Self::has_false`].
-    /// Using `chunks_exact` with this size lets the compiler fully unroll the inner
-    /// fold (no inner branch/loop), enabling short-circuit exits every N chunks.
-    const CHUNK_FOLD_BLOCK_SIZE: usize = 16;
-
-    /// Returns an [`UnalignedBitChunk`] over this array's values.
-    fn unaligned_bit_chunks(&self) -> UnalignedBitChunk<'_> {
-        UnalignedBitChunk::new(self.values().values(), self.values().offset(), self.len())
-    }
-
     /// Returns the number of non null, true values within this array.
     /// If you only need to check if there is at least one true value, consider using `has_true()` which can short-circuit and be more efficient.
     pub fn true_count(&self) -> usize {
@@ -202,16 +191,7 @@ impl BooleanArray {
                 let value_chunks = self.values().bit_chunks().iter_padded();
                 null_chunks.zip(value_chunks).any(|(n, v)| (n & v) != 0)
             }
-            None => {
-                let bit_chunks = self.unaligned_bit_chunks();
-                let chunks = bit_chunks.chunks();
-                let mut exact = chunks.chunks_exact(Self::CHUNK_FOLD_BLOCK_SIZE);
-                let found = bit_chunks.prefix().unwrap_or(0) != 0
-                    || exact.any(|block| block.iter().fold(0u64, |acc, &c| acc | c) != 0);
-                found
-                    || exact.remainder().iter().any(|&c| c != 0)
-                    || bit_chunks.suffix().unwrap_or(0) != 0
-            }
+            None => self.values().has_true(),
         }
     }
 
@@ -228,35 +208,7 @@ impl BooleanArray {
                 let value_chunks = self.values().bit_chunks().iter_padded();
                 null_chunks.zip(value_chunks).any(|(n, v)| (n & !v) != 0)
             }
-            None => {
-                let bit_chunks = self.unaligned_bit_chunks();
-                // UnalignedBitChunk zeros padding bits; fill them with 1s so
-                // they don't appear as false values.
-                let lead_mask = !((1u64 << bit_chunks.lead_padding()) - 1);
-                let trail_mask = if bit_chunks.trailing_padding() == 0 {
-                    u64::MAX
-                } else {
-                    (1u64 << (64 - bit_chunks.trailing_padding())) - 1
-                };
-                let (prefix_fill, suffix_fill) = match (bit_chunks.prefix(), bit_chunks.suffix()) {
-                    (Some(_), Some(_)) => (!lead_mask, !trail_mask),
-                    (Some(_), None) => (!lead_mask | !trail_mask, 0),
-                    (None, Some(_)) => (0, !trail_mask),
-                    (None, None) => (0, 0),
-                };
-                let chunks = bit_chunks.chunks();
-                let mut exact = chunks.chunks_exact(Self::CHUNK_FOLD_BLOCK_SIZE);
-                let found = bit_chunks
-                    .prefix()
-                    .is_some_and(|v| (v | prefix_fill) != u64::MAX)
-                    || exact
-                        .any(|block| block.iter().fold(u64::MAX, |acc, &c| acc & c) != u64::MAX);
-                found
-                    || exact.remainder().iter().any(|&c| c != u64::MAX)
-                    || bit_chunks
-                        .suffix()
-                        .is_some_and(|v| (v | suffix_fill) != u64::MAX)
-            }
+            None => self.values().has_false(),
         }
     }