diff --git a/Cargo.lock b/Cargo.lock index a3465ba9e458..11979eb5836b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -170,7 +170,7 @@ dependencies = [ [[package]] name = "arrow" -version = "58.2.0" +version = "58.3.0" dependencies = [ "arrow-arith", "arrow-array", @@ -197,7 +197,7 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "58.2.0" +version = "58.3.0" dependencies = [ "arrow-array", "arrow-buffer", @@ -209,7 +209,7 @@ dependencies = [ [[package]] name = "arrow-array" -version = "58.2.0" +version = "58.3.0" dependencies = [ "ahash", "arrow-buffer", @@ -229,7 +229,7 @@ dependencies = [ [[package]] name = "arrow-avro" -version = "58.2.0" +version = "58.3.0" dependencies = [ "apache-avro", "arrow", @@ -266,7 +266,7 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "58.2.0" +version = "58.3.0" dependencies = [ "bytes", "criterion", @@ -278,7 +278,7 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "58.2.0" +version = "58.3.0" dependencies = [ "arrow-array", "arrow-buffer", @@ -301,7 +301,7 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "58.2.0" +version = "58.3.0" dependencies = [ "arrow-array", "arrow-buffer", @@ -319,7 +319,7 @@ dependencies = [ [[package]] name = "arrow-data" -version = "58.2.0" +version = "58.3.0" dependencies = [ "arrow-buffer", "arrow-schema", @@ -330,7 +330,7 @@ dependencies = [ [[package]] name = "arrow-flight" -version = "58.2.0" +version = "58.3.0" dependencies = [ "anyhow", "arrow-arith", @@ -370,7 +370,7 @@ dependencies = [ [[package]] name = "arrow-integration-test" -version = "58.2.0" +version = "58.3.0" dependencies = [ "arrow", "arrow-buffer", @@ -383,7 +383,7 @@ dependencies = [ [[package]] name = "arrow-integration-testing" -version = "58.2.0" +version = "58.3.0" dependencies = [ "arrow", "arrow-buffer", @@ -402,7 +402,7 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "58.2.0" +version = "58.3.0" dependencies = [ "arrow-array", "arrow-buffer", @@ -421,7 +421,7 @@ dependencies = [ [[package]] name = "arrow-json" -version = "58.2.0" +version = "58.3.0" dependencies = [ "arbitrary", "arrow-array", @@ -452,7 +452,7 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "58.2.0" +version = "58.3.0" dependencies = [ "arrow-array", "arrow-buffer", @@ -465,7 +465,7 @@ dependencies = [ [[package]] name = "arrow-pyarrow" -version = "58.2.0" +version = "58.3.0" dependencies = [ "arrow-array", "arrow-data", @@ -475,7 +475,7 @@ dependencies = [ [[package]] name = "arrow-row" -version = "58.2.0" +version = "58.3.0" dependencies = [ "arrow-array", "arrow-buffer", @@ -490,7 +490,7 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "58.2.0" +version = "58.3.0" dependencies = [ "bitflags", "criterion", @@ -503,7 +503,7 @@ dependencies = [ [[package]] name = "arrow-select" -version = "58.2.0" +version = "58.3.0" dependencies = [ "ahash", "arrow-array", @@ -516,7 +516,7 @@ dependencies = [ [[package]] name = "arrow-string" -version = "58.2.0" +version = "58.3.0" dependencies = [ "arrow-array", "arrow-buffer", @@ -2327,7 +2327,7 @@ dependencies = [ [[package]] name = "parquet" -version = "58.2.0" +version = "58.3.0" dependencies = [ "ahash", "arrow", @@ -2378,7 +2378,7 @@ dependencies = [ [[package]] name = "parquet-geospatial" -version = "58.2.0" +version = "58.3.0" dependencies = [ "arrow-schema", "geo-traits", @@ -2390,7 +2390,7 @@ dependencies = [ [[package]] name = "parquet-variant" -version = "58.2.0" +version = "58.3.0" dependencies = [ "arrow", "arrow-schema", @@ -2407,7 +2407,7 @@ dependencies = [ [[package]] name = "parquet-variant-compute" -version = "58.2.0" +version = "58.3.0" dependencies = [ "arrow", "arrow-schema", @@ -2424,7 +2424,7 @@ dependencies = [ [[package]] name = "parquet-variant-json" -version = "58.2.0" +version = "58.3.0" dependencies = [ "arrow-schema", "base64", @@ -2436,7 +2436,7 @@ dependencies = [ [[package]] name = "parquet_derive" -version = "58.2.0" +version = "58.3.0" dependencies = [ "parquet", "proc-macro2", @@ -2446,7 +2446,7 @@ dependencies = [ [[package]] name = "parquet_derive_test" -version = "58.2.0" +version = "58.3.0" dependencies = [ "chrono", "parquet", diff --git a/arrow-string/src/like.rs b/arrow-string/src/like.rs index 2ad7e1cfd961..e9077d4b090d 100644 --- a/arrow-string/src/like.rs +++ b/arrow-string/src/like.rs @@ -28,8 +28,6 @@ use arrow_array::*; use arrow_schema::*; use arrow_select::take::take; -use std::sync::Arc; - use crate::binary_like::binary_apply; pub use arrow_array::StringArrayType; @@ -433,118 +431,7 @@ fn binary_predicate<'a>( .collect() } -// Deprecated kernels - -fn make_scalar(data_type: &DataType, scalar: &str) -> Result { - match data_type { - DataType::Utf8 => Ok(Arc::new(StringArray::from_iter_values([scalar]))), - DataType::LargeUtf8 => Ok(Arc::new(LargeStringArray::from_iter_values([scalar]))), - DataType::Dictionary(_, v) => make_scalar(v.as_ref(), scalar), - d => Err(ArrowError::InvalidArgumentError(format!( - "Unsupported string scalar data type {d:?}", - ))), - } -} - -macro_rules! legacy_kernels { - ($fn_datum:ident, $fn_array:ident, $fn_scalar:ident, $fn_array_dyn:ident, $fn_scalar_dyn:ident, $deprecation:expr) => { - #[doc(hidden)] - #[deprecated(note = $deprecation)] - pub fn $fn_array( - left: &GenericStringArray, - right: &GenericStringArray, - ) -> Result { - $fn_datum(left, right) - } - - #[doc(hidden)] - #[deprecated(note = $deprecation)] - pub fn $fn_scalar( - left: &GenericStringArray, - right: &str, - ) -> Result { - let scalar = GenericStringArray::::from_iter_values([right]); - $fn_datum(left, &Scalar::new(&scalar)) - } - - #[doc(hidden)] - #[deprecated(note = $deprecation)] - pub fn $fn_array_dyn( - left: &dyn Array, - right: &dyn Array, - ) -> Result { - $fn_datum(&left, &right) - } - - #[doc(hidden)] - #[deprecated(note = $deprecation)] - pub fn $fn_scalar_dyn(left: &dyn Array, right: &str) -> Result { - let scalar = make_scalar(left.data_type(), right)?; - $fn_datum(&left, &Scalar::new(&scalar)) - } - }; -} - -legacy_kernels!( - like, - like_utf8, - like_utf8_scalar, - like_dyn, - like_utf8_scalar_dyn, - "Use arrow_string::like::like" -); -legacy_kernels!( - ilike, - ilike_utf8, - ilike_utf8_scalar, - ilike_dyn, - ilike_utf8_scalar_dyn, - "Use arrow_string::like::ilike" -); -legacy_kernels!( - nlike, - nlike_utf8, - nlike_utf8_scalar, - nlike_dyn, - nlike_utf8_scalar_dyn, - "Use arrow_string::like::nlike" -); -legacy_kernels!( - nilike, - nilike_utf8, - nilike_utf8_scalar, - nilike_dyn, - nilike_utf8_scalar_dyn, - "Use arrow_string::like::nilike" -); -legacy_kernels!( - contains, - contains_utf8, - contains_utf8_scalar, - contains_dyn, - contains_utf8_scalar_dyn, - "Use arrow_string::like::contains" -); -legacy_kernels!( - starts_with, - starts_with_utf8, - starts_with_utf8_scalar, - starts_with_dyn, - starts_with_utf8_scalar_dyn, - "Use arrow_string::like::starts_with" -); - -legacy_kernels!( - ends_with, - ends_with_utf8, - ends_with_utf8_scalar, - ends_with_dyn, - ends_with_utf8_scalar_dyn, - "Use arrow_string::like::ends_with" -); - #[cfg(test)] -#[allow(deprecated)] mod tests { use super::*; use arrow_array::builder::BinaryDictionaryBuilder; @@ -688,6 +575,10 @@ mod tests { assert_eq!(res, expected); let left: DictionaryArray = $left.into_iter().collect(); + let right = StringArray::from_iter_values([$right]); + let res = $op(&left, &Scalar::new(&right)).unwrap(); + assert_eq!(res, expected); + let right: DictionaryArray = [$right].into_iter().collect(); let res = $op(&left, &Scalar::new(&right)).unwrap(); assert_eq!(res, expected); @@ -1429,601 +1320,102 @@ mod tests { vec![true, false, true, true, true] ); - test_utf8!( - test_utf8_array_eq_ignore_ascii_case, - vec!["arrow", "arrow", "arrow", "arrow", "parquet", "parquet"], - vec!["arrow", "ARROW", "arro", "aRrOw", "arrow", "ARROW"], - eq_ignore_ascii_case, - vec![true, true, false, true, false, false] - ); - + // Nullable, repeated values exercise dictionary remapping with a plain UTF8 scalar RHS. test_utf8_scalar!( - test_utf8_array_eq_ignore_ascii_case_scalar, - vec!["arrow", "aRrOW", "arro", "ARROW", "parquet", "PARQUET"], - "arrow", - eq_ignore_ascii_case, - vec![true, true, false, true, false, false] - ); - - #[test] - fn test_dict_like_kernels() { - let data = vec![ + test_utf8_scalar_nullable_like, + vec![ Some("Earth"), Some("Fire"), Some("Water"), Some("Air"), None, Some("Air"), - Some("bbbbb\nAir"), - ]; - - let dict_array: DictionaryArray = data.into_iter().collect(); - - assert_eq!( - like_utf8_scalar_dyn(&dict_array, "Air").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(false), - Some(false), - Some(true), - None, - Some(true), - Some(false), - ]), - ); - - assert_eq!( - like_utf8_scalar_dyn(&dict_array, "Air").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(false), - Some(false), - Some(true), - None, - Some(true), - Some(false), - ]), - ); - - assert_eq!( - like_utf8_scalar_dyn(&dict_array, "Wa%").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(false), - Some(true), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - - assert_eq!( - like_utf8_scalar_dyn(&dict_array, "Wa%").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(false), - Some(true), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - - assert_eq!( - like_utf8_scalar_dyn(&dict_array, "%r").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(false), - Some(true), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - - assert_eq!( - like_utf8_scalar_dyn(&dict_array, "%r").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(false), - Some(true), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - - assert_eq!( - like_utf8_scalar_dyn(&dict_array, "%i%").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(true), - Some(false), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - - assert_eq!( - like_utf8_scalar_dyn(&dict_array, "%i%").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(true), - Some(false), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - - assert_eq!( - like_utf8_scalar_dyn(&dict_array, "%a%r%").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(false), - Some(true), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - - assert_eq!( - like_utf8_scalar_dyn(&dict_array, "%a%r%").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(false), - Some(true), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - } + Some("bbbbb\nAir") + ], + "Air", + like, + vec![ + Some(false), + Some(false), + Some(false), + Some(true), + None, + Some(true), + Some(false) + ] + ); - #[test] - fn test_dict_nlike_kernels() { - let data = vec![ + test_utf8_scalar!( + test_utf8_scalar_nullable_nlike, + vec![ Some("Earth"), Some("Fire"), Some("Water"), Some("Air"), None, Some("Air"), - Some("bbbbb\nAir"), - ]; - - let dict_array: DictionaryArray = data.into_iter().collect(); - - assert_eq!( - nlike_utf8_scalar_dyn(&dict_array, "Air").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(true), - Some(true), - Some(false), - None, - Some(false), - Some(true), - ]), - ); - - assert_eq!( - nlike_utf8_scalar_dyn(&dict_array, "Air").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(true), - Some(true), - Some(false), - None, - Some(false), - Some(true), - ]), - ); - - assert_eq!( - nlike_utf8_scalar_dyn(&dict_array, "Wa%").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(true), - Some(false), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - - assert_eq!( - nlike_utf8_scalar_dyn(&dict_array, "Wa%").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(true), - Some(false), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - - assert_eq!( - nlike_utf8_scalar_dyn(&dict_array, "%r").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(true), - Some(false), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - - assert_eq!( - nlike_utf8_scalar_dyn(&dict_array, "%r").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(true), - Some(false), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - - assert_eq!( - nlike_utf8_scalar_dyn(&dict_array, "%i%").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(false), - Some(true), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - - assert_eq!( - nlike_utf8_scalar_dyn(&dict_array, "%i%").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(false), - Some(true), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - - assert_eq!( - nlike_utf8_scalar_dyn(&dict_array, "%a%r%").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(true), - Some(false), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - - assert_eq!( - nlike_utf8_scalar_dyn(&dict_array, "%a%r%").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(true), - Some(false), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - } + Some("bbbbb\nAir") + ], + "%a%r%", + nlike, + vec![ + Some(false), + Some(true), + Some(false), + Some(true), + None, + Some(true), + Some(true) + ] + ); - #[test] - fn test_dict_ilike_kernels() { - let data = vec![ + test_utf8_scalar!( + test_utf8_scalar_nullable_ilike, + vec![ Some("Earth"), Some("Fire"), Some("Water"), Some("Air"), None, Some("Air"), - Some("bbbbb\nAir"), - ]; - - let dict_array: DictionaryArray = data.into_iter().collect(); - - assert_eq!( - ilike_utf8_scalar_dyn(&dict_array, "air").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(false), - Some(false), - Some(true), - None, - Some(true), - Some(false), - ]), - ); - - assert_eq!( - ilike_utf8_scalar_dyn(&dict_array, "air").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(false), - Some(false), - Some(true), - None, - Some(true), - Some(false), - ]), - ); - - assert_eq!( - ilike_utf8_scalar_dyn(&dict_array, "wa%").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(false), - Some(true), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - - assert_eq!( - ilike_utf8_scalar_dyn(&dict_array, "wa%").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(false), - Some(true), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - - assert_eq!( - ilike_utf8_scalar_dyn(&dict_array, "%R").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(false), - Some(true), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - - assert_eq!( - ilike_utf8_scalar_dyn(&dict_array, "%R").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(false), - Some(true), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - - assert_eq!( - ilike_utf8_scalar_dyn(&dict_array, "%I%").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(true), - Some(false), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - - assert_eq!( - ilike_utf8_scalar_dyn(&dict_array, "%I%").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(true), - Some(false), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - - assert_eq!( - ilike_utf8_scalar_dyn(&dict_array, "%A%r%").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(false), - Some(true), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - - assert_eq!( - ilike_utf8_scalar_dyn(&dict_array, "%A%r%").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(false), - Some(true), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - } + Some("bbbbb\nAir") + ], + "%I%", + ilike, + vec![ + Some(false), + Some(true), + Some(false), + Some(true), + None, + Some(true), + Some(true) + ] + ); - #[test] - fn test_dict_nilike_kernels() { - let data = vec![ + test_utf8_scalar!( + test_utf8_scalar_nullable_nilike, + vec![ Some("Earth"), Some("Fire"), Some("Water"), Some("Air"), None, Some("Air"), - Some("bbbbb\nAir"), - ]; - - let dict_array: DictionaryArray = data.into_iter().collect(); - - assert_eq!( - nilike_utf8_scalar_dyn(&dict_array, "air").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(true), - Some(true), - Some(false), - None, - Some(false), - Some(true), - ]), - ); - - assert_eq!( - nilike_utf8_scalar_dyn(&dict_array, "air").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(true), - Some(true), - Some(false), - None, - Some(false), - Some(true), - ]), - ); - - assert_eq!( - nilike_utf8_scalar_dyn(&dict_array, "wa%").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(true), - Some(false), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - - assert_eq!( - nilike_utf8_scalar_dyn(&dict_array, "wa%").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(true), - Some(false), - Some(true), - None, - Some(true), - Some(true), - ]), - ); - - assert_eq!( - nilike_utf8_scalar_dyn(&dict_array, "%R").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(true), - Some(false), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - - assert_eq!( - nilike_utf8_scalar_dyn(&dict_array, "%R").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(true), - Some(false), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - - assert_eq!( - nilike_utf8_scalar_dyn(&dict_array, "%I%").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(false), - Some(true), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - - assert_eq!( - nilike_utf8_scalar_dyn(&dict_array, "%I%").unwrap(), - BooleanArray::from(vec![ - Some(true), - Some(false), - Some(true), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - - assert_eq!( - nilike_utf8_scalar_dyn(&dict_array, "%A%r%").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(true), - Some(false), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - - assert_eq!( - nilike_utf8_scalar_dyn(&dict_array, "%A%r%").unwrap(), - BooleanArray::from(vec![ - Some(false), - Some(true), - Some(false), - Some(false), - None, - Some(false), - Some(false), - ]), - ); - } + Some("bbbbb\nAir") + ], + "%R", + nilike, + vec![ + Some(true), + Some(true), + Some(false), + Some(false), + None, + Some(false), + Some(false) + ] + ); #[test] fn string_null_like_pattern() {