From 04b9405a6f51721382748b590a1176d928bd69c5 Mon Sep 17 00:00:00 2001 From: rich-T-kid Date: Mon, 20 Jul 2026 14:27:29 -0400 Subject: [PATCH 1/9] support suffix white space in parse --- arrow-cast/src/parse.rs | 60 +++++++++++++++++++++++++++-------------- 1 file changed, 40 insertions(+), 20 deletions(-) diff --git a/arrow-cast/src/parse.rs b/arrow-cast/src/parse.rs index 57fbdd40acf8..973f553e810b 100644 --- a/arrow-cast/src/parse.rs +++ b/arrow-cast/src/parse.rs @@ -445,7 +445,7 @@ pub trait Parser: ArrowPrimitiveType { impl Parser for Float16Type { fn parse(string: &str) -> Option { - let string = truncate_white_space(string); + let string = trim_pre_and_post_whitespace(string); lexical_core::parse(string.as_bytes()) .ok() .map(f16::from_f32) @@ -454,44 +454,40 @@ impl Parser for Float16Type { impl Parser for Float32Type { fn parse(string: &str) -> Option { - let string = truncate_white_space(string); + let string = trim_pre_and_post_whitespace(string); lexical_core::parse(string.as_bytes()).ok() } } impl Parser for Float64Type { fn parse(string: &str) -> Option { - let string = truncate_white_space(string); + let string = trim_pre_and_post_whitespace(string); lexical_core::parse(string.as_bytes()).ok() } } + +/// this is a no-op if the string starts and ends with a digit, otherwise it will trim whitespace from the start and end of the string. #[inline] -fn truncate_white_space(string: &str) -> &str { - if string - .as_bytes() - .first() - .is_some_and(|first_byte| !first_byte.is_ascii_digit()) - { - string.trim_ascii_start() - } else { - string +fn trim_pre_and_post_whitespace(string: &str) -> &str { + let bytes = string.as_bytes(); + let prefix = bytes.first().is_some_and(|b| !b.is_ascii_digit()); + let suffix = bytes.last().is_some_and(|b| !b.is_ascii_digit()); + match (prefix, suffix) { + (false, false) => string, + (true, false) => string.trim_ascii_start(), + (false, true) => string.trim_ascii_end(), + (true, true) => string.trim_ascii(), } } macro_rules! parser_primitive { ($t:ty) => { impl Parser for $t { - fn parse(mut string: &str) -> Option { + fn parse(string: &str) -> Option { + let string = trim_pre_and_post_whitespace(string); if !string.as_bytes().last().is_some_and(|x| x.is_ascii_digit()) { return None; } - if string - .as_bytes() - .first() - .is_some_and(|first| !first.is_ascii_digit()) - { - string = string.trim_ascii_start(); - }; match atoi::FromRadix10SignedChecked::from_radix_10_signed_checked( string.as_bytes(), ) { @@ -2913,5 +2909,29 @@ mod tests { assert_eq!(Int32Type::parse("\t\n\t\n\n\n\t1"), Some(1)); assert_eq!(Int32Type::parse(" \n-25"), Some(-25)); assert_eq!(Int32Type::parse("\t-800"), Some(-800)); + + // suffix whitespace + assert_eq!(Float64Type::parse("1.5 "), Some(1.5)); + assert_eq!(Float64Type::parse("40.5123\n"), Some(40.5123)); + assert_eq!(Float64Type::parse("40.5123\n\t\n\t\n"), Some(40.5123)); + assert_eq!(Float64Type::parse("-942.5423\t"), Some(-942.5423)); + assert_eq!(Int32Type::parse("3 "), Some(3)); + assert_eq!(Int32Type::parse("30 "), Some(30)); + assert_eq!(Int32Type::parse("-25 \n"), Some(-25)); + assert_eq!(Int32Type::parse("800\t"), Some(800)); + // whitespace on both sides + assert_eq!(Float64Type::parse(" 1.5 "), Some(1.5)); + assert_eq!(Float64Type::parse("\t\n 20.54 \t"), Some(20.54)); + assert_eq!(Float64Type::parse("\n-942.5423\n"), Some(-942.5423)); + assert_eq!(Int32Type::parse(" 3 "), Some(3)); + assert_eq!(Int32Type::parse("\n \n 100 \n"), Some(100)); + assert_eq!(Int32Type::parse("\t-800\t\n"), Some(-800)); + + // trailing non-whitespace chars should not parse + assert_eq!(Float64Type::parse("1.5abc"), None); + assert_eq!(Float64Type::parse("40.5123x"), None); + assert_eq!(Int32Type::parse("30x"), None); + assert_eq!(Int32Type::parse("100px"), None); + assert_eq!(Int32Type::parse("-25!"), None); } } From 9038bafb157e099cc902b5b256ba28df5f69aa31 Mon Sep 17 00:00:00 2001 From: rich-T-kid Date: Mon, 20 Jul 2026 16:10:41 -0400 Subject: [PATCH 2/9] replace buffer look up comparission with two boolean comps --- arrow-cast/src/parse.rs | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/arrow-cast/src/parse.rs b/arrow-cast/src/parse.rs index 973f553e810b..5ff611e75689 100644 --- a/arrow-cast/src/parse.rs +++ b/arrow-cast/src/parse.rs @@ -445,7 +445,7 @@ pub trait Parser: ArrowPrimitiveType { impl Parser for Float16Type { fn parse(string: &str) -> Option { - let string = trim_pre_and_post_whitespace(string); + let (string, _) = trim_pre_and_post_whitespace(string); lexical_core::parse(string.as_bytes()) .ok() .map(f16::from_f32) @@ -454,29 +454,30 @@ impl Parser for Float16Type { impl Parser for Float32Type { fn parse(string: &str) -> Option { - let string = trim_pre_and_post_whitespace(string); + let (string, _) = trim_pre_and_post_whitespace(string); lexical_core::parse(string.as_bytes()).ok() } } impl Parser for Float64Type { fn parse(string: &str) -> Option { - let string = trim_pre_and_post_whitespace(string); + let (string, _) = trim_pre_and_post_whitespace(string); lexical_core::parse(string.as_bytes()).ok() } } /// this is a no-op if the string starts and ends with a digit, otherwise it will trim whitespace from the start and end of the string. #[inline] -fn trim_pre_and_post_whitespace(string: &str) -> &str { +fn trim_pre_and_post_whitespace(string: &str) -> (&str, bool) { let bytes = string.as_bytes(); let prefix = bytes.first().is_some_and(|b| !b.is_ascii_digit()); let suffix = bytes.last().is_some_and(|b| !b.is_ascii_digit()); + println!("bytes : {:?}, len: {}", bytes, bytes.len()); match (prefix, suffix) { - (false, false) => string, - (true, false) => string.trim_ascii_start(), - (false, true) => string.trim_ascii_end(), - (true, true) => string.trim_ascii(), + (false, false) => (string, bytes.len() >= 1), + (true, false) => (string.trim_ascii_start(), false), + (false, true) => (string.trim_ascii_end(), false), + (true, true) => (string.trim_ascii(), false), } } @@ -484,8 +485,8 @@ macro_rules! parser_primitive { ($t:ty) => { impl Parser for $t { fn parse(string: &str) -> Option { - let string = trim_pre_and_post_whitespace(string); - if !string.as_bytes().last().is_some_and(|x| x.is_ascii_digit()) { + let (string, not_empty) = trim_pre_and_post_whitespace(string); + if !not_empty { return None; } match atoi::FromRadix10SignedChecked::from_radix_10_signed_checked( @@ -2892,6 +2893,7 @@ mod tests { } #[test] fn test_parse_prefix_white_space() { + /* assert_eq!(Float64Type::parse(" 1.5"), Some(1.5)); assert_eq!(Float64Type::parse("\t\n 20.54"), Some(20.54)); assert_eq!(Float64Type::parse("\n2.5"), Some(2.5)); @@ -2933,5 +2935,8 @@ mod tests { assert_eq!(Int32Type::parse("30x"), None); assert_eq!(Int32Type::parse("100px"), None); assert_eq!(Int32Type::parse("-25!"), None); + */ + assert_eq!(Int32Type::parse("3j"), None); + assert_eq!(Int32Type::parse("3"), Some(3)); } } From 534f92ef480ef4b09cde587f8de4ce3af259f652 Mon Sep 17 00:00:00 2001 From: rich-T-kid Date: Mon, 20 Jul 2026 16:38:26 -0400 Subject: [PATCH 3/9] exist early for empty cases --- arrow-cast/src/parse.rs | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/arrow-cast/src/parse.rs b/arrow-cast/src/parse.rs index 5ff611e75689..9848cc20b749 100644 --- a/arrow-cast/src/parse.rs +++ b/arrow-cast/src/parse.rs @@ -445,7 +445,7 @@ pub trait Parser: ArrowPrimitiveType { impl Parser for Float16Type { fn parse(string: &str) -> Option { - let (string, _) = trim_pre_and_post_whitespace(string); + let string = trim_pre_and_post_whitespace(string); lexical_core::parse(string.as_bytes()) .ok() .map(f16::from_f32) @@ -454,30 +454,29 @@ impl Parser for Float16Type { impl Parser for Float32Type { fn parse(string: &str) -> Option { - let (string, _) = trim_pre_and_post_whitespace(string); + let string = trim_pre_and_post_whitespace(string); lexical_core::parse(string.as_bytes()).ok() } } impl Parser for Float64Type { fn parse(string: &str) -> Option { - let (string, _) = trim_pre_and_post_whitespace(string); + let string = trim_pre_and_post_whitespace(string); lexical_core::parse(string.as_bytes()).ok() } } /// this is a no-op if the string starts and ends with a digit, otherwise it will trim whitespace from the start and end of the string. #[inline] -fn trim_pre_and_post_whitespace(string: &str) -> (&str, bool) { +fn trim_pre_and_post_whitespace(string: &str) -> &str { let bytes = string.as_bytes(); let prefix = bytes.first().is_some_and(|b| !b.is_ascii_digit()); let suffix = bytes.last().is_some_and(|b| !b.is_ascii_digit()); - println!("bytes : {:?}, len: {}", bytes, bytes.len()); match (prefix, suffix) { - (false, false) => (string, bytes.len() >= 1), - (true, false) => (string.trim_ascii_start(), false), - (false, true) => (string.trim_ascii_end(), false), - (true, true) => (string.trim_ascii(), false), + (false, false) => string, + (true, false) => string.trim_ascii_start(), + (false, true) => string.trim_ascii_end(), + (true, true) => string.trim_ascii(), } } @@ -485,10 +484,10 @@ macro_rules! parser_primitive { ($t:ty) => { impl Parser for $t { fn parse(string: &str) -> Option { - let (string, not_empty) = trim_pre_and_post_whitespace(string); - if !not_empty { + if string.is_empty() { return None; } + let string = trim_pre_and_post_whitespace(string); match atoi::FromRadix10SignedChecked::from_radix_10_signed_checked( string.as_bytes(), ) { From 750fe858b20dfef81c59dfd1791fd548dddb461b Mon Sep 17 00:00:00 2001 From: rich-T-kid Date: Mon, 20 Jul 2026 16:41:41 -0400 Subject: [PATCH 4/9] re-introduce check --- arrow-cast/src/parse.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/arrow-cast/src/parse.rs b/arrow-cast/src/parse.rs index 9848cc20b749..9e031a98891b 100644 --- a/arrow-cast/src/parse.rs +++ b/arrow-cast/src/parse.rs @@ -484,13 +484,11 @@ macro_rules! parser_primitive { ($t:ty) => { impl Parser for $t { fn parse(string: &str) -> Option { - if string.is_empty() { + let string_bytes = trim_pre_and_post_whitespace(string).as_bytes(); + if !string_bytes.last().is_some_and(|x| x.is_ascii_digit()) { return None; } - let string = trim_pre_and_post_whitespace(string); - match atoi::FromRadix10SignedChecked::from_radix_10_signed_checked( - string.as_bytes(), - ) { + match atoi::FromRadix10SignedChecked::from_radix_10_signed_checked(string_bytes) { (Some(n), x) if x == string.len() => Some(n), _ => None, } From d8a346de8252b976ec1731b7f39c80b3adc3a446 Mon Sep 17 00:00:00 2001 From: rich-T-kid Date: Thu, 23 Jul 2026 09:45:52 -0400 Subject: [PATCH 5/9] leave fast path untouched, add prefix/suffix trim on failure only --- arrow-cast/src/parse.rs | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/arrow-cast/src/parse.rs b/arrow-cast/src/parse.rs index 9e031a98891b..289700f2062c 100644 --- a/arrow-cast/src/parse.rs +++ b/arrow-cast/src/parse.rs @@ -445,6 +445,9 @@ pub trait Parser: ArrowPrimitiveType { impl Parser for Float16Type { fn parse(string: &str) -> Option { + if let Some(raw_float) = lexical_core::parse(string.as_bytes()).ok() { + return Some(f16::from_f32(raw_float)); + } let string = trim_pre_and_post_whitespace(string); lexical_core::parse(string.as_bytes()) .ok() @@ -454,6 +457,9 @@ impl Parser for Float16Type { impl Parser for Float32Type { fn parse(string: &str) -> Option { + if let Some(raw_float) = lexical_core::parse(string.as_bytes()).ok() { + return Some(raw_float); + } let string = trim_pre_and_post_whitespace(string); lexical_core::parse(string.as_bytes()).ok() } @@ -484,13 +490,24 @@ macro_rules! parser_primitive { ($t:ty) => { impl Parser for $t { fn parse(string: &str) -> Option { - let string_bytes = trim_pre_and_post_whitespace(string).as_bytes(); - if !string_bytes.last().is_some_and(|x| x.is_ascii_digit()) { - return None; + // let string_bytes = trim_pre_and_post_whitespace(string).as_bytes(); + let mut raw_bytes = string.as_bytes(); + if !raw_bytes.last().is_some_and(|x| x.is_ascii_digit()) { + raw_bytes = raw_bytes.trim_ascii_end(); + if !raw_bytes.last().is_some_and(|x| x.is_ascii_digit()) { + return None; + } } - match atoi::FromRadix10SignedChecked::from_radix_10_signed_checked(string_bytes) { - (Some(n), x) if x == string.len() => Some(n), - _ => None, + match atoi::FromRadix10SignedChecked::from_radix_10_signed_checked(raw_bytes) { + (Some(n), x) if x == raw_bytes.len() => Some(n), + _ => { + let trimmed = raw_bytes.trim_ascii_start(); + match atoi::FromRadix10SignedChecked::from_radix_10_signed_checked(trimmed) + { + (Some(n), x) if x == trimmed.len() => Some(n), + _ => None, + } + } } } } From 483b5955873066d1aa5391410daef58a3ffd5936 Mon Sep 17 00:00:00 2001 From: rich-T-kid Date: Thu, 23 Jul 2026 09:47:26 -0400 Subject: [PATCH 6/9] remove comment --- arrow-cast/src/parse.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/arrow-cast/src/parse.rs b/arrow-cast/src/parse.rs index 289700f2062c..bfbe80025e18 100644 --- a/arrow-cast/src/parse.rs +++ b/arrow-cast/src/parse.rs @@ -490,7 +490,6 @@ macro_rules! parser_primitive { ($t:ty) => { impl Parser for $t { fn parse(string: &str) -> Option { - // let string_bytes = trim_pre_and_post_whitespace(string).as_bytes(); let mut raw_bytes = string.as_bytes(); if !raw_bytes.last().is_some_and(|x| x.is_ascii_digit()) { raw_bytes = raw_bytes.trim_ascii_end(); From 769a124283cbf0d56c0bb6d5bc9bc8e2292adca1 Mon Sep 17 00:00:00 2001 From: rich-T-kid Date: Thu, 23 Jul 2026 09:48:17 -0400 Subject: [PATCH 7/9] uncomment test --- arrow-cast/src/parse.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/arrow-cast/src/parse.rs b/arrow-cast/src/parse.rs index bfbe80025e18..80e52daf310b 100644 --- a/arrow-cast/src/parse.rs +++ b/arrow-cast/src/parse.rs @@ -2906,7 +2906,6 @@ mod tests { } #[test] fn test_parse_prefix_white_space() { - /* assert_eq!(Float64Type::parse(" 1.5"), Some(1.5)); assert_eq!(Float64Type::parse("\t\n 20.54"), Some(20.54)); assert_eq!(Float64Type::parse("\n2.5"), Some(2.5)); @@ -2948,7 +2947,6 @@ mod tests { assert_eq!(Int32Type::parse("30x"), None); assert_eq!(Int32Type::parse("100px"), None); assert_eq!(Int32Type::parse("-25!"), None); - */ assert_eq!(Int32Type::parse("3j"), None); assert_eq!(Int32Type::parse("3"), Some(3)); } From ec198ad3c16968a9917c48f60fe4bc104e6ae3d3 Mon Sep 17 00:00:00 2001 From: rich-T-kid Date: Thu, 23 Jul 2026 10:20:01 -0400 Subject: [PATCH 8/9] fix clippy --- arrow-cast/src/parse.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arrow-cast/src/parse.rs b/arrow-cast/src/parse.rs index 80e52daf310b..e37a51f62f6f 100644 --- a/arrow-cast/src/parse.rs +++ b/arrow-cast/src/parse.rs @@ -445,7 +445,7 @@ pub trait Parser: ArrowPrimitiveType { impl Parser for Float16Type { fn parse(string: &str) -> Option { - if let Some(raw_float) = lexical_core::parse(string.as_bytes()).ok() { + if let Ok(raw_float) = lexical_core::parse(string.as_bytes()) { return Some(f16::from_f32(raw_float)); } let string = trim_pre_and_post_whitespace(string); @@ -457,7 +457,7 @@ impl Parser for Float16Type { impl Parser for Float32Type { fn parse(string: &str) -> Option { - if let Some(raw_float) = lexical_core::parse(string.as_bytes()).ok() { + if let Ok(raw_float) = lexical_core::parse(string.as_bytes()) { return Some(raw_float); } let string = trim_pre_and_post_whitespace(string); From 5c1d048d72863dfd48c3013a3545f39393a4ff3d Mon Sep 17 00:00:00 2001 From: rich-T-kid Date: Thu, 23 Jul 2026 23:44:30 -0400 Subject: [PATCH 9/9] add changes for f64 --- arrow-cast/src/parse.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arrow-cast/src/parse.rs b/arrow-cast/src/parse.rs index e37a51f62f6f..2db047359ab6 100644 --- a/arrow-cast/src/parse.rs +++ b/arrow-cast/src/parse.rs @@ -467,6 +467,9 @@ impl Parser for Float32Type { impl Parser for Float64Type { fn parse(string: &str) -> Option { + if let Ok(raw_float) = lexical_core::parse(string.as_bytes()) { + return Some(raw_float); + } let string = trim_pre_and_post_whitespace(string); lexical_core::parse(string.as_bytes()).ok() }