From e9287b7b4b4927e217c54ded8609e6dab9a57fba Mon Sep 17 00:00:00 2001 From: peterxcli Date: Wed, 29 Jul 2026 23:37:12 +0800 Subject: [PATCH 1/3] fix: support null calendar interval literals --- native/core/src/execution/planner.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/native/core/src/execution/planner.rs b/native/core/src/execution/planner.rs index 077df787c8..181b19bb79 100644 --- a/native/core/src/execution/planner.rs +++ b/native/core/src/execution/planner.rs @@ -415,6 +415,9 @@ impl PhysicalPlanner { DataType::Duration(TimeUnit::Microsecond) => { ScalarValue::DurationMicrosecond(None) } + DataType::Interval(arrow::datatypes::IntervalUnit::MonthDayNano) => { + ScalarValue::IntervalMonthDayNano(None) + } dt => { return Err(GeneralError(format!("{dt:?} is not supported in Comet"))) } From f831cbe2de0e3bc63bc1055e4d7660f260156df1 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Sat, 1 Aug 2026 10:42:56 +0800 Subject: [PATCH 2/3] andy's review --- native/core/src/execution/planner.rs | 34 +------------------ .../datetime/calendar_interval.sql | 14 ++++++++ 2 files changed, 15 insertions(+), 33 deletions(-) diff --git a/native/core/src/execution/planner.rs b/native/core/src/execution/planner.rs index 181b19bb79..050f543910 100644 --- a/native/core/src/execution/planner.rs +++ b/native/core/src/execution/planner.rs @@ -85,7 +85,6 @@ use crate::execution::operators::ExecutionError::GeneralError; use crate::execution::shuffle::{CometPartitioning, CompressionCodec}; use crate::execution::spark_plan::SparkPlan; use crate::parquet::parquet_support::prepare_object_store_with_configs; -use datafusion::common::scalar::ScalarStructBuilder; use datafusion::common::{ tree_node::{Transformed, TransformedResult, TreeNode, TreeNodeRecursion, TreeNodeRewriter}, JoinType as DFJoinType, NullEquality, ScalarValue, @@ -390,38 +389,7 @@ impl PhysicalPlanner { ExprStruct::Literal(literal) => { let data_type = to_arrow_datatype(literal.datatype.as_ref().unwrap()); let scalar_value = if literal.is_null { - match data_type { - DataType::Boolean => ScalarValue::Boolean(None), - DataType::Int8 => ScalarValue::Int8(None), - DataType::Int16 => ScalarValue::Int16(None), - DataType::Int32 => ScalarValue::Int32(None), - DataType::Int64 => ScalarValue::Int64(None), - DataType::Float32 => ScalarValue::Float32(None), - DataType::Float64 => ScalarValue::Float64(None), - DataType::Utf8 => ScalarValue::Utf8(None), - DataType::Date32 => ScalarValue::Date32(None), - DataType::Timestamp(TimeUnit::Microsecond, timezone) => { - ScalarValue::TimestampMicrosecond(None, timezone) - } - DataType::Binary => ScalarValue::Binary(None), - DataType::Decimal128(p, s) => ScalarValue::Decimal128(None, p, s), - DataType::Struct(fields) => ScalarStructBuilder::new_null(fields), - DataType::Map(f, s) => DataType::Map(f, s).try_into()?, - DataType::List(f) => DataType::List(f).try_into()?, - DataType::Null => ScalarValue::Null, - DataType::Time64(TimeUnit::Nanosecond) => { - ScalarValue::Time64Nanosecond(None) - } - DataType::Duration(TimeUnit::Microsecond) => { - ScalarValue::DurationMicrosecond(None) - } - DataType::Interval(arrow::datatypes::IntervalUnit::MonthDayNano) => { - ScalarValue::IntervalMonthDayNano(None) - } - dt => { - return Err(GeneralError(format!("{dt:?} is not supported in Comet"))) - } - } + ScalarValue::try_new_null(&data_type)? } else { match literal.value.as_ref().unwrap() { Value::BoolVal(value) => ScalarValue::Boolean(Some(*value)), diff --git a/spark/src/test/resources/sql-tests/expressions/datetime/calendar_interval.sql b/spark/src/test/resources/sql-tests/expressions/datetime/calendar_interval.sql index 796e6354ac..6eaa9648f2 100644 --- a/spark/src/test/resources/sql-tests/expressions/datetime/calendar_interval.sql +++ b/spark/src/test/resources/sql-tests/expressions/datetime/calendar_interval.sql @@ -18,6 +18,20 @@ -- Config: spark.comet.exec.localTableScan.enabled=true -- Config: spark.comet.shuffle.mode=native +statement +CREATE TABLE test_null_interval(id int) USING parquet + +statement +INSERT INTO test_null_interval VALUES (1) + +-- Null calendar interval literal in a projection over a real scan. +query +SELECT CAST(NULL AS INTERVAL) FROM test_null_interval + +-- NullPropagation folds make_interval with a null argument to a null interval literal. +query +SELECT make_interval(NULL, 2, 3, 4, 5, 6, 7.008009) FROM test_null_interval + query SELECT * FROM VALUES (make_interval(1, 2, 3, 4, 5, 6, 7.008009)), From db9076fb58da03c375f40664b05e39893dc67bd6 Mon Sep 17 00:00:00 2001 From: peterxcli Date: Sat, 1 Aug 2026 10:54:22 +0800 Subject: [PATCH 3/3] switch back to explicit matching for planner with Literal expr --- native/core/src/execution/planner.rs | 34 +++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/native/core/src/execution/planner.rs b/native/core/src/execution/planner.rs index 56f8eaf01d..fcd146d4a3 100644 --- a/native/core/src/execution/planner.rs +++ b/native/core/src/execution/planner.rs @@ -85,6 +85,7 @@ use crate::execution::operators::ExecutionError::GeneralError; use crate::execution::shuffle::{CometPartitioning, CompressionCodec}; use crate::execution::spark_plan::SparkPlan; use crate::parquet::parquet_support::prepare_object_store_with_configs; +use datafusion::common::scalar::ScalarStructBuilder; use datafusion::common::{ tree_node::{Transformed, TransformedResult, TreeNode, TreeNodeRecursion, TreeNodeRewriter}, JoinType as DFJoinType, NullEquality, ScalarValue, @@ -391,7 +392,38 @@ impl PhysicalPlanner { ExprStruct::Literal(literal) => { let data_type = to_arrow_datatype(literal.datatype.as_ref().unwrap()); let scalar_value = if literal.is_null { - ScalarValue::try_new_null(&data_type)? + match data_type { + DataType::Boolean => ScalarValue::Boolean(None), + DataType::Int8 => ScalarValue::Int8(None), + DataType::Int16 => ScalarValue::Int16(None), + DataType::Int32 => ScalarValue::Int32(None), + DataType::Int64 => ScalarValue::Int64(None), + DataType::Float32 => ScalarValue::Float32(None), + DataType::Float64 => ScalarValue::Float64(None), + DataType::Utf8 => ScalarValue::Utf8(None), + DataType::Date32 => ScalarValue::Date32(None), + DataType::Timestamp(TimeUnit::Microsecond, timezone) => { + ScalarValue::TimestampMicrosecond(None, timezone) + } + DataType::Binary => ScalarValue::Binary(None), + DataType::Decimal128(p, s) => ScalarValue::Decimal128(None, p, s), + DataType::Struct(fields) => ScalarStructBuilder::new_null(fields), + DataType::Map(f, s) => DataType::Map(f, s).try_into()?, + DataType::List(f) => DataType::List(f).try_into()?, + DataType::Null => ScalarValue::Null, + DataType::Time64(TimeUnit::Nanosecond) => { + ScalarValue::Time64Nanosecond(None) + } + DataType::Duration(TimeUnit::Microsecond) => { + ScalarValue::DurationMicrosecond(None) + } + DataType::Interval(arrow::datatypes::IntervalUnit::MonthDayNano) => { + ScalarValue::IntervalMonthDayNano(None) + } + dt => { + return Err(GeneralError(format!("{dt:?} is not supported in Comet"))) + } + } } else { match literal.value.as_ref().unwrap() { Value::BoolVal(value) => ScalarValue::Boolean(Some(*value)),