feat: support timestampadd, timestampdiff and make_interval via codegen dispatch

docs/source/user-guide/latest/expressions.md

@@ -254,15 +254,15 @@ The type-name conversion functions (`bigint`, `binary`, `boolean`, `date`, `deci
 | `current_time` | 🔜 | — | Blocked on Spark 4.1 TIME type support ([#4288](https://github.com/apache/datafusion-comet/issues/4288)) |
 | `current_timestamp` | ✅ | — | Constant-folded to a literal before Comet sees the plan |
 | `current_timezone` | ✅ | — |  |
-| `date_add` | ✅ | Native |  |
-| `date_diff` | ✅ | Native |  |
+| `date_add` | ✅ | Native | The 2-argument form is native; the `date_add(UNIT, n, ts)` form (Spark 3.5+) parses to `timestampadd` and runs through codegen dispatch |
+| `date_diff` | ✅ | Native | The 2-argument form is native; the `date_diff(UNIT, start, end)` form (Spark 3.5+) parses to `timestampdiff` and runs through codegen dispatch |
 | `date_format` | ✅ | Hybrid |  |
 | `date_from_unix_date` | ✅ | Native |  |
 | `date_part` | ✅ | — |  |
 | `date_sub` | ✅ | Native |  |
 | `date_trunc` | ✅ | Hybrid |  |
-| `dateadd` | ✅ | Native |  |
-| `datediff` | ✅ | Native |  |
+| `dateadd` | ✅ | Native | The 2-argument form is native; the `dateadd(UNIT, n, ts)` form parses to `timestampadd` and runs through codegen dispatch |
+| `datediff` | ✅ | Native | The 2-argument form is native; the `datediff(UNIT, start, end)` form parses to `timestampdiff` and runs through codegen dispatch |
 | `datepart` | ✅ | — |  |
 | `day` | ✅ | Native |  |
 | `dayname` | ✅ | — | Abbreviated day name (Spark 4.0+) |
@@ -277,7 +277,7 @@ The type-name conversion functions (`bigint`, `binary`, `boolean`, `date`, `deci
 | `localtimestamp` | ✅ | — |  |
 | `make_date` | ✅ | Native |  |
 | `make_dt_interval` | ✅ | Codegen dispatch |  |
-| `make_interval` | 🔜 | — | Produces legacy CalendarInterval; tracked by [#4540](https://github.com/apache/datafusion-comet/issues/4540) |
+| `make_interval` | ✅ | Codegen dispatch |  |
 | `make_time` | 🔜 | — | Spark 4.1 TIME type; tracked by [#4288](https://github.com/apache/datafusion-comet/issues/4288) |
 | `make_timestamp` | ✅ | Hybrid |  |
 | `make_timestamp_ltz` | ✅ | — | 2-arg TIME form falls back |
@@ -294,9 +294,12 @@ The type-name conversion functions (`bigint`, `binary`, `boolean`, `date`, `deci
 | `session_window` | 🔜 | — | Batch session-window grouping falls back (`UpdatingSessionsExec` is not yet native); tracked by [#4785](https://github.com/apache/datafusion-comet/issues/4785) |
 | `time_diff` | 🔜 | — | Spark 4.1 TIME type; tracked by [#4288](https://github.com/apache/datafusion-comet/issues/4288) |
 | `time_trunc` | 🔜 | — | Spark 4.1 TIME type; tracked by [#4288](https://github.com/apache/datafusion-comet/issues/4288) |
+| `timediff` | ✅ | — | Spark 4.0+ grammar alias that parses to `timestampdiff`; runs through codegen dispatch |
 | `timestamp_micros` | ✅ | Codegen dispatch |  |
 | `timestamp_millis` | ✅ | Codegen dispatch |  |
 | `timestamp_seconds` | ✅ | Native |  |
+| `timestampadd` | ✅ | — | Reached through the grammar rather than the function registry; runs through codegen dispatch |
+| `timestampdiff` | ✅ | — | Reached through the grammar rather than the function registry; runs through codegen dispatch |
 | `to_date` | ✅ | — | Rewrites to `Cast` (or `Cast(GetTimestamp)` with a format) before Comet sees the plan |
 | `to_time` | 🔜 | — | Spark 4.1 TIME type; tracked by [#4288](https://github.com/apache/datafusion-comet/issues/4288) |
 | `to_timestamp` | ✅ | — | Rewrites to `Cast` (or `GetTimestamp` with a format) before Comet sees the plan |

spark/src/main/scala/org/apache/comet/serde/QueryPlanSerde.scala

@@ -301,7 +301,10 @@ object QueryPlanSerde extends Logging with CometExprShim with CometTypeShim {
       classOf[MakeTimestamp] -> CometMakeTimestamp,
       classOf[MakeYMInterval] -> CometMakeYMInterval,
       classOf[MakeDTInterval] -> CometMakeDTInterval,
+      classOf[MakeInterval] -> CometMakeInterval,
       classOf[MultiplyDTInterval] -> CometMultiplyDTInterval,
+      classOf[TimestampAdd] -> CometTimestampAdd,
+      classOf[TimestampDiff] -> CometTimestampDiff,
       classOf[MicrosToTimestamp] -> CometMicrosToTimestamp,
       classOf[MillisToTimestamp] -> CometMillisToTimestamp,
       classOf[MonthsBetween] -> CometMonthsBetween,

spark/src/main/scala/org/apache/comet/serde/datetime.scala

@@ -21,7 +21,7 @@ package org.apache.comet.serde
 
 import java.util.Locale
 
-import org.apache.spark.sql.catalyst.expressions.{AddMonths, Attribute, ConvertTimezone, DateAdd, DateDiff, DateFormatClass, DateFromUnixDate, DateSub, DayOfMonth, DayOfWeek, DayOfYear, Days, Expression, FromUTCTimestamp, GetDateField, GetTimestamp, Hour, Hours, LastDay, Literal, MakeDate, MakeDTInterval, MakeTimestamp, MakeYMInterval, MicrosToTimestamp, MillisToTimestamp, Minute, Month, MonthsBetween, MultiplyDTInterval, NextDay, PreciseTimestampConversion, Quarter, Second, SecondsToTimestamp, ToUnixTimestamp, ToUTCTimestamp, TruncDate, TruncTimestamp, UnixDate, UnixMicros, UnixMillis, UnixSeconds, UnixTimestamp, WeekDay, WeekOfYear, Year}
+import org.apache.spark.sql.catalyst.expressions.{AddMonths, Attribute, ConvertTimezone, DateAdd, DateDiff, DateFormatClass, DateFromUnixDate, DateSub, DayOfMonth, DayOfWeek, DayOfYear, Days, Expression, FromUTCTimestamp, GetDateField, GetTimestamp, Hour, Hours, LastDay, Literal, MakeDate, MakeDTInterval, MakeInterval, MakeTimestamp, MakeYMInterval, MicrosToTimestamp, MillisToTimestamp, Minute, Month, MonthsBetween, MultiplyDTInterval, NextDay, PreciseTimestampConversion, Quarter, Second, SecondsToTimestamp, TimestampAdd, TimestampDiff, ToUnixTimestamp, ToUTCTimestamp, TruncDate, TruncTimestamp, UnixDate, UnixMicros, UnixMillis, UnixSeconds, UnixTimestamp, WeekDay, WeekOfYear, Year}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types.{DataType, DateType, DoubleType, FloatType, IntegerType, LongType, StringType, TimestampNTZType, TimestampType}
 import org.apache.spark.unsafe.types.UTF8String
@@ -954,8 +954,14 @@ object CometMakeYMInterval extends CometCodegenDispatch[MakeYMInterval]
 
 object CometMakeDTInterval extends CometCodegenDispatch[MakeDTInterval]
 
+object CometMakeInterval extends CometCodegenDispatch[MakeInterval]
+
 object CometMultiplyDTInterval extends CometCodegenDispatch[MultiplyDTInterval]
 
+object CometTimestampAdd extends CometCodegenDispatch[TimestampAdd]
+
+object CometTimestampDiff extends CometCodegenDispatch[TimestampDiff]
+
 /**
  * Spark's internal `PreciseTimestampConversion` reinterprets a value between the timestamp types
  * (`TimestampType` / `TimestampNTZType`) and `LongType` without losing microsecond precision. It

spark/src/test/resources/sql-tests/expressions/datetime/date_add_unit_alias.sql

@@ -0,0 +1,61 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- DATE_ADD and DATE_DIFF joined the timestampadd/timestampdiff grammar rules in Spark 3.5.
+-- With a datetimeUnit keyword as the first argument they parse to TimestampAdd/TimestampDiff
+-- and run through the codegen dispatcher, not to the two-argument DateAdd/DateDiff. The
+-- two-argument forms stay native and are covered here so both spellings are pinned together.
+-- See timestampadd.sql and timestampdiff.sql for the unit and DST coverage.
+-- MinSparkVersion: 3.5
+-- Config: spark.sql.session.timeZone=America/Los_Angeles
+-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true
+
+statement
+CREATE TABLE test_date_add_unit(ts timestamp, d date, q int) USING parquet
+
+statement
+INSERT INTO test_date_add_unit VALUES
+  (timestamp'2024-01-15 10:30:45', date'2024-01-15', 3),
+  (timestamp'2024-01-31 23:00:00', date'2024-01-31', 1),
+  (timestamp'2024-02-29 12:00:00', date'2024-02-29', -5),
+  (NULL, NULL, 1),
+  (timestamp'2024-06-15 00:00:00', date'2024-06-15', NULL)
+
+-- unit form parses to TimestampAdd
+query
+SELECT
+  date_add(DAY, q, ts),
+  date_add(MONTH, 1, ts),
+  date_add(HOUR, 6, ts),
+  date_add(MICROSECOND, 500, ts)
+FROM test_date_add_unit
+
+-- unit form parses to TimestampDiff
+query
+SELECT
+  date_diff(DAY, ts, timestamp'2024-07-01 00:00:00'),
+  date_diff(MONTH, ts, timestamp'2024-07-01 00:00:00'),
+  date_diff(HOUR, ts, timestamp'2024-07-01 00:00:00'),
+  date_diff(QUARTER, ts, timestamp'2024-07-01 00:00:00')
+FROM test_date_add_unit
+
+-- the two-argument forms are unaffected and stay on DateAdd / DateDiff
+query
+SELECT
+  date_add(d, q),
+  date_diff(d, date'2024-07-01')
+FROM test_date_add_unit

spark/src/test/resources/sql-tests/expressions/datetime/make_interval.sql

@@ -0,0 +1,83 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- make_interval runs through the codegen dispatcher and produces CalendarIntervalType.
+-- The suite disables ANSI mode, so MakeInterval.failOnError is false here and overflow
+-- yields NULL. See make_interval_ansi.sql for the throwing path.
+-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true
+
+statement
+CREATE TABLE test_make_interval(y int, mo int, w int, d int, h int, mi int, s decimal(18,6)) USING parquet
+
+statement
+INSERT INTO test_make_interval VALUES
+  (1, 2, 3, 4, 5, 6, 7.008009),
+  (30, 25, 0, -100, 40, 80, 299.889987),
+  (0, -1, 0, 1, 0, 0, -1.000000),
+  (-1, -2, -3, -4, -5, -6, -7.500000),
+  (0, 0, 0, 0, 0, 0, 0.000000),
+  (NULL, 2, 3, 4, 5, 6, 7.008009),
+  (1, 2, 3, 4, 5, 6, NULL)
+
+-- all seven arguments as columns
+query
+SELECT make_interval(y, mo, w, d, h, mi, s) FROM test_make_interval
+
+-- the shorter arities filled in by MakeInterval's auxiliary constructors
+query
+SELECT
+  make_interval(y),
+  make_interval(y, mo),
+  make_interval(y, mo, w),
+  make_interval(y, mo, w, d),
+  make_interval(y, mo, w, d, h),
+  make_interval(y, mo, w, d, h, mi)
+FROM test_make_interval
+
+-- mixed literal and column arguments
+query
+SELECT
+  make_interval(1, mo, 3, d, 5, mi, 7.008009),
+  make_interval(y, 2, w, 4, h, 6, s)
+FROM test_make_interval
+
+-- literal arguments (constant folding is disabled by the test suite)
+query
+SELECT
+  make_interval(1, 2, 3, 4, 5, 6, 7.008009),
+  make_interval(0, 0, 0, 0, 0, 0, 0),
+  make_interval(-1, -2, -3, -4, -5, -6, -7.5),
+  make_interval(0, 13, 0, 0, 25, 61, 61.5)
+
+-- NULL arguments propagate. These have to come from nullable columns rather than NULL
+-- literals: MakeInterval is NullIntolerant, so NullPropagation rewrites any call with a
+-- literal NULL argument to a null interval literal and the expression never reaches Comet.
+-- Such a literal currently fails natively rather than falling back, tracked by #5058.
+query
+SELECT
+  make_interval(y, 2, 3, 4, 5, 6, 7.008009),
+  make_interval(1, 2, 3, 4, 5, 6, s),
+  make_interval(y)
+FROM test_make_interval
+
+-- years * 12 exceeds the int range. Outside ANSI mode MakeInterval swallows the
+-- ArithmeticException and returns NULL.
+query
+SELECT
+  make_interval(200000000, 0, 0, 0, 0, 0, 0),
+  make_interval(y + 200000000, mo, w, d, h, mi, s)
+FROM test_make_interval

spark/src/test/resources/sql-tests/expressions/datetime/make_interval_ansi.sql

@@ -0,0 +1,49 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- MakeInterval.failOnError defaults to SQLConf.get.ansiEnabled, so in ANSI mode overflow
+-- raises ARITHMETIC_OVERFLOW instead of returning NULL. The exception has to cross out of
+-- the generated kernel and surface as the same Spark error.
+-- See make_interval.sql for the non-ANSI coverage.
+-- Config: spark.sql.ansi.enabled=true
+-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true
+
+statement
+CREATE TABLE test_make_interval_ansi(y int, mo int, w int, d int, h int, mi int, s decimal(18,6)) USING parquet
+
+statement
+INSERT INTO test_make_interval_ansi VALUES
+  (1, 2, 3, 4, 5, 6, 7.008009),
+  (-1, -2, -3, -4, -5, -6, -7.500000),
+  (NULL, 2, 3, 4, 5, 6, 7.008009)
+
+-- valid inputs still evaluate normally under ANSI, and act as the sentinel proving the
+-- expression is not silently falling back to Spark
+query
+SELECT make_interval(y, mo, w, d, h, mi, s) FROM test_make_interval_ansi
+
+query
+SELECT make_interval(1, 2, 3, 4, 5, 6, 7.008009)
+
+-- years * 12 overflows the int range. Spark 3.5 renders ARITHMETIC_OVERFLOW without the
+-- condition name in the message, so match on the shared "overflow" text instead.
+query expect_error(overflow)
+SELECT make_interval(200000000, 0, 0, 0, 0, 0, 0)
+
+-- weeks * 7 overflows the int range
+query expect_error(overflow)
+SELECT make_interval(0, 0, 400000000, 0, 0, 0, 0)

spark/src/test/resources/sql-tests/expressions/datetime/timediff.sql

@@ -0,0 +1,48 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--   http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied.  See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- TIMEDIFF was added to the timestampdiff grammar rule in Spark 4.0, so it parses to the
+-- same TimestampDiff node and runs through the codegen dispatcher. See timestampdiff.sql
+-- for the unit and DST coverage that applies to every spelling.
+-- MinSparkVersion: 4.0
+-- Config: spark.sql.session.timeZone=America/Los_Angeles
+-- Config: spark.comet.exec.scalaUDF.codegen.enabled=true
+
+statement
+CREATE TABLE test_timediff(a timestamp, b timestamp) USING parquet
+
+statement
+INSERT INTO test_timediff VALUES
+  (timestamp'2024-01-01 00:00:00', timestamp'2024-03-15 12:30:00'),
+  (timestamp'2024-03-15 12:30:00', timestamp'2024-01-01 00:00:00'),
+  (NULL, timestamp'2024-01-01 00:00:00'),
+  (timestamp'2024-01-01 00:00:00', NULL)
+
+query
+SELECT
+  timediff(YEAR, a, b),
+  timediff(MONTH, a, b),
+  timediff(DAY, a, b),
+  timediff(HOUR, a, b),
+  timediff(MICROSECOND, a, b)
+FROM test_timediff
+
+-- literal arguments (constant folding is disabled by the test suite)
+query
+SELECT
+  timediff(HOUR, timestamp'2024-03-09 12:00:00', timestamp'2024-03-10 12:00:00'),
+  timediff(QUARTER, timestamp'2024-01-01 00:00:00', timestamp'2024-08-15 00:00:00')