diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4f38a2c515..476dddd3bb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -260,7 +260,7 @@ jobs: uses: ./.github/workflows/spark_sql_test_reusable.yml with: spark-short: '4.1' - spark-full: '4.1.2' + spark-full: '4.1.3' java: 17 iceberg_1_8: @@ -332,5 +332,5 @@ jobs: iceberg-short: '1.11' iceberg-full: '1.11.0' spark-short: '4.1' - spark-full: '4.1.2' + spark-full: '4.1.3' java: 17 diff --git a/.github/workflows/pyarrow_udf_test.yml b/.github/workflows/pyarrow_udf_test.yml index 807056cf9e..8493ebb9f0 100644 --- a/.github/workflows/pyarrow_udf_test.yml +++ b/.github/workflows/pyarrow_udf_test.yml @@ -73,7 +73,7 @@ jobs: pyspark: "4.0.2" - name: Spark 4.1 maven_profiles: "-Pspark-4.1" - pyspark: "4.1.2" + pyspark: "4.1.3" container: # Pinned to the Debian 12 (bookworm) base so the system `python3` is 3.11. The default # `amd64/rust` image is Debian 13 (trixie) which ships Python 3.13 and no python3.11 apt diff --git a/.github/workflows/spark_sql_writer_tests.yml b/.github/workflows/spark_sql_writer_tests.yml index 0406aa7ff8..c813a8b39e 100644 --- a/.github/workflows/spark_sql_writer_tests.yml +++ b/.github/workflows/spark_sql_writer_tests.yml @@ -71,7 +71,7 @@ jobs: # versions only; 3.4 and 4.0 are label-gated and not offered here). case "${{ inputs.spark-version }}" in 3.5) spark_full=3.5.9; java=17 ;; - 4.1) spark_full=4.1.2; java=17 ;; + 4.1) spark_full=4.1.3; java=17 ;; *) echo "Unsupported spark-version: ${{ inputs.spark-version }}" >&2; exit 1 ;; esac echo "spark-full=$spark_full" >> "$GITHUB_OUTPUT" diff --git a/dev/ci/compute-changes.py b/dev/ci/compute-changes.py index 195d27e828..bc28e0e789 100644 --- a/dev/ci/compute-changes.py +++ b/dev/ci/compute-changes.py @@ -169,7 +169,7 @@ "!spark/src/main/spark-4.2/**", "!spark/src/main/scala/org/apache/comet/GenerateDocs.scala", "spark/pom.xml", - "dev/diffs/4.1.2.diff", + "dev/diffs/4.1.3.diff", "pom.xml", "rust-toolchain.toml", ".github/workflows/ci.yml", diff --git a/dev/diffs/4.1.2.diff b/dev/diffs/4.1.3.diff similarity index 99% rename from dev/diffs/4.1.2.diff rename to dev/diffs/4.1.3.diff index b0a32baaba..79d008c989 100644 --- a/dev/diffs/4.1.2.diff +++ b/dev/diffs/4.1.3.diff @@ -39,7 +39,7 @@ index 6df8bc85b51..dabb75e2b75 100644 withSpark(sc) { sc => TestUtils.waitUntilExecutorsUp(sc, 2, 60000) diff --git a/pom.xml b/pom.xml -index dc201151999..20ee0e7482a 100644 +index 370bfa4397f..8447b5c375e 100644 --- a/pom.xml +++ b/pom.xml @@ -152,6 +152,8 @@ @@ -78,7 +78,7 @@ index dc201151999..20ee0e7482a 100644 org.apache.datasketches diff --git a/sql/core/pom.xml b/sql/core/pom.xml -index c25b83c355b..5e23b863dcf 100644 +index febc7d747bc..63c8e4e22c0 100644 --- a/sql/core/pom.xml +++ b/sql/core/pom.xml @@ -97,6 +97,10 @@ @@ -514,7 +514,7 @@ index 93ff7becaec..87537a25b3b 100644 assert(unionExec.size == 1) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala -index 5b88eeefeca..d4f07bc182a 100644 +index d9ce3000a0c..f2d044ed6b8 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala @@ -36,11 +36,12 @@ import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, @@ -574,13 +574,13 @@ index 5b88eeefeca..d4f07bc182a 100644 assert(exchanges.size == 2) } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala -index 6e9f3385571..5af115c9a7d 100644 +index 4a070becfa6..61d515d127e 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameWindowFunctionsSuite.scala -@@ -24,8 +24,10 @@ import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression - import org.apache.spark.sql.catalyst.optimizer.TransposeWindow +@@ -25,8 +25,10 @@ import org.apache.spark.sql.catalyst.optimizer.TransposeWindow import org.apache.spark.sql.catalyst.plans.logical.{Window => LogicalWindow} import org.apache.spark.sql.catalyst.plans.physical.HashPartitioning + import org.apache.spark.sql.catalyst.trees.UnaryLike +import org.apache.spark.sql.comet.CometWindowExec +import org.apache.spark.sql.comet.execution.shuffle.CometShuffleExchangeExec import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper @@ -589,7 +589,7 @@ index 6e9f3385571..5af115c9a7d 100644 import org.apache.spark.sql.execution.window.WindowExec import org.apache.spark.sql.expressions.{Aggregator, MutableAggregationBuffer, UserDefinedAggregateFunction, Window} import org.apache.spark.sql.functions._ -@@ -951,7 +953,8 @@ class DataFrameWindowFunctionsSuite extends QueryTest +@@ -974,7 +976,8 @@ class DataFrameWindowFunctionsSuite extends QueryTest } } @@ -599,7 +599,7 @@ index 6e9f3385571..5af115c9a7d 100644 val df = Seq((1, "1"), (2, "2"), (1, "3"), (2, "4")).toDF("key", "value") val window = Window.partitionBy($"key").orderBy($"value") -@@ -963,7 +966,7 @@ class DataFrameWindowFunctionsSuite extends QueryTest +@@ -986,7 +989,7 @@ class DataFrameWindowFunctionsSuite extends QueryTest } } @@ -608,7 +608,7 @@ index 6e9f3385571..5af115c9a7d 100644 // Make sure we trigger the spilling path. withSQLConf(SQLConf.WINDOW_EXEC_BUFFER_IN_MEMORY_THRESHOLD.key -> "1", SQLConf.WINDOW_EXEC_BUFFER_SPILL_THRESHOLD.key -> "17") { -@@ -1142,10 +1145,12 @@ class DataFrameWindowFunctionsSuite extends QueryTest +@@ -1165,10 +1168,12 @@ class DataFrameWindowFunctionsSuite extends QueryTest } def isShuffleExecByRequirement( @@ -622,7 +622,7 @@ index 6e9f3385571..5af115c9a7d 100644 case _ => false } -@@ -1168,7 +1173,12 @@ class DataFrameWindowFunctionsSuite extends QueryTest +@@ -1191,7 +1196,12 @@ class DataFrameWindowFunctionsSuite extends QueryTest val shuffleByRequirement = windowed.queryExecution.executedPlan.exists { case w: WindowExec => w.child.exists { @@ -636,7 +636,7 @@ index 6e9f3385571..5af115c9a7d 100644 case _ => false } case _ => false -@@ -1620,7 +1630,8 @@ class DataFrameWindowFunctionsSuite extends QueryTest +@@ -1643,7 +1653,8 @@ class DataFrameWindowFunctionsSuite extends QueryTest } } @@ -1660,7 +1660,7 @@ index a09b7e0827c..ffc29f764bc 100644 } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/FileDataSourceV2FallBackSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/FileDataSourceV2FallBackSuite.scala -index 2a0ab21ddb0..6030e7c2b9b 100644 +index 370cd931809..d90bc1e8d54 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/connector/FileDataSourceV2FallBackSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/FileDataSourceV2FallBackSuite.scala @@ -21,6 +21,7 @@ import scala.collection.mutable.ArrayBuffer @@ -1671,7 +1671,7 @@ index 2a0ab21ddb0..6030e7c2b9b 100644 import org.apache.spark.sql.connector.catalog.{SupportsRead, SupportsWrite, Table, TableCapability} import org.apache.spark.sql.connector.read.ScanBuilder import org.apache.spark.sql.connector.write.{LogicalWriteInfo, WriteBuilder} -@@ -188,7 +189,11 @@ class FileDataSourceV2FallBackSuite extends QueryTest with SharedSparkSession { +@@ -191,7 +192,11 @@ class FileDataSourceV2FallBackSuite extends QueryTest with SharedSparkSession { val df = spark.read.format(format).load(path.getCanonicalPath) checkAnswer(df, inputData.toDF()) assert( @@ -2298,7 +2298,7 @@ index a3cfdc5a240..3793b6191bf 100644 }) checkAnswer(distinctWithId, Seq(Row(1, 0), Row(1, 0))) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala -index 3e7d26f74bd..79232dc3664 100644 +index 188a28ff1c0..3cacc4efbdf 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/adaptive/AdaptiveQueryExecSuite.scala @@ -27,12 +27,14 @@ import org.apache.spark.SparkException @@ -2566,7 +2566,7 @@ index 3e7d26f74bd..79232dc3664 100644 Seq("SHUFFLE_MERGE", "SHUFFLE_HASH").foreach { joinHint => def getJoinNode(plan: SparkPlan): Seq[ShuffledJoin] = if (joinHint == "SHUFFLE_MERGE") { findTopLevelSortMergeJoin(plan) -@@ -1257,7 +1286,8 @@ class AdaptiveQueryExecSuite +@@ -1266,7 +1295,8 @@ class AdaptiveQueryExecSuite } } @@ -2576,7 +2576,7 @@ index 3e7d26f74bd..79232dc3664 100644 withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true") { val (_, adaptivePlan) = runAdaptiveAndVerifyResult( "SELECT key FROM testData GROUP BY key") -@@ -1891,7 +1921,7 @@ class AdaptiveQueryExecSuite +@@ -1900,7 +1930,7 @@ class AdaptiveQueryExecSuite val (_, adaptivePlan) = runAdaptiveAndVerifyResult( "SELECT id FROM v1 GROUP BY id DISTRIBUTE BY id") assert(collect(adaptivePlan) { @@ -2585,7 +2585,7 @@ index 3e7d26f74bd..79232dc3664 100644 }.length == 1) } } -@@ -1972,7 +2002,8 @@ class AdaptiveQueryExecSuite +@@ -1981,7 +2011,8 @@ class AdaptiveQueryExecSuite } } @@ -2595,7 +2595,7 @@ index 3e7d26f74bd..79232dc3664 100644 def hasRepartitionShuffle(plan: SparkPlan): Boolean = { find(plan) { case s: ShuffleExchangeLike => -@@ -2159,6 +2190,9 @@ class AdaptiveQueryExecSuite +@@ -2168,6 +2199,9 @@ class AdaptiveQueryExecSuite def checkNoCoalescePartitions(ds: Dataset[Row], origin: ShuffleOrigin): Unit = { assert(collect(ds.queryExecution.executedPlan) { case s: ShuffleExchangeExec if s.shuffleOrigin == origin && s.numPartitions == 2 => s @@ -2605,7 +2605,7 @@ index 3e7d26f74bd..79232dc3664 100644 }.size == 1) ds.collect() val plan = ds.queryExecution.executedPlan -@@ -2167,6 +2201,9 @@ class AdaptiveQueryExecSuite +@@ -2176,6 +2210,9 @@ class AdaptiveQueryExecSuite }.isEmpty) assert(collect(plan) { case s: ShuffleExchangeExec if s.shuffleOrigin == origin && s.numPartitions == 2 => s @@ -2615,7 +2615,7 @@ index 3e7d26f74bd..79232dc3664 100644 }.size == 1) checkAnswer(ds, testData) } -@@ -2331,7 +2368,8 @@ class AdaptiveQueryExecSuite +@@ -2340,7 +2377,8 @@ class AdaptiveQueryExecSuite } } @@ -2625,7 +2625,7 @@ index 3e7d26f74bd..79232dc3664 100644 withTempView("t1", "t2") { def checkJoinStrategy(shouldShuffleHashJoin: Boolean): Unit = { Seq("100", "100000").foreach { size => -@@ -2417,7 +2455,8 @@ class AdaptiveQueryExecSuite +@@ -2426,7 +2464,8 @@ class AdaptiveQueryExecSuite } } @@ -2635,7 +2635,7 @@ index 3e7d26f74bd..79232dc3664 100644 withTempView("v") { withSQLConf( SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", -@@ -2516,7 +2555,7 @@ class AdaptiveQueryExecSuite +@@ -2525,7 +2564,7 @@ class AdaptiveQueryExecSuite runAdaptiveAndVerifyResult(s"SELECT $repartition key1 FROM skewData1 " + s"JOIN skewData2 ON key1 = key2 GROUP BY key1") val shuffles1 = collect(adaptive1) { @@ -2644,7 +2644,7 @@ index 3e7d26f74bd..79232dc3664 100644 } assert(shuffles1.size == 3) // shuffles1.head is the top-level shuffle under the Aggregate operator -@@ -2529,7 +2568,7 @@ class AdaptiveQueryExecSuite +@@ -2538,7 +2577,7 @@ class AdaptiveQueryExecSuite runAdaptiveAndVerifyResult(s"SELECT $repartition key1 FROM skewData1 " + s"JOIN skewData2 ON key1 = key2") val shuffles2 = collect(adaptive2) { @@ -2653,7 +2653,7 @@ index 3e7d26f74bd..79232dc3664 100644 } if (hasRequiredDistribution) { assert(shuffles2.size == 3) -@@ -2563,7 +2602,8 @@ class AdaptiveQueryExecSuite +@@ -2572,7 +2611,8 @@ class AdaptiveQueryExecSuite } } @@ -2663,7 +2663,7 @@ index 3e7d26f74bd..79232dc3664 100644 CostEvaluator.instantiate( classOf[SimpleShuffleSortCostEvaluator].getCanonicalName, spark.sparkContext.getConf) intercept[IllegalArgumentException] { -@@ -2729,6 +2769,7 @@ class AdaptiveQueryExecSuite +@@ -2738,6 +2778,7 @@ class AdaptiveQueryExecSuite val (_, adaptive) = runAdaptiveAndVerifyResult(query) assert(adaptive.collect { case sort: SortExec => sort @@ -2671,7 +2671,7 @@ index 3e7d26f74bd..79232dc3664 100644 }.size == 1) val read = collect(adaptive) { case read: AQEShuffleReadExec => read -@@ -2746,7 +2787,8 @@ class AdaptiveQueryExecSuite +@@ -2755,7 +2796,8 @@ class AdaptiveQueryExecSuite } } @@ -2681,7 +2681,7 @@ index 3e7d26f74bd..79232dc3664 100644 withTempView("v") { withSQLConf( SQLConf.ADAPTIVE_OPTIMIZE_SKEWS_IN_REBALANCE_PARTITIONS_ENABLED.key -> "true", -@@ -2858,7 +2900,7 @@ class AdaptiveQueryExecSuite +@@ -2867,7 +2909,7 @@ class AdaptiveQueryExecSuite runAdaptiveAndVerifyResult("SELECT key1 FROM skewData1 JOIN skewData2 ON key1 = key2 " + "JOIN skewData3 ON value2 = value3") val shuffles1 = collect(adaptive1) { @@ -2690,7 +2690,7 @@ index 3e7d26f74bd..79232dc3664 100644 } assert(shuffles1.size == 4) val smj1 = findTopLevelSortMergeJoin(adaptive1) -@@ -2869,7 +2911,7 @@ class AdaptiveQueryExecSuite +@@ -2878,7 +2920,7 @@ class AdaptiveQueryExecSuite runAdaptiveAndVerifyResult("SELECT key1 FROM skewData1 JOIN skewData2 ON key1 = key2 " + "JOIN skewData3 ON value1 = value3") val shuffles2 = collect(adaptive2) { @@ -2699,7 +2699,7 @@ index 3e7d26f74bd..79232dc3664 100644 } assert(shuffles2.size == 4) val smj2 = findTopLevelSortMergeJoin(adaptive2) -@@ -3127,6 +3169,7 @@ class AdaptiveQueryExecSuite +@@ -3136,6 +3178,7 @@ class AdaptiveQueryExecSuite }.size == (if (firstAccess) 1 else 0)) assert(collect(initialExecutedPlan) { case s: SortExec => s @@ -2707,7 +2707,7 @@ index 3e7d26f74bd..79232dc3664 100644 }.size == (if (firstAccess) 2 else 0)) assert(collect(initialExecutedPlan) { case i: InMemoryTableScanLike => i -@@ -3139,6 +3182,7 @@ class AdaptiveQueryExecSuite +@@ -3148,6 +3191,7 @@ class AdaptiveQueryExecSuite }.isEmpty) assert(collect(finalExecutedPlan) { case s: SortExec => s @@ -2715,7 +2715,7 @@ index 3e7d26f74bd..79232dc3664 100644 }.isEmpty) assert(collect(initialExecutedPlan) { case i: InMemoryTableScanLike => i -@@ -3220,7 +3264,8 @@ class AdaptiveQueryExecSuite +@@ -3229,7 +3273,8 @@ class AdaptiveQueryExecSuite } } @@ -2725,7 +2725,7 @@ index 3e7d26f74bd..79232dc3664 100644 val emptyDf = spark.range(1).where("false") val aggDf1 = emptyDf.agg(sum("id").as("id")).withColumn("name", lit("df1")) val aggDf2 = emptyDf.agg(sum("id").as("id")).withColumn("name", lit("df2")) -@@ -3310,7 +3355,8 @@ class AdaptiveQueryExecSuite +@@ -3319,7 +3364,8 @@ class AdaptiveQueryExecSuite val plan = df.queryExecution.executedPlan.asInstanceOf[AdaptiveSparkPlanExec] assert(plan.inputPlan.isInstanceOf[TakeOrderedAndProjectExec]) @@ -3060,10 +3060,10 @@ index 6ba790deddf..34b2f424c8f 100644 checkAnswer( // "fruit" column in this file is encoded using DELTA_LENGTH_BYTE_ARRAY. diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala -index c530dc0d3df..d23069689da 100644 +index 97aad5b30ea..77de969b80b 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetQuerySuite.scala -@@ -996,7 +996,11 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS +@@ -1032,7 +1032,11 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS Seq(Some("A"), Some("A"), None).toDF().repartition(1) .write.parquet(path.getAbsolutePath) val df = spark.read.parquet(path.getAbsolutePath) @@ -3076,7 +3076,7 @@ index c530dc0d3df..d23069689da 100644 } } } -@@ -1060,7 +1064,8 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS +@@ -1096,7 +1100,8 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS checkAnswer(readParquet(schema2, path), df) } @@ -3086,7 +3086,7 @@ index c530dc0d3df..d23069689da 100644 val schema1 = "a DECIMAL(3, 2), b DECIMAL(18, 3), c DECIMAL(37, 3)" checkAnswer(readParquet(schema1, path), df) val schema2 = "a DECIMAL(3, 0), b DECIMAL(18, 1), c DECIMAL(37, 1)" -@@ -1084,7 +1089,8 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS +@@ -1120,7 +1125,8 @@ abstract class ParquetQuerySuite extends QueryTest with ParquetTest with SharedS val df = sql(s"SELECT 1 a, 123456 b, ${Int.MaxValue.toLong * 10} c, CAST('1.2' AS BINARY) d") df.write.parquet(path.toString) @@ -3392,7 +3392,7 @@ index e31e0e70cf3..034ad5b953e 100644 import testImplicits._ diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreIntegrationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreIntegrationSuite.scala -index 38e5b15465b..ca3e8fef27a 100644 +index a32084bf74b..39eca1173b7 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreIntegrationSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreIntegrationSuite.scala @@ -24,6 +24,7 @@ import scala.jdk.CollectionConverters.SetHasAsScala @@ -3681,7 +3681,7 @@ index 4c06a0109e3..0b0e4990c0a 100644 fail(s"No FileScan in query\n${df.queryExecution}") } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamRealTimeModeAllowlistSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamRealTimeModeAllowlistSuite.scala -index 4306e5a8601..ee80e65ec26 100644 +index a9b8d54a6c6..01b5a13d735 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamRealTimeModeAllowlistSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/StreamRealTimeModeAllowlistSuite.scala @@ -20,11 +20,13 @@ package org.apache.spark.sql.streaming @@ -3689,7 +3689,7 @@ index 4306e5a8601..ee80e65ec26 100644 import org.apache.spark.SparkIllegalArgumentException +import org.apache.spark.sql.IgnoreCometSuite - import org.apache.spark.sql.execution.streaming.LowLatencyMemoryStream + import org.apache.spark.sql.execution.streaming.sources.LowLatencyMemoryStream import org.apache.spark.sql.functions._ import org.apache.spark.sql.internal.SQLConf diff --git a/docs/source/contributor-guide/benchmarking_macos.md b/docs/source/contributor-guide/benchmarking_macos.md index 07e4b27105..916fa25940 100644 --- a/docs/source/contributor-guide/benchmarking_macos.md +++ b/docs/source/contributor-guide/benchmarking_macos.md @@ -55,13 +55,13 @@ export DF_BENCH=`pwd` ## Install Spark -Install Apache Spark. This example refers to 4.1.2 version. +Install Apache Spark. This example refers to 4.1.3 version. ```shell -wget https://archive.apache.org/dist/spark/spark-4.1.2/spark-4.1.2-bin-hadoop3.tgz -tar xzf spark-4.1.2-bin-hadoop3.tgz -sudo mv spark-4.1.2-bin-hadoop3 /opt -export SPARK_HOME=/opt/spark-4.1.2-bin-hadoop3/ +wget https://archive.apache.org/dist/spark/spark-4.1.3/spark-4.1.3-bin-hadoop3.tgz +tar xzf spark-4.1.3-bin-hadoop3.tgz +sudo mv spark-4.1.3-bin-hadoop3 /opt +export SPARK_HOME=/opt/spark-4.1.3-bin-hadoop3/ ``` Start Spark in standalone mode: diff --git a/docs/source/contributor-guide/iceberg-spark-tests.md b/docs/source/contributor-guide/iceberg-spark-tests.md index f7468e4240..fccbdeb54b 100644 --- a/docs/source/contributor-guide/iceberg-spark-tests.md +++ b/docs/source/contributor-guide/iceberg-spark-tests.md @@ -95,7 +95,7 @@ diff must be generated against its own tag. The `iceberg_spark_test_.yml` workflows apply these diffs and run the three Gradle targets above against each Iceberg version. Iceberg 1.8.1 runs against Spark 3.4.3 with Java 11; Iceberg 1.9.1 and 1.10.0 -run against Spark 3.5.9 with Java 17; Iceberg 1.11.0 runs against Spark 4.1.2 with Java 17. Iceberg 1.11 +run against Spark 3.5.9 with Java 17; Iceberg 1.11.0 runs against Spark 4.1.3 with Java 17. Iceberg 1.11 (the only version testing Spark 4.1) runs on every pull request and on pushes to main; the older versions (1.8, 1.9, 1.10) run only on pushes to main, or on a pull request labeled `run-iceberg-tests`. All caller workflows delegate to `iceberg_spark_test_reusable.yml`, which holds the build and test job logic. diff --git a/docs/source/user-guide/latest/compatibility/spark-versions.md b/docs/source/user-guide/latest/compatibility/spark-versions.md index d96294b5c9..9fe7a2b339 100644 --- a/docs/source/user-guide/latest/compatibility/spark-versions.md +++ b/docs/source/user-guide/latest/compatibility/spark-versions.md @@ -76,7 +76,7 @@ Spark 4.0.2 is supported with Java 17 and Scala 2.13. ## Spark 4.1 -Spark 4.1.2 is supported with Java 17/21 and Scala 2.13. +Spark 4.1.3 is supported with Java 17/21 and Scala 2.13. ### Known Limitations diff --git a/docs/source/user-guide/latest/installation.md b/docs/source/user-guide/latest/installation.md index c25ce8da75..93da456421 100644 --- a/docs/source/user-guide/latest/installation.md +++ b/docs/source/user-guide/latest/installation.md @@ -55,7 +55,7 @@ We recommend moving to JDK 17 or later and Spark 3.5 or later. | 3.4.3 | 11/17 | 2.12/2.13 | Yes | Yes | | 3.5.9 | 11/17 | 2.12/2.13 | Yes | Yes | | 4.0.2 | 17/21 | 2.13 | Yes | Yes | -| 4.1.2 | 17/21 | 2.13 | Yes | Yes | +| 4.1.3 | 17/21 | 2.13 | Yes | Yes | Note that we do not test the full matrix of supported Java and Scala versions in CI for every Spark version. diff --git a/docs/source/user-guide/latest/kubernetes.md b/docs/source/user-guide/latest/kubernetes.md index 5dfb723e65..b91bb93e80 100644 --- a/docs/source/user-guide/latest/kubernetes.md +++ b/docs/source/user-guide/latest/kubernetes.md @@ -72,7 +72,7 @@ spec: image: apache/datafusion-comet:$COMET_VERSION-spark3.5.5-scala2.12-java11 imagePullPolicy: IfNotPresent mainClass: org.apache.spark.examples.SparkPi - mainApplicationFile: local:///opt/spark/examples/jars/spark-examples_2.13-4.1.2.jar + mainApplicationFile: local:///opt/spark/examples/jars/spark-examples_2.13-4.1.3.jar sparkConf: "spark.executor.extraClassPath": "/opt/spark/jars/comet-spark-spark3.5_2.12-$COMET_VERSION.jar" "spark.driver.extraClassPath": "/opt/spark/jars/comet-spark-spark3.5_2.12-$COMET_VERSION.jar" @@ -82,17 +82,17 @@ spec: "spark.comet.shuffle.enabled": "true" "spark.comet.shuffle.mode": "auto" "spark.shuffle.manager": "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager" - sparkVersion: 4.1.2 + sparkVersion: 4.1.3 driver: labels: - version: 4.1.2 + version: 4.1.3 cores: 1 coreLimit: 1200m memory: 512m serviceAccount: spark-operator-spark executor: labels: - version: 4.1.2 + version: 4.1.3 instances: 1 cores: 1 coreLimit: 1200m diff --git a/kube/Dockerfile b/kube/Dockerfile index 5ee3eac35c..f28514681e 100644 --- a/kube/Dockerfile +++ b/kube/Dockerfile @@ -15,7 +15,7 @@ # limitations under the License. # -FROM apache/spark:4.1.2 AS builder +FROM apache/spark:4.1.3 AS builder USER root @@ -69,7 +69,7 @@ RUN mkdir -p /root/.m2 && \ RUN cd /comet \ && JAVA_HOME=$(readlink -f $(which javac) | sed "s/\/bin\/javac//") make release-nogit PROFILES="-Pspark-$SPARK_VERSION -Pscala-$SCALA_VERSION" -FROM apache/spark:4.1.2 +FROM apache/spark:4.1.3 ENV SPARK_VERSION=4.1 ENV SCALA_VERSION=2.13 USER root diff --git a/pom.xml b/pom.xml index 927c07ca9c..1ab9a6d69a 100644 --- a/pom.xml +++ b/pom.xml @@ -69,7 +69,7 @@ under the License. 4.9.6 3.2.16 2.2.0 - 4.1.2 + 4.1.3 4.1 provided 3.25.5 @@ -717,7 +717,7 @@ under the License. yet published, so the -Psemanticdb / scalafix lint job is skipped for spark-4.1. --> 2.13.17 2.13 - 4.1.2 + 4.1.3 4.1 1.16.0 4.13.6 diff --git a/spark/pom.xml b/spark/pom.xml index fbe059c739..33e6d102c1 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -297,7 +297,7 @@ under the License. 1.11.0 test - + org.eclipse.jetty jetty-server