feat: Iceberg table format V3: native table decryption, fall back for other V3 features

.github/actions/java-test/action.yaml

@@ -50,6 +50,18 @@ runs:
         cd native
         cargo build --release
 
+    # Evaluate hashFiles now, on the clean checkout, and reuse the result in the
+    # cache key below. The cache step's inputs are re-evaluated when its post
+    # (cache-save) phase runs, and by then the test run has left trees under the
+    # workspace (e.g. the Hive-catalog warehouse from CometIcebergEncryptionSuite)
+    # that make a fresh '**/pom.xml' walk throw "Fail to hash files under
+    # directory". A precomputed string keeps the save phase from re-walking them.
+    - name: Compute Maven cache key
+      id: maven-cache-key
+      if: ${{ runner.os != 'macOS' }}
+      shell: bash
+      run: echo "hash=${{ hashFiles('**/pom.xml') }}" >> "$GITHUB_OUTPUT"
+
     - name: Cache Maven dependencies
       # TODO: remove next line after working again
       #  temporarily work around https://github.com/actions/runner-images/issues/13341
@@ -60,7 +72,7 @@ runs:
         path: |
           ~/.m2/repository
           /root/.m2/repository
-        key: ${{ runner.os }}-java-maven-${{ hashFiles('**/pom.xml') }}
+        key: ${{ runner.os }}-java-maven-${{ steps.maven-cache-key.outputs.hash }}
         restore-keys: |
           ${{ runner.os }}-java-maven-
 

.github/workflows/pr_build_linux.yml

@@ -312,6 +312,7 @@ jobs:
               org.apache.spark.sql.comet.ParquetEncryptionITCase
               org.apache.comet.exec.CometNativeReaderSuite
               org.apache.comet.CometIcebergNativeSuite
+              org.apache.comet.CometIcebergEncryptionSuite
               org.apache.comet.CometIcebergRewriteActionSuite
               org.apache.comet.iceberg.IcebergReflectionSuite
               org.apache.comet.csv.CometCsvNativeReadSuite

.github/workflows/pr_build_macos.yml

@@ -128,6 +128,7 @@ jobs:
               org.apache.spark.sql.comet.ParquetEncryptionITCase
               org.apache.comet.exec.CometNativeReaderSuite
               org.apache.comet.CometIcebergNativeSuite
+              org.apache.comet.CometIcebergEncryptionSuite
               org.apache.comet.CometIcebergRewriteActionSuite
               org.apache.comet.iceberg.IcebergReflectionSuite
               org.apache.comet.csv.CometCsvNativeReadSuite

dev/diffs/iceberg/1.11.0.diff

@@ -251,6 +251,31 @@ index 507d7b313b..3b73dcc014 100644
              .config("spark.ui.enabled", "false")
              .config(DISABLE_UI)
              .enableHiveSupport()
+diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/AvroDataTestBase.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/AvroDataTestBase.java
+index 8ce60f6275..75112b4354 100644
+--- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/AvroDataTestBase.java
++++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/AvroDataTestBase.java
+@@ -48,7 +48,6 @@ import org.apache.iceberg.types.Types.LongType;
+ import org.apache.iceberg.types.Types.MapType;
+ import org.apache.iceberg.types.Types.StructType;
+ import org.apache.iceberg.util.DateTimeUtil;
+-import org.assertj.core.api.Condition;
+ import org.junit.jupiter.api.Test;
+ import org.junit.jupiter.api.io.TempDir;
+ import org.junit.jupiter.params.ParameterizedTest;
+@@ -385,12 +384,6 @@ public abstract class AvroDataTestBase {
+                 .build());
+
+     assertThatThrownBy(() -> writeAndValidate(writeSchema, expectedSchema))
+-        .has(
+-            new Condition<>(
+-                t ->
+-                    IllegalArgumentException.class.isInstance(t)
+-                        || IllegalArgumentException.class.isInstance(t.getCause()),
+-                "Expecting a throwable or cause that is an instance of IllegalArgumentException"))
+         .hasMessageContaining("Missing required field: missing_str");
+   }
+
 diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/vectorized/parquet/TestParquetDictionaryEncodedVectorizedReads.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/vectorized/parquet/TestParquetDictionaryEncodedVectorizedReads.java
 index b61ecfa2f4..d696e85139 100644
 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/vectorized/parquet/TestParquetDictionaryEncodedVectorizedReads.java

docs/source/contributor-guide/roadmap.md

@@ -52,14 +52,14 @@ significant family of Spark expressions in one effort.
 
 Native Parquet scans (`CometNativeScanExec`) support Dynamic Partition Pruning (DPP) both with and without
 Adaptive Query Execution. Non-AQE DPP landed in [#4011] and AQE DPP with broadcast reuse landed in [#4112].
-Iceberg native scans currently support non-AQE DPP only ([#3349], [#3511]); extending broadcast reuse to AQE
-DPP for Iceberg is tracked at [#3510].
+Iceberg native scans support non-AQE DPP ([#3349], [#3511]) and, on Spark 3.5+, AQE DPP with broadcast reuse
+([#4215]); on Spark 3.4 Iceberg AQE DPP falls back to Spark without reuse.
 
 [#3349]: https://github.com/apache/datafusion-comet/pull/3349
-[#3510]: https://github.com/apache/datafusion-comet/issues/3510
 [#3511]: https://github.com/apache/datafusion-comet/pull/3511
 [#4011]: https://github.com/apache/datafusion-comet/pull/4011
 [#4112]: https://github.com/apache/datafusion-comet/pull/4112
+[#4215]: https://github.com/apache/datafusion-comet/pull/4215
 
 ## TPC-H and TPC-DS Performance
 

docs/source/user-guide/latest/iceberg.md

@@ -26,7 +26,7 @@ then serialized to Comet's native execution engine (see
 [PR #2528](https://github.com/apache/datafusion-comet/pull/2528)).
 
 The example below uses Spark's package downloader to retrieve Comet $COMET_VERSION and Iceberg
-1.8.1, but Comet has been tested with Iceberg 1.5, 1.7, 1.8, 1.9, and 1.10. The native Iceberg
+1.8.1, but Comet has been tested with Iceberg 1.5, 1.7, 1.8, 1.9, 1.10, and 1.11. The native Iceberg
 reader is enabled by default. To disable it, set `spark.comet.scan.icebergNative.enabled=false`.
 
 The example uses the Spark 3.5 / Scala 2.12 build of Comet; substitute the Comet artifact
@@ -63,7 +63,15 @@ The native Iceberg reader supports the following features:
 
 **Table specifications:**
 
-- Iceberg table spec v1 and v2 (v3 will fall back to Spark)
+- Iceberg table spec v1, v2, and v3
+
+**Encryption:**
+
+- Encrypted v3 tables using 128-bit or 256-bit AES-GCM data keys (requires Iceberg 1.11 or newer).
+  Iceberg-Java unwraps the key envelope on the driver during planning and stores the plaintext data
+  key in each file's `key_metadata`, which the native reader uses directly, so no KMS integration is
+  needed on the native side. Iceberg-Java also permits 192-bit data keys; those tables fall back to
+  Spark (no AES-192-GCM in the underlying crypto).
 
 **Schema and data types:**
 
@@ -156,17 +164,19 @@ These `s3.*` storage properties are not specific to the Hive catalog shown here.
 
 ### Current limitations
 
-The following scenarios will fall back to Spark's native Iceberg reader:
+The following scenarios will fall back to the JVM Iceberg reader:
 
-- Iceberg table spec v3 scans
+- Iceberg table spec v4 or newer
+- v3 tables with columns that declare an initial default value
+- v3 column types the native reader cannot read (`variant`, `geometry`, `geography`, `unknown`)
+- Encrypted tables with 192-bit data keys (no AES-192-GCM in the underlying crypto)
+- Deletion vectors (v3 Puffin deletes); positional and equality deletes in Parquet are supported
 - Iceberg writes (reads are accelerated, writes use Spark)
 - Tables backed by Avro or ORC data files (only Parquet is accelerated)
 - Tables partitioned on `BINARY` or `DECIMAL` (with precision >28) columns
 - Scans with residual filters using `truncate`, `bucket`, `year`, `month`, `day`, or `hour`
   transform functions (partition pruning still works, but row-level filtering of these
   transforms falls back)
-- Dynamic Partition Pruning under Adaptive Query Execution (non-AQE DPP is supported);
-  see [#3510](https://github.com/apache/datafusion-comet/issues/3510)
 
 ### Iceberg UDFs