From b70b31d221374900651702b4f3474852d1e1676d Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Wed, 24 Jun 2026 10:02:43 -0700 Subject: [PATCH 01/26] Core: Add v4 manifest reader supports: - column projection - partition pruning via a row filter, including multi-spec manifests Content stats reading and metadata inheritance are not yet implemented. --- .../java/org/apache/iceberg/Partitioning.java | 15 +- .../org/apache/iceberg/V4ManifestReader.java | 283 +++++++++ .../apache/iceberg/TestV4ManifestReader.java | 571 ++++++++++++++++++ 3 files changed, 866 insertions(+), 3 deletions(-) create mode 100644 core/src/main/java/org/apache/iceberg/V4ManifestReader.java create mode 100644 core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java diff --git a/core/src/main/java/org/apache/iceberg/Partitioning.java b/core/src/main/java/org/apache/iceberg/Partitioning.java index c708d39f523e..7cfb0bef9beb 100644 --- a/core/src/main/java/org/apache/iceberg/Partitioning.java +++ b/core/src/main/java/org/apache/iceberg/Partitioning.java @@ -238,9 +238,18 @@ public static StructType groupingKeyType(Schema schema, Collection specs = table.specs().values(); - return buildPartitionProjectionType( - "table partition", specs, allActiveFieldIds(table.schema(), specs)); + return partitionType(table.schema(), table.specs().values()); + } + + /** + * Builds a unified partition type from a schema and its specs, unioning every partition field + * whose source column is present in the schema. + * + * @param schema the schema used to determine which partition fields are active + * @param specs the partition specs to unify + */ + static StructType partitionType(Schema schema, Collection specs) { + return buildPartitionProjectionType("table partition", specs, allActiveFieldIds(schema, specs)); } /** diff --git a/core/src/main/java/org/apache/iceberg/V4ManifestReader.java b/core/src/main/java/org/apache/iceberg/V4ManifestReader.java new file mode 100644 index 000000000000..aec57a318cc5 --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/V4ManifestReader.java @@ -0,0 +1,283 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.iceberg.expressions.Evaluator; +import org.apache.iceberg.expressions.Expression; +import org.apache.iceberg.expressions.Expressions; +import org.apache.iceberg.expressions.Projections; +import org.apache.iceberg.io.CloseableGroup; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.io.CloseableIterator; +import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.metrics.ScanMetrics; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.StructProjection; + +/** Reader that reads a v4 manifest file as {@link TrackedFile}s. */ +class V4ManifestReader extends CloseableGroup implements CloseableIterable { + private final InputFile file; + private final Types.StructType partitionType; + private final Schema fileProjection; + private final ScanMetrics scanMetrics; + + // partition pruning state, keyed by spec ID; empty when no filtering is required + private final Map partitionEvaluators; + private final Map partitionProjections; + + private V4ManifestReader( + InputFile file, + Types.StructType partitionType, + Map partitionEvaluators, + Map partitionProjections, + Schema fileProjection, + ScanMetrics scanMetrics) { + this.file = file; + this.partitionType = partitionType; + this.partitionEvaluators = partitionEvaluators; + this.partitionProjections = partitionProjections; + this.fileProjection = fileProjection; + this.scanMetrics = scanMetrics; + } + + static Builder builder( + InputFile file, Schema tableSchema, Map specsById) { + return new Builder(file, tableSchema, specsById); + } + + /** Returns all tracked files in this manifest, regardless of status. */ + CloseableIterable allFiles() { + return files(false /* all files */); + } + + /** Returns tracked files whose tracking {@link Tracking#isLive() is live}. */ + CloseableIterable liveFiles() { + return files(true /* only live files */); + } + + /** Returns live tracked files, each as an independent copy. */ + @Override + public CloseableIterator iterator() { + return CloseableIterable.transform(liveFiles(), TrackedFile::copy).iterator(); + } + + private CloseableIterable files(boolean onlyLive) { + CloseableIterable entries = CloseableIterable.transform(open(), this::prepare); + if (!partitionEvaluators.isEmpty()) { + entries = CloseableIterable.filter(entries, this::matchesPartition); + } + + if (onlyLive) { + entries = CloseableIterable.filter(entries, entry -> entry.tracking().isLive()); + } + + return entries; + } + + private boolean matchesPartition(TrackedFile trackedFile) { + FileContent content = trackedFile.contentType(); + if (content == FileContent.DATA_MANIFEST || content == FileContent.DELETE_MANIFEST) { + // manifest references are expanded later and are not pruned by the partition filter + return true; + } + + Integer specId = trackedFile.specId(); + Evaluator evaluator = specId != null ? partitionEvaluators.get(specId) : null; + StructProjection projection = specId != null ? partitionProjections.get(specId) : null; + Preconditions.checkState( + evaluator != null && projection != null, + "Cannot apply partition filter: file %s has spec ID %s, not one of the known specs %s " + + "in manifest %s", + trackedFile.location(), + specId, + partitionEvaluators.keySet(), + file.location()); + + boolean matches = evaluator.eval(projection.wrap(trackedFile.partition())); + if (!matches) { + if (content == FileContent.DATA) { + scanMetrics.skippedDataFiles().increment(); + } else { + scanMetrics.skippedDeleteFiles().increment(); + } + } + + return matches; + } + + private CloseableIterable open() { + FileFormat format = FileFormat.fromFileName(file.location()); + Preconditions.checkArgument( + format != null, "Unable to determine format of manifest: %s", file.location()); + + CloseableIterable reader = + InternalData.read(format, file) + .project(readSchema()) + .setRootType(TrackedFileStruct.class) + .setCustomType(TrackedFile.TRACKING.fieldId(), TrackingStruct.class) + .setCustomType(TrackedFile.DELETION_VECTOR.fieldId(), DeletionVectorStruct.class) + .setCustomType(TrackedFile.MANIFEST_INFO.fieldId(), ManifestInfoStruct.class) + .setCustomType(TrackedFile.PARTITION_ID, PartitionData.class) + .reuseContainers() + .build(); + addCloseable(reader); + return reader; + } + + private TrackedFile prepare(TrackedFile trackedFile) { + Tracking tracking = trackedFile.tracking(); + Preconditions.checkState( + tracking != null, + "Invalid tracked file: tracking is required but was missing in manifest %s", + file.location()); + + // manifestLocation is not stored in the manifest; the reader fills it from the file location. + // manifestPos is filled from ROW_POSITION while reading the tracking struct. + if (tracking instanceof TrackingStruct) { + ((TrackingStruct) tracking).setManifestLocation(file.location()); + } + + return trackedFile; + } + + private Schema readSchema() { + // content_stats is not projected yet, so build the schema with an empty stats type + Types.StructType fullType = + TrackedFile.schemaWithContentStats(partitionType, Types.StructType.of()); + boolean unpartitioned = partitionType.fields().isEmpty(); + + Set projectedIds = null; + if (fileProjection != null) { + projectedIds = Sets.newHashSet(); + for (Types.NestedField field : fileProjection.asStruct().fields()) { + projectedIds.add(field.fieldId()); + } + + // tracking carries the status used to filter live files and is always projected + projectedIds.add(TrackedFile.TRACKING.fieldId()); + + // spec_id is required to resolve each entry's partition spec when pruning + if (!partitionEvaluators.isEmpty()) { + projectedIds.add(TrackedFile.SPEC_ID.fieldId()); + } + } + + List fields = Lists.newArrayList(); + for (Types.NestedField field : fullType.fields()) { + if (projectedIds != null && !projectedIds.contains(field.fieldId())) { + continue; + } + + if (field.fieldId() == TrackedFile.TRACKING.fieldId()) { + fields.add(trackingWithRowPosition()); + } else if (field.fieldId() == TrackedFile.CONTENT_STATS_ID) { + // content_stats are omitted for now + } else if (field.fieldId() == TrackedFile.PARTITION_ID && unpartitioned) { + // unpartitioned manifests omit the partition field + } else { + fields.add(field); + } + } + + return new Schema(fields); + } + + /** + * Builds the tracking field with {@link MetadataColumns#ROW_POSITION} appended so the reader + * populates the manifest position of each entry. + */ + private static Types.NestedField trackingWithRowPosition() { + List trackingFields = Lists.newArrayList(Tracking.schema().fields()); + trackingFields.add(MetadataColumns.ROW_POSITION); + return Types.NestedField.required( + TrackedFile.TRACKING.fieldId(), + TrackedFile.TRACKING.name(), + Types.StructType.of(trackingFields), + TrackedFile.TRACKING.doc()); + } + + static class Builder { + private final InputFile file; + private final Schema tableSchema; + private final Map specsById; + private Expression rowFilter = Expressions.alwaysTrue(); + private boolean caseSensitive = true; + private Schema fileProjection = null; + private ScanMetrics scanMetrics = ScanMetrics.noop(); + + private Builder(InputFile file, Schema tableSchema, Map specsById) { + this.file = file; + this.tableSchema = tableSchema; + this.specsById = specsById; + } + + /** Sets a row filter; files that cannot match the expression are skipped. */ + Builder filterRows(Expression expr) { + Preconditions.checkNotNull(expr, "Row filter cannot be null"); + this.rowFilter = expr; + return this; + } + + Builder caseSensitive(boolean isCaseSensitive) { + this.caseSensitive = isCaseSensitive; + return this; + } + + Builder project(Schema newFileProjection) { + this.fileProjection = newFileProjection; + return this; + } + + Builder scanMetrics(ScanMetrics newScanMetrics) { + Preconditions.checkNotNull(newScanMetrics, "Scan metrics cannot be null"); + this.scanMetrics = newScanMetrics; + return this; + } + + V4ManifestReader build() { + Types.StructType partitionType = Partitioning.partitionType(tableSchema, specsById.values()); + Map partitionEvaluators = Maps.newHashMap(); + Map partitionProjections = Maps.newHashMap(); + if (rowFilter != Expressions.alwaysTrue() && !partitionType.fields().isEmpty()) { + for (PartitionSpec spec : specsById.values()) { + Expression partFilter = Projections.inclusive(spec, caseSensitive).project(rowFilter); + partitionEvaluators.put( + spec.specId(), new Evaluator(spec.partitionType(), partFilter, caseSensitive)); + partitionProjections.put( + spec.specId(), StructProjection.create(partitionType, spec.partitionType())); + } + } + + return new V4ManifestReader( + file, + partitionType, + partitionEvaluators, + partitionProjections, + fileProjection, + scanMetrics); + } + } +} diff --git a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java new file mode 100644 index 000000000000..382fa4fe3e83 --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java @@ -0,0 +1,571 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import static org.apache.iceberg.types.Types.NestedField.optional; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import org.apache.iceberg.expressions.Expressions; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.io.FileAppender; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.io.OutputFile; +import org.apache.iceberg.metrics.DefaultMetricsContext; +import org.apache.iceberg.metrics.ScanMetrics; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.transforms.Transforms; +import org.apache.iceberg.types.Types; +import org.junit.jupiter.api.TestTemplate; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; + +@ExtendWith(ParameterizedTestExtension.class) +public class TestV4ManifestReader { + private static final long SNAPSHOT_ID = 42L; + private static final int WRITER_FORMAT_VERSION_V4 = 4; + + private static final Schema TABLE_SCHEMA = + new Schema( + optional(1, "id", Types.IntegerType.get()), optional(2, "data", Types.StringType.get())); + private static final PartitionSpec SPEC = + PartitionSpec.builderFor(TABLE_SCHEMA).identity("id").build(); + private static final Types.StructType PARTITION_TYPE = SPEC.partitionType(); + private static final Types.StructType EMPTY_PARTITION = Types.StructType.of(); + private static final PartitionData EMPTY_PARTITION_DATA = new PartitionData(EMPTY_PARTITION); + private static final Map PARTITIONED_SPECS = + ImmutableMap.of(SPEC.specId(), SPEC); + private static final Map UNPARTITIONED_SPECS = + ImmutableMap.of(PartitionSpec.unpartitioned().specId(), PartitionSpec.unpartitioned()); + + private static final List SCHEMA_FIELDS = + TrackedFile.schemaWithContentStats(Types.StructType.of(), Types.StructType.of()).fields(); + + @Parameter private FileFormat format; + + @Parameters(name = "format = {0}") + protected static List parameters() { + return Arrays.asList(FileFormat.AVRO, FileFormat.PARQUET); + } + + @TempDir private Path tempDir; + + private final FileIO fileIO = new TestTables.LocalFileIO(); + + @TestTemplate + public void testRoundTrip() { + DeletionVector dv = + DeletionVectorStruct.builder() + .location("s3://bucket/dv.puffin") + .offset(100L) + .sizeInBytes(50L) + .cardinality(5L) + .build(); + + TrackedFile file = + dataFileBuilder("s3://bucket/data/file.parquet", partition(7)) + .sortOrderId(1) + .deletionVector(dv) + .keyMetadata(ByteBuffer.wrap(new byte[] {1, 2, 3})) + .splitOffsets(ImmutableList.of(50L, 100L)) + .build(); + + InputFile manifest = writeManifest(PARTITION_TYPE, ImmutableList.of(file)); + + List read = read(manifest, PARTITIONED_SPECS); + assertThat(read).hasSize(1); + TrackedFile actual = read.get(0); + + assertThat(actual.contentType()).isEqualTo(file.contentType()); + assertThat(actual.writerFormatVersion()).isEqualTo(file.writerFormatVersion()); + assertThat(actual.location()).isEqualTo(file.location()); + assertThat(actual.fileFormat()).isEqualTo(file.fileFormat()); + assertThat(actual.recordCount()).isEqualTo(file.recordCount()); + assertThat(actual.fileSizeInBytes()).isEqualTo(file.fileSizeInBytes()); + assertThat(actual.specId()).isEqualTo(file.specId()); + assertThat(actual.sortOrderId()).isEqualTo(file.sortOrderId()); + assertThat(actual.keyMetadata()).isEqualTo(file.keyMetadata()); + assertThat(actual.splitOffsets()).isEqualTo(file.splitOffsets()); + assertThat(actual.partition().get(0, Integer.class)) + .isEqualTo(file.partition().get(0, Integer.class)); + + assertThat(actual.tracking()).isNotNull(); + assertThat(actual.tracking().status()).isEqualTo(file.tracking().status()); + assertThat(actual.tracking().snapshotId()).isEqualTo(file.tracking().snapshotId()); + + assertThat(actual.deletionVector()).isNotNull(); + assertThat(actual.deletionVector().location()).isEqualTo(file.deletionVector().location()); + assertThat(actual.deletionVector().offset()).isEqualTo(file.deletionVector().offset()); + assertThat(actual.deletionVector().sizeInBytes()) + .isEqualTo(file.deletionVector().sizeInBytes()); + assertThat(actual.deletionVector().cardinality()) + .isEqualTo(file.deletionVector().cardinality()); + } + + @TestTemplate + public void testEqualityDeleteRoundTrip() { + TrackedFile delete = + TrackedFileBuilder.equalityDelete(SNAPSHOT_ID) + .writerFormatVersion(WRITER_FORMAT_VERSION_V4) + .location("s3://bucket/eq-delete.parquet") + .fileFormat(FileFormat.PARQUET) + .recordCount(10L) + .fileSizeInBytes(128L) + .partition(EMPTY_PARTITION_DATA) + .specId(0) + .equalityIds(ImmutableList.of(1, 2)) + .build(); + + InputFile manifest = writeManifest(EMPTY_PARTITION, ImmutableList.of(delete)); + + TrackedFile actual = read(manifest, UNPARTITIONED_SPECS).get(0); + assertThat(actual.contentType()).isEqualTo(FileContent.EQUALITY_DELETES); + assertThat(actual.equalityIds()).containsExactly(1, 2); + } + + @TestTemplate + public void testLiveFilesExcludesDeletedAndReplaced() { + List files = + ImmutableList.of( + fileWithStatus(EntryStatus.ADDED, "s3://bucket/added.parquet"), + fileWithStatus(EntryStatus.EXISTING, "s3://bucket/existing.parquet"), + fileWithStatus(EntryStatus.MODIFIED, "s3://bucket/modified.parquet"), + fileWithStatus(EntryStatus.DELETED, "s3://bucket/deleted.parquet"), + fileWithStatus(EntryStatus.REPLACED, "s3://bucket/replaced.parquet")); + + InputFile manifest = writeManifest(EMPTY_PARTITION, files); + + try (V4ManifestReader reader = newReader(manifest, UNPARTITIONED_SPECS).build()) { + assertThat(reader.allFiles()) + .extracting(file -> file.tracking().status()) + .containsExactly( + EntryStatus.ADDED, + EntryStatus.EXISTING, + EntryStatus.MODIFIED, + EntryStatus.DELETED, + EntryStatus.REPLACED); + + assertThat(reader.liveFiles()) + .extracting(file -> file.tracking().status()) + .containsExactly(EntryStatus.ADDED, EntryStatus.EXISTING, EntryStatus.MODIFIED); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @TestTemplate + public void testManifestLocationAndPosition() { + List files = + ImmutableList.of( + dataFile("s3://bucket/a.parquet", EMPTY_PARTITION_DATA), + dataFile("s3://bucket/b.parquet", EMPTY_PARTITION_DATA), + dataFile("s3://bucket/c.parquet", EMPTY_PARTITION_DATA)); + + InputFile manifest = writeManifest(EMPTY_PARTITION, files); + + List read = read(manifest, UNPARTITIONED_SPECS); + assertThat(read) + .allSatisfy( + file -> assertThat(file.tracking().manifestLocation()).isEqualTo(manifest.location())); + assertThat(read).extracting(file -> file.tracking().manifestPos()).containsExactly(0L, 1L, 2L); + } + + @TestTemplate + public void testProjectionRestrictsFields() { + // sort_order_id is written but not projected below, so it must not be read back + TrackedFile file = + dataFileBuilder("s3://bucket/file.parquet", EMPTY_PARTITION_DATA).sortOrderId(7).build(); + + InputFile manifest = writeManifest(EMPTY_PARTITION, ImmutableList.of(file)); + + Schema projection = new Schema(TrackedFile.LOCATION); + try (V4ManifestReader reader = + newReader(manifest, UNPARTITIONED_SPECS).project(projection).build()) { + TrackedFile actual = Lists.newArrayList(reader.allFiles()).get(0); + assertThat(actual.location()).isEqualTo("s3://bucket/file.parquet"); + // tracking is always projected even when the caller omits it + assertThat(actual.tracking()).isNotNull(); + assertThat(actual.tracking().status()).isEqualTo(EntryStatus.ADDED); + assertThat(actual.tracking().manifestPos()).isEqualTo(0L); + // sort_order_id was written but not projected, so it should not be read + assertThat(actual.sortOrderId()).isNull(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @TestTemplate + public void testUnpartitioned() { + TrackedFile file = dataFile("s3://bucket/file.parquet", EMPTY_PARTITION_DATA); + + InputFile manifest = writeManifest(EMPTY_PARTITION, ImmutableList.of(file)); + + TrackedFile actual = read(manifest, UNPARTITIONED_SPECS).get(0); + assertThat(actual.partition()).isNotNull(); + assertThat(actual.partition().size()).isEqualTo(0); + } + + @TestTemplate + public void testPartitionFilterPrunesNonMatchingFiles() { + TrackedFile keep = dataFile("keep.parquet", partition(1)); + TrackedFile prune = dataFile("prune.parquet", partition(2)); + + InputFile manifest = writeManifest(PARTITION_TYPE, ImmutableList.of(keep, prune)); + + ScanMetrics metrics = ScanMetrics.of(new DefaultMetricsContext()); + try (V4ManifestReader reader = + newReader(manifest, PARTITIONED_SPECS) + .filterRows(Expressions.equal("id", 1)) + .scanMetrics(metrics) + .build()) { + assertThat(reader.allFiles()) + .extracting(TrackedFile::location) + .containsExactly("keep.parquet"); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + + assertThat(metrics.skippedDataFiles().value()).isEqualTo(1L); + } + + @TestTemplate + public void testPartitionFilterCountsSkippedDeleteFiles() { + TrackedFile delete = + TrackedFileBuilder.equalityDelete(SNAPSHOT_ID) + .writerFormatVersion(WRITER_FORMAT_VERSION_V4) + .location("delete.parquet") + .fileFormat(FileFormat.PARQUET) + .recordCount(100L) + .fileSizeInBytes(1024L) + .partition(partition(2)) + .specId(0) + .equalityIds(ImmutableList.of(1)) + .build(); + + InputFile manifest = writeManifest(PARTITION_TYPE, ImmutableList.of(delete)); + + ScanMetrics metrics = ScanMetrics.of(new DefaultMetricsContext()); + try (V4ManifestReader reader = + newReader(manifest, PARTITIONED_SPECS) + .filterRows(Expressions.equal("id", 1)) + .scanMetrics(metrics) + .build()) { + assertThat(reader.allFiles()).isEmpty(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + + assertThat(metrics.skippedDeleteFiles().value()).isEqualTo(1L); + assertThat(metrics.skippedDataFiles().value()).isEqualTo(0L); + } + + @TestTemplate + public void testPartitionFilterKeepsManifestReferences() { + TrackedFile keep = dataFile("data-1.parquet", partition(1)); + TrackedFile prune = dataFile("data-2.parquet", partition(2)); + ManifestInfo info = + ManifestInfoStruct.builder() + .addedFilesCount(1) + .existingFilesCount(0) + .deletedFilesCount(0) + .replacedFilesCount(0) + .addedRowsCount(1L) + .existingRowsCount(0L) + .deletedRowsCount(0L) + .replacedRowsCount(0L) + .minSequenceNumber(1L) + .build(); + TrackedFile manifestRef = + TrackedFileBuilder.dataManifest(SNAPSHOT_ID) + .writerFormatVersion(WRITER_FORMAT_VERSION_V4) + .location("leaf.parquet") + .fileFormat(FileFormat.PARQUET) + .recordCount(1L) + .fileSizeInBytes(100L) + .partition(partition(2)) + .specId(0) + .manifestInfo(info) + .build(); + + InputFile manifest = writeManifest(PARTITION_TYPE, ImmutableList.of(keep, prune, manifestRef)); + + try (V4ManifestReader reader = + newReader(manifest, PARTITIONED_SPECS).filterRows(Expressions.equal("id", 1)).build()) { + assertThat(reader.allFiles()) + .extracting(TrackedFile::location) + .containsExactlyInAnyOrder("data-1.parquet", "leaf.parquet"); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @TestTemplate + public void testCaseInsensitivePartitionFilter() { + TrackedFile keep = dataFile("keep.parquet", partition(1)); + TrackedFile prune = dataFile("prune.parquet", partition(2)); + + InputFile manifest = writeManifest(PARTITION_TYPE, ImmutableList.of(keep, prune)); + + try (V4ManifestReader reader = + newReader(manifest, PARTITIONED_SPECS) + .filterRows(Expressions.equal("ID", 1)) + .caseSensitive(false) + .build()) { + assertThat(reader.allFiles()) + .extracting(TrackedFile::location) + .containsExactly("keep.parquet"); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @TestTemplate + public void testMultiSpecPartitionPruning() { + PartitionSpec spec0 = + PartitionSpec.builderFor(TABLE_SCHEMA).withSpecId(0).identity("id").build(); + PartitionSpec spec1 = + PartitionSpec.builderFor(TABLE_SCHEMA) + .withSpecId(1) + .add(2, 1001, "data", Transforms.identity()) + .build(); + Map specsById = ImmutableMap.of(0, spec0, 1, spec1); + Types.StructType unionType = Partitioning.partitionType(TABLE_SCHEMA, specsById.values()); + + TrackedFile keepById = + dataFileBuilder("spec0-id1.parquet", unionPartition(unionType, 1, null)).specId(0).build(); + TrackedFile prunedById = + dataFileBuilder("spec0-id2.parquet", unionPartition(unionType, 2, null)).specId(0).build(); + TrackedFile keptOtherSpec = + dataFileBuilder("spec1-data.parquet", unionPartition(unionType, null, "x")) + .specId(1) + .build(); + + InputFile manifest = + writeManifest(unionType, ImmutableList.of(keepById, prunedById, keptOtherSpec)); + + try (V4ManifestReader reader = + V4ManifestReader.builder(manifest, TABLE_SCHEMA, specsById) + .filterRows(Expressions.equal("id", 1)) + .build()) { + // spec0 entries are pruned by id; the spec1 entry is not partitioned by id so it survives + assertThat(reader.allFiles()) + .extracting(TrackedFile::location) + .containsExactlyInAnyOrder("spec0-id1.parquet", "spec1-data.parquet"); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @TestTemplate + public void testIteratorReturnsLiveCopies() { + List files = + ImmutableList.of( + dataFile("s3://bucket/added-1.parquet", EMPTY_PARTITION_DATA), + dataFile("s3://bucket/added-2.parquet", EMPTY_PARTITION_DATA), + fileWithStatus(EntryStatus.DELETED, "s3://bucket/deleted.parquet")); + + InputFile manifest = writeManifest(EMPTY_PARTITION, files); + + try (V4ManifestReader reader = newReader(manifest, UNPARTITIONED_SPECS).build()) { + List read = Lists.newArrayList(reader); + assertThat(read) + .hasSize(2) + .extracting(TrackedFile::location) + .containsExactly("s3://bucket/added-1.parquet", "s3://bucket/added-2.parquet"); + // iterator() copies each entry, so the collected instances are independent of the reused + // container (they would be the same object if iterator() did not copy) + assertThat(read.get(0)).isNotSameAs(read.get(1)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @TestTemplate + public void testUnknownManifestFormatThrows() throws IOException { + InputFile badFile = + fileIO.newInputFile(tempDir.resolve("manifest-" + System.nanoTime() + ".txt").toString()); + + try (V4ManifestReader reader = newReader(badFile, UNPARTITIONED_SPECS).build()) { + assertThatThrownBy(reader::allFiles) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Unable to determine format of manifest"); + } + } + + @TestTemplate + public void testFileWithUnknownSpecThrows() throws IOException { + // spec ID 5 is not in PARTITIONED_SPECS, so pruning cannot resolve a spec for this file + TrackedFile file = dataFileBuilder("orphan.parquet", partition(1)).specId(5).build(); + + InputFile manifest = writeManifest(PARTITION_TYPE, ImmutableList.of(file)); + + try (V4ManifestReader reader = + newReader(manifest, PARTITIONED_SPECS).filterRows(Expressions.equal("id", 1)).build()) { + assertThatThrownBy(() -> Lists.newArrayList(reader.allFiles())) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("not one of the known specs"); + } + } + + private static TrackedFile dataFile(String location, PartitionData partition) { + return dataFileBuilder(location, partition).build(); + } + + private static TrackedFileBuilder dataFileBuilder(String location, PartitionData partition) { + return TrackedFileBuilder.data(SNAPSHOT_ID) + .writerFormatVersion(WRITER_FORMAT_VERSION_V4) + .location(location) + .fileFormat(FileFormat.PARQUET) + .recordCount(100L) + .fileSizeInBytes(1024L) + .partition(partition) + .specId(0); + } + + private static TrackedFile fileWithStatus(EntryStatus status, String location) { + Tracking tracking = new TrackingStruct(status, SNAPSHOT_ID, 3L, 3L, null, null, null, null); + return new TrackedFileStruct( + tracking, + FileContent.DATA, + WRITER_FORMAT_VERSION_V4, + location, + FileFormat.PARQUET, + EMPTY_PARTITION_DATA, + 100L, + 1024L, + 0, + null, + null, + null, + null, + null, + null, + null); + } + + private static PartitionData partition(int id) { + PartitionData partition = new PartitionData(PARTITION_TYPE); + partition.set(0, id); + return partition; + } + + + private static PartitionData unionPartition(Types.StructType unionType, Integer id, String data) { + PartitionData partition = new PartitionData(unionType); + partition.set(0, id); + partition.set(1, data); + return partition; + } + + private InputFile writeManifest(Types.StructType partitionType, Iterable files) { + // Parquet cannot write empty groups, so v4 writers omit the partition and content_stats fields + // entirely when they would be empty (unpartitioned tables, no stats). + List writeFields = Lists.newArrayList(); + for (Types.NestedField field : + TrackedFile.schemaWithContentStats(partitionType, Types.StructType.of()).fields()) { + if (field.type().isStructType() && field.type().asStructType().fields().isEmpty()) { + continue; + } + + writeFields.add(field); + } + + Schema writeSchema = new Schema(writeFields); + OutputFile out = + fileIO.newOutputFile( + tempDir + .resolve( + "manifest-" + System.nanoTime() + "." + format.name().toLowerCase(Locale.ROOT)) + .toString()); + try (FileAppender appender = + InternalData.write(format, out).schema(writeSchema).named("tracked_file").build()) { + for (TrackedFile file : files) { + appender.add(toWriteRow(file, writeSchema)); + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + + return fileIO.newInputFile(out.location()); + } + + /** + * Adapts a fully-populated tracked file to a write schema that may omit fields (partition and + * content_stats are omitted when empty). + */ + private static StructLike toWriteRow(TrackedFile file, Schema writeSchema) { + StructLike struct = (StructLike) file; + int[] toBase = new int[writeSchema.columns().size()]; + for (int i = 0; i < writeSchema.columns().size(); i++) { + toBase[i] = ordinalOf(writeSchema.columns().get(i).fieldId()); + } + + return new StructLike() { + @Override + public int size() { + return toBase.length; + } + + @Override + public T get(int pos, Class javaClass) { + return struct.get(toBase[pos], javaClass); + } + + @Override + public void set(int pos, T value) { + throw new UnsupportedOperationException("Cannot modify write row"); + } + }; + } + + private V4ManifestReader.Builder newReader( + InputFile manifest, Map specsById) { + return V4ManifestReader.builder(manifest, TABLE_SCHEMA, specsById); + } + + private List read(InputFile manifest, Map specsById) { + // allFiles() returns reused instances, so copy each entry before collecting. + try (V4ManifestReader reader = newReader(manifest, specsById).build()) { + return Lists.newArrayList(CloseableIterable.transform(reader.allFiles(), TrackedFile::copy)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private static int ordinalOf(int fieldId) { + for (int i = 0; i < SCHEMA_FIELDS.size(); i++) { + if (SCHEMA_FIELDS.get(i).fieldId() == fieldId) { + return i; + } + } + + throw new IllegalArgumentException("Field not found in TrackedFile schema: " + fieldId); + } +} From e45c65eb1df3336243f473c336c349d638e2f82f Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Fri, 26 Jun 2026 17:38:10 -0700 Subject: [PATCH 02/26] PR feedback --- .../java/org/apache/iceberg/TrackingStruct.java | 2 +- .../java/org/apache/iceberg/V4ManifestReader.java | 13 +++---------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/TrackingStruct.java b/core/src/main/java/org/apache/iceberg/TrackingStruct.java index 8ae4b7e4ce88..46d03bf88a2a 100644 --- a/core/src/main/java/org/apache/iceberg/TrackingStruct.java +++ b/core/src/main/java/org/apache/iceberg/TrackingStruct.java @@ -30,7 +30,7 @@ /** Mutable {@link StructLike} implementation of {@link Tracking}. */ class TrackingStruct extends SupportsIndexProjection implements Tracking, Serializable { - private static final Types.StructType BASE_TYPE = + static final Types.StructType BASE_TYPE = Types.StructType.of( Tracking.STATUS, Tracking.SNAPSHOT_ID, diff --git a/core/src/main/java/org/apache/iceberg/V4ManifestReader.java b/core/src/main/java/org/apache/iceberg/V4ManifestReader.java index aec57a318cc5..d58ebcd338b4 100644 --- a/core/src/main/java/org/apache/iceberg/V4ManifestReader.java +++ b/core/src/main/java/org/apache/iceberg/V4ManifestReader.java @@ -149,11 +149,6 @@ private CloseableIterable open() { private TrackedFile prepare(TrackedFile trackedFile) { Tracking tracking = trackedFile.tracking(); - Preconditions.checkState( - tracking != null, - "Invalid tracked file: tracking is required but was missing in manifest %s", - file.location()); - // manifestLocation is not stored in the manifest; the reader fills it from the file location. // manifestPos is filled from ROW_POSITION while reading the tracking struct. if (tracking instanceof TrackingStruct) { @@ -206,16 +201,14 @@ private Schema readSchema() { } /** - * Builds the tracking field with {@link MetadataColumns#ROW_POSITION} appended so the reader - * populates the manifest position of each entry. + * Builds the tracking field from the read schema, which includes {@code ROW_POSITION} so the + * reader populates the manifest position of each entry. */ private static Types.NestedField trackingWithRowPosition() { - List trackingFields = Lists.newArrayList(Tracking.schema().fields()); - trackingFields.add(MetadataColumns.ROW_POSITION); return Types.NestedField.required( TrackedFile.TRACKING.fieldId(), TrackedFile.TRACKING.name(), - Types.StructType.of(trackingFields), + TrackingStruct.BASE_TYPE, TrackedFile.TRACKING.doc()); } From 085349261b76cac0a6938d335161a30430fe89c4 Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Sat, 27 Jun 2026 10:22:41 -0700 Subject: [PATCH 03/26] Rename format version --- .../org/apache/iceberg/TestV4ManifestReader.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java index 382fa4fe3e83..2254f0b511ec 100644 --- a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java +++ b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java @@ -50,7 +50,7 @@ @ExtendWith(ParameterizedTestExtension.class) public class TestV4ManifestReader { private static final long SNAPSHOT_ID = 42L; - private static final int WRITER_FORMAT_VERSION_V4 = 4; + private static final int FORMAT_VERSION_V4 = 4; private static final Schema TABLE_SCHEMA = new Schema( @@ -104,7 +104,7 @@ public void testRoundTrip() { TrackedFile actual = read.get(0); assertThat(actual.contentType()).isEqualTo(file.contentType()); - assertThat(actual.writerFormatVersion()).isEqualTo(file.writerFormatVersion()); + assertThat(actual.formatVersion()).isEqualTo(file.formatVersion()); assertThat(actual.location()).isEqualTo(file.location()); assertThat(actual.fileFormat()).isEqualTo(file.fileFormat()); assertThat(actual.recordCount()).isEqualTo(file.recordCount()); @@ -133,7 +133,7 @@ public void testRoundTrip() { public void testEqualityDeleteRoundTrip() { TrackedFile delete = TrackedFileBuilder.equalityDelete(SNAPSHOT_ID) - .writerFormatVersion(WRITER_FORMAT_VERSION_V4) + .formatVersion(FORMAT_VERSION_V4) .location("s3://bucket/eq-delete.parquet") .fileFormat(FileFormat.PARQUET) .recordCount(10L) @@ -259,7 +259,7 @@ public void testPartitionFilterPrunesNonMatchingFiles() { public void testPartitionFilterCountsSkippedDeleteFiles() { TrackedFile delete = TrackedFileBuilder.equalityDelete(SNAPSHOT_ID) - .writerFormatVersion(WRITER_FORMAT_VERSION_V4) + .formatVersion(FORMAT_VERSION_V4) .location("delete.parquet") .fileFormat(FileFormat.PARQUET) .recordCount(100L) @@ -304,7 +304,7 @@ public void testPartitionFilterKeepsManifestReferences() { .build(); TrackedFile manifestRef = TrackedFileBuilder.dataManifest(SNAPSHOT_ID) - .writerFormatVersion(WRITER_FORMAT_VERSION_V4) + .formatVersion(FORMAT_VERSION_V4) .location("leaf.parquet") .fileFormat(FileFormat.PARQUET) .recordCount(1L) @@ -440,7 +440,7 @@ private static TrackedFile dataFile(String location, PartitionData partition) { private static TrackedFileBuilder dataFileBuilder(String location, PartitionData partition) { return TrackedFileBuilder.data(SNAPSHOT_ID) - .writerFormatVersion(WRITER_FORMAT_VERSION_V4) + .formatVersion(FORMAT_VERSION_V4) .location(location) .fileFormat(FileFormat.PARQUET) .recordCount(100L) @@ -454,7 +454,7 @@ private static TrackedFile fileWithStatus(EntryStatus status, String location) { return new TrackedFileStruct( tracking, FileContent.DATA, - WRITER_FORMAT_VERSION_V4, + FORMAT_VERSION_V4, location, FileFormat.PARQUET, EMPTY_PARTITION_DATA, @@ -476,7 +476,6 @@ private static PartitionData partition(int id) { return partition; } - private static PartitionData unionPartition(Types.StructType unionType, Integer id, String data) { PartitionData partition = new PartitionData(unionType); partition.set(0, id); From f1821b3440b9e56d627cbebcdc915b4877082290 Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Mon, 29 Jun 2026 10:38:54 -0700 Subject: [PATCH 04/26] Comment update --- core/src/main/java/org/apache/iceberg/V4ManifestReader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/iceberg/V4ManifestReader.java b/core/src/main/java/org/apache/iceberg/V4ManifestReader.java index d58ebcd338b4..f6eb40285f7e 100644 --- a/core/src/main/java/org/apache/iceberg/V4ManifestReader.java +++ b/core/src/main/java/org/apache/iceberg/V4ManifestReader.java @@ -37,7 +37,7 @@ import org.apache.iceberg.types.Types; import org.apache.iceberg.util.StructProjection; -/** Reader that reads a v4 manifest file as {@link TrackedFile}s. */ +/** Reader that reads a v4+ manifest file as {@link TrackedFile}s. */ class V4ManifestReader extends CloseableGroup implements CloseableIterable { private final InputFile file; private final Types.StructType partitionType; From d872a76f4e3d92b2dce9044f9c462e5c5da7e078 Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Tue, 30 Jun 2026 14:00:04 -0700 Subject: [PATCH 05/26] PR feedback --- .../org/apache/iceberg/V4ManifestReader.java | 49 ++++++------ .../apache/iceberg/TestV4ManifestReader.java | 77 +++++++------------ 2 files changed, 53 insertions(+), 73 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/V4ManifestReader.java b/core/src/main/java/org/apache/iceberg/V4ManifestReader.java index f6eb40285f7e..4d46ab929bd8 100644 --- a/core/src/main/java/org/apache/iceberg/V4ManifestReader.java +++ b/core/src/main/java/org/apache/iceberg/V4ManifestReader.java @@ -21,6 +21,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import org.apache.iceberg.expressions.Evaluator; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.expressions.Expressions; @@ -39,6 +40,9 @@ /** Reader that reads a v4+ manifest file as {@link TrackedFile}s. */ class V4ManifestReader extends CloseableGroup implements CloseableIterable { + // minimal tracking projection used when the caller does not request tracking + private static final Types.StructType STATUS_TRACKING = Types.StructType.of(Tracking.STATUS); + private final InputFile file; private final Types.StructType partitionType; private final Schema fileProjection; @@ -165,16 +169,19 @@ private Schema readSchema() { boolean unpartitioned = partitionType.fields().isEmpty(); Set projectedIds = null; + boolean fullTracking = true; if (fileProjection != null) { - projectedIds = Sets.newHashSet(); - for (Types.NestedField field : fileProjection.asStruct().fields()) { - projectedIds.add(field.fieldId()); - } - - // tracking carries the status used to filter live files and is always projected + projectedIds = + fileProjection.asStruct().fields().stream() + .map(Types.NestedField::fieldId) + .collect(Collectors.toCollection(Sets::newHashSet)); + + // read the full tracking struct only when the caller requests it; otherwise force-add a + // minimal tracking carrying just the status used to filter live files + fullTracking = projectedIds.contains(TrackedFile.TRACKING.fieldId()); projectedIds.add(TrackedFile.TRACKING.fieldId()); - // spec_id is required to resolve each entry's partition spec when pruning + // project spec_id for partition filtering if (!partitionEvaluators.isEmpty()) { projectedIds.add(TrackedFile.SPEC_ID.fieldId()); } @@ -187,7 +194,12 @@ private Schema readSchema() { } if (field.fieldId() == TrackedFile.TRACKING.fieldId()) { - fields.add(trackingWithRowPosition()); + fields.add( + Types.NestedField.required( + TrackedFile.TRACKING.fieldId(), + TrackedFile.TRACKING.name(), + fullTracking ? TrackingStruct.BASE_TYPE : STATUS_TRACKING, + TrackedFile.TRACKING.doc())); } else if (field.fieldId() == TrackedFile.CONTENT_STATS_ID) { // content_stats are omitted for now } else if (field.fieldId() == TrackedFile.PARTITION_ID && unpartitioned) { @@ -200,21 +212,9 @@ private Schema readSchema() { return new Schema(fields); } - /** - * Builds the tracking field from the read schema, which includes {@code ROW_POSITION} so the - * reader populates the manifest position of each entry. - */ - private static Types.NestedField trackingWithRowPosition() { - return Types.NestedField.required( - TrackedFile.TRACKING.fieldId(), - TrackedFile.TRACKING.name(), - TrackingStruct.BASE_TYPE, - TrackedFile.TRACKING.doc()); - } - static class Builder { private final InputFile file; - private final Schema tableSchema; + private final Types.StructType partitionType; private final Map specsById; private Expression rowFilter = Expressions.alwaysTrue(); private boolean caseSensitive = true; @@ -223,13 +223,13 @@ static class Builder { private Builder(InputFile file, Schema tableSchema, Map specsById) { this.file = file; - this.tableSchema = tableSchema; + this.partitionType = Partitioning.partitionType(tableSchema, specsById.values()); this.specsById = specsById; } /** Sets a row filter; files that cannot match the expression are skipped. */ Builder filterRows(Expression expr) { - Preconditions.checkNotNull(expr, "Row filter cannot be null"); + Preconditions.checkArgument(expr != null, "Invalid row filter: null"); this.rowFilter = expr; return this; } @@ -245,13 +245,12 @@ Builder project(Schema newFileProjection) { } Builder scanMetrics(ScanMetrics newScanMetrics) { - Preconditions.checkNotNull(newScanMetrics, "Scan metrics cannot be null"); + Preconditions.checkArgument(newScanMetrics != null, "Invalid scan metrics: null"); this.scanMetrics = newScanMetrics; return this; } V4ManifestReader build() { - Types.StructType partitionType = Partitioning.partitionType(tableSchema, specsById.values()); Map partitionEvaluators = Maps.newHashMap(); Map partitionProjections = Maps.newHashMap(); if (rowFilter != Expressions.alwaysTrue() && !partitionType.fields().isEmpty()) { diff --git a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java index 2254f0b511ec..17addd847236 100644 --- a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java +++ b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java @@ -23,7 +23,6 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import java.io.IOException; -import java.io.UncheckedIOException; import java.nio.ByteBuffer; import java.nio.file.Path; import java.util.Arrays; @@ -80,7 +79,7 @@ protected static List parameters() { private final FileIO fileIO = new TestTables.LocalFileIO(); @TestTemplate - public void testRoundTrip() { + public void testRoundTrip() throws IOException { DeletionVector dv = DeletionVectorStruct.builder() .location("s3://bucket/dv.puffin") @@ -130,7 +129,7 @@ public void testRoundTrip() { } @TestTemplate - public void testEqualityDeleteRoundTrip() { + public void testEqualityDeleteRoundTrip() throws IOException { TrackedFile delete = TrackedFileBuilder.equalityDelete(SNAPSHOT_ID) .formatVersion(FORMAT_VERSION_V4) @@ -151,7 +150,7 @@ public void testEqualityDeleteRoundTrip() { } @TestTemplate - public void testLiveFilesExcludesDeletedAndReplaced() { + public void testLiveFilesExcludesDeletedAndReplaced() throws IOException { List files = ImmutableList.of( fileWithStatus(EntryStatus.ADDED, "s3://bucket/added.parquet"), @@ -175,13 +174,11 @@ public void testLiveFilesExcludesDeletedAndReplaced() { assertThat(reader.liveFiles()) .extracting(file -> file.tracking().status()) .containsExactly(EntryStatus.ADDED, EntryStatus.EXISTING, EntryStatus.MODIFIED); - } catch (IOException e) { - throw new UncheckedIOException(e); } } @TestTemplate - public void testManifestLocationAndPosition() { + public void testManifestLocationAndPosition() throws IOException { List files = ImmutableList.of( dataFile("s3://bucket/a.parquet", EMPTY_PARTITION_DATA), @@ -198,8 +195,7 @@ public void testManifestLocationAndPosition() { } @TestTemplate - public void testProjectionRestrictsFields() { - // sort_order_id is written but not projected below, so it must not be read back + public void testProjectionRestrictsFields() throws IOException { TrackedFile file = dataFileBuilder("s3://bucket/file.parquet", EMPTY_PARTITION_DATA).sortOrderId(7).build(); @@ -209,20 +205,19 @@ public void testProjectionRestrictsFields() { try (V4ManifestReader reader = newReader(manifest, UNPARTITIONED_SPECS).project(projection).build()) { TrackedFile actual = Lists.newArrayList(reader.allFiles()).get(0); - assertThat(actual.location()).isEqualTo("s3://bucket/file.parquet"); - // tracking is always projected even when the caller omits it + assertThat(actual.location()).isEqualTo(file.location()); + // a minimal status-only tracking is force-added when the caller omits tracking assertThat(actual.tracking()).isNotNull(); assertThat(actual.tracking().status()).isEqualTo(EntryStatus.ADDED); - assertThat(actual.tracking().manifestPos()).isEqualTo(0L); - // sort_order_id was written but not projected, so it should not be read + // sort_order_id, file_format, and spec_id are null because they were not projected assertThat(actual.sortOrderId()).isNull(); - } catch (IOException e) { - throw new UncheckedIOException(e); + assertThat(actual.fileFormat()).isNull(); + assertThat(actual.specId()).isNull(); } } @TestTemplate - public void testUnpartitioned() { + public void testUnpartitioned() throws IOException { TrackedFile file = dataFile("s3://bucket/file.parquet", EMPTY_PARTITION_DATA); InputFile manifest = writeManifest(EMPTY_PARTITION, ImmutableList.of(file)); @@ -233,7 +228,7 @@ public void testUnpartitioned() { } @TestTemplate - public void testPartitionFilterPrunesNonMatchingFiles() { + public void testPartitionFilterPrunesNonMatchingFiles() throws IOException { TrackedFile keep = dataFile("keep.parquet", partition(1)); TrackedFile prune = dataFile("prune.parquet", partition(2)); @@ -247,16 +242,14 @@ public void testPartitionFilterPrunesNonMatchingFiles() { .build()) { assertThat(reader.allFiles()) .extracting(TrackedFile::location) - .containsExactly("keep.parquet"); - } catch (IOException e) { - throw new UncheckedIOException(e); + .containsExactly(keep.location()); } assertThat(metrics.skippedDataFiles().value()).isEqualTo(1L); } @TestTemplate - public void testPartitionFilterCountsSkippedDeleteFiles() { + public void testPartitionFilterCountsSkippedDeleteFiles() throws IOException { TrackedFile delete = TrackedFileBuilder.equalityDelete(SNAPSHOT_ID) .formatVersion(FORMAT_VERSION_V4) @@ -278,8 +271,6 @@ public void testPartitionFilterCountsSkippedDeleteFiles() { .scanMetrics(metrics) .build()) { assertThat(reader.allFiles()).isEmpty(); - } catch (IOException e) { - throw new UncheckedIOException(e); } assertThat(metrics.skippedDeleteFiles().value()).isEqualTo(1L); @@ -287,7 +278,7 @@ public void testPartitionFilterCountsSkippedDeleteFiles() { } @TestTemplate - public void testPartitionFilterKeepsManifestReferences() { + public void testPartitionFilterKeepsManifestReferences() throws IOException { TrackedFile keep = dataFile("data-1.parquet", partition(1)); TrackedFile prune = dataFile("data-2.parquet", partition(2)); ManifestInfo info = @@ -320,14 +311,12 @@ public void testPartitionFilterKeepsManifestReferences() { newReader(manifest, PARTITIONED_SPECS).filterRows(Expressions.equal("id", 1)).build()) { assertThat(reader.allFiles()) .extracting(TrackedFile::location) - .containsExactlyInAnyOrder("data-1.parquet", "leaf.parquet"); - } catch (IOException e) { - throw new UncheckedIOException(e); + .containsExactlyInAnyOrder(keep.location(), manifestRef.location()); } } @TestTemplate - public void testCaseInsensitivePartitionFilter() { + public void testCaseInsensitivePartitionFilter() throws IOException { TrackedFile keep = dataFile("keep.parquet", partition(1)); TrackedFile prune = dataFile("prune.parquet", partition(2)); @@ -340,14 +329,12 @@ public void testCaseInsensitivePartitionFilter() { .build()) { assertThat(reader.allFiles()) .extracting(TrackedFile::location) - .containsExactly("keep.parquet"); - } catch (IOException e) { - throw new UncheckedIOException(e); + .containsExactly(keep.location()); } } @TestTemplate - public void testMultiSpecPartitionPruning() { + public void testMultiSpecPartitionPruning() throws IOException { PartitionSpec spec0 = PartitionSpec.builderFor(TABLE_SCHEMA).withSpecId(0).identity("id").build(); PartitionSpec spec1 = @@ -377,19 +364,17 @@ public void testMultiSpecPartitionPruning() { // spec0 entries are pruned by id; the spec1 entry is not partitioned by id so it survives assertThat(reader.allFiles()) .extracting(TrackedFile::location) - .containsExactlyInAnyOrder("spec0-id1.parquet", "spec1-data.parquet"); - } catch (IOException e) { - throw new UncheckedIOException(e); + .containsExactlyInAnyOrder(keepById.location(), keptOtherSpec.location()); } } @TestTemplate - public void testIteratorReturnsLiveCopies() { + public void testIteratorReturnsLiveCopies() throws IOException { + TrackedFile added1 = dataFile("s3://bucket/added-1.parquet", EMPTY_PARTITION_DATA); + TrackedFile added2 = dataFile("s3://bucket/added-2.parquet", EMPTY_PARTITION_DATA); List files = ImmutableList.of( - dataFile("s3://bucket/added-1.parquet", EMPTY_PARTITION_DATA), - dataFile("s3://bucket/added-2.parquet", EMPTY_PARTITION_DATA), - fileWithStatus(EntryStatus.DELETED, "s3://bucket/deleted.parquet")); + added1, added2, fileWithStatus(EntryStatus.DELETED, "s3://bucket/deleted.parquet")); InputFile manifest = writeManifest(EMPTY_PARTITION, files); @@ -398,12 +383,10 @@ public void testIteratorReturnsLiveCopies() { assertThat(read) .hasSize(2) .extracting(TrackedFile::location) - .containsExactly("s3://bucket/added-1.parquet", "s3://bucket/added-2.parquet"); + .containsExactly(added1.location(), added2.location()); // iterator() copies each entry, so the collected instances are independent of the reused // container (they would be the same object if iterator() did not copy) assertThat(read.get(0)).isNotSameAs(read.get(1)); - } catch (IOException e) { - throw new UncheckedIOException(e); } } @@ -483,7 +466,8 @@ private static PartitionData unionPartition(Types.StructType unionType, Integer return partition; } - private InputFile writeManifest(Types.StructType partitionType, Iterable files) { + private InputFile writeManifest(Types.StructType partitionType, Iterable files) + throws IOException { // Parquet cannot write empty groups, so v4 writers omit the partition and content_stats fields // entirely when they would be empty (unpartitioned tables, no stats). List writeFields = Lists.newArrayList(); @@ -508,8 +492,6 @@ private InputFile writeManifest(Types.StructType partitionType, Iterable read(InputFile manifest, Map specsById) { + private List read(InputFile manifest, Map specsById) + throws IOException { // allFiles() returns reused instances, so copy each entry before collecting. try (V4ManifestReader reader = newReader(manifest, specsById).build()) { return Lists.newArrayList(CloseableIterable.transform(reader.allFiles(), TrackedFile::copy)); - } catch (IOException e) { - throw new UncheckedIOException(e); } } From 67579d40e5289a3bfc5c34c33cb6fb5061d1c251 Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Wed, 1 Jul 2026 09:22:16 -0700 Subject: [PATCH 06/26] Remove builder from tests --- .../apache/iceberg/TestV4ManifestReader.java | 233 ++++++++++-------- 1 file changed, 136 insertions(+), 97 deletions(-) diff --git a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java index 17addd847236..f39e06080424 100644 --- a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java +++ b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java @@ -50,6 +50,13 @@ public class TestV4ManifestReader { private static final long SNAPSHOT_ID = 42L; private static final int FORMAT_VERSION_V4 = 4; + private static final long RECORD_COUNT = 100L; + private static final long FILE_SIZE_IN_BYTES = 1024L; + private static final int SORT_ORDER_ID = 1; + private static final String DV_LOCATION = "s3://bucket/dv.puffin"; + private static final long DV_OFFSET = 100L; + private static final long DV_SIZE_IN_BYTES = 50L; + private static final long DV_CARDINALITY = 5L; private static final Schema TABLE_SCHEMA = new Schema( @@ -66,6 +73,7 @@ public class TestV4ManifestReader { private static final List SCHEMA_FIELDS = TrackedFile.schemaWithContentStats(Types.StructType.of(), Types.StructType.of()).fields(); + private static final int SORT_ORDER_ID_ORDINAL = ordinalOf(TrackedFile.SORT_ORDER_ID.fieldId()); @Parameter private FileFormat format; @@ -80,21 +88,26 @@ protected static List parameters() { @TestTemplate public void testRoundTrip() throws IOException { - DeletionVector dv = - DeletionVectorStruct.builder() - .location("s3://bucket/dv.puffin") - .offset(100L) - .sizeInBytes(50L) - .cardinality(5L) - .build(); + DeletionVector dv = deletionVector(DV_LOCATION, DV_OFFSET, DV_SIZE_IN_BYTES, DV_CARDINALITY); TrackedFile file = - dataFileBuilder("s3://bucket/data/file.parquet", partition(7)) - .sortOrderId(1) - .deletionVector(dv) - .keyMetadata(ByteBuffer.wrap(new byte[] {1, 2, 3})) - .splitOffsets(ImmutableList.of(50L, 100L)) - .build(); + new TrackedFileStruct( + addedTracking(), + FileContent.DATA, + FORMAT_VERSION_V4, + "s3://bucket/data/file.parquet", + FileFormat.PARQUET, + partition(7), + RECORD_COUNT, + FILE_SIZE_IN_BYTES, + 0, + null, + SORT_ORDER_ID, + dv, + null, + ByteBuffer.wrap(new byte[] {1, 2, 3}), + ImmutableList.of(50L, 100L), + null); InputFile manifest = writeManifest(PARTITION_TYPE, ImmutableList.of(file)); @@ -102,45 +115,49 @@ public void testRoundTrip() throws IOException { assertThat(read).hasSize(1); TrackedFile actual = read.get(0); - assertThat(actual.contentType()).isEqualTo(file.contentType()); - assertThat(actual.formatVersion()).isEqualTo(file.formatVersion()); - assertThat(actual.location()).isEqualTo(file.location()); - assertThat(actual.fileFormat()).isEqualTo(file.fileFormat()); - assertThat(actual.recordCount()).isEqualTo(file.recordCount()); - assertThat(actual.fileSizeInBytes()).isEqualTo(file.fileSizeInBytes()); - assertThat(actual.specId()).isEqualTo(file.specId()); - assertThat(actual.sortOrderId()).isEqualTo(file.sortOrderId()); - assertThat(actual.keyMetadata()).isEqualTo(file.keyMetadata()); - assertThat(actual.splitOffsets()).isEqualTo(file.splitOffsets()); - assertThat(actual.partition().get(0, Integer.class)) - .isEqualTo(file.partition().get(0, Integer.class)); + assertThat(actual.contentType()).isEqualTo(FileContent.DATA); + assertThat(actual.formatVersion()).isEqualTo(FORMAT_VERSION_V4); + assertThat(actual.location()).isEqualTo("s3://bucket/data/file.parquet"); + assertThat(actual.fileFormat()).isEqualTo(FileFormat.PARQUET); + assertThat(actual.recordCount()).isEqualTo(RECORD_COUNT); + assertThat(actual.fileSizeInBytes()).isEqualTo(FILE_SIZE_IN_BYTES); + assertThat(actual.specId()).isEqualTo(0); + assertThat(actual.sortOrderId()).isEqualTo(SORT_ORDER_ID); + assertThat(actual.keyMetadata()).isEqualTo(ByteBuffer.wrap(new byte[] {1, 2, 3})); + assertThat(actual.splitOffsets()).containsExactly(50L, 100L); + assertThat(actual.partition().get(0, Integer.class)).isEqualTo(7); assertThat(actual.tracking()).isNotNull(); - assertThat(actual.tracking().status()).isEqualTo(file.tracking().status()); - assertThat(actual.tracking().snapshotId()).isEqualTo(file.tracking().snapshotId()); + assertThat(actual.tracking().status()).isEqualTo(EntryStatus.ADDED); + assertThat(actual.tracking().snapshotId()).isEqualTo(SNAPSHOT_ID); assertThat(actual.deletionVector()).isNotNull(); - assertThat(actual.deletionVector().location()).isEqualTo(file.deletionVector().location()); - assertThat(actual.deletionVector().offset()).isEqualTo(file.deletionVector().offset()); - assertThat(actual.deletionVector().sizeInBytes()) - .isEqualTo(file.deletionVector().sizeInBytes()); - assertThat(actual.deletionVector().cardinality()) - .isEqualTo(file.deletionVector().cardinality()); + assertThat(actual.deletionVector().location()).isEqualTo(DV_LOCATION); + assertThat(actual.deletionVector().offset()).isEqualTo(DV_OFFSET); + assertThat(actual.deletionVector().sizeInBytes()).isEqualTo(DV_SIZE_IN_BYTES); + assertThat(actual.deletionVector().cardinality()).isEqualTo(DV_CARDINALITY); } @TestTemplate public void testEqualityDeleteRoundTrip() throws IOException { TrackedFile delete = - TrackedFileBuilder.equalityDelete(SNAPSHOT_ID) - .formatVersion(FORMAT_VERSION_V4) - .location("s3://bucket/eq-delete.parquet") - .fileFormat(FileFormat.PARQUET) - .recordCount(10L) - .fileSizeInBytes(128L) - .partition(EMPTY_PARTITION_DATA) - .specId(0) - .equalityIds(ImmutableList.of(1, 2)) - .build(); + new TrackedFileStruct( + addedTracking(), + FileContent.EQUALITY_DELETES, + FORMAT_VERSION_V4, + "s3://bucket/eq-delete.parquet", + FileFormat.PARQUET, + EMPTY_PARTITION_DATA, + RECORD_COUNT, + FILE_SIZE_IN_BYTES, + 0, + null, + null, + null, + null, + null, + null, + ImmutableList.of(1, 2)); InputFile manifest = writeManifest(EMPTY_PARTITION, ImmutableList.of(delete)); @@ -196,8 +213,8 @@ public void testManifestLocationAndPosition() throws IOException { @TestTemplate public void testProjectionRestrictsFields() throws IOException { - TrackedFile file = - dataFileBuilder("s3://bucket/file.parquet", EMPTY_PARTITION_DATA).sortOrderId(7).build(); + TrackedFile file = dataFile("s3://bucket/file.parquet", EMPTY_PARTITION_DATA); + ((StructLike) file).set(SORT_ORDER_ID_ORDINAL, SORT_ORDER_ID); InputFile manifest = writeManifest(EMPTY_PARTITION, ImmutableList.of(file)); @@ -251,16 +268,23 @@ public void testPartitionFilterPrunesNonMatchingFiles() throws IOException { @TestTemplate public void testPartitionFilterCountsSkippedDeleteFiles() throws IOException { TrackedFile delete = - TrackedFileBuilder.equalityDelete(SNAPSHOT_ID) - .formatVersion(FORMAT_VERSION_V4) - .location("delete.parquet") - .fileFormat(FileFormat.PARQUET) - .recordCount(100L) - .fileSizeInBytes(1024L) - .partition(partition(2)) - .specId(0) - .equalityIds(ImmutableList.of(1)) - .build(); + new TrackedFileStruct( + addedTracking(), + FileContent.EQUALITY_DELETES, + FORMAT_VERSION_V4, + "delete.parquet", + FileFormat.PARQUET, + partition(2), + RECORD_COUNT, + FILE_SIZE_IN_BYTES, + 0, + null, + null, + null, + null, + null, + null, + ImmutableList.of(1)); InputFile manifest = writeManifest(PARTITION_TYPE, ImmutableList.of(delete)); @@ -281,29 +305,25 @@ public void testPartitionFilterCountsSkippedDeleteFiles() throws IOException { public void testPartitionFilterKeepsManifestReferences() throws IOException { TrackedFile keep = dataFile("data-1.parquet", partition(1)); TrackedFile prune = dataFile("data-2.parquet", partition(2)); - ManifestInfo info = - ManifestInfoStruct.builder() - .addedFilesCount(1) - .existingFilesCount(0) - .deletedFilesCount(0) - .replacedFilesCount(0) - .addedRowsCount(1L) - .existingRowsCount(0L) - .deletedRowsCount(0L) - .replacedRowsCount(0L) - .minSequenceNumber(1L) - .build(); + ManifestInfo info = new ManifestInfoStruct(1, 0, 0, 0, 1L, 0L, 0L, 0L, 1L, null, null); TrackedFile manifestRef = - TrackedFileBuilder.dataManifest(SNAPSHOT_ID) - .formatVersion(FORMAT_VERSION_V4) - .location("leaf.parquet") - .fileFormat(FileFormat.PARQUET) - .recordCount(1L) - .fileSizeInBytes(100L) - .partition(partition(2)) - .specId(0) - .manifestInfo(info) - .build(); + new TrackedFileStruct( + addedTracking(), + FileContent.DATA_MANIFEST, + FORMAT_VERSION_V4, + "leaf.parquet", + FileFormat.PARQUET, + partition(2), + RECORD_COUNT, + FILE_SIZE_IN_BYTES, + 0, + null, + null, + null, + info, + null, + null, + null); InputFile manifest = writeManifest(PARTITION_TYPE, ImmutableList.of(keep, prune, manifestRef)); @@ -345,14 +365,10 @@ public void testMultiSpecPartitionPruning() throws IOException { Map specsById = ImmutableMap.of(0, spec0, 1, spec1); Types.StructType unionType = Partitioning.partitionType(TABLE_SCHEMA, specsById.values()); - TrackedFile keepById = - dataFileBuilder("spec0-id1.parquet", unionPartition(unionType, 1, null)).specId(0).build(); - TrackedFile prunedById = - dataFileBuilder("spec0-id2.parquet", unionPartition(unionType, 2, null)).specId(0).build(); + TrackedFile keepById = dataFile("spec0-id1.parquet", unionPartition(unionType, 1, null), 0); + TrackedFile prunedById = dataFile("spec0-id2.parquet", unionPartition(unionType, 2, null), 0); TrackedFile keptOtherSpec = - dataFileBuilder("spec1-data.parquet", unionPartition(unionType, null, "x")) - .specId(1) - .build(); + dataFile("spec1-data.parquet", unionPartition(unionType, null, "x"), 1); InputFile manifest = writeManifest(unionType, ImmutableList.of(keepById, prunedById, keptOtherSpec)); @@ -405,7 +421,7 @@ public void testUnknownManifestFormatThrows() throws IOException { @TestTemplate public void testFileWithUnknownSpecThrows() throws IOException { // spec ID 5 is not in PARTITIONED_SPECS, so pruning cannot resolve a spec for this file - TrackedFile file = dataFileBuilder("orphan.parquet", partition(1)).specId(5).build(); + TrackedFile file = dataFile("orphan.parquet", partition(1), 5); InputFile manifest = writeManifest(PARTITION_TYPE, ImmutableList.of(file)); @@ -418,18 +434,27 @@ public void testFileWithUnknownSpecThrows() throws IOException { } private static TrackedFile dataFile(String location, PartitionData partition) { - return dataFileBuilder(location, partition).build(); + return dataFile(location, partition, 0); } - private static TrackedFileBuilder dataFileBuilder(String location, PartitionData partition) { - return TrackedFileBuilder.data(SNAPSHOT_ID) - .formatVersion(FORMAT_VERSION_V4) - .location(location) - .fileFormat(FileFormat.PARQUET) - .recordCount(100L) - .fileSizeInBytes(1024L) - .partition(partition) - .specId(0); + private static TrackedFile dataFile(String location, PartitionData partition, int specId) { + return new TrackedFileStruct( + addedTracking(), + FileContent.DATA, + FORMAT_VERSION_V4, + location, + FileFormat.PARQUET, + partition, + RECORD_COUNT, + FILE_SIZE_IN_BYTES, + specId, + null, + null, + null, + null, + null, + null, + null); } private static TrackedFile fileWithStatus(EntryStatus status, String location) { @@ -441,8 +466,8 @@ private static TrackedFile fileWithStatus(EntryStatus status, String location) { location, FileFormat.PARQUET, EMPTY_PARTITION_DATA, - 100L, - 1024L, + RECORD_COUNT, + FILE_SIZE_IN_BYTES, 0, null, null, @@ -453,6 +478,20 @@ private static TrackedFile fileWithStatus(EntryStatus status, String location) { null); } + private static Tracking addedTracking() { + return new TrackingStruct(EntryStatus.ADDED, SNAPSHOT_ID, null, null, null, null, null, null); + } + + private static DeletionVector deletionVector( + String location, long offset, long sizeInBytes, long cardinality) { + DeletionVectorStruct dv = new DeletionVectorStruct(DeletionVector.schema()); + dv.set(0, location); + dv.set(1, offset); + dv.set(2, sizeInBytes); + dv.set(3, cardinality); + return dv; + } + private static PartitionData partition(int id) { PartitionData partition = new PartitionData(PARTITION_TYPE); partition.set(0, id); From c78b6642f0a403e3da4111bea2cd43f1d1eab65d Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Fri, 3 Jul 2026 10:40:56 -0700 Subject: [PATCH 07/26] PR feedback --- .../org/apache/iceberg/V4ManifestReader.java | 37 ++++++---- .../apache/iceberg/TestV4ManifestReader.java | 72 ++++++++++++++++++- 2 files changed, 92 insertions(+), 17 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/V4ManifestReader.java b/core/src/main/java/org/apache/iceberg/V4ManifestReader.java index 4d46ab929bd8..b5a45e6a11ca 100644 --- a/core/src/main/java/org/apache/iceberg/V4ManifestReader.java +++ b/core/src/main/java/org/apache/iceberg/V4ManifestReader.java @@ -40,8 +40,17 @@ /** Reader that reads a v4+ manifest file as {@link TrackedFile}s. */ class V4ManifestReader extends CloseableGroup implements CloseableIterable { - // minimal tracking projection used when the caller does not request tracking - private static final Types.StructType STATUS_TRACKING = Types.StructType.of(Tracking.STATUS); + // Tracking fields read on the scan path. Omits the change-tracking fields (dv_snapshot_id, + // deleted_positions, replaced_positions) that a scan does not need. row_position backs + // Tracking.manifestPos. + private static final Types.StructType SCAN_TRACKING = + Types.StructType.of( + Tracking.STATUS, + Tracking.SNAPSHOT_ID, + Tracking.SEQUENCE_NUMBER, + Tracking.FILE_SEQUENCE_NUMBER, + Tracking.FIRST_ROW_ID, + MetadataColumns.ROW_POSITION); private final InputFile file; private final Types.StructType partitionType; @@ -82,7 +91,7 @@ CloseableIterable liveFiles() { return files(true /* only live files */); } - /** Returns live tracked files, each as an independent copy. */ + /** Returns live tracked files. Makes defensive copies before returning. */ @Override public CloseableIterator iterator() { return CloseableIterable.transform(liveFiles(), TrackedFile::copy).iterator(); @@ -113,9 +122,7 @@ private boolean matchesPartition(TrackedFile trackedFile) { StructProjection projection = specId != null ? partitionProjections.get(specId) : null; Preconditions.checkState( evaluator != null && projection != null, - "Cannot apply partition filter: file %s has spec ID %s, not one of the known specs %s " - + "in manifest %s", - trackedFile.location(), + "Cannot apply partition filter: spec ID %s is not one of the known specs %s in manifest %s", specId, partitionEvaluators.keySet(), file.location()); @@ -135,7 +142,7 @@ private boolean matchesPartition(TrackedFile trackedFile) { private CloseableIterable open() { FileFormat format = FileFormat.fromFileName(file.location()); Preconditions.checkArgument( - format != null, "Unable to determine format of manifest: %s", file.location()); + format != null, "Cannot determine format of manifest: %s", file.location()); CloseableIterable reader = InternalData.read(format, file) @@ -163,27 +170,27 @@ private TrackedFile prepare(TrackedFile trackedFile) { } private Schema readSchema() { - // content_stats is not projected yet, so build the schema with an empty stats type Types.StructType fullType = TrackedFile.schemaWithContentStats(partitionType, Types.StructType.of()); boolean unpartitioned = partitionType.fields().isEmpty(); Set projectedIds = null; - boolean fullTracking = true; if (fileProjection != null) { projectedIds = fileProjection.asStruct().fields().stream() .map(Types.NestedField::fieldId) .collect(Collectors.toCollection(Sets::newHashSet)); - // read the full tracking struct only when the caller requests it; otherwise force-add a - // minimal tracking carrying just the status used to filter live files - fullTracking = projectedIds.contains(TrackedFile.TRACKING.fieldId()); + // Always project tracking and content type. status drives live-file filtering, and content + // type distinguishes data, delete, and manifest-reference entries. projectedIds.add(TrackedFile.TRACKING.fieldId()); + projectedIds.add(TrackedFile.CONTENT_TYPE.fieldId()); - // project spec_id for partition filtering + // Force-project the remaining fields the partition filter reads, regardless of caller + // projection. if (!partitionEvaluators.isEmpty()) { projectedIds.add(TrackedFile.SPEC_ID.fieldId()); + projectedIds.add(TrackedFile.PARTITION_ID); } } @@ -198,10 +205,10 @@ private Schema readSchema() { Types.NestedField.required( TrackedFile.TRACKING.fieldId(), TrackedFile.TRACKING.name(), - fullTracking ? TrackingStruct.BASE_TYPE : STATUS_TRACKING, + SCAN_TRACKING, TrackedFile.TRACKING.doc())); } else if (field.fieldId() == TrackedFile.CONTENT_STATS_ID) { - // content_stats are omitted for now + // content_stats are not projected yet } else if (field.fieldId() == TrackedFile.PARTITION_ID && unpartitioned) { // unpartitioned manifests omit the partition field } else { diff --git a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java index f39e06080424..4243a5da104e 100644 --- a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java +++ b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java @@ -223,9 +223,10 @@ public void testProjectionRestrictsFields() throws IOException { newReader(manifest, UNPARTITIONED_SPECS).project(projection).build()) { TrackedFile actual = Lists.newArrayList(reader.allFiles()).get(0); assertThat(actual.location()).isEqualTo(file.location()); - // a minimal status-only tracking is force-added when the caller omits tracking + // tracking and content_type are always projected, even though the caller omitted them assertThat(actual.tracking()).isNotNull(); assertThat(actual.tracking().status()).isEqualTo(EntryStatus.ADDED); + assertThat(actual.contentType()).isEqualTo(FileContent.DATA); // sort_order_id, file_format, and spec_id are null because they were not projected assertThat(actual.sortOrderId()).isNull(); assertThat(actual.fileFormat()).isNull(); @@ -233,6 +234,73 @@ public void testProjectionRestrictsFields() throws IOException { } } + @TestTemplate + public void testTrackingProjectionOmitsChangeTrackingFields() throws IOException { + Tracking tracking = + new TrackingStruct( + EntryStatus.ADDED, + SNAPSHOT_ID, + 5L, // data sequence number + 6L, // file sequence number + 7L, // dv snapshot id + 8L, // first row id + new byte[] {1, 2}, // deleted positions + new byte[] {3, 4}); // replaced positions + TrackedFile file = + new TrackedFileStruct( + tracking, + FileContent.DATA, + FORMAT_VERSION_V4, + "s3://bucket/file.parquet", + FileFormat.PARQUET, + EMPTY_PARTITION_DATA, + RECORD_COUNT, + FILE_SIZE_IN_BYTES, + 0, + null, + null, + null, + null, + null, + null, + null); + + InputFile manifest = writeManifest(EMPTY_PARTITION, ImmutableList.of(file)); + + Tracking actual = read(manifest, UNPARTITIONED_SPECS).get(0).tracking(); + // scan-relevant tracking fields are projected + assertThat(actual.status()).isEqualTo(EntryStatus.ADDED); + assertThat(actual.snapshotId()).isEqualTo(SNAPSHOT_ID); + assertThat(actual.dataSequenceNumber()).isEqualTo(5L); + assertThat(actual.fileSequenceNumber()).isEqualTo(6L); + assertThat(actual.firstRowId()).isEqualTo(8L); + // change-tracking fields are omitted from the scan projection + assertThat(actual.dvSnapshotId()).isNull(); + assertThat(actual.deletedPositions()).isNull(); + assertThat(actual.replacedPositions()).isNull(); + } + + @TestTemplate + public void testPartitionFilterForceProjectsFilterFields() throws IOException { + TrackedFile keep = dataFile("keep.parquet", partition(1)); + TrackedFile prune = dataFile("prune.parquet", partition(2)); + + InputFile manifest = writeManifest(PARTITION_TYPE, ImmutableList.of(keep, prune)); + + // the caller projects only location; the reader must still project the fields the partition + // filter reads (content_type, spec_id, partition) or every row would be pruned + Schema projection = new Schema(TrackedFile.LOCATION); + try (V4ManifestReader reader = + newReader(manifest, PARTITIONED_SPECS) + .project(projection) + .filterRows(Expressions.equal("id", 1)) + .build()) { + assertThat(reader.allFiles()) + .extracting(TrackedFile::location) + .containsExactly(keep.location()); + } + } + @TestTemplate public void testUnpartitioned() throws IOException { TrackedFile file = dataFile("s3://bucket/file.parquet", EMPTY_PARTITION_DATA); @@ -414,7 +482,7 @@ public void testUnknownManifestFormatThrows() throws IOException { try (V4ManifestReader reader = newReader(badFile, UNPARTITIONED_SPECS).build()) { assertThatThrownBy(reader::allFiles) .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining("Unable to determine format of manifest"); + .hasMessageContaining("Cannot determine format of manifest"); } } From cb918d69ea940145d23dabd35a14f168452057c1 Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Fri, 3 Jul 2026 10:54:17 -0700 Subject: [PATCH 08/26] Revert change --- core/src/main/java/org/apache/iceberg/TrackingStruct.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/java/org/apache/iceberg/TrackingStruct.java b/core/src/main/java/org/apache/iceberg/TrackingStruct.java index 46d03bf88a2a..8ae4b7e4ce88 100644 --- a/core/src/main/java/org/apache/iceberg/TrackingStruct.java +++ b/core/src/main/java/org/apache/iceberg/TrackingStruct.java @@ -30,7 +30,7 @@ /** Mutable {@link StructLike} implementation of {@link Tracking}. */ class TrackingStruct extends SupportsIndexProjection implements Tracking, Serializable { - static final Types.StructType BASE_TYPE = + private static final Types.StructType BASE_TYPE = Types.StructType.of( Tracking.STATUS, Tracking.SNAPSHOT_ID, From 765b5e76d088ec140011037c75efcce3a84880eb Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Mon, 13 Jul 2026 20:27:04 -0700 Subject: [PATCH 09/26] Review feedback --- .../java/org/apache/iceberg/Partitioning.java | 22 +- .../java/org/apache/iceberg/TrackedFile.java | 51 +++-- .../org/apache/iceberg/TrackedFileStruct.java | 5 +- .../org/apache/iceberg/V4ManifestReader.java | 202 ++++++++++-------- .../org/apache/iceberg/TestPartitioning.java | 19 ++ .../org/apache/iceberg/TestTrackedFile.java | 14 ++ .../iceberg/TestTrackedFileAdapters.java | 7 + .../apache/iceberg/TestTrackedFileStruct.java | 9 +- .../apache/iceberg/TestV4ManifestReader.java | 81 +++---- 9 files changed, 247 insertions(+), 163 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/Partitioning.java b/core/src/main/java/org/apache/iceberg/Partitioning.java index 7cfb0bef9beb..ee47a92e112c 100644 --- a/core/src/main/java/org/apache/iceberg/Partitioning.java +++ b/core/src/main/java/org/apache/iceberg/Partitioning.java @@ -238,18 +238,20 @@ public static StructType groupingKeyType(Schema schema, Collection specs = table.specs().values(); + return buildPartitionProjectionType( + "table partition", specs, allActiveFieldIds(table.schema(), specs)); } /** - * Builds a unified partition type from a schema and its specs, unioning every partition field - * whose source column is present in the schema. + * Builds a unified partition type containing all partition fields from the given specs, including + * fields whose source columns are no longer present in the table schema. * - * @param schema the schema used to determine which partition fields are active * @param specs the partition specs to unify + * @return the constructed unified partition type */ - static StructType partitionType(Schema schema, Collection specs) { - return buildPartitionProjectionType("table partition", specs, allActiveFieldIds(schema, specs)); + static StructType partitionType(Collection specs) { + return buildPartitionProjectionType("table partition", specs, allFieldIds(specs)); } /** @@ -356,6 +358,14 @@ private static boolean compatibleTransforms(Transform t1, Transform || t2.equals(Transforms.alwaysNull()); } + // collects IDs of all partition fields used across specs + private static Set allFieldIds(Collection specs) { + return FluentIterable.from(specs) + .transformAndConcat(PartitionSpec::fields) + .transform(PartitionField::fieldId) + .toSet(); + } + // collects IDs of all partition field used across specs that are in the current schema private static Set allActiveFieldIds(Schema schema, Collection specs) { return FluentIterable.from(specs) diff --git a/core/src/main/java/org/apache/iceberg/TrackedFile.java b/core/src/main/java/org/apache/iceberg/TrackedFile.java index 9aaeae48d688..0be1c7a9262b 100644 --- a/core/src/main/java/org/apache/iceberg/TrackedFile.java +++ b/core/src/main/java/org/apache/iceberg/TrackedFile.java @@ -22,6 +22,7 @@ import java.util.Collections; import java.util.List; import java.util.Set; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.types.Types; /** A file tracked by a manifest. */ @@ -95,26 +96,40 @@ interface TrackedFile { Types.ListType.ofRequired(136, Types.IntegerType.get()), "Field ids used to determine row equality in equality delete files"); + /** + * Returns the schema for the given partition and content stats types. + * + *

The partition and content stats fields are omitted when their types have no fields. + */ static Types.StructType schemaWithContentStats( Types.StructType partitionType, Types.StructType contentStatsType) { - return Types.StructType.of( - TRACKING, - CONTENT_TYPE, - FORMAT_VERSION, - LOCATION, - FILE_FORMAT, - RECORD_COUNT, - FILE_SIZE_IN_BYTES, - SPEC_ID, - Types.NestedField.optional(PARTITION_ID, PARTITION_NAME, partitionType, PARTITION_DOC), - Types.NestedField.optional( - CONTENT_STATS_ID, CONTENT_STATS_NAME, contentStatsType, CONTENT_STATS_DOC), - SORT_ORDER_ID, - DELETION_VECTOR, - MANIFEST_INFO, - KEY_METADATA, - SPLIT_OFFSETS, - EQUALITY_IDS); + List fields = Lists.newArrayList(); + fields.add(TRACKING); + fields.add(CONTENT_TYPE); + fields.add(FORMAT_VERSION); + fields.add(LOCATION); + fields.add(FILE_FORMAT); + fields.add(RECORD_COUNT); + fields.add(FILE_SIZE_IN_BYTES); + fields.add(SPEC_ID); + if (!partitionType.fields().isEmpty()) { + fields.add( + Types.NestedField.optional(PARTITION_ID, PARTITION_NAME, partitionType, PARTITION_DOC)); + } + + if (!contentStatsType.fields().isEmpty()) { + fields.add( + Types.NestedField.optional( + CONTENT_STATS_ID, CONTENT_STATS_NAME, contentStatsType, CONTENT_STATS_DOC)); + } + + fields.add(SORT_ORDER_ID); + fields.add(DELETION_VECTOR); + fields.add(MANIFEST_INFO); + fields.add(KEY_METADATA); + fields.add(SPLIT_OFFSETS); + fields.add(EQUALITY_IDS); + return Types.StructType.of(fields); } /** Returns the tracking information for this entry. */ diff --git a/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java b/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java index 4c7e8cda3c50..188da8ea9128 100644 --- a/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java +++ b/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java @@ -34,7 +34,10 @@ class TrackedFileStruct extends SupportsIndexProjection implements TrackedFile, Serializable { private static final Types.StructType EMPTY_STRUCT_TYPE = Types.StructType.of(); - private static final Types.StructType BASE_TYPE = + // Package-private only so tests can look up positional ordinals. Unlike + // TrackedFile.schemaWithContentStats, the base layout always includes the partition and + // content_stats positions. + static final Types.StructType BASE_TYPE = Types.StructType.of( TrackedFile.TRACKING, TrackedFile.CONTENT_TYPE, diff --git a/core/src/main/java/org/apache/iceberg/V4ManifestReader.java b/core/src/main/java/org/apache/iceberg/V4ManifestReader.java index b5a45e6a11ca..4e182cf8a17f 100644 --- a/core/src/main/java/org/apache/iceberg/V4ManifestReader.java +++ b/core/src/main/java/org/apache/iceberg/V4ManifestReader.java @@ -21,7 +21,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; import org.apache.iceberg.expressions.Evaluator; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.expressions.Expressions; @@ -35,14 +34,13 @@ import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types; import org.apache.iceberg.util.StructProjection; /** Reader that reads a v4+ manifest file as {@link TrackedFile}s. */ class V4ManifestReader extends CloseableGroup implements CloseableIterable { - // Tracking fields read on the scan path. Omits the change-tracking fields (dv_snapshot_id, - // deleted_positions, replaced_positions) that a scan does not need. row_position backs - // Tracking.manifestPos. + // tracking fields read on the scan path; row_position backs Tracking.manifestPos private static final Types.StructType SCAN_TRACKING = Types.StructType.of( Tracking.STATUS, @@ -53,51 +51,42 @@ class V4ManifestReader extends CloseableGroup implements CloseableIterable partitionEvaluators; private final Map partitionProjections; private V4ManifestReader( InputFile file, - Types.StructType partitionType, + Schema readSchema, Map partitionEvaluators, Map partitionProjections, - Schema fileProjection, + boolean onlyLive, + boolean reuseContainers, ScanMetrics scanMetrics) { this.file = file; - this.partitionType = partitionType; + this.readSchema = readSchema; this.partitionEvaluators = partitionEvaluators; this.partitionProjections = partitionProjections; - this.fileProjection = fileProjection; + this.onlyLive = onlyLive; + this.reuseContainers = reuseContainers; this.scanMetrics = scanMetrics; } - static Builder builder( - InputFile file, Schema tableSchema, Map specsById) { - return new Builder(file, tableSchema, specsById); + static Builder builder(InputFile file, Map specsById) { + return new Builder(file, specsById); } - /** Returns all tracked files in this manifest, regardless of status. */ - CloseableIterable allFiles() { - return files(false /* all files */); - } - - /** Returns tracked files whose tracking {@link Tracking#isLive() is live}. */ - CloseableIterable liveFiles() { - return files(true /* only live files */); - } - - /** Returns live tracked files. Makes defensive copies before returning. */ + /** + * Returns tracked files that match this reader's configured filters. Files are copied unless the + * reader was built with {@link Builder#reuseContainers()}. + */ @Override public CloseableIterator iterator() { - return CloseableIterable.transform(liveFiles(), TrackedFile::copy).iterator(); - } - - private CloseableIterable files(boolean onlyLive) { CloseableIterable entries = CloseableIterable.transform(open(), this::prepare); if (!partitionEvaluators.isEmpty()) { entries = CloseableIterable.filter(entries, this::matchesPartition); @@ -107,7 +96,11 @@ private CloseableIterable files(boolean onlyLive) { entries = CloseableIterable.filter(entries, entry -> entry.tracking().isLive()); } - return entries; + if (reuseContainers) { + return entries.iterator(); + } + + return CloseableIterable.transform(entries, TrackedFile::copy).iterator(); } private boolean matchesPartition(TrackedFile trackedFile) { @@ -129,16 +122,31 @@ private boolean matchesPartition(TrackedFile trackedFile) { boolean matches = evaluator.eval(projection.wrap(trackedFile.partition())); if (!matches) { - if (content == FileContent.DATA) { - scanMetrics.skippedDataFiles().increment(); - } else { - scanMetrics.skippedDeleteFiles().increment(); - } + incrementSkipCount(content); } return matches; } + private void incrementSkipCount(FileContent content) { + switch (content) { + case DATA: + scanMetrics.skippedDataFiles().increment(); + break; + case EQUALITY_DELETES: + scanMetrics.skippedDeleteFiles().increment(); + break; + case DATA_MANIFEST: + scanMetrics.skippedDataManifests().increment(); + break; + case DELETE_MANIFEST: + scanMetrics.skippedDeleteManifests().increment(); + break; + default: + throw new UnsupportedOperationException("Unsupported content type: " + content); + } + } + private CloseableIterable open() { FileFormat format = FileFormat.fromFileName(file.location()); Preconditions.checkArgument( @@ -146,7 +154,7 @@ private CloseableIterable open() { CloseableIterable reader = InternalData.read(format, file) - .project(readSchema()) + .project(readSchema) .setRootType(TrackedFileStruct.class) .setCustomType(TrackedFile.TRACKING.fieldId(), TrackingStruct.class) .setCustomType(TrackedFile.DELETION_VECTOR.fieldId(), DeletionVectorStruct.class) @@ -160,8 +168,7 @@ private CloseableIterable open() { private TrackedFile prepare(TrackedFile trackedFile) { Tracking tracking = trackedFile.tracking(); - // manifestLocation is not stored in the manifest; the reader fills it from the file location. - // manifestPos is filled from ROW_POSITION while reading the tracking struct. + // manifestLocation is not stored in the manifest; the reader fills it in if (tracking instanceof TrackingStruct) { ((TrackingStruct) tracking).setManifestLocation(file.location()); } @@ -169,68 +176,20 @@ private TrackedFile prepare(TrackedFile trackedFile) { return trackedFile; } - private Schema readSchema() { - Types.StructType fullType = - TrackedFile.schemaWithContentStats(partitionType, Types.StructType.of()); - boolean unpartitioned = partitionType.fields().isEmpty(); - - Set projectedIds = null; - if (fileProjection != null) { - projectedIds = - fileProjection.asStruct().fields().stream() - .map(Types.NestedField::fieldId) - .collect(Collectors.toCollection(Sets::newHashSet)); - - // Always project tracking and content type. status drives live-file filtering, and content - // type distinguishes data, delete, and manifest-reference entries. - projectedIds.add(TrackedFile.TRACKING.fieldId()); - projectedIds.add(TrackedFile.CONTENT_TYPE.fieldId()); - - // Force-project the remaining fields the partition filter reads, regardless of caller - // projection. - if (!partitionEvaluators.isEmpty()) { - projectedIds.add(TrackedFile.SPEC_ID.fieldId()); - projectedIds.add(TrackedFile.PARTITION_ID); - } - } - - List fields = Lists.newArrayList(); - for (Types.NestedField field : fullType.fields()) { - if (projectedIds != null && !projectedIds.contains(field.fieldId())) { - continue; - } - - if (field.fieldId() == TrackedFile.TRACKING.fieldId()) { - fields.add( - Types.NestedField.required( - TrackedFile.TRACKING.fieldId(), - TrackedFile.TRACKING.name(), - SCAN_TRACKING, - TrackedFile.TRACKING.doc())); - } else if (field.fieldId() == TrackedFile.CONTENT_STATS_ID) { - // content_stats are not projected yet - } else if (field.fieldId() == TrackedFile.PARTITION_ID && unpartitioned) { - // unpartitioned manifests omit the partition field - } else { - fields.add(field); - } - } - - return new Schema(fields); - } - static class Builder { private final InputFile file; private final Types.StructType partitionType; private final Map specsById; private Expression rowFilter = Expressions.alwaysTrue(); private boolean caseSensitive = true; + private boolean onlyLive = false; + private boolean reuseContainers = false; private Schema fileProjection = null; private ScanMetrics scanMetrics = ScanMetrics.noop(); - private Builder(InputFile file, Schema tableSchema, Map specsById) { + private Builder(InputFile file, Map specsById) { this.file = file; - this.partitionType = Partitioning.partitionType(tableSchema, specsById.values()); + this.partitionType = Partitioning.partitionType(specsById.values()); this.specsById = specsById; } @@ -246,6 +205,21 @@ Builder caseSensitive(boolean isCaseSensitive) { return this; } + /** Returns only files whose tracking {@link Tracking#isLive() is live}. */ + Builder liveOnly() { + this.onlyLive = true; + return this; + } + + /** + * Reuses file instances while iterating; each file is valid only until the iterator advances. + * Callers must {@link TrackedFile#copy() copy} files that are retained. + */ + Builder reuseContainers() { + this.reuseContainers = true; + return this; + } + Builder project(Schema newFileProjection) { this.fileProjection = newFileProjection; return this; @@ -260,7 +234,7 @@ Builder scanMetrics(ScanMetrics newScanMetrics) { V4ManifestReader build() { Map partitionEvaluators = Maps.newHashMap(); Map partitionProjections = Maps.newHashMap(); - if (rowFilter != Expressions.alwaysTrue() && !partitionType.fields().isEmpty()) { + if (hasPartitionFilter()) { for (PartitionSpec spec : specsById.values()) { Expression partFilter = Projections.inclusive(spec, caseSensitive).project(rowFilter); partitionEvaluators.put( @@ -272,11 +246,53 @@ V4ManifestReader build() { return new V4ManifestReader( file, - partitionType, + readSchema(), partitionEvaluators, partitionProjections, - fileProjection, + onlyLive, + reuseContainers, scanMetrics); } + + private boolean hasPartitionFilter() { + return rowFilter != Expressions.alwaysTrue() && !partitionType.fields().isEmpty(); + } + + private Schema readSchema() { + Types.StructType fullType = + TrackedFile.schemaWithContentStats(partitionType, Types.StructType.of()); + + // replace tracking with the subset of fields read on the scan path + List fields = Lists.newArrayList(); + for (Types.NestedField field : fullType.fields()) { + if (field.fieldId() == TrackedFile.TRACKING.fieldId()) { + fields.add( + Types.NestedField.required( + field.fieldId(), field.name(), SCAN_TRACKING, field.doc())); + } else { + fields.add(field); + } + } + + Schema fullSchema = new Schema(fields); + if (fileProjection == null) { + return fullSchema; + } + + Set projectedIds = Sets.newHashSet(); + for (Types.NestedField field : fileProjection.asStruct().fields()) { + projectedIds.add(field.fieldId()); + } + + // status drives live-file filtering and content type distinguishes entry kinds + projectedIds.add(TrackedFile.TRACKING.fieldId()); + projectedIds.add(TrackedFile.CONTENT_TYPE.fieldId()); + if (hasPartitionFilter()) { + projectedIds.add(TrackedFile.SPEC_ID.fieldId()); + projectedIds.add(TrackedFile.PARTITION_ID); + } + + return TypeUtil.select(fullSchema, projectedIds); + } } } diff --git a/core/src/test/java/org/apache/iceberg/TestPartitioning.java b/core/src/test/java/org/apache/iceberg/TestPartitioning.java index dc362d33c335..577750fbb948 100644 --- a/core/src/test/java/org/apache/iceberg/TestPartitioning.java +++ b/core/src/test/java/org/apache/iceberg/TestPartitioning.java @@ -211,6 +211,25 @@ public void testPartitionTypeIgnoreInactiveFields() { assertThat(actualType).isEqualTo(StructType.of()); } + @Test + public void testPartitionTypeFromSpecsRetainsDroppedSourceFields() { + TestTables.TestTable table = + TestTables.create( + tableDir, "test", SCHEMA, BY_DATA_CATEGORY_BUCKET_SPEC, V2_FORMAT_VERSION); + + table.updateSpec().removeField("category_bucket").commit(); + table.updateSchema().deleteColumn("category").commit(); + + // fields with dropped source columns are retained to preserve partition tuple equality; + // their type is unknown because it cannot be determined without the source column + StructType actualType = Partitioning.partitionType(table.specs().values()); + assertThat(actualType) + .isEqualTo( + StructType.of( + NestedField.optional(1000, "data", Types.StringType.get()), + NestedField.optional(1001, "category_bucket", Types.UnknownType.get()))); + } + @Test public void testGroupingKeyTypeWithSpecEvolutionInV1Tables() { TestTables.TestTable table = diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFile.java b/core/src/test/java/org/apache/iceberg/TestTrackedFile.java index e5c3ba8a247c..415bc6b4323d 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackedFile.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackedFile.java @@ -117,4 +117,18 @@ public void schemaWithContentStatsPartitionIsOptional() { assertThat(partitionField.name()).isEqualTo(TrackedFile.PARTITION_NAME); assertThat(partitionField.doc()).isEqualTo(TrackedFile.PARTITION_DOC); } + + @Test + public void schemaWithContentStatsOmitsEmptyStructs() { + Types.StructType type = + TrackedFile.schemaWithContentStats(Types.StructType.of(), Types.StructType.of()); + + assertThat(type.field(TrackedFile.PARTITION_ID)).isNull(); + assertThat(type.field(TrackedFile.CONTENT_STATS_ID)).isNull(); + + Types.StructType partitionedType = + TrackedFile.schemaWithContentStats(PARTITION_TYPE, Types.StructType.of()); + assertThat(partitionedType.field(TrackedFile.PARTITION_ID)).isNotNull(); + assertThat(partitionedType.field(TrackedFile.CONTENT_STATS_ID)).isNull(); + } } diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java index 6223dc0d1676..533d8d0cb0aa 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java @@ -62,6 +62,13 @@ class TestTrackedFileAdapters { // manifestPos is populated by readers using the setter with the position of the field. private static final int MANIFEST_POS_ORDINAL = Tracking.schema().fields().size(); + // TrackedFile optional field ordinals, looked up from the schema. + private static final Types.StructType TRACKED_FILE_SCHEMA = TrackedFileStruct.BASE_TYPE; + private static final int CONTENT_TYPE_ORDINAL = ordinalOf(TRACKED_FILE_SCHEMA, "content_type"); + private static final int SPEC_ID_ORDINAL = ordinalOf(TRACKED_FILE_SCHEMA, "spec_id"); + private static final int DELETION_VECTOR_ORDINAL = + ordinalOf(TRACKED_FILE_SCHEMA, "deletion_vector"); + @Test void testDataFileAdapterDelegation() { TrackingStruct tracking = diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java b/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java index 92453ff28f7c..7eeb184c1c6b 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java @@ -32,13 +32,10 @@ class TestTrackedFileStruct { private static final int FORMAT_VERSION_V4 = 4; - private static final Types.StructType PARTITION_TYPE = - Types.StructType.of( - Types.NestedField.optional(1000, "id_bucket", Types.IntegerType.get()), - Types.NestedField.optional(1001, "category", Types.StringType.get())); - private static final List FIELDS = - TrackedFile.schemaWithContentStats(PARTITION_TYPE, Types.StructType.of()).fields(); + // The struct's base layout always includes partition and content_stats positions, unlike + // TrackedFile.schemaWithContentStats which omits them when their types are empty. + private static final List FIELDS = TrackedFileStruct.BASE_TYPE.fields(); private static final Tracking TRACKING = Mockito.mock(Tracking.class); private static final Tracking TRACKING_COPY = Mockito.mock(Tracking.class); diff --git a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java index 4243a5da104e..1cab0f07b89f 100644 --- a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java +++ b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java @@ -30,7 +30,7 @@ import java.util.Locale; import java.util.Map; import org.apache.iceberg.expressions.Expressions; -import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.io.CloseableIterator; import org.apache.iceberg.io.FileAppender; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.InputFile; @@ -71,8 +71,7 @@ public class TestV4ManifestReader { private static final Map UNPARTITIONED_SPECS = ImmutableMap.of(PartitionSpec.unpartitioned().specId(), PartitionSpec.unpartitioned()); - private static final List SCHEMA_FIELDS = - TrackedFile.schemaWithContentStats(Types.StructType.of(), Types.StructType.of()).fields(); + private static final List SCHEMA_FIELDS = TrackedFileStruct.BASE_TYPE.fields(); private static final int SORT_ORDER_ID_ORDINAL = ordinalOf(TrackedFile.SORT_ORDER_ID.fieldId()); @Parameter private FileFormat format; @@ -179,7 +178,7 @@ public void testLiveFilesExcludesDeletedAndReplaced() throws IOException { InputFile manifest = writeManifest(EMPTY_PARTITION, files); try (V4ManifestReader reader = newReader(manifest, UNPARTITIONED_SPECS).build()) { - assertThat(reader.allFiles()) + assertThat(reader) .extracting(file -> file.tracking().status()) .containsExactly( EntryStatus.ADDED, @@ -187,8 +186,10 @@ public void testLiveFilesExcludesDeletedAndReplaced() throws IOException { EntryStatus.MODIFIED, EntryStatus.DELETED, EntryStatus.REPLACED); + } - assertThat(reader.liveFiles()) + try (V4ManifestReader reader = newReader(manifest, UNPARTITIONED_SPECS).liveOnly().build()) { + assertThat(reader) .extracting(file -> file.tracking().status()) .containsExactly(EntryStatus.ADDED, EntryStatus.EXISTING, EntryStatus.MODIFIED); } @@ -221,7 +222,7 @@ public void testProjectionRestrictsFields() throws IOException { Schema projection = new Schema(TrackedFile.LOCATION); try (V4ManifestReader reader = newReader(manifest, UNPARTITIONED_SPECS).project(projection).build()) { - TrackedFile actual = Lists.newArrayList(reader.allFiles()).get(0); + TrackedFile actual = Lists.newArrayList(reader).get(0); assertThat(actual.location()).isEqualTo(file.location()); // tracking and content_type are always projected, even though the caller omitted them assertThat(actual.tracking()).isNotNull(); @@ -295,9 +296,7 @@ public void testPartitionFilterForceProjectsFilterFields() throws IOException { .project(projection) .filterRows(Expressions.equal("id", 1)) .build()) { - assertThat(reader.allFiles()) - .extracting(TrackedFile::location) - .containsExactly(keep.location()); + assertThat(reader).extracting(TrackedFile::location).containsExactly(keep.location()); } } @@ -325,9 +324,7 @@ public void testPartitionFilterPrunesNonMatchingFiles() throws IOException { .filterRows(Expressions.equal("id", 1)) .scanMetrics(metrics) .build()) { - assertThat(reader.allFiles()) - .extracting(TrackedFile::location) - .containsExactly(keep.location()); + assertThat(reader).extracting(TrackedFile::location).containsExactly(keep.location()); } assertThat(metrics.skippedDataFiles().value()).isEqualTo(1L); @@ -362,7 +359,7 @@ public void testPartitionFilterCountsSkippedDeleteFiles() throws IOException { .filterRows(Expressions.equal("id", 1)) .scanMetrics(metrics) .build()) { - assertThat(reader.allFiles()).isEmpty(); + assertThat(reader).isEmpty(); } assertThat(metrics.skippedDeleteFiles().value()).isEqualTo(1L); @@ -397,7 +394,7 @@ public void testPartitionFilterKeepsManifestReferences() throws IOException { try (V4ManifestReader reader = newReader(manifest, PARTITIONED_SPECS).filterRows(Expressions.equal("id", 1)).build()) { - assertThat(reader.allFiles()) + assertThat(reader) .extracting(TrackedFile::location) .containsExactlyInAnyOrder(keep.location(), manifestRef.location()); } @@ -415,9 +412,7 @@ public void testCaseInsensitivePartitionFilter() throws IOException { .filterRows(Expressions.equal("ID", 1)) .caseSensitive(false) .build()) { - assertThat(reader.allFiles()) - .extracting(TrackedFile::location) - .containsExactly(keep.location()); + assertThat(reader).extracting(TrackedFile::location).containsExactly(keep.location()); } } @@ -431,7 +426,7 @@ public void testMultiSpecPartitionPruning() throws IOException { .add(2, 1001, "data", Transforms.identity()) .build(); Map specsById = ImmutableMap.of(0, spec0, 1, spec1); - Types.StructType unionType = Partitioning.partitionType(TABLE_SCHEMA, specsById.values()); + Types.StructType unionType = Partitioning.partitionType(specsById.values()); TrackedFile keepById = dataFile("spec0-id1.parquet", unionPartition(unionType, 1, null), 0); TrackedFile prunedById = dataFile("spec0-id2.parquet", unionPartition(unionType, 2, null), 0); @@ -442,11 +437,11 @@ public void testMultiSpecPartitionPruning() throws IOException { writeManifest(unionType, ImmutableList.of(keepById, prunedById, keptOtherSpec)); try (V4ManifestReader reader = - V4ManifestReader.builder(manifest, TABLE_SCHEMA, specsById) + V4ManifestReader.builder(manifest, specsById) .filterRows(Expressions.equal("id", 1)) .build()) { // spec0 entries are pruned by id; the spec1 entry is not partitioned by id so it survives - assertThat(reader.allFiles()) + assertThat(reader) .extracting(TrackedFile::location) .containsExactlyInAnyOrder(keepById.location(), keptOtherSpec.location()); } @@ -462,7 +457,7 @@ public void testIteratorReturnsLiveCopies() throws IOException { InputFile manifest = writeManifest(EMPTY_PARTITION, files); - try (V4ManifestReader reader = newReader(manifest, UNPARTITIONED_SPECS).build()) { + try (V4ManifestReader reader = newReader(manifest, UNPARTITIONED_SPECS).liveOnly().build()) { List read = Lists.newArrayList(reader); assertThat(read) .hasSize(2) @@ -474,13 +469,32 @@ public void testIteratorReturnsLiveCopies() throws IOException { } } + @TestTemplate + public void testReuseContainersReturnsReusedInstances() throws IOException { + TrackedFile file1 = dataFile("s3://bucket/file-1.parquet", EMPTY_PARTITION_DATA); + TrackedFile file2 = dataFile("s3://bucket/file-2.parquet", EMPTY_PARTITION_DATA); + + InputFile manifest = writeManifest(EMPTY_PARTITION, ImmutableList.of(file1, file2)); + + try (V4ManifestReader reader = + newReader(manifest, UNPARTITIONED_SPECS).reuseContainers().build()) { + CloseableIterator files = reader.iterator(); + TrackedFile first = files.next(); + assertThat(first.location()).isEqualTo(file1.location()); + + TrackedFile second = files.next(); + assertThat(second).isSameAs(first); + assertThat(second.location()).isEqualTo(file2.location()); + } + } + @TestTemplate public void testUnknownManifestFormatThrows() throws IOException { InputFile badFile = fileIO.newInputFile(tempDir.resolve("manifest-" + System.nanoTime() + ".txt").toString()); try (V4ManifestReader reader = newReader(badFile, UNPARTITIONED_SPECS).build()) { - assertThatThrownBy(reader::allFiles) + assertThatThrownBy(reader::iterator) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("Cannot determine format of manifest"); } @@ -495,7 +509,7 @@ public void testFileWithUnknownSpecThrows() throws IOException { try (V4ManifestReader reader = newReader(manifest, PARTITIONED_SPECS).filterRows(Expressions.equal("id", 1)).build()) { - assertThatThrownBy(() -> Lists.newArrayList(reader.allFiles())) + assertThatThrownBy(() -> Lists.newArrayList(reader)) .isInstanceOf(IllegalStateException.class) .hasMessageContaining("not one of the known specs"); } @@ -575,19 +589,9 @@ private static PartitionData unionPartition(Types.StructType unionType, Integer private InputFile writeManifest(Types.StructType partitionType, Iterable files) throws IOException { - // Parquet cannot write empty groups, so v4 writers omit the partition and content_stats fields - // entirely when they would be empty (unpartitioned tables, no stats). - List writeFields = Lists.newArrayList(); - for (Types.NestedField field : - TrackedFile.schemaWithContentStats(partitionType, Types.StructType.of()).fields()) { - if (field.type().isStructType() && field.type().asStructType().fields().isEmpty()) { - continue; - } - - writeFields.add(field); - } - - Schema writeSchema = new Schema(writeFields); + Schema writeSchema = + new Schema( + TrackedFile.schemaWithContentStats(partitionType, Types.StructType.of()).fields()); OutputFile out = fileIO.newOutputFile( tempDir @@ -635,14 +639,13 @@ public void set(int pos, T value) { private V4ManifestReader.Builder newReader( InputFile manifest, Map specsById) { - return V4ManifestReader.builder(manifest, TABLE_SCHEMA, specsById); + return V4ManifestReader.builder(manifest, specsById); } private List read(InputFile manifest, Map specsById) throws IOException { - // allFiles() returns reused instances, so copy each entry before collecting. try (V4ManifestReader reader = newReader(manifest, specsById).build()) { - return Lists.newArrayList(CloseableIterable.transform(reader.allFiles(), TrackedFile::copy)); + return Lists.newArrayList(reader); } } From 08f5888d9290a98bbb09b8c94971f7777e2c128a Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Tue, 14 Jul 2026 10:44:03 -0700 Subject: [PATCH 10/26] Rebase with #17000 --- .../test/java/org/apache/iceberg/TestV4ManifestReader.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java index 1cab0f07b89f..591c03f3b5ee 100644 --- a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java +++ b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java @@ -307,8 +307,8 @@ public void testUnpartitioned() throws IOException { InputFile manifest = writeManifest(EMPTY_PARTITION, ImmutableList.of(file)); TrackedFile actual = read(manifest, UNPARTITIONED_SPECS).get(0); - assertThat(actual.partition()).isNotNull(); - assertThat(actual.partition().size()).isEqualTo(0); + // unpartitioned manifests omit the partition field, which is read as null + assertThat(actual.partition()).isNull(); } @TestTemplate From a951e11210a5ea5eb844925a3a62b05860ff19d5 Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Wed, 15 Jul 2026 09:59:25 -0700 Subject: [PATCH 11/26] PR feedback from Ryan --- .../java/org/apache/iceberg/Partitioning.java | 2 +- .../java/org/apache/iceberg/TrackedFile.java | 60 +++++++++---------- .../org/apache/iceberg/TrackedFileStruct.java | 12 ++-- .../org/apache/iceberg/V4ManifestReader.java | 50 ++++------------ .../org/apache/iceberg/TestPartitioning.java | 4 +- .../org/apache/iceberg/TestTrackedFile.java | 39 ++++++------ .../iceberg/TestTrackedFileAdapters.java | 3 +- .../apache/iceberg/TestTrackedFileStruct.java | 5 +- .../apache/iceberg/TestV4ManifestReader.java | 28 ++------- 9 files changed, 77 insertions(+), 126 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/Partitioning.java b/core/src/main/java/org/apache/iceberg/Partitioning.java index ee47a92e112c..def7827419db 100644 --- a/core/src/main/java/org/apache/iceberg/Partitioning.java +++ b/core/src/main/java/org/apache/iceberg/Partitioning.java @@ -250,7 +250,7 @@ public static StructType partitionType(Table table) { * @param specs the partition specs to unify * @return the constructed unified partition type */ - static StructType partitionType(Collection specs) { + static StructType unionPartitionTypes(Collection specs) { return buildPartitionProjectionType("table partition", specs, allFieldIds(specs)); } diff --git a/core/src/main/java/org/apache/iceberg/TrackedFile.java b/core/src/main/java/org/apache/iceberg/TrackedFile.java index 0be1c7a9262b..1cf453d03dc8 100644 --- a/core/src/main/java/org/apache/iceberg/TrackedFile.java +++ b/core/src/main/java/org/apache/iceberg/TrackedFile.java @@ -22,7 +22,7 @@ import java.util.Collections; import java.util.List; import java.util.Set; -import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; /** A file tracked by a manifest. */ @@ -99,37 +99,37 @@ interface TrackedFile { /** * Returns the schema for the given partition and content stats types. * - *

The partition and content stats fields are omitted when their types have no fields. + *

The partition and content stats fields use {@link Types.UnknownType} when their types have + * no fields, so that they are not stored in manifest files. */ - static Types.StructType schemaWithContentStats( + static Types.StructType schema( Types.StructType partitionType, Types.StructType contentStatsType) { - List fields = Lists.newArrayList(); - fields.add(TRACKING); - fields.add(CONTENT_TYPE); - fields.add(FORMAT_VERSION); - fields.add(LOCATION); - fields.add(FILE_FORMAT); - fields.add(RECORD_COUNT); - fields.add(FILE_SIZE_IN_BYTES); - fields.add(SPEC_ID); - if (!partitionType.fields().isEmpty()) { - fields.add( - Types.NestedField.optional(PARTITION_ID, PARTITION_NAME, partitionType, PARTITION_DOC)); - } - - if (!contentStatsType.fields().isEmpty()) { - fields.add( - Types.NestedField.optional( - CONTENT_STATS_ID, CONTENT_STATS_NAME, contentStatsType, CONTENT_STATS_DOC)); - } - - fields.add(SORT_ORDER_ID); - fields.add(DELETION_VECTOR); - fields.add(MANIFEST_INFO); - fields.add(KEY_METADATA); - fields.add(SPLIT_OFFSETS); - fields.add(EQUALITY_IDS); - return Types.StructType.of(fields); + return Types.StructType.of( + TRACKING, + CONTENT_TYPE, + FORMAT_VERSION, + LOCATION, + FILE_FORMAT, + RECORD_COUNT, + FILE_SIZE_IN_BYTES, + SPEC_ID, + Types.NestedField.optional( + PARTITION_ID, PARTITION_NAME, typeOrUnknown(partitionType), PARTITION_DOC), + Types.NestedField.optional( + CONTENT_STATS_ID, + CONTENT_STATS_NAME, + typeOrUnknown(contentStatsType), + CONTENT_STATS_DOC), + SORT_ORDER_ID, + DELETION_VECTOR, + MANIFEST_INFO, + KEY_METADATA, + SPLIT_OFFSETS, + EQUALITY_IDS); + } + + private static Type typeOrUnknown(Types.StructType structType) { + return structType.fields().isEmpty() ? Types.UnknownType.get() : structType; } /** Returns the tracking information for this entry. */ diff --git a/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java b/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java index 188da8ea9128..b49c52c8d244 100644 --- a/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java +++ b/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java @@ -34,10 +34,7 @@ class TrackedFileStruct extends SupportsIndexProjection implements TrackedFile, Serializable { private static final Types.StructType EMPTY_STRUCT_TYPE = Types.StructType.of(); - // Package-private only so tests can look up positional ordinals. Unlike - // TrackedFile.schemaWithContentStats, the base layout always includes the partition and - // content_stats positions. - static final Types.StructType BASE_TYPE = + private static final Types.StructType BASE_TYPE = Types.StructType.of( TrackedFile.TRACKING, TrackedFile.CONTENT_TYPE, @@ -86,10 +83,11 @@ class TrackedFileStruct extends SupportsIndexProjection implements TrackedFile, /** Used by internal readers to instantiate this class with a projection schema. */ TrackedFileStruct(Types.StructType projection) { super(BASE_TYPE, projection); - // partition type may be null if the field was not projected + // partition type may be null if the field was not projected, or unknown for unpartitioned + // manifests Type partType = projection.fieldType("partition"); - if (partType != null) { - this.partitionData = new PartitionData(partType.asNestedType().asStructType()); + if (partType != null && partType.isStructType()) { + this.partitionData = new PartitionData(partType.asStructType()); } } diff --git a/core/src/main/java/org/apache/iceberg/V4ManifestReader.java b/core/src/main/java/org/apache/iceberg/V4ManifestReader.java index 4e182cf8a17f..cbee098ceed6 100644 --- a/core/src/main/java/org/apache/iceberg/V4ManifestReader.java +++ b/core/src/main/java/org/apache/iceberg/V4ManifestReader.java @@ -53,7 +53,6 @@ class V4ManifestReader extends CloseableGroup implements CloseableIterable partitionEvaluators, Map partitionProjections, boolean onlyLive, - boolean reuseContainers, ScanMetrics scanMetrics) { this.file = file; this.readSchema = readSchema; this.partitionEvaluators = partitionEvaluators; this.partitionProjections = partitionProjections; this.onlyLive = onlyLive; - this.reuseContainers = reuseContainers; this.scanMetrics = scanMetrics; } @@ -81,35 +78,29 @@ static Builder builder(InputFile file, Map specsById) { return new Builder(file, specsById); } - /** - * Returns tracked files that match this reader's configured filters. Files are copied unless the - * reader was built with {@link Builder#reuseContainers()}. - */ + /** Returns copies of the tracked files that match this reader's configured filters. */ @Override public CloseableIterator iterator() { CloseableIterable entries = CloseableIterable.transform(open(), this::prepare); if (!partitionEvaluators.isEmpty()) { - entries = CloseableIterable.filter(entries, this::matchesPartition); + // manifest references are expanded later and are not pruned by the partition filter + entries = + CloseableIterable.filter(entries, entry -> isManifest(entry) || matchesPartition(entry)); } if (onlyLive) { entries = CloseableIterable.filter(entries, entry -> entry.tracking().isLive()); } - if (reuseContainers) { - return entries.iterator(); - } - return CloseableIterable.transform(entries, TrackedFile::copy).iterator(); } - private boolean matchesPartition(TrackedFile trackedFile) { + private static boolean isManifest(TrackedFile trackedFile) { FileContent content = trackedFile.contentType(); - if (content == FileContent.DATA_MANIFEST || content == FileContent.DELETE_MANIFEST) { - // manifest references are expanded later and are not pruned by the partition filter - return true; - } + return content == FileContent.DATA_MANIFEST || content == FileContent.DELETE_MANIFEST; + } + private boolean matchesPartition(TrackedFile trackedFile) { Integer specId = trackedFile.specId(); Evaluator evaluator = specId != null ? partitionEvaluators.get(specId) : null; StructProjection projection = specId != null ? partitionProjections.get(specId) : null; @@ -122,7 +113,7 @@ private boolean matchesPartition(TrackedFile trackedFile) { boolean matches = evaluator.eval(projection.wrap(trackedFile.partition())); if (!matches) { - incrementSkipCount(content); + incrementSkipCount(trackedFile.contentType()); } return matches; @@ -183,13 +174,12 @@ static class Builder { private Expression rowFilter = Expressions.alwaysTrue(); private boolean caseSensitive = true; private boolean onlyLive = false; - private boolean reuseContainers = false; private Schema fileProjection = null; private ScanMetrics scanMetrics = ScanMetrics.noop(); private Builder(InputFile file, Map specsById) { this.file = file; - this.partitionType = Partitioning.partitionType(specsById.values()); + this.partitionType = Partitioning.unionPartitionTypes(specsById.values()); this.specsById = specsById; } @@ -211,15 +201,6 @@ Builder liveOnly() { return this; } - /** - * Reuses file instances while iterating; each file is valid only until the iterator advances. - * Callers must {@link TrackedFile#copy() copy} files that are retained. - */ - Builder reuseContainers() { - this.reuseContainers = true; - return this; - } - Builder project(Schema newFileProjection) { this.fileProjection = newFileProjection; return this; @@ -245,13 +226,7 @@ V4ManifestReader build() { } return new V4ManifestReader( - file, - readSchema(), - partitionEvaluators, - partitionProjections, - onlyLive, - reuseContainers, - scanMetrics); + file, readSchema(), partitionEvaluators, partitionProjections, onlyLive, scanMetrics); } private boolean hasPartitionFilter() { @@ -259,8 +234,7 @@ private boolean hasPartitionFilter() { } private Schema readSchema() { - Types.StructType fullType = - TrackedFile.schemaWithContentStats(partitionType, Types.StructType.of()); + Types.StructType fullType = TrackedFile.schema(partitionType, Types.StructType.of()); // replace tracking with the subset of fields read on the scan path List fields = Lists.newArrayList(); diff --git a/core/src/test/java/org/apache/iceberg/TestPartitioning.java b/core/src/test/java/org/apache/iceberg/TestPartitioning.java index 577750fbb948..b955455bfe6d 100644 --- a/core/src/test/java/org/apache/iceberg/TestPartitioning.java +++ b/core/src/test/java/org/apache/iceberg/TestPartitioning.java @@ -212,7 +212,7 @@ public void testPartitionTypeIgnoreInactiveFields() { } @Test - public void testPartitionTypeFromSpecsRetainsDroppedSourceFields() { + public void testUnionPartitionTypesRetainsDroppedSourceFields() { TestTables.TestTable table = TestTables.create( tableDir, "test", SCHEMA, BY_DATA_CATEGORY_BUCKET_SPEC, V2_FORMAT_VERSION); @@ -222,7 +222,7 @@ public void testPartitionTypeFromSpecsRetainsDroppedSourceFields() { // fields with dropped source columns are retained to preserve partition tuple equality; // their type is unknown because it cannot be determined without the source column - StructType actualType = Partitioning.partitionType(table.specs().values()); + StructType actualType = Partitioning.unionPartitionTypes(table.specs().values()); assertThat(actualType) .isEqualTo( StructType.of( diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFile.java b/core/src/test/java/org/apache/iceberg/TestTrackedFile.java index 415bc6b4323d..d68d1decf0f7 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackedFile.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackedFile.java @@ -38,8 +38,8 @@ public class TestTrackedFile { PartitionSpec.builderFor(TABLE_SCHEMA).identity("id").build().partitionType(); @Test - public void schemaWithContentStatsFieldOrder() { - Types.StructType type = TrackedFile.schemaWithContentStats(PARTITION_TYPE, CONTENT_STATS_TYPE); + public void schemaFieldOrder() { + Types.StructType type = TrackedFile.schema(PARTITION_TYPE, CONTENT_STATS_TYPE); List fields = type.fields(); assertThat(fields) @@ -64,8 +64,8 @@ public void schemaWithContentStatsFieldOrder() { } @Test - public void schemaWithContentStatsFieldIds() { - Types.StructType type = TrackedFile.schemaWithContentStats(PARTITION_TYPE, CONTENT_STATS_TYPE); + public void schemaFieldIds() { + Types.StructType type = TrackedFile.schema(PARTITION_TYPE, CONTENT_STATS_TYPE); List fields = type.fields(); assertThat(fields) @@ -75,8 +75,8 @@ public void schemaWithContentStatsFieldIds() { } @Test - public void schemaWithContentStatsUsesProvidedType() { - Types.StructType type = TrackedFile.schemaWithContentStats(PARTITION_TYPE, CONTENT_STATS_TYPE); + public void schemaUsesProvidedType() { + Types.StructType type = TrackedFile.schema(PARTITION_TYPE, CONTENT_STATS_TYPE); Types.NestedField contentStatsField = type.field(TrackedFile.CONTENT_STATS_ID); Types.NestedField partitionField = type.field(TrackedFile.PARTITION_ID); @@ -85,7 +85,7 @@ public void schemaWithContentStatsUsesProvidedType() { } @Test - public void schemaWithContentStatsReflectsInput() { + public void schemaReflectsInput() { Schema smallSchema = new Schema(optional(1, "id", Types.IntegerType.get())); Schema largeSchema = new Schema( @@ -96,8 +96,8 @@ public void schemaWithContentStatsReflectsInput() { Types.StructType smallStats = StatsUtil.contentStatsFor(smallSchema).type().asStructType(); Types.StructType largeStats = StatsUtil.contentStatsFor(largeSchema).type().asStructType(); - Types.StructType smallType = TrackedFile.schemaWithContentStats(PARTITION_TYPE, smallStats); - Types.StructType largeType = TrackedFile.schemaWithContentStats(PARTITION_TYPE, largeStats); + Types.StructType smallType = TrackedFile.schema(PARTITION_TYPE, smallStats); + Types.StructType largeType = TrackedFile.schema(PARTITION_TYPE, largeStats); Types.StructType smallResult = smallType.field(TrackedFile.CONTENT_STATS_ID).type().asStructType(); @@ -109,8 +109,8 @@ public void schemaWithContentStatsReflectsInput() { } @Test - public void schemaWithContentStatsPartitionIsOptional() { - Types.StructType type = TrackedFile.schemaWithContentStats(PARTITION_TYPE, CONTENT_STATS_TYPE); + public void schemaPartitionIsOptional() { + Types.StructType type = TrackedFile.schema(PARTITION_TYPE, CONTENT_STATS_TYPE); Types.NestedField partitionField = type.field(TrackedFile.PARTITION_ID); assertThat(partitionField.isOptional()).isTrue(); @@ -119,16 +119,15 @@ public void schemaWithContentStatsPartitionIsOptional() { } @Test - public void schemaWithContentStatsOmitsEmptyStructs() { - Types.StructType type = - TrackedFile.schemaWithContentStats(Types.StructType.of(), Types.StructType.of()); + public void schemaUsesUnknownForEmptyStructs() { + Types.StructType type = TrackedFile.schema(Types.StructType.of(), Types.StructType.of()); - assertThat(type.field(TrackedFile.PARTITION_ID)).isNull(); - assertThat(type.field(TrackedFile.CONTENT_STATS_ID)).isNull(); + assertThat(type.field(TrackedFile.PARTITION_ID).type()).isEqualTo(Types.UnknownType.get()); + assertThat(type.field(TrackedFile.CONTENT_STATS_ID).type()).isEqualTo(Types.UnknownType.get()); - Types.StructType partitionedType = - TrackedFile.schemaWithContentStats(PARTITION_TYPE, Types.StructType.of()); - assertThat(partitionedType.field(TrackedFile.PARTITION_ID)).isNotNull(); - assertThat(partitionedType.field(TrackedFile.CONTENT_STATS_ID)).isNull(); + Types.StructType partitionedType = TrackedFile.schema(PARTITION_TYPE, Types.StructType.of()); + assertThat(partitionedType.field(TrackedFile.PARTITION_ID).type()).isEqualTo(PARTITION_TYPE); + assertThat(partitionedType.field(TrackedFile.CONTENT_STATS_ID).type()) + .isEqualTo(Types.UnknownType.get()); } } diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java index 533d8d0cb0aa..184e58f0839e 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java @@ -63,7 +63,8 @@ class TestTrackedFileAdapters { private static final int MANIFEST_POS_ORDINAL = Tracking.schema().fields().size(); // TrackedFile optional field ordinals, looked up from the schema. - private static final Types.StructType TRACKED_FILE_SCHEMA = TrackedFileStruct.BASE_TYPE; + private static final Types.StructType TRACKED_FILE_SCHEMA = + TrackedFile.schema(Types.StructType.of(), Types.StructType.of()); private static final int CONTENT_TYPE_ORDINAL = ordinalOf(TRACKED_FILE_SCHEMA, "content_type"); private static final int SPEC_ID_ORDINAL = ordinalOf(TRACKED_FILE_SCHEMA, "spec_id"); private static final int DELETION_VECTOR_ORDINAL = diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java b/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java index 7eeb184c1c6b..e0f6e5531744 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java @@ -33,9 +33,8 @@ class TestTrackedFileStruct { private static final int FORMAT_VERSION_V4 = 4; - // The struct's base layout always includes partition and content_stats positions, unlike - // TrackedFile.schemaWithContentStats which omits them when their types are empty. - private static final List FIELDS = TrackedFileStruct.BASE_TYPE.fields(); + private static final List FIELDS = + TrackedFile.schema(Types.StructType.of(), Types.StructType.of()).fields(); private static final Tracking TRACKING = Mockito.mock(Tracking.class); private static final Tracking TRACKING_COPY = Mockito.mock(Tracking.class); diff --git a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java index 591c03f3b5ee..b1949326757b 100644 --- a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java +++ b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java @@ -30,7 +30,6 @@ import java.util.Locale; import java.util.Map; import org.apache.iceberg.expressions.Expressions; -import org.apache.iceberg.io.CloseableIterator; import org.apache.iceberg.io.FileAppender; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.InputFile; @@ -71,7 +70,8 @@ public class TestV4ManifestReader { private static final Map UNPARTITIONED_SPECS = ImmutableMap.of(PartitionSpec.unpartitioned().specId(), PartitionSpec.unpartitioned()); - private static final List SCHEMA_FIELDS = TrackedFileStruct.BASE_TYPE.fields(); + private static final List SCHEMA_FIELDS = + TrackedFile.schema(Types.StructType.of(), Types.StructType.of()).fields(); private static final int SORT_ORDER_ID_ORDINAL = ordinalOf(TrackedFile.SORT_ORDER_ID.fieldId()); @Parameter private FileFormat format; @@ -426,7 +426,7 @@ public void testMultiSpecPartitionPruning() throws IOException { .add(2, 1001, "data", Transforms.identity()) .build(); Map specsById = ImmutableMap.of(0, spec0, 1, spec1); - Types.StructType unionType = Partitioning.partitionType(specsById.values()); + Types.StructType unionType = Partitioning.unionPartitionTypes(specsById.values()); TrackedFile keepById = dataFile("spec0-id1.parquet", unionPartition(unionType, 1, null), 0); TrackedFile prunedById = dataFile("spec0-id2.parquet", unionPartition(unionType, 2, null), 0); @@ -469,25 +469,6 @@ public void testIteratorReturnsLiveCopies() throws IOException { } } - @TestTemplate - public void testReuseContainersReturnsReusedInstances() throws IOException { - TrackedFile file1 = dataFile("s3://bucket/file-1.parquet", EMPTY_PARTITION_DATA); - TrackedFile file2 = dataFile("s3://bucket/file-2.parquet", EMPTY_PARTITION_DATA); - - InputFile manifest = writeManifest(EMPTY_PARTITION, ImmutableList.of(file1, file2)); - - try (V4ManifestReader reader = - newReader(manifest, UNPARTITIONED_SPECS).reuseContainers().build()) { - CloseableIterator files = reader.iterator(); - TrackedFile first = files.next(); - assertThat(first.location()).isEqualTo(file1.location()); - - TrackedFile second = files.next(); - assertThat(second).isSameAs(first); - assertThat(second.location()).isEqualTo(file2.location()); - } - } - @TestTemplate public void testUnknownManifestFormatThrows() throws IOException { InputFile badFile = @@ -590,8 +571,7 @@ private static PartitionData unionPartition(Types.StructType unionType, Integer private InputFile writeManifest(Types.StructType partitionType, Iterable files) throws IOException { Schema writeSchema = - new Schema( - TrackedFile.schemaWithContentStats(partitionType, Types.StructType.of()).fields()); + new Schema(TrackedFile.schema(partitionType, Types.StructType.of()).fields()); OutputFile out = fileIO.newOutputFile( tempDir From f0925d6d72a95c65f0f513e17f5e095700686752 Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Thu, 16 Jul 2026 12:45:11 -0700 Subject: [PATCH 12/26] Rebase from main --- .../java/org/apache/iceberg/TestTrackedFileAdapters.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java index 184e58f0839e..6223dc0d1676 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java @@ -62,14 +62,6 @@ class TestTrackedFileAdapters { // manifestPos is populated by readers using the setter with the position of the field. private static final int MANIFEST_POS_ORDINAL = Tracking.schema().fields().size(); - // TrackedFile optional field ordinals, looked up from the schema. - private static final Types.StructType TRACKED_FILE_SCHEMA = - TrackedFile.schema(Types.StructType.of(), Types.StructType.of()); - private static final int CONTENT_TYPE_ORDINAL = ordinalOf(TRACKED_FILE_SCHEMA, "content_type"); - private static final int SPEC_ID_ORDINAL = ordinalOf(TRACKED_FILE_SCHEMA, "spec_id"); - private static final int DELETION_VECTOR_ORDINAL = - ordinalOf(TRACKED_FILE_SCHEMA, "deletion_vector"); - @Test void testDataFileAdapterDelegation() { TrackingStruct tracking = From f4a75b2479340a3cc8464c44ad4fdf2f2f041aac Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Mon, 20 Jul 2026 14:00:19 -0700 Subject: [PATCH 13/26] PR feedback --- .../java/org/apache/iceberg/TrackedFile.java | 11 +- .../java/org/apache/iceberg/Tracking.java | 11 + .../org/apache/iceberg/V4ManifestReader.java | 118 ++++---- .../apache/iceberg/TestV4ManifestReader.java | 259 ++++++++++++------ 4 files changed, 264 insertions(+), 135 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/TrackedFile.java b/core/src/main/java/org/apache/iceberg/TrackedFile.java index 1cf453d03dc8..fd73646e9fc5 100644 --- a/core/src/main/java/org/apache/iceberg/TrackedFile.java +++ b/core/src/main/java/org/apache/iceberg/TrackedFile.java @@ -104,8 +104,17 @@ interface TrackedFile { */ static Types.StructType schema( Types.StructType partitionType, Types.StructType contentStatsType) { + return schema(Tracking.schema(), partitionType, contentStatsType); + } + + /** Returns the schema with the given tracking, partition, and content stats types. */ + static Types.StructType schema( + Types.StructType trackingType, + Types.StructType partitionType, + Types.StructType contentStatsType) { return Types.StructType.of( - TRACKING, + Types.NestedField.required( + TRACKING.fieldId(), TRACKING.name(), trackingType, TRACKING.doc()), CONTENT_TYPE, FORMAT_VERSION, LOCATION, diff --git a/core/src/main/java/org/apache/iceberg/Tracking.java b/core/src/main/java/org/apache/iceberg/Tracking.java index fcdc4e50b236..7b415cfb049d 100644 --- a/core/src/main/java/org/apache/iceberg/Tracking.java +++ b/core/src/main/java/org/apache/iceberg/Tracking.java @@ -78,6 +78,17 @@ static Types.StructType schema() { REPLACED_POSITIONS); } + /** Returns the tracking fields read on the scan path; row_position backs {@link #manifestPos}. */ + static Types.StructType scanSchema() { + return Types.StructType.of( + STATUS, + SNAPSHOT_ID, + SEQUENCE_NUMBER, + FILE_SEQUENCE_NUMBER, + FIRST_ROW_ID, + MetadataColumns.ROW_POSITION); + } + /** Returns the status of the entry. */ EntryStatus status(); diff --git a/core/src/main/java/org/apache/iceberg/V4ManifestReader.java b/core/src/main/java/org/apache/iceberg/V4ManifestReader.java index cbee098ceed6..1c9a9e1ad615 100644 --- a/core/src/main/java/org/apache/iceberg/V4ManifestReader.java +++ b/core/src/main/java/org/apache/iceberg/V4ManifestReader.java @@ -18,7 +18,7 @@ */ package org.apache.iceberg; -import java.util.List; +import java.util.Collection; import java.util.Map; import java.util.Set; import org.apache.iceberg.expressions.Evaluator; @@ -31,7 +31,6 @@ import org.apache.iceberg.io.InputFile; import org.apache.iceberg.metrics.ScanMetrics; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; -import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.relocated.com.google.common.collect.Sets; import org.apache.iceberg.types.TypeUtil; @@ -40,16 +39,6 @@ /** Reader that reads a v4+ manifest file as {@link TrackedFile}s. */ class V4ManifestReader extends CloseableGroup implements CloseableIterable { - // tracking fields read on the scan path; row_position backs Tracking.manifestPos - private static final Types.StructType SCAN_TRACKING = - Types.StructType.of( - Tracking.STATUS, - Tracking.SNAPSHOT_ID, - Tracking.SEQUENCE_NUMBER, - Tracking.FILE_SEQUENCE_NUMBER, - Tracking.FIRST_ROW_ID, - MetadataColumns.ROW_POSITION); - private final InputFile file; private final Schema readSchema; private final boolean onlyLive; @@ -95,20 +84,24 @@ public CloseableIterator iterator() { return CloseableIterable.transform(entries, TrackedFile::copy).iterator(); } - private static boolean isManifest(TrackedFile trackedFile) { - FileContent content = trackedFile.contentType(); - return content == FileContent.DATA_MANIFEST || content == FileContent.DELETE_MANIFEST; - } - private boolean matchesPartition(TrackedFile trackedFile) { Integer specId = trackedFile.specId(); - Evaluator evaluator = specId != null ? partitionEvaluators.get(specId) : null; - StructProjection projection = specId != null ? partitionProjections.get(specId) : null; + if (specId == null) { + // a file without a spec is not partitioned and may match the filter + return true; + } + + Evaluator evaluator = partitionEvaluators.get(specId); + if (evaluator == null) { + // the row filter does not project to a partition filter for this spec + return true; + } + + StructProjection projection = partitionProjections.get(specId); Preconditions.checkState( - evaluator != null && projection != null, - "Cannot apply partition filter: spec ID %s is not one of the known specs %s in manifest %s", + projection != null, + "Cannot produce partition tuple for spec ID %s in manifest %s", specId, - partitionEvaluators.keySet(), file.location()); boolean matches = evaluator.eval(projection.wrap(trackedFile.partition())); @@ -167,19 +160,25 @@ private TrackedFile prepare(TrackedFile trackedFile) { return trackedFile; } + private static boolean isManifest(TrackedFile trackedFile) { + FileContent content = trackedFile.contentType(); + return content == FileContent.DATA_MANIFEST || content == FileContent.DELETE_MANIFEST; + } + static class Builder { private final InputFile file; - private final Types.StructType partitionType; + private final Types.StructType unionPartitionType; private final Map specsById; private Expression rowFilter = Expressions.alwaysTrue(); private boolean caseSensitive = true; - private boolean onlyLive = false; + private boolean onlyLive = true; + private Collection columns = null; private Schema fileProjection = null; private ScanMetrics scanMetrics = ScanMetrics.noop(); private Builder(InputFile file, Map specsById) { this.file = file; - this.partitionType = Partitioning.unionPartitionTypes(specsById.values()); + this.unionPartitionType = Partitioning.unionPartitionTypes(specsById.values()); this.specsById = specsById; } @@ -195,13 +194,27 @@ Builder caseSensitive(boolean isCaseSensitive) { return this; } - /** Returns only files whose tracking {@link Tracking#isLive() is live}. */ - Builder liveOnly() { - this.onlyLive = true; + /** Returns deleted and replaced files in addition to {@link Tracking#isLive() live} files. */ + Builder includeTombstones() { + this.onlyLive = false; return this; } + /** Selects columns to read by name; fields needed by the reader are always read. */ + Builder select(Collection newColumns) { + Preconditions.checkArgument(newColumns != null, "Invalid columns: null"); + Preconditions.checkState( + fileProjection == null, + "Cannot select columns using both select(Collection) and project(Schema)"); + this.columns = newColumns; + return this; + } + + /** Sets the exact schema to read; used in place of {@link #select(Collection)}. */ Builder project(Schema newFileProjection) { + Preconditions.checkState( + columns == null, + "Cannot select columns using both select(Collection) and project(Schema)"); this.fileProjection = newFileProjection; return this; } @@ -218,10 +231,12 @@ V4ManifestReader build() { if (hasPartitionFilter()) { for (PartitionSpec spec : specsById.values()) { Expression partFilter = Projections.inclusive(spec, caseSensitive).project(rowFilter); - partitionEvaluators.put( - spec.specId(), new Evaluator(spec.partitionType(), partFilter, caseSensitive)); - partitionProjections.put( - spec.specId(), StructProjection.create(partitionType, spec.partitionType())); + if (partFilter != Expressions.alwaysTrue()) { + partitionEvaluators.put( + spec.specId(), new Evaluator(spec.partitionType(), partFilter, caseSensitive)); + partitionProjections.put( + spec.specId(), StructProjection.create(unionPartitionType, spec.partitionType())); + } } } @@ -230,33 +245,20 @@ V4ManifestReader build() { } private boolean hasPartitionFilter() { - return rowFilter != Expressions.alwaysTrue() && !partitionType.fields().isEmpty(); + return rowFilter != Expressions.alwaysTrue() && !unionPartitionType.fields().isEmpty(); } private Schema readSchema() { - Types.StructType fullType = TrackedFile.schema(partitionType, Types.StructType.of()); - - // replace tracking with the subset of fields read on the scan path - List fields = Lists.newArrayList(); - for (Types.NestedField field : fullType.fields()) { - if (field.fieldId() == TrackedFile.TRACKING.fieldId()) { - fields.add( - Types.NestedField.required( - field.fieldId(), field.name(), SCAN_TRACKING, field.doc())); - } else { - fields.add(field); - } - } - - Schema fullSchema = new Schema(fields); - if (fileProjection == null) { + Schema fullSchema = + new Schema( + TrackedFile.schema(Tracking.scanSchema(), unionPartitionType, Types.StructType.of()) + .fields()); + Schema projection = projection(fullSchema); + if (projection == null) { return fullSchema; } - Set projectedIds = Sets.newHashSet(); - for (Types.NestedField field : fileProjection.asStruct().fields()) { - projectedIds.add(field.fieldId()); - } + Set projectedIds = Sets.newHashSet(TypeUtil.getProjectedIds(projection)); // status drives live-file filtering and content type distinguishes entry kinds projectedIds.add(TrackedFile.TRACKING.fieldId()); @@ -268,5 +270,15 @@ private Schema readSchema() { return TypeUtil.select(fullSchema, projectedIds); } + + private Schema projection(Schema fullSchema) { + if (columns != null) { + return caseSensitive + ? fullSchema.select(columns) + : fullSchema.caseInsensitiveSelect(columns); + } + + return fileProjection; + } } } diff --git a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java index b1949326757b..dd440ecca2ec 100644 --- a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java +++ b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java @@ -70,10 +70,6 @@ public class TestV4ManifestReader { private static final Map UNPARTITIONED_SPECS = ImmutableMap.of(PartitionSpec.unpartitioned().specId(), PartitionSpec.unpartitioned()); - private static final List SCHEMA_FIELDS = - TrackedFile.schema(Types.StructType.of(), Types.StructType.of()).fields(); - private static final int SORT_ORDER_ID_ORDINAL = ordinalOf(TrackedFile.SORT_ORDER_ID.fieldId()); - @Parameter private FileFormat format; @Parameters(name = "format = {0}") @@ -178,6 +174,13 @@ public void testLiveFilesExcludesDeletedAndReplaced() throws IOException { InputFile manifest = writeManifest(EMPTY_PARTITION, files); try (V4ManifestReader reader = newReader(manifest, UNPARTITIONED_SPECS).build()) { + assertThat(reader) + .extracting(file -> file.tracking().status()) + .containsExactly(EntryStatus.ADDED, EntryStatus.EXISTING, EntryStatus.MODIFIED); + } + + try (V4ManifestReader reader = + newReader(manifest, UNPARTITIONED_SPECS).includeTombstones().build()) { assertThat(reader) .extracting(file -> file.tracking().status()) .containsExactly( @@ -187,12 +190,6 @@ public void testLiveFilesExcludesDeletedAndReplaced() throws IOException { EntryStatus.DELETED, EntryStatus.REPLACED); } - - try (V4ManifestReader reader = newReader(manifest, UNPARTITIONED_SPECS).liveOnly().build()) { - assertThat(reader) - .extracting(file -> file.tracking().status()) - .containsExactly(EntryStatus.ADDED, EntryStatus.EXISTING, EntryStatus.MODIFIED); - } } @TestTemplate @@ -214,8 +211,24 @@ public void testManifestLocationAndPosition() throws IOException { @TestTemplate public void testProjectionRestrictsFields() throws IOException { - TrackedFile file = dataFile("s3://bucket/file.parquet", EMPTY_PARTITION_DATA); - ((StructLike) file).set(SORT_ORDER_ID_ORDINAL, SORT_ORDER_ID); + TrackedFile file = + new TrackedFileStruct( + addedTracking(), + FileContent.DATA, + FORMAT_VERSION_V4, + "s3://bucket/file.parquet", + FileFormat.PARQUET, + EMPTY_PARTITION_DATA, + RECORD_COUNT, + FILE_SIZE_IN_BYTES, + 0, + null, + SORT_ORDER_ID, + null, + null, + null, + null, + null); InputFile manifest = writeManifest(EMPTY_PARTITION, ImmutableList.of(file)); @@ -235,6 +248,69 @@ public void testProjectionRestrictsFields() throws IOException { } } + @TestTemplate + public void testSelectRestrictsFields() throws IOException { + TrackedFile file = dataFile("s3://bucket/file.parquet", EMPTY_PARTITION_DATA); + + InputFile manifest = writeManifest(EMPTY_PARTITION, ImmutableList.of(file)); + + try (V4ManifestReader reader = + newReader(manifest, UNPARTITIONED_SPECS) + .select(ImmutableList.of("location", "record_count")) + .build()) { + TrackedFile actual = Lists.newArrayList(reader).get(0); + assertThat(actual.location()).isEqualTo(file.location()); + assertThat(actual.recordCount()).isEqualTo(RECORD_COUNT); + // tracking and content_type are always projected, even though the caller omitted them + assertThat(actual.tracking()).isNotNull(); + assertThat(actual.tracking().status()).isEqualTo(EntryStatus.ADDED); + assertThat(actual.contentType()).isEqualTo(FileContent.DATA); + // file_format and spec_id are null because they were not selected + assertThat(actual.fileFormat()).isNull(); + assertThat(actual.specId()).isNull(); + } + } + + @TestTemplate + public void testCaseInsensitiveSelect() throws IOException { + TrackedFile file = dataFile("s3://bucket/file.parquet", EMPTY_PARTITION_DATA); + + InputFile manifest = writeManifest(EMPTY_PARTITION, ImmutableList.of(file)); + + try (V4ManifestReader reader = + newReader(manifest, UNPARTITIONED_SPECS) + .select(ImmutableList.of("LOCATION")) + .caseSensitive(false) + .build()) { + TrackedFile actual = Lists.newArrayList(reader).get(0); + assertThat(actual.location()).isEqualTo(file.location()); + assertThat(actual.fileFormat()).isNull(); + } + } + + @TestTemplate + public void testSelectAndProjectAreMutuallyExclusive() { + InputFile manifest = fileIO.newInputFile(tempDir.resolve("manifest.avro").toString()); + + assertThatThrownBy( + () -> + newReader(manifest, UNPARTITIONED_SPECS) + .select(ImmutableList.of("location")) + .project(new Schema(TrackedFile.LOCATION))) + .isInstanceOf(IllegalStateException.class) + .hasMessage( + "Cannot select columns using both select(Collection) and project(Schema)"); + + assertThatThrownBy( + () -> + newReader(manifest, UNPARTITIONED_SPECS) + .project(new Schema(TrackedFile.LOCATION)) + .select(ImmutableList.of("location"))) + .isInstanceOf(IllegalStateException.class) + .hasMessage( + "Cannot select columns using both select(Collection) and project(Schema)"); + } + @TestTemplate public void testTrackingProjectionOmitsChangeTrackingFields() throws IOException { Tracking tracking = @@ -370,34 +446,62 @@ public void testPartitionFilterCountsSkippedDeleteFiles() throws IOException { public void testPartitionFilterKeepsManifestReferences() throws IOException { TrackedFile keep = dataFile("data-1.parquet", partition(1)); TrackedFile prune = dataFile("data-2.parquet", partition(2)); - ManifestInfo info = new ManifestInfoStruct(1, 0, 0, 0, 1L, 0L, 0L, 0L, 1L, null, null); - TrackedFile manifestRef = - new TrackedFileStruct( - addedTracking(), - FileContent.DATA_MANIFEST, - FORMAT_VERSION_V4, - "leaf.parquet", - FileFormat.PARQUET, - partition(2), - RECORD_COUNT, - FILE_SIZE_IN_BYTES, - 0, - null, - null, - null, - info, - null, - null, - null); + // a real manifest reference has a null spec_id and no partition tuple; these refs carry a + // spec and a tuple that fails the filter so that pruning would be detected if the manifest + // passthrough broke + TrackedFile dataManifestRef = manifestRef(FileContent.DATA_MANIFEST, "data-leaf.parquet"); + TrackedFile deleteManifestRef = manifestRef(FileContent.DELETE_MANIFEST, "delete-leaf.parquet"); - InputFile manifest = writeManifest(PARTITION_TYPE, ImmutableList.of(keep, prune, manifestRef)); + InputFile manifest = + writeManifest( + PARTITION_TYPE, ImmutableList.of(keep, prune, dataManifestRef, deleteManifestRef)); try (V4ManifestReader reader = newReader(manifest, PARTITIONED_SPECS).filterRows(Expressions.equal("id", 1)).build()) { assertThat(reader) .extracting(TrackedFile::location) - .containsExactlyInAnyOrder(keep.location(), manifestRef.location()); + .containsExactlyInAnyOrder( + keep.location(), dataManifestRef.location(), deleteManifestRef.location()); + } + } + + @TestTemplate + public void testRowFilterOnUnpartitionedTableKeepsAllFiles() throws IOException { + TrackedFile file1 = dataFile("s3://bucket/a.parquet", EMPTY_PARTITION_DATA); + TrackedFile file2 = dataFile("s3://bucket/b.parquet", EMPTY_PARTITION_DATA); + + InputFile manifest = writeManifest(EMPTY_PARTITION, ImmutableList.of(file1, file2)); + + ScanMetrics metrics = ScanMetrics.of(new DefaultMetricsContext()); + try (V4ManifestReader reader = + newReader(manifest, UNPARTITIONED_SPECS) + .filterRows(Expressions.equal("id", 1)) + .scanMetrics(metrics) + .build()) { + assertThat(reader) + .extracting(TrackedFile::location) + .containsExactly(file1.location(), file2.location()); } + + assertThat(metrics.skippedDataFiles().value()).isEqualTo(0L); + assertThat(metrics.skippedDeleteFiles().value()).isEqualTo(0L); + } + + @TestTemplate + public void testInvalidBuilderArguments() { + InputFile manifest = fileIO.newInputFile(tempDir.resolve("manifest.avro").toString()); + + assertThatThrownBy(() -> newReader(manifest, UNPARTITIONED_SPECS).filterRows(null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid row filter: null"); + + assertThatThrownBy(() -> newReader(manifest, UNPARTITIONED_SPECS).scanMetrics(null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid scan metrics: null"); + + assertThatThrownBy(() -> newReader(manifest, UNPARTITIONED_SPECS).select(null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid columns: null"); } @TestTemplate @@ -457,7 +561,7 @@ public void testIteratorReturnsLiveCopies() throws IOException { InputFile manifest = writeManifest(EMPTY_PARTITION, files); - try (V4ManifestReader reader = newReader(manifest, UNPARTITIONED_SPECS).liveOnly().build()) { + try (V4ManifestReader reader = newReader(manifest, UNPARTITIONED_SPECS).build()) { List read = Lists.newArrayList(reader); assertThat(read) .hasSize(2) @@ -482,17 +586,28 @@ public void testUnknownManifestFormatThrows() throws IOException { } @TestTemplate - public void testFileWithUnknownSpecThrows() throws IOException { - // spec ID 5 is not in PARTITIONED_SPECS, so pruning cannot resolve a spec for this file + public void testPartitionFilterKeepsFileWithUnknownSpec() throws IOException { + // spec ID 5 is not in PARTITIONED_SPECS, so no partition filter applies to this file TrackedFile file = dataFile("orphan.parquet", partition(1), 5); InputFile manifest = writeManifest(PARTITION_TYPE, ImmutableList.of(file)); + // the filter would prune partition id=1 under spec 0, but cannot be applied to spec 5 try (V4ManifestReader reader = - newReader(manifest, PARTITIONED_SPECS).filterRows(Expressions.equal("id", 1)).build()) { - assertThatThrownBy(() -> Lists.newArrayList(reader)) - .isInstanceOf(IllegalStateException.class) - .hasMessageContaining("not one of the known specs"); + newReader(manifest, PARTITIONED_SPECS).filterRows(Expressions.equal("id", 2)).build()) { + assertThat(reader).extracting(TrackedFile::location).containsExactly(file.location()); + } + } + + @TestTemplate + public void testPartitionFilterKeepsFileWithNullSpecId() throws IOException { + TrackedFile file = dataFile("no-spec.parquet", null, null); + + InputFile manifest = writeManifest(PARTITION_TYPE, ImmutableList.of(file)); + + try (V4ManifestReader reader = + newReader(manifest, PARTITIONED_SPECS).filterRows(Expressions.equal("id", 2)).build()) { + assertThat(reader).extracting(TrackedFile::location).containsExactly(file.location()); } } @@ -500,7 +615,7 @@ private static TrackedFile dataFile(String location, PartitionData partition) { return dataFile(location, partition, 0); } - private static TrackedFile dataFile(String location, PartitionData partition, int specId) { + private static TrackedFile dataFile(String location, PartitionData partition, Integer specId) { return new TrackedFileStruct( addedTracking(), FileContent.DATA, @@ -520,6 +635,27 @@ private static TrackedFile dataFile(String location, PartitionData partition, in null); } + private static TrackedFile manifestRef(FileContent content, String location) { + ManifestInfo info = new ManifestInfoStruct(1, 0, 0, 0, 1L, 0L, 0L, 0L, 1L, null, null); + return new TrackedFileStruct( + addedTracking(), + content, + FORMAT_VERSION_V4, + location, + FileFormat.PARQUET, + partition(2), + RECORD_COUNT, + FILE_SIZE_IN_BYTES, + 0, + null, + null, + null, + info, + null, + null, + null); + } + private static TrackedFile fileWithStatus(EntryStatus status, String location) { Tracking tracking = new TrackingStruct(status, SNAPSHOT_ID, 3L, 3L, null, null, null, null); return new TrackedFileStruct( @@ -581,42 +717,13 @@ private InputFile writeManifest(Types.StructType partitionType, Iterable appender = InternalData.write(format, out).schema(writeSchema).named("tracked_file").build()) { for (TrackedFile file : files) { - appender.add(toWriteRow(file, writeSchema)); + appender.add((StructLike) file); } } return fileIO.newInputFile(out.location()); } - /** - * Adapts a fully-populated tracked file to a write schema that may omit fields (partition and - * content_stats are omitted when empty). - */ - private static StructLike toWriteRow(TrackedFile file, Schema writeSchema) { - StructLike struct = (StructLike) file; - int[] toBase = new int[writeSchema.columns().size()]; - for (int i = 0; i < writeSchema.columns().size(); i++) { - toBase[i] = ordinalOf(writeSchema.columns().get(i).fieldId()); - } - - return new StructLike() { - @Override - public int size() { - return toBase.length; - } - - @Override - public T get(int pos, Class javaClass) { - return struct.get(toBase[pos], javaClass); - } - - @Override - public void set(int pos, T value) { - throw new UnsupportedOperationException("Cannot modify write row"); - } - }; - } - private V4ManifestReader.Builder newReader( InputFile manifest, Map specsById) { return V4ManifestReader.builder(manifest, specsById); @@ -628,14 +735,4 @@ private List read(InputFile manifest, Map s return Lists.newArrayList(reader); } } - - private static int ordinalOf(int fieldId) { - for (int i = 0; i < SCHEMA_FIELDS.size(); i++) { - if (SCHEMA_FIELDS.get(i).fieldId() == fieldId) { - return i; - } - } - - throw new IllegalArgumentException("Field not found in TrackedFile schema: " + fieldId); - } } From 9656a4d7085016bce78d6c1459b4b5b9add6ae8c Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Mon, 20 Jul 2026 16:57:53 -0700 Subject: [PATCH 14/26] Cleanup --- .../java/org/apache/iceberg/Tracking.java | 11 ---------- .../org/apache/iceberg/TrackingStruct.java | 11 ++++++++++ .../org/apache/iceberg/V4ManifestReader.java | 22 ++++++++++++------- 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/Tracking.java b/core/src/main/java/org/apache/iceberg/Tracking.java index 7b415cfb049d..fcdc4e50b236 100644 --- a/core/src/main/java/org/apache/iceberg/Tracking.java +++ b/core/src/main/java/org/apache/iceberg/Tracking.java @@ -78,17 +78,6 @@ static Types.StructType schema() { REPLACED_POSITIONS); } - /** Returns the tracking fields read on the scan path; row_position backs {@link #manifestPos}. */ - static Types.StructType scanSchema() { - return Types.StructType.of( - STATUS, - SNAPSHOT_ID, - SEQUENCE_NUMBER, - FILE_SEQUENCE_NUMBER, - FIRST_ROW_ID, - MetadataColumns.ROW_POSITION); - } - /** Returns the status of the entry. */ EntryStatus status(); diff --git a/core/src/main/java/org/apache/iceberg/TrackingStruct.java b/core/src/main/java/org/apache/iceberg/TrackingStruct.java index 8ae4b7e4ce88..151ed8737c2f 100644 --- a/core/src/main/java/org/apache/iceberg/TrackingStruct.java +++ b/core/src/main/java/org/apache/iceberg/TrackingStruct.java @@ -42,6 +42,17 @@ class TrackingStruct extends SupportsIndexProjection implements Tracking, Serial Tracking.REPLACED_POSITIONS, MetadataColumns.ROW_POSITION); + // tracking fields read on the scan path; row_position backs manifestPos. + // Package-private only for V4ManifestReader's read projection. + static final Types.StructType SCAN_TYPE = + Types.StructType.of( + Tracking.STATUS, + Tracking.SNAPSHOT_ID, + Tracking.SEQUENCE_NUMBER, + Tracking.FILE_SEQUENCE_NUMBER, + Tracking.FIRST_ROW_ID, + MetadataColumns.ROW_POSITION); + private EntryStatus status = null; private Long snapshotId = null; private Long dataSequenceNumber = null; diff --git a/core/src/main/java/org/apache/iceberg/V4ManifestReader.java b/core/src/main/java/org/apache/iceberg/V4ManifestReader.java index 1c9a9e1ad615..983170be895f 100644 --- a/core/src/main/java/org/apache/iceberg/V4ManifestReader.java +++ b/core/src/main/java/org/apache/iceberg/V4ManifestReader.java @@ -41,7 +41,7 @@ class V4ManifestReader extends CloseableGroup implements CloseableIterable { private final InputFile file; private final Schema readSchema; - private final boolean onlyLive; + private final boolean includeTombstones; private final ScanMetrics scanMetrics; // partition pruning state, keyed by spec ID @@ -53,13 +53,13 @@ private V4ManifestReader( Schema readSchema, Map partitionEvaluators, Map partitionProjections, - boolean onlyLive, + boolean includeTombstones, ScanMetrics scanMetrics) { this.file = file; this.readSchema = readSchema; this.partitionEvaluators = partitionEvaluators; this.partitionProjections = partitionProjections; - this.onlyLive = onlyLive; + this.includeTombstones = includeTombstones; this.scanMetrics = scanMetrics; } @@ -77,7 +77,7 @@ public CloseableIterator iterator() { CloseableIterable.filter(entries, entry -> isManifest(entry) || matchesPartition(entry)); } - if (onlyLive) { + if (!includeTombstones) { entries = CloseableIterable.filter(entries, entry -> entry.tracking().isLive()); } @@ -171,7 +171,7 @@ static class Builder { private final Map specsById; private Expression rowFilter = Expressions.alwaysTrue(); private boolean caseSensitive = true; - private boolean onlyLive = true; + private boolean includeTombstones = false; private Collection columns = null; private Schema fileProjection = null; private ScanMetrics scanMetrics = ScanMetrics.noop(); @@ -196,7 +196,7 @@ Builder caseSensitive(boolean isCaseSensitive) { /** Returns deleted and replaced files in addition to {@link Tracking#isLive() live} files. */ Builder includeTombstones() { - this.onlyLive = false; + this.includeTombstones = true; return this; } @@ -241,7 +241,12 @@ V4ManifestReader build() { } return new V4ManifestReader( - file, readSchema(), partitionEvaluators, partitionProjections, onlyLive, scanMetrics); + file, + readSchema(), + partitionEvaluators, + partitionProjections, + includeTombstones, + scanMetrics); } private boolean hasPartitionFilter() { @@ -251,7 +256,8 @@ private boolean hasPartitionFilter() { private Schema readSchema() { Schema fullSchema = new Schema( - TrackedFile.schema(Tracking.scanSchema(), unionPartitionType, Types.StructType.of()) + TrackedFile.schema( + TrackingStruct.SCAN_TYPE, unionPartitionType, Types.StructType.of()) .fields()); Schema projection = projection(fullSchema); if (projection == null) { From 607628d164f036233b43c444f913f64db5008431 Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Mon, 20 Jul 2026 17:06:59 -0700 Subject: [PATCH 15/26] Revamp the reader API The API now supports four read modes: the default returns the full schema for copying records to other manifests, forScanPlanning() selects the minimal fields for planning, select() projects columns by name with reader-required fields joined in, and project() reads an exact schema. The three configuration methods are mutually exclusive --- .../org/apache/iceberg/TrackingStruct.java | 5 +- .../org/apache/iceberg/V4ManifestReader.java | 26 +++- .../apache/iceberg/TestV4ManifestReader.java | 135 ++++++++++++------ 3 files changed, 119 insertions(+), 47 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/TrackingStruct.java b/core/src/main/java/org/apache/iceberg/TrackingStruct.java index 151ed8737c2f..283e0a51ffb3 100644 --- a/core/src/main/java/org/apache/iceberg/TrackingStruct.java +++ b/core/src/main/java/org/apache/iceberg/TrackingStruct.java @@ -30,7 +30,8 @@ /** Mutable {@link StructLike} implementation of {@link Tracking}. */ class TrackingStruct extends SupportsIndexProjection implements Tracking, Serializable { - private static final Types.StructType BASE_TYPE = + // Package-private only for read projection. + static final Types.StructType BASE_TYPE = Types.StructType.of( Tracking.STATUS, Tracking.SNAPSHOT_ID, @@ -43,7 +44,7 @@ class TrackingStruct extends SupportsIndexProjection implements Tracking, Serial MetadataColumns.ROW_POSITION); // tracking fields read on the scan path; row_position backs manifestPos. - // Package-private only for V4ManifestReader's read projection. + // Package-private only for read projection. static final Types.StructType SCAN_TYPE = Types.StructType.of( Tracking.STATUS, diff --git a/core/src/main/java/org/apache/iceberg/V4ManifestReader.java b/core/src/main/java/org/apache/iceberg/V4ManifestReader.java index 983170be895f..41263e430680 100644 --- a/core/src/main/java/org/apache/iceberg/V4ManifestReader.java +++ b/core/src/main/java/org/apache/iceberg/V4ManifestReader.java @@ -172,6 +172,7 @@ static class Builder { private Expression rowFilter = Expressions.alwaysTrue(); private boolean caseSensitive = true; private boolean includeTombstones = false; + private boolean scanPlanning = false; private Collection columns = null; private Schema fileProjection = null; private ScanMetrics scanMetrics = ScanMetrics.noop(); @@ -200,9 +201,20 @@ Builder includeTombstones() { return this; } + /** Configures the reader to select the minimal fields needed for scan planning. */ + Builder forScanPlanning() { + Preconditions.checkState( + columns == null && fileProjection == null, + "Cannot use forScanPlanning() with select(Collection) or project(Schema)"); + this.scanPlanning = true; + return this; + } + /** Selects columns to read by name; fields needed by the reader are always read. */ Builder select(Collection newColumns) { Preconditions.checkArgument(newColumns != null, "Invalid columns: null"); + Preconditions.checkState( + !scanPlanning, "Cannot use select(Collection) with forScanPlanning()"); Preconditions.checkState( fileProjection == null, "Cannot select columns using both select(Collection) and project(Schema)"); @@ -212,6 +224,7 @@ Builder select(Collection newColumns) { /** Sets the exact schema to read; used in place of {@link #select(Collection)}. */ Builder project(Schema newFileProjection) { + Preconditions.checkState(!scanPlanning, "Cannot use project(Schema) with forScanPlanning()"); Preconditions.checkState( columns == null, "Cannot select columns using both select(Collection) and project(Schema)"); @@ -254,11 +267,11 @@ private boolean hasPartitionFilter() { } private Schema readSchema() { + Types.StructType trackingType = + scanPlanning ? TrackingStruct.SCAN_TYPE : TrackingStruct.BASE_TYPE; Schema fullSchema = new Schema( - TrackedFile.schema( - TrackingStruct.SCAN_TYPE, unionPartitionType, Types.StructType.of()) - .fields()); + TrackedFile.schema(trackingType, unionPartitionType, Types.StructType.of()).fields()); Schema projection = projection(fullSchema); if (projection == null) { return fullSchema; @@ -266,8 +279,11 @@ private Schema readSchema() { Set projectedIds = Sets.newHashSet(TypeUtil.getProjectedIds(projection)); - // status drives live-file filtering and content type distinguishes entry kinds - projectedIds.add(TrackedFile.TRACKING.fieldId()); + // fields the reader itself needs: status for live filtering, row_position for manifestPos, + // record count, and content type to distinguish entry kinds + projectedIds.add(Tracking.STATUS.fieldId()); + projectedIds.add(MetadataColumns.ROW_POSITION.fieldId()); + projectedIds.add(TrackedFile.RECORD_COUNT.fieldId()); projectedIds.add(TrackedFile.CONTENT_TYPE.fieldId()); if (hasPartitionFilter()) { projectedIds.add(TrackedFile.SPEC_ID.fieldId()); diff --git a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java index dd440ecca2ec..a093b208690a 100644 --- a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java +++ b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java @@ -255,16 +255,16 @@ public void testSelectRestrictsFields() throws IOException { InputFile manifest = writeManifest(EMPTY_PARTITION, ImmutableList.of(file)); try (V4ManifestReader reader = - newReader(manifest, UNPARTITIONED_SPECS) - .select(ImmutableList.of("location", "record_count")) - .build()) { + newReader(manifest, UNPARTITIONED_SPECS).select(ImmutableList.of("location")).build()) { TrackedFile actual = Lists.newArrayList(reader).get(0); assertThat(actual.location()).isEqualTo(file.location()); + // record_count, tracking status, and content_type are joined in even though not selected assertThat(actual.recordCount()).isEqualTo(RECORD_COUNT); - // tracking and content_type are always projected, even though the caller omitted them assertThat(actual.tracking()).isNotNull(); assertThat(actual.tracking().status()).isEqualTo(EntryStatus.ADDED); assertThat(actual.contentType()).isEqualTo(FileContent.DATA); + // only status is joined from tracking, not the other tracking fields + assertThat(actual.tracking().snapshotId()).isNull(); // file_format and spec_id are null because they were not selected assertThat(actual.fileFormat()).isNull(); assertThat(actual.specId()).isNull(); @@ -289,7 +289,7 @@ public void testCaseInsensitiveSelect() throws IOException { } @TestTemplate - public void testSelectAndProjectAreMutuallyExclusive() { + public void testProjectionModesAreMutuallyExclusive() { InputFile manifest = fileIO.newInputFile(tempDir.resolve("manifest.avro").toString()); assertThatThrownBy( @@ -309,52 +309,77 @@ public void testSelectAndProjectAreMutuallyExclusive() { .isInstanceOf(IllegalStateException.class) .hasMessage( "Cannot select columns using both select(Collection) and project(Schema)"); + + assertThatThrownBy( + () -> + newReader(manifest, UNPARTITIONED_SPECS) + .forScanPlanning() + .select(ImmutableList.of("location"))) + .isInstanceOf(IllegalStateException.class) + .hasMessage("Cannot use select(Collection) with forScanPlanning()"); + + assertThatThrownBy( + () -> + newReader(manifest, UNPARTITIONED_SPECS) + .select(ImmutableList.of("location")) + .forScanPlanning()) + .isInstanceOf(IllegalStateException.class) + .hasMessage( + "Cannot use forScanPlanning() with select(Collection) or project(Schema)"); + + assertThatThrownBy( + () -> + newReader(manifest, UNPARTITIONED_SPECS) + .forScanPlanning() + .project(new Schema(TrackedFile.LOCATION))) + .isInstanceOf(IllegalStateException.class) + .hasMessage("Cannot use project(Schema) with forScanPlanning()"); + + assertThatThrownBy( + () -> + newReader(manifest, UNPARTITIONED_SPECS) + .project(new Schema(TrackedFile.LOCATION)) + .forScanPlanning()) + .isInstanceOf(IllegalStateException.class) + .hasMessage( + "Cannot use forScanPlanning() with select(Collection) or project(Schema)"); } @TestTemplate - public void testTrackingProjectionOmitsChangeTrackingFields() throws IOException { - Tracking tracking = - new TrackingStruct( - EntryStatus.ADDED, - SNAPSHOT_ID, - 5L, // data sequence number - 6L, // file sequence number - 7L, // dv snapshot id - 8L, // first row id - new byte[] {1, 2}, // deleted positions - new byte[] {3, 4}); // replaced positions - TrackedFile file = - new TrackedFileStruct( - tracking, - FileContent.DATA, - FORMAT_VERSION_V4, - "s3://bucket/file.parquet", - FileFormat.PARQUET, - EMPTY_PARTITION_DATA, - RECORD_COUNT, - FILE_SIZE_IN_BYTES, - 0, - null, - null, - null, - null, - null, - null, - null); + public void testForScanPlanningOmitsChangeTrackingFields() throws IOException { + InputFile manifest = writeManifest(EMPTY_PARTITION, ImmutableList.of(fileWithFullTracking())); - InputFile manifest = writeManifest(EMPTY_PARTITION, ImmutableList.of(file)); + try (V4ManifestReader reader = + newReader(manifest, UNPARTITIONED_SPECS).forScanPlanning().build()) { + Tracking actual = Lists.newArrayList(reader).get(0).tracking(); + // scan-relevant tracking fields are projected + assertThat(actual.status()).isEqualTo(EntryStatus.ADDED); + assertThat(actual.snapshotId()).isEqualTo(SNAPSHOT_ID); + assertThat(actual.dataSequenceNumber()).isEqualTo(5L); + assertThat(actual.fileSequenceNumber()).isEqualTo(6L); + assertThat(actual.firstRowId()).isEqualTo(8L); + // change-tracking fields are omitted from the scan projection + assertThat(actual.dvSnapshotId()).isNull(); + assertThat(actual.deletedPositions()).isNull(); + assertThat(actual.replacedPositions()).isNull(); + } + } + @TestTemplate + public void testDefaultReadsFullTracking() throws IOException { + InputFile manifest = writeManifest(EMPTY_PARTITION, ImmutableList.of(fileWithFullTracking())); + + // without a projection, the reader returns the full schema for copying to other manifests, + // including the change-tracking fields Tracking actual = read(manifest, UNPARTITIONED_SPECS).get(0).tracking(); - // scan-relevant tracking fields are projected assertThat(actual.status()).isEqualTo(EntryStatus.ADDED); assertThat(actual.snapshotId()).isEqualTo(SNAPSHOT_ID); assertThat(actual.dataSequenceNumber()).isEqualTo(5L); assertThat(actual.fileSequenceNumber()).isEqualTo(6L); assertThat(actual.firstRowId()).isEqualTo(8L); - // change-tracking fields are omitted from the scan projection - assertThat(actual.dvSnapshotId()).isNull(); - assertThat(actual.deletedPositions()).isNull(); - assertThat(actual.replacedPositions()).isNull(); + assertThat(actual.dvSnapshotId()).isEqualTo(7L); + assertThat(actual.deletedPositions()).isEqualTo(ByteBuffer.wrap(new byte[] {1, 2})); + assertThat(actual.replacedPositions()).isEqualTo(ByteBuffer.wrap(new byte[] {3, 4})); } @TestTemplate @@ -635,6 +660,36 @@ private static TrackedFile dataFile(String location, PartitionData partition, In null); } + private static TrackedFile fileWithFullTracking() { + Tracking tracking = + new TrackingStruct( + EntryStatus.ADDED, + SNAPSHOT_ID, + 5L, // data sequence number + 6L, // file sequence number + 7L, // dv snapshot id + 8L, // first row id + new byte[] {1, 2}, // deleted positions + new byte[] {3, 4}); // replaced positions + return new TrackedFileStruct( + tracking, + FileContent.DATA, + FORMAT_VERSION_V4, + "s3://bucket/file.parquet", + FileFormat.PARQUET, + EMPTY_PARTITION_DATA, + RECORD_COUNT, + FILE_SIZE_IN_BYTES, + 0, + null, + null, + null, + null, + null, + null, + null); + } + private static TrackedFile manifestRef(FileContent content, String location) { ManifestInfo info = new ManifestInfoStruct(1, 0, 0, 0, 1L, 0L, 0L, 0L, 1L, null, null); return new TrackedFileStruct( From 7a11c800cc8d92a5d9797f588902587b5aa3b220 Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Mon, 20 Jul 2026 22:57:40 -0700 Subject: [PATCH 16/26] Feedback from Steven --- .../java/org/apache/iceberg/Partitioning.java | 2 +- .../org/apache/iceberg/V4ManifestReader.java | 11 ++- .../apache/iceberg/TestV4ManifestReader.java | 86 ++++++++++++++++++- 3 files changed, 94 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/Partitioning.java b/core/src/main/java/org/apache/iceberg/Partitioning.java index def7827419db..66a681f88486 100644 --- a/core/src/main/java/org/apache/iceberg/Partitioning.java +++ b/core/src/main/java/org/apache/iceberg/Partitioning.java @@ -251,7 +251,7 @@ public static StructType partitionType(Table table) { * @return the constructed unified partition type */ static StructType unionPartitionTypes(Collection specs) { - return buildPartitionProjectionType("table partition", specs, allFieldIds(specs)); + return buildPartitionProjectionType("union partition", specs, allFieldIds(specs)); } /** diff --git a/core/src/main/java/org/apache/iceberg/V4ManifestReader.java b/core/src/main/java/org/apache/iceberg/V4ManifestReader.java index 41263e430680..8ae284c19548 100644 --- a/core/src/main/java/org/apache/iceberg/V4ManifestReader.java +++ b/core/src/main/java/org/apache/iceberg/V4ManifestReader.java @@ -288,9 +288,18 @@ private Schema readSchema() { if (hasPartitionFilter()) { projectedIds.add(TrackedFile.SPEC_ID.fieldId()); projectedIds.add(TrackedFile.PARTITION_ID); + projectedIds.addAll(TypeUtil.getProjectedIds(unionPartitionType)); } - return TypeUtil.select(fullSchema, projectedIds); + // list and map fields cannot be projected by ID; their element IDs carry the selection + projectedIds.removeIf( + id -> { + Types.NestedField field = fullSchema.findField(id); + return field != null && (field.type().isListType() || field.type().isMapType()); + }); + + // project instead of select to preserve narrow struct projections from the caller + return TypeUtil.project(fullSchema, projectedIds); } private Schema projection(Schema fullSchema) { diff --git a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java index a093b208690a..30ada7581599 100644 --- a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java +++ b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java @@ -25,7 +25,6 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.file.Path; -import java.util.Arrays; import java.util.List; import java.util.Locale; import java.util.Map; @@ -74,7 +73,7 @@ public class TestV4ManifestReader { @Parameters(name = "format = {0}") protected static List parameters() { - return Arrays.asList(FileFormat.AVRO, FileFormat.PARQUET); + return ImmutableList.of(FileFormat.AVRO, FileFormat.PARQUET); } @TempDir private Path tempDir; @@ -345,6 +344,78 @@ public void testProjectionModesAreMutuallyExclusive() { "Cannot use forScanPlanning() with select(Collection) or project(Schema)"); } + @TestTemplate + public void testProjectionPreservesNarrowTrackingProjection() throws IOException { + InputFile manifest = writeManifest(EMPTY_PARTITION, ImmutableList.of(fileWithFullTracking())); + + Schema projection = + new Schema( + Types.NestedField.required( + TrackedFile.TRACKING.fieldId(), "tracking", Types.StructType.of(Tracking.STATUS))); + + try (V4ManifestReader reader = + newReader(manifest, UNPARTITIONED_SPECS).project(projection).build()) { + Tracking actual = Lists.newArrayList(reader).get(0).tracking(); + assertThat(actual.status()).isEqualTo(EntryStatus.ADDED); + // the narrow tracking projection is not widened to the full tracking type + assertThat(actual.snapshotId()).isNull(); + assertThat(actual.dvSnapshotId()).isNull(); + assertThat(actual.deletedPositions()).isNull(); + assertThat(actual.replacedPositions()).isNull(); + } + } + + @TestTemplate + public void testSelectListColumn() throws IOException { + TrackedFile file = + new TrackedFileStruct( + addedTracking(), + FileContent.DATA, + FORMAT_VERSION_V4, + "s3://bucket/file.parquet", + FileFormat.PARQUET, + EMPTY_PARTITION_DATA, + RECORD_COUNT, + FILE_SIZE_IN_BYTES, + 0, + null, + null, + null, + null, + null, + ImmutableList.of(50L, 100L), + null); + + InputFile manifest = writeManifest(EMPTY_PARTITION, ImmutableList.of(file)); + + try (V4ManifestReader reader = + newReader(manifest, UNPARTITIONED_SPECS) + .select(ImmutableList.of("split_offsets")) + .build()) { + TrackedFile actual = Lists.newArrayList(reader).get(0); + assertThat(actual.splitOffsets()).containsExactly(50L, 100L); + assertThat(actual.location()).isNull(); + } + } + + @TestTemplate + public void testSelectWithPartitionFilterProjectsFilterFields() throws IOException { + TrackedFile keep = dataFile("keep.parquet", partition(1)); + TrackedFile prune = dataFile("prune.parquet", partition(2)); + + InputFile manifest = writeManifest(PARTITION_TYPE, ImmutableList.of(keep, prune)); + + // the caller selects only location; the reader must still project spec_id and partition + // for the partition filter or every row would be pruned + try (V4ManifestReader reader = + newReader(manifest, PARTITIONED_SPECS) + .select(ImmutableList.of("location")) + .filterRows(Expressions.equal("id", 1)) + .build()) { + assertThat(reader).extracting(TrackedFile::location).containsExactly(keep.location()); + } + } + @TestTemplate public void testForScanPlanningOmitsChangeTrackingFields() throws IOException { InputFile manifest = writeManifest(EMPTY_PARTITION, ImmutableList.of(fileWithFullTracking())); @@ -481,13 +552,22 @@ public void testPartitionFilterKeepsManifestReferences() throws IOException { writeManifest( PARTITION_TYPE, ImmutableList.of(keep, prune, dataManifestRef, deleteManifestRef)); + ScanMetrics metrics = ScanMetrics.of(new DefaultMetricsContext()); try (V4ManifestReader reader = - newReader(manifest, PARTITIONED_SPECS).filterRows(Expressions.equal("id", 1)).build()) { + newReader(manifest, PARTITIONED_SPECS) + .filterRows(Expressions.equal("id", 1)) + .scanMetrics(metrics) + .build()) { assertThat(reader) .extracting(TrackedFile::location) .containsExactlyInAnyOrder( keep.location(), dataManifestRef.location(), deleteManifestRef.location()); } + + // the manifest references bypass the filter instead of being evaluated and skipped + assertThat(metrics.skippedDataFiles().value()).isEqualTo(1L); + assertThat(metrics.skippedDataManifests().value()).isEqualTo(0L); + assertThat(metrics.skippedDeleteManifests().value()).isEqualTo(0L); } @TestTemplate From a807c9af732e47e993fd24120b442ab48cdb572a Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Tue, 21 Jul 2026 10:25:30 -0700 Subject: [PATCH 17/26] Remove record count from projection --- core/src/main/java/org/apache/iceberg/V4ManifestReader.java | 5 ++--- .../test/java/org/apache/iceberg/TestV4ManifestReader.java | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/V4ManifestReader.java b/core/src/main/java/org/apache/iceberg/V4ManifestReader.java index 8ae284c19548..18fa36af433e 100644 --- a/core/src/main/java/org/apache/iceberg/V4ManifestReader.java +++ b/core/src/main/java/org/apache/iceberg/V4ManifestReader.java @@ -279,11 +279,10 @@ private Schema readSchema() { Set projectedIds = Sets.newHashSet(TypeUtil.getProjectedIds(projection)); - // fields the reader itself needs: status for live filtering, row_position for manifestPos, - // record count, and content type to distinguish entry kinds + // fields the reader consumes internally: status for live filtering, row_position for + // manifestPos, and content type to distinguish entry kinds projectedIds.add(Tracking.STATUS.fieldId()); projectedIds.add(MetadataColumns.ROW_POSITION.fieldId()); - projectedIds.add(TrackedFile.RECORD_COUNT.fieldId()); projectedIds.add(TrackedFile.CONTENT_TYPE.fieldId()); if (hasPartitionFilter()) { projectedIds.add(TrackedFile.SPEC_ID.fieldId()); diff --git a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java index 30ada7581599..67504bb7a411 100644 --- a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java +++ b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java @@ -257,8 +257,7 @@ public void testSelectRestrictsFields() throws IOException { newReader(manifest, UNPARTITIONED_SPECS).select(ImmutableList.of("location")).build()) { TrackedFile actual = Lists.newArrayList(reader).get(0); assertThat(actual.location()).isEqualTo(file.location()); - // record_count, tracking status, and content_type are joined in even though not selected - assertThat(actual.recordCount()).isEqualTo(RECORD_COUNT); + // tracking status and content_type are joined in even though not selected assertThat(actual.tracking()).isNotNull(); assertThat(actual.tracking().status()).isEqualTo(EntryStatus.ADDED); assertThat(actual.contentType()).isEqualTo(FileContent.DATA); From b18836912504504345b4463409baa373390335ad Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Thu, 23 Jul 2026 12:23:41 -0700 Subject: [PATCH 18/26] Feedback from Ryan --- .../apache/iceberg/types/ReplaceTypeById.java | 107 ++++++++++++++++++ .../org/apache/iceberg/types/TypeUtil.java | 18 +++ .../apache/iceberg/types/TestTypeUtil.java | 38 +++++++ .../java/org/apache/iceberg/TrackedFile.java | 11 +- .../org/apache/iceberg/V4ManifestReader.java | 71 ++++++------ .../apache/iceberg/TestV4ManifestReader.java | 26 ++++- 6 files changed, 226 insertions(+), 45 deletions(-) create mode 100644 api/src/main/java/org/apache/iceberg/types/ReplaceTypeById.java diff --git a/api/src/main/java/org/apache/iceberg/types/ReplaceTypeById.java b/api/src/main/java/org/apache/iceberg/types/ReplaceTypeById.java new file mode 100644 index 000000000000..1c94bd57c114 --- /dev/null +++ b/api/src/main/java/org/apache/iceberg/types/ReplaceTypeById.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.types; + +import java.util.List; +import java.util.Map; +import org.apache.iceberg.Schema; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; + +class ReplaceTypeById extends TypeUtil.SchemaVisitor { + private final Map replacementsById; + + ReplaceTypeById(Map replacementsById) { + this.replacementsById = replacementsById; + } + + @Override + public Type schema(Schema schema, Type structResult) { + return structResult; + } + + @Override + public Type struct(Types.StructType struct, List fieldResults) { + List fields = struct.fields(); + List newFields = Lists.newArrayListWithExpectedSize(fields.size()); + boolean hasChanged = false; + + for (int i = 0; i < fields.size(); i += 1) { + Type fieldReplacement = fieldResults.get(i); + Types.NestedField field = fields.get(i); + if (field.type() != fieldReplacement) { + hasChanged = true; + newFields.add(Types.NestedField.from(field).ofType(fieldReplacement).build()); + } else { + newFields.add(field); + } + } + + if (hasChanged) { + return Types.StructType.of(newFields); + } + + return struct; + } + + @Override + public Type field(Types.NestedField field, Type fieldResult) { + return replacementsById.getOrDefault(field.fieldId(), fieldResult); + } + + @Override + public Type list(Types.ListType list, Type elementResult) { + Type elementReplacement = replacementsById.getOrDefault(list.elementId(), elementResult); + if (list.elementType() != elementReplacement) { + if (list.isElementRequired()) { + return Types.ListType.ofRequired(list.elementId(), elementReplacement); + } else { + return Types.ListType.ofOptional(list.elementId(), elementReplacement); + } + } + + return list; + } + + @Override + public Type map(Types.MapType map, Type keyResult, Type valueResult) { + Type keyReplacement = replacementsById.getOrDefault(map.keyId(), keyResult); + Type valueReplacement = replacementsById.getOrDefault(map.valueId(), valueResult); + if (map.keyType() != keyReplacement || map.valueType() != valueReplacement) { + if (map.isValueRequired()) { + return Types.MapType.ofRequired( + map.keyId(), map.valueId(), keyReplacement, valueReplacement); + } else { + return Types.MapType.ofOptional( + map.keyId(), map.valueId(), keyReplacement, valueReplacement); + } + } + + return map; + } + + @Override + public Type variant(Types.VariantType variant) { + return variant; + } + + @Override + public Type primitive(Type.PrimitiveType primitive) { + return primitive; + } +} diff --git a/api/src/main/java/org/apache/iceberg/types/TypeUtil.java b/api/src/main/java/org/apache/iceberg/types/TypeUtil.java index a3bee3e3d860..8e39ae7a43bc 100644 --- a/api/src/main/java/org/apache/iceberg/types/TypeUtil.java +++ b/api/src/main/java/org/apache/iceberg/types/TypeUtil.java @@ -157,6 +157,24 @@ public static Schema selectNot(Schema schema, Set fieldIds) { return project(schema, projectedIds); } + /** + * Returns a copy of the schema with the type of each field in {@code replacementsById} replaced + * by its mapped type. Fields not in the map are unchanged. + * + * @param schema a schema + * @param replacementsById a map from field ID to the type that should replace the field's type + * @return a schema with the replaced field types + */ + public static Schema replaceFieldTypes(Schema schema, Map replacementsById) { + Types.StructType struct = visit(schema, new ReplaceTypeById(replacementsById)).asStructType(); + if (struct.equals(schema.asStruct())) { + return schema; + } + + return new Schema( + schema.schemaId(), struct.fields(), schema.getAliases(), schema.identifierFieldIds()); + } + public static Schema join(Schema left, Schema right) { List joinedColumns = Lists.newArrayList(left.columns()); for (Types.NestedField rightColumn : right.columns()) { diff --git a/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java b/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java index b7da4b3108e6..3c25e930a3fb 100644 --- a/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java +++ b/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java @@ -29,6 +29,7 @@ import java.util.stream.Stream; import org.apache.iceberg.Schema; import org.apache.iceberg.expressions.Literal; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Sets; import org.apache.iceberg.types.Types.IntegerType; @@ -1037,4 +1038,41 @@ public void testIndexStatsNames() { .containsEntry(24, "addresses_value") // the leaf takes precedence .hasSize(22); } + + @Test + public void testReplaceFieldTypes() { + Types.StructType replacement = Types.StructType.of(required(10, "x", IntegerType.get())); + Schema schema = + new Schema( + required(1, "id", IntegerType.get()), + required(2, "s", Types.StructType.of(required(3, "a", Types.LongType.get())))); + + Schema result = TypeUtil.replaceFieldTypes(schema, ImmutableMap.of(2, (Type) replacement)); + + assertThat(result.findField(1).type()).isEqualTo(IntegerType.get()); + assertThat(result.findField(2).type()).isEqualTo(replacement); + } + + @Test + public void testReplaceFieldTypesListElement() { + Schema schema = + new Schema(required(1, "list", Types.ListType.ofRequired(2, Types.StructType.of()))); + + Schema result = + TypeUtil.replaceFieldTypes( + schema, + ImmutableMap.of(2, (Type) Types.StructType.of(required(3, "x", IntegerType.get())))); + + Types.ListType list = (Types.ListType) result.findField(1).type(); + assertThat(list.elementType().asStructType().field(3).name()).isEqualTo("x"); + assertThat(list.isElementRequired()).isTrue(); + } + + @Test + public void testReplaceFieldTypesNoMatchReturnsSameSchema() { + Schema schema = new Schema(required(1, "id", IntegerType.get())); + Schema result = + TypeUtil.replaceFieldTypes(schema, ImmutableMap.of(99, (Type) Types.LongType.get())); + assertThat(result).isSameAs(schema); + } } diff --git a/core/src/main/java/org/apache/iceberg/TrackedFile.java b/core/src/main/java/org/apache/iceberg/TrackedFile.java index fd73646e9fc5..1cf453d03dc8 100644 --- a/core/src/main/java/org/apache/iceberg/TrackedFile.java +++ b/core/src/main/java/org/apache/iceberg/TrackedFile.java @@ -104,17 +104,8 @@ interface TrackedFile { */ static Types.StructType schema( Types.StructType partitionType, Types.StructType contentStatsType) { - return schema(Tracking.schema(), partitionType, contentStatsType); - } - - /** Returns the schema with the given tracking, partition, and content stats types. */ - static Types.StructType schema( - Types.StructType trackingType, - Types.StructType partitionType, - Types.StructType contentStatsType) { return Types.StructType.of( - Types.NestedField.required( - TRACKING.fieldId(), TRACKING.name(), trackingType, TRACKING.doc()), + TRACKING, CONTENT_TYPE, FORMAT_VERSION, LOCATION, diff --git a/core/src/main/java/org/apache/iceberg/V4ManifestReader.java b/core/src/main/java/org/apache/iceberg/V4ManifestReader.java index 18fa36af433e..36419a46c61d 100644 --- a/core/src/main/java/org/apache/iceberg/V4ManifestReader.java +++ b/core/src/main/java/org/apache/iceberg/V4ManifestReader.java @@ -31,6 +31,7 @@ import org.apache.iceberg.io.InputFile; import org.apache.iceberg.metrics.ScanMetrics; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.relocated.com.google.common.collect.Sets; import org.apache.iceberg.types.TypeUtil; @@ -41,7 +42,7 @@ class V4ManifestReader extends CloseableGroup implements CloseableIterable { private final InputFile file; private final Schema readSchema; - private final boolean includeTombstones; + private final boolean includeNonLive; private final ScanMetrics scanMetrics; // partition pruning state, keyed by spec ID @@ -53,13 +54,13 @@ private V4ManifestReader( Schema readSchema, Map partitionEvaluators, Map partitionProjections, - boolean includeTombstones, + boolean includeNonLive, ScanMetrics scanMetrics) { this.file = file; this.readSchema = readSchema; this.partitionEvaluators = partitionEvaluators; this.partitionProjections = partitionProjections; - this.includeTombstones = includeTombstones; + this.includeNonLive = includeNonLive; this.scanMetrics = scanMetrics; } @@ -77,7 +78,7 @@ public CloseableIterator iterator() { CloseableIterable.filter(entries, entry -> isManifest(entry) || matchesPartition(entry)); } - if (!includeTombstones) { + if (!includeNonLive) { entries = CloseableIterable.filter(entries, entry -> entry.tracking().isLive()); } @@ -169,9 +170,10 @@ static class Builder { private final InputFile file; private final Types.StructType unionPartitionType; private final Map specsById; + private final Schema fullSchema; private Expression rowFilter = Expressions.alwaysTrue(); private boolean caseSensitive = true; - private boolean includeTombstones = false; + private boolean includeNonLive = false; private boolean scanPlanning = false; private Collection columns = null; private Schema fileProjection = null; @@ -179,8 +181,14 @@ static class Builder { private Builder(InputFile file, Map specsById) { this.file = file; - this.unionPartitionType = Partitioning.unionPartitionTypes(specsById.values()); this.specsById = specsById; + this.unionPartitionType = Partitioning.unionPartitionTypes(specsById.values()); + Schema base = + new Schema(TrackedFile.schema(unionPartitionType, Types.StructType.of()).fields()); + // the read schema carries row_position (via BASE_TYPE) so the reader can fill manifestPos + this.fullSchema = + TypeUtil.replaceFieldTypes( + base, ImmutableMap.of(TrackedFile.TRACKING.fieldId(), TrackingStruct.BASE_TYPE)); } /** Sets a row filter; files that cannot match the expression are skipped. */ @@ -195,9 +203,9 @@ Builder caseSensitive(boolean isCaseSensitive) { return this; } - /** Returns deleted and replaced files in addition to {@link Tracking#isLive() live} files. */ - Builder includeTombstones() { - this.includeTombstones = true; + /** Returns entries that are not {@link Tracking#isLive() live} in addition to live entries. */ + Builder includeNonLive() { + this.includeNonLive = true; return this; } @@ -241,7 +249,7 @@ Builder scanMetrics(ScanMetrics newScanMetrics) { V4ManifestReader build() { Map partitionEvaluators = Maps.newHashMap(); Map partitionProjections = Maps.newHashMap(); - if (hasPartitionFilter()) { + if (rowFilter != Expressions.alwaysTrue() && !unionPartitionType.fields().isEmpty()) { for (PartitionSpec spec : specsById.values()) { Expression partFilter = Projections.inclusive(spec, caseSensitive).project(rowFilter); if (partFilter != Expressions.alwaysTrue()) { @@ -253,55 +261,52 @@ V4ManifestReader build() { } } + boolean hasPartitionFilter = !partitionEvaluators.isEmpty(); return new V4ManifestReader( file, - readSchema(), + readSchema(hasPartitionFilter), partitionEvaluators, partitionProjections, - includeTombstones, + includeNonLive, scanMetrics); } - private boolean hasPartitionFilter() { - return rowFilter != Expressions.alwaysTrue() && !unionPartitionType.fields().isEmpty(); - } - - private Schema readSchema() { - Types.StructType trackingType = - scanPlanning ? TrackingStruct.SCAN_TYPE : TrackingStruct.BASE_TYPE; - Schema fullSchema = - new Schema( - TrackedFile.schema(trackingType, unionPartitionType, Types.StructType.of()).fields()); - Schema projection = projection(fullSchema); + private Schema readSchema(boolean hasPartitionFilter) { + Schema projection = projection(); if (projection == null) { + if (scanPlanning) { + // scan planning does not read the change-tracking fields omitted by SCAN_TYPE + return TypeUtil.replaceFieldTypes( + fullSchema, + ImmutableMap.of(TrackedFile.TRACKING.fieldId(), TrackingStruct.SCAN_TYPE)); + } + return fullSchema; } Set projectedIds = Sets.newHashSet(TypeUtil.getProjectedIds(projection)); - // fields the reader consumes internally: status for live filtering, row_position for + // fields the reader consumes internally: status for liveness filtering, row_position for // manifestPos, and content type to distinguish entry kinds projectedIds.add(Tracking.STATUS.fieldId()); projectedIds.add(MetadataColumns.ROW_POSITION.fieldId()); projectedIds.add(TrackedFile.CONTENT_TYPE.fieldId()); - if (hasPartitionFilter()) { + if (rowFilter != Expressions.alwaysTrue()) { + // record_count is read when evaluating a filter against file metrics + projectedIds.add(TrackedFile.RECORD_COUNT.fieldId()); + } + + if (hasPartitionFilter) { projectedIds.add(TrackedFile.SPEC_ID.fieldId()); projectedIds.add(TrackedFile.PARTITION_ID); projectedIds.addAll(TypeUtil.getProjectedIds(unionPartitionType)); } - // list and map fields cannot be projected by ID; their element IDs carry the selection - projectedIds.removeIf( - id -> { - Types.NestedField field = fullSchema.findField(id); - return field != null && (field.type().isListType() || field.type().isMapType()); - }); - // project instead of select to preserve narrow struct projections from the caller return TypeUtil.project(fullSchema, projectedIds); } - private Schema projection(Schema fullSchema) { + private Schema projection() { if (columns != null) { return caseSensitive ? fullSchema.select(columns) diff --git a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java index 67504bb7a411..db8d6d478d4a 100644 --- a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java +++ b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java @@ -179,7 +179,7 @@ public void testLiveFilesExcludesDeletedAndReplaced() throws IOException { } try (V4ManifestReader reader = - newReader(manifest, UNPARTITIONED_SPECS).includeTombstones().build()) { + newReader(manifest, UNPARTITIONED_SPECS).includeNonLive().build()) { assertThat(reader) .extracting(file -> file.tracking().status()) .containsExactly( @@ -240,10 +240,32 @@ public void testProjectionRestrictsFields() throws IOException { assertThat(actual.tracking()).isNotNull(); assertThat(actual.tracking().status()).isEqualTo(EntryStatus.ADDED); assertThat(actual.contentType()).isEqualTo(FileContent.DATA); - // sort_order_id, file_format, and spec_id are null because they were not projected + // sort_order_id, file_format, spec_id, and record_count are null because they were not + // projected and no row filter forces them assertThat(actual.sortOrderId()).isNull(); assertThat(actual.fileFormat()).isNull(); assertThat(actual.specId()).isNull(); + assertThat(actual.recordCount()).isEqualTo(-1L); + } + } + + @TestTemplate + public void testRowFilterForcesRecordCount() throws IOException { + TrackedFile file = dataFile("s3://bucket/file.parquet", EMPTY_PARTITION_DATA); + + InputFile manifest = writeManifest(EMPTY_PARTITION, ImmutableList.of(file)); + + // record_count is read when evaluating a row filter against file metrics, so it is projected + // even though the caller selected only location + Schema projection = new Schema(TrackedFile.LOCATION); + try (V4ManifestReader reader = + newReader(manifest, UNPARTITIONED_SPECS) + .project(projection) + .filterRows(Expressions.equal("id", 1)) + .build()) { + TrackedFile actual = Lists.newArrayList(reader).get(0); + assertThat(actual.location()).isEqualTo(file.location()); + assertThat(actual.recordCount()).isEqualTo(RECORD_COUNT); } } From 638e1d8cfd3eb6a3bbfa09f29f3726c66c66b3ca Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Thu, 23 Jul 2026 15:30:24 -0700 Subject: [PATCH 19/26] Rename to includeAll(), inline projection() --- .../org/apache/iceberg/V4ManifestReader.java | 35 ++++++++----------- .../apache/iceberg/TestV4ManifestReader.java | 3 +- 2 files changed, 16 insertions(+), 22 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/V4ManifestReader.java b/core/src/main/java/org/apache/iceberg/V4ManifestReader.java index 36419a46c61d..df5f52647351 100644 --- a/core/src/main/java/org/apache/iceberg/V4ManifestReader.java +++ b/core/src/main/java/org/apache/iceberg/V4ManifestReader.java @@ -42,7 +42,7 @@ class V4ManifestReader extends CloseableGroup implements CloseableIterable { private final InputFile file; private final Schema readSchema; - private final boolean includeNonLive; + private final boolean includeAll; private final ScanMetrics scanMetrics; // partition pruning state, keyed by spec ID @@ -54,13 +54,13 @@ private V4ManifestReader( Schema readSchema, Map partitionEvaluators, Map partitionProjections, - boolean includeNonLive, + boolean includeAll, ScanMetrics scanMetrics) { this.file = file; this.readSchema = readSchema; this.partitionEvaluators = partitionEvaluators; this.partitionProjections = partitionProjections; - this.includeNonLive = includeNonLive; + this.includeAll = includeAll; this.scanMetrics = scanMetrics; } @@ -78,7 +78,7 @@ public CloseableIterator iterator() { CloseableIterable.filter(entries, entry -> isManifest(entry) || matchesPartition(entry)); } - if (!includeNonLive) { + if (!includeAll) { entries = CloseableIterable.filter(entries, entry -> entry.tracking().isLive()); } @@ -173,7 +173,7 @@ static class Builder { private final Schema fullSchema; private Expression rowFilter = Expressions.alwaysTrue(); private boolean caseSensitive = true; - private boolean includeNonLive = false; + private boolean includeAll = false; private boolean scanPlanning = false; private Collection columns = null; private Schema fileProjection = null; @@ -203,9 +203,9 @@ Builder caseSensitive(boolean isCaseSensitive) { return this; } - /** Returns entries that are not {@link Tracking#isLive() live} in addition to live entries. */ - Builder includeNonLive() { - this.includeNonLive = true; + /** Returns all entries without filtering by {@link Tracking#isLive() liveness}. */ + Builder includeAll() { + this.includeAll = true; return this; } @@ -267,12 +267,17 @@ V4ManifestReader build() { readSchema(hasPartitionFilter), partitionEvaluators, partitionProjections, - includeNonLive, + includeAll, scanMetrics); } private Schema readSchema(boolean hasPartitionFilter) { - Schema projection = projection(); + Schema projection = fileProjection; + if (columns != null) { + projection = + caseSensitive ? fullSchema.select(columns) : fullSchema.caseInsensitiveSelect(columns); + } + if (projection == null) { if (scanPlanning) { // scan planning does not read the change-tracking fields omitted by SCAN_TYPE @@ -305,15 +310,5 @@ private Schema readSchema(boolean hasPartitionFilter) { // project instead of select to preserve narrow struct projections from the caller return TypeUtil.project(fullSchema, projectedIds); } - - private Schema projection() { - if (columns != null) { - return caseSensitive - ? fullSchema.select(columns) - : fullSchema.caseInsensitiveSelect(columns); - } - - return fileProjection; - } } } diff --git a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java index db8d6d478d4a..b755d382a464 100644 --- a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java +++ b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java @@ -178,8 +178,7 @@ public void testLiveFilesExcludesDeletedAndReplaced() throws IOException { .containsExactly(EntryStatus.ADDED, EntryStatus.EXISTING, EntryStatus.MODIFIED); } - try (V4ManifestReader reader = - newReader(manifest, UNPARTITIONED_SPECS).includeNonLive().build()) { + try (V4ManifestReader reader = newReader(manifest, UNPARTITIONED_SPECS).includeAll().build()) { assertThat(reader) .extracting(file -> file.tracking().status()) .containsExactly( From 3c6be36ee24572e580d7b87ebecc928afa39a4da Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Fri, 24 Jul 2026 11:00:10 -0700 Subject: [PATCH 20/26] PR feedback from Ryan and Steven --- .../java/org/apache/iceberg/TrackedFile.java | 5 +- .../org/apache/iceberg/TrackedFileStruct.java | 5 +- .../org/apache/iceberg/V4ManifestReader.java | 94 ++--- .../org/apache/iceberg/TestTrackedFile.java | 18 +- .../iceberg/TestTrackedFileAdapters.java | 14 +- .../apache/iceberg/TestTrackedFileStruct.java | 16 +- .../apache/iceberg/TestV4ManifestReader.java | 395 ++++++++++-------- 7 files changed, 285 insertions(+), 262 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/TrackedFile.java b/core/src/main/java/org/apache/iceberg/TrackedFile.java index 1cf453d03dc8..1a457fb048e2 100644 --- a/core/src/main/java/org/apache/iceberg/TrackedFile.java +++ b/core/src/main/java/org/apache/iceberg/TrackedFile.java @@ -102,9 +102,8 @@ interface TrackedFile { *

The partition and content stats fields use {@link Types.UnknownType} when their types have * no fields, so that they are not stored in manifest files. */ - static Types.StructType schema( - Types.StructType partitionType, Types.StructType contentStatsType) { - return Types.StructType.of( + static Schema schema(Types.StructType partitionType, Types.StructType contentStatsType) { + return new Schema( TRACKING, CONTENT_TYPE, FORMAT_VERSION, diff --git a/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java b/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java index 4bd24b39d090..f5ce03c7eb62 100644 --- a/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java +++ b/core/src/main/java/org/apache/iceberg/TrackedFileStruct.java @@ -102,10 +102,10 @@ class TrackedFileStruct extends SupportsIndexProjection implements TrackedFile, int formatVersion, String location, FileFormat fileFormat, - PartitionData partition, long recordCount, long fileSizeInBytes, Integer specId, + PartitionData partition, ContentStats contentStats, Integer sortOrderId, DeletionVector deletionVector, @@ -121,9 +121,8 @@ class TrackedFileStruct extends SupportsIndexProjection implements TrackedFile, this.fileFormat = fileFormat; this.recordCount = recordCount; this.fileSizeInBytes = fileSizeInBytes; - this.partitionData = partition; - this.specId = specId; + this.partitionData = partition; this.contentStats = contentStats; this.sortOrderId = sortOrderId; this.deletionVector = deletionVector; diff --git a/core/src/main/java/org/apache/iceberg/V4ManifestReader.java b/core/src/main/java/org/apache/iceberg/V4ManifestReader.java index df5f52647351..808cd4f82412 100644 --- a/core/src/main/java/org/apache/iceberg/V4ManifestReader.java +++ b/core/src/main/java/org/apache/iceberg/V4ManifestReader.java @@ -36,6 +36,7 @@ import org.apache.iceberg.relocated.com.google.common.collect.Sets; import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.Pair; import org.apache.iceberg.util.StructProjection; /** Reader that reads a v4+ manifest file as {@link TrackedFile}s. */ @@ -45,21 +46,18 @@ class V4ManifestReader extends CloseableGroup implements CloseableIterable partitionEvaluators; - private final Map partitionProjections; + // partition filters keyed by spec ID; empty when no partition filter applies + private final Map> partitionFilters; private V4ManifestReader( InputFile file, Schema readSchema, - Map partitionEvaluators, - Map partitionProjections, + Map> partitionFilters, boolean includeAll, ScanMetrics scanMetrics) { this.file = file; this.readSchema = readSchema; - this.partitionEvaluators = partitionEvaluators; - this.partitionProjections = partitionProjections; + this.partitionFilters = partitionFilters; this.includeAll = includeAll; this.scanMetrics = scanMetrics; } @@ -72,8 +70,8 @@ static Builder builder(InputFile file, Map specsById) { @Override public CloseableIterator iterator() { CloseableIterable entries = CloseableIterable.transform(open(), this::prepare); - if (!partitionEvaluators.isEmpty()) { - // manifest references are expanded later and are not pruned by the partition filter + if (!partitionFilters.isEmpty()) { + // manifests have no partition, so the partition filter cannot apply to them entries = CloseableIterable.filter(entries, entry -> isManifest(entry) || matchesPartition(entry)); } @@ -92,19 +90,14 @@ private boolean matchesPartition(TrackedFile trackedFile) { return true; } - Evaluator evaluator = partitionEvaluators.get(specId); - if (evaluator == null) { + Pair partitionFilter = partitionFilters.get(specId); + if (partitionFilter == null) { // the row filter does not project to a partition filter for this spec return true; } - StructProjection projection = partitionProjections.get(specId); - Preconditions.checkState( - projection != null, - "Cannot produce partition tuple for spec ID %s in manifest %s", - specId, - file.location()); - + Evaluator evaluator = partitionFilter.first(); + StructProjection projection = partitionFilter.second(); boolean matches = evaluator.eval(projection.wrap(trackedFile.partition())); if (!matches) { incrementSkipCount(trackedFile.contentType()); @@ -176,24 +169,23 @@ static class Builder { private boolean includeAll = false; private boolean scanPlanning = false; private Collection columns = null; - private Schema fileProjection = null; + private Schema requestedProjection = null; private ScanMetrics scanMetrics = ScanMetrics.noop(); private Builder(InputFile file, Map specsById) { this.file = file; this.specsById = specsById; this.unionPartitionType = Partitioning.unionPartitionTypes(specsById.values()); - Schema base = - new Schema(TrackedFile.schema(unionPartitionType, Types.StructType.of()).fields()); + Schema base = TrackedFile.schema(unionPartitionType, Types.StructType.of()); // the read schema carries row_position (via BASE_TYPE) so the reader can fill manifestPos this.fullSchema = TypeUtil.replaceFieldTypes( base, ImmutableMap.of(TrackedFile.TRACKING.fieldId(), TrackingStruct.BASE_TYPE)); } - /** Sets a row filter; files that cannot match the expression are skipped. */ - Builder filterRows(Expression expr) { - Preconditions.checkArgument(expr != null, "Invalid row filter: null"); + /** Sets a filter; files that cannot match the expression are skipped. */ + Builder filter(Expression expr) { + Preconditions.checkArgument(expr != null, "Invalid filter: null"); this.rowFilter = expr; return this; } @@ -212,7 +204,7 @@ Builder includeAll() { /** Configures the reader to select the minimal fields needed for scan planning. */ Builder forScanPlanning() { Preconditions.checkState( - columns == null && fileProjection == null, + columns == null && requestedProjection == null, "Cannot use forScanPlanning() with select(Collection) or project(Schema)"); this.scanPlanning = true; return this; @@ -224,19 +216,19 @@ Builder select(Collection newColumns) { Preconditions.checkState( !scanPlanning, "Cannot use select(Collection) with forScanPlanning()"); Preconditions.checkState( - fileProjection == null, + requestedProjection == null, "Cannot select columns using both select(Collection) and project(Schema)"); this.columns = newColumns; return this; } /** Sets the exact schema to read; used in place of {@link #select(Collection)}. */ - Builder project(Schema newFileProjection) { + Builder project(Schema newProjection) { Preconditions.checkState(!scanPlanning, "Cannot use project(Schema) with forScanPlanning()"); Preconditions.checkState( columns == null, "Cannot select columns using both select(Collection) and project(Schema)"); - this.fileProjection = newFileProjection; + this.requestedProjection = newProjection; return this; } @@ -247,48 +239,45 @@ Builder scanMetrics(ScanMetrics newScanMetrics) { } V4ManifestReader build() { - Map partitionEvaluators = Maps.newHashMap(); - Map partitionProjections = Maps.newHashMap(); + Map> partitionFilters = Maps.newHashMap(); if (rowFilter != Expressions.alwaysTrue() && !unionPartitionType.fields().isEmpty()) { for (PartitionSpec spec : specsById.values()) { Expression partFilter = Projections.inclusive(spec, caseSensitive).project(rowFilter); if (partFilter != Expressions.alwaysTrue()) { - partitionEvaluators.put( - spec.specId(), new Evaluator(spec.partitionType(), partFilter, caseSensitive)); - partitionProjections.put( - spec.specId(), StructProjection.create(unionPartitionType, spec.partitionType())); + Evaluator evaluator = new Evaluator(spec.partitionType(), partFilter, caseSensitive); + StructProjection projection = + StructProjection.create(unionPartitionType, spec.partitionType()); + partitionFilters.put(spec.specId(), Pair.of(evaluator, projection)); } } } - boolean hasPartitionFilter = !partitionEvaluators.isEmpty(); + boolean hasPartitionFilter = !partitionFilters.isEmpty(); return new V4ManifestReader( - file, - readSchema(hasPartitionFilter), - partitionEvaluators, - partitionProjections, - includeAll, - scanMetrics); + file, readSchema(hasPartitionFilter), partitionFilters, includeAll, scanMetrics); } private Schema readSchema(boolean hasPartitionFilter) { - Schema projection = fileProjection; + if (scanPlanning) { + // scan planning does not read the change-tracking fields omitted by SCAN_TYPE + return TypeUtil.replaceFieldTypes( + fullSchema, ImmutableMap.of(TrackedFile.TRACKING.fieldId(), TrackingStruct.SCAN_TYPE)); + } + if (columns != null) { - projection = + Schema selected = caseSensitive ? fullSchema.select(columns) : fullSchema.caseInsensitiveSelect(columns); + return addRequiredColumns(selected, hasPartitionFilter); } - if (projection == null) { - if (scanPlanning) { - // scan planning does not read the change-tracking fields omitted by SCAN_TYPE - return TypeUtil.replaceFieldTypes( - fullSchema, - ImmutableMap.of(TrackedFile.TRACKING.fieldId(), TrackingStruct.SCAN_TYPE)); - } - - return fullSchema; + if (requestedProjection != null) { + return addRequiredColumns(requestedProjection, hasPartitionFilter); } + return fullSchema; + } + + private Schema addRequiredColumns(Schema projection, boolean hasPartitionFilter) { Set projectedIds = Sets.newHashSet(TypeUtil.getProjectedIds(projection)); // fields the reader consumes internally: status for liveness filtering, row_position for @@ -301,6 +290,7 @@ private Schema readSchema(boolean hasPartitionFilter) { projectedIds.add(TrackedFile.RECORD_COUNT.fieldId()); } + // add the partition tuple only when it is needed to evaluate a partition filter if (hasPartitionFilter) { projectedIds.add(TrackedFile.SPEC_ID.fieldId()); projectedIds.add(TrackedFile.PARTITION_ID); diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFile.java b/core/src/test/java/org/apache/iceberg/TestTrackedFile.java index f3c5adb58fac..9a32243cc12f 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackedFile.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackedFile.java @@ -40,7 +40,7 @@ public class TestTrackedFile { @Test public void schemaFieldOrder() { - Types.StructType type = TrackedFile.schema(PARTITION_TYPE, CONTENT_STATS_TYPE); + Types.StructType type = TrackedFile.schema(PARTITION_TYPE, CONTENT_STATS_TYPE).asStruct(); List fields = type.fields(); assertThat(fields) @@ -66,7 +66,7 @@ public void schemaFieldOrder() { @Test public void schemaFieldIds() { - Types.StructType type = TrackedFile.schema(PARTITION_TYPE, CONTENT_STATS_TYPE); + Types.StructType type = TrackedFile.schema(PARTITION_TYPE, CONTENT_STATS_TYPE).asStruct(); List fields = type.fields(); assertThat(fields) @@ -77,7 +77,7 @@ public void schemaFieldIds() { @Test public void schemaUsesProvidedType() { - Types.StructType type = TrackedFile.schema(PARTITION_TYPE, CONTENT_STATS_TYPE); + Types.StructType type = TrackedFile.schema(PARTITION_TYPE, CONTENT_STATS_TYPE).asStruct(); Types.NestedField contentStatsField = type.field(TrackedFile.CONTENT_STATS_ID); Types.NestedField partitionField = type.field(TrackedFile.PARTITION_ID); @@ -97,8 +97,8 @@ public void schemaReflectsInput() { Types.StructType smallStats = StatsUtil.statsReadSchema(smallSchema, ImmutableList.of(1)); Types.StructType largeStats = StatsUtil.statsReadSchema(largeSchema, ImmutableList.of(1, 3)); - Types.StructType smallType = TrackedFile.schema(PARTITION_TYPE, smallStats); - Types.StructType largeType = TrackedFile.schema(PARTITION_TYPE, largeStats); + Types.StructType smallType = TrackedFile.schema(PARTITION_TYPE, smallStats).asStruct(); + Types.StructType largeType = TrackedFile.schema(PARTITION_TYPE, largeStats).asStruct(); Types.StructType smallResult = smallType.field(TrackedFile.CONTENT_STATS_ID).type().asStructType(); @@ -111,7 +111,7 @@ public void schemaReflectsInput() { @Test public void schemaPartitionIsOptional() { - Types.StructType type = TrackedFile.schema(PARTITION_TYPE, CONTENT_STATS_TYPE); + Types.StructType type = TrackedFile.schema(PARTITION_TYPE, CONTENT_STATS_TYPE).asStruct(); Types.NestedField partitionField = type.field(TrackedFile.PARTITION_ID); assertThat(partitionField.isOptional()).isTrue(); @@ -121,12 +121,14 @@ public void schemaPartitionIsOptional() { @Test public void schemaUsesUnknownForEmptyStructs() { - Types.StructType type = TrackedFile.schema(Types.StructType.of(), Types.StructType.of()); + Types.StructType type = + TrackedFile.schema(Types.StructType.of(), Types.StructType.of()).asStruct(); assertThat(type.field(TrackedFile.PARTITION_ID).type()).isEqualTo(Types.UnknownType.get()); assertThat(type.field(TrackedFile.CONTENT_STATS_ID).type()).isEqualTo(Types.UnknownType.get()); - Types.StructType partitionedType = TrackedFile.schema(PARTITION_TYPE, Types.StructType.of()); + Types.StructType partitionedType = + TrackedFile.schema(PARTITION_TYPE, Types.StructType.of()).asStruct(); assertThat(partitionedType.field(TrackedFile.PARTITION_ID).type()).isEqualTo(PARTITION_TYPE); assertThat(partitionedType.field(TrackedFile.CONTENT_STATS_ID).type()) .isEqualTo(Types.UnknownType.get()); diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java index 37b8beb112a2..96422c126411 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackedFileAdapters.java @@ -110,10 +110,10 @@ void testDataFileAdapterDelegation() { FORMAT_VERSION_V4, DATA_FILE_LOCATION, FileFormat.PARQUET, - PARTITION, 100L, 1024L, PARTITIONED_SPEC_ID, + PARTITION, CONTENT_STATS, 3, null, @@ -186,10 +186,10 @@ void testEqualityDeleteFileAdapterDelegation() { FORMAT_VERSION_V4, "s3://bucket/eq-delete.avro", FileFormat.AVRO, - PARTITION, 50L, 512L, PARTITIONED_SPEC_ID, + PARTITION, CONTENT_STATS, 5, null, @@ -271,10 +271,10 @@ void testDVDeleteFileAdapterDelegation() { FORMAT_VERSION_V4, DATA_FILE_LOCATION, FileFormat.PARQUET, - PARTITION, 100L, 1024L, PARTITIONED_SPEC_ID, + PARTITION, null, null, dv, @@ -368,12 +368,12 @@ void testNullTrackingReturnsNullTrackingFields() { 0, null, null, - null, 0L, 0L, null, null, null, + null, deletionVector(), null, null, @@ -422,7 +422,6 @@ void testUnknownSpecIdThrows() { 0, null, null, - null, 0L, 0L, 99, @@ -432,6 +431,7 @@ void testUnknownSpecIdThrows() { null, null, null, + null, null); assertThatThrownBy(() -> TrackedFileAdapters.asDataFile(file, ImmutableMap.of())) @@ -448,7 +448,6 @@ void testSpecIdMismatchThrows() { 0, null, null, - null, 0L, 0L, PARTITIONED_SPEC_ID, @@ -458,6 +457,7 @@ void testSpecIdMismatchThrows() { null, null, null, + null, null); int mismatchedSpecId = PARTITIONED_SPEC_ID + 1; PartitionSpec mismatched = @@ -503,7 +503,6 @@ private static TrackedFileStruct dummyTrackedFile(FileContent contentType) { FORMAT_VERSION_V4, DATA_FILE_LOCATION, FileFormat.PARQUET, - null, 1L, 1L, null, @@ -513,6 +512,7 @@ private static TrackedFileStruct dummyTrackedFile(FileContent contentType) { null, null, null, + null, null); } diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java b/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java index b04302f7f6a2..010695e43c08 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java @@ -35,7 +35,7 @@ class TestTrackedFileStruct { private static final int FORMAT_VERSION_V4 = 4; private static final List FIELDS = - TrackedFile.schema(Types.StructType.of(), Types.StructType.of()).fields(); + TrackedFile.schema(Types.StructType.of(), Types.StructType.of()).asStruct().fields(); private static final Tracking TRACKING = Mockito.mock(Tracking.class); private static final Tracking TRACKING_COPY = Mockito.mock(Tracking.class); @@ -69,10 +69,10 @@ void fieldAccess() { FORMAT_VERSION_V4, "s3://bucket/data/00000-0-file.parquet", FileFormat.PARQUET, - PARTITION, 50L, 512L, 1, + PARTITION, CONTENT_STATS, 5, DELETION_VECTOR, @@ -146,10 +146,10 @@ void getByPosition() { FORMAT_VERSION_V4, "s3://bucket/data/00000-0-file.parquet", FileFormat.PARQUET, - PARTITION, 50L, 512L, 1, + PARTITION, CONTENT_STATS, 5, DELETION_VECTOR, @@ -187,10 +187,10 @@ void copy() { FORMAT_VERSION_V4, "s3://bucket/data/00000-0-file.parquet", FileFormat.PARQUET, - PARTITION, 50L, 512L, 1, + PARTITION, CONTENT_STATS, 5, DELETION_VECTOR, @@ -236,10 +236,10 @@ void copyWithStats() { FORMAT_VERSION_V4, "s3://bucket/data/00000-0-file.parquet", FileFormat.PARQUET, - PARTITION, 50L, 512L, 1, + PARTITION, stats, 5, DELETION_VECTOR, @@ -283,10 +283,10 @@ void copyWithoutStats() { FORMAT_VERSION_V4, "s3://bucket/data/00000-0-file.parquet", FileFormat.PARQUET, - PARTITION, 50L, 512L, 1, + PARTITION, stats, 5, DELETION_VECTOR, @@ -355,12 +355,12 @@ void serializationRoundTrip(RoundTripSerializer serializer) t FileContent.DATA, FORMAT_VERSION_V4, "s3://bucket/data/file.parquet", - FileFormat.PARQUET, - null, // PartitionData has its own serialization tests + FileFormat.PARQUET, // PartitionData has its own serialization tests 100L, 1024L, 7, null, + null, 1, null, // DeletionVector has its own serialization tests null, // ManifestInfo has its own serialization tests diff --git a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java index b755d382a464..f304fe43a926 100644 --- a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java +++ b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java @@ -29,6 +29,7 @@ import java.util.Locale; import java.util.Map; import org.apache.iceberg.expressions.Expressions; +import org.apache.iceberg.inmemory.InMemoryOutputFile; import org.apache.iceberg.io.FileAppender; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.InputFile; @@ -39,12 +40,14 @@ import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.transforms.Transforms; +import org.apache.iceberg.types.Comparators; +import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types; -import org.junit.jupiter.api.TestTemplate; -import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.FieldSource; -@ExtendWith(ParameterizedTestExtension.class) public class TestV4ManifestReader { private static final long SNAPSHOT_ID = 42L; private static final int FORMAT_VERSION_V4 = 4; @@ -69,19 +72,19 @@ public class TestV4ManifestReader { private static final Map UNPARTITIONED_SPECS = ImmutableMap.of(PartitionSpec.unpartitioned().specId(), PartitionSpec.unpartitioned()); - @Parameter private FileFormat format; + private static final List FORMATS = + ImmutableList.of(FileFormat.AVRO, FileFormat.PARQUET); - @Parameters(name = "format = {0}") - protected static List parameters() { - return ImmutableList.of(FileFormat.AVRO, FileFormat.PARQUET); - } + // row_position is appended after the tracking schema fields by the reader + private static final int MANIFEST_POS_ORDINAL = Tracking.schema().fields().size(); @TempDir private Path tempDir; private final FileIO fileIO = new TestTables.LocalFileIO(); - @TestTemplate - public void testRoundTrip() throws IOException { + @ParameterizedTest + @FieldSource("FORMATS") + public void testRoundTrip(FileFormat format) throws IOException { DeletionVector dv = deletionVector(DV_LOCATION, DV_OFFSET, DV_SIZE_IN_BYTES, DV_CARDINALITY); TrackedFile file = @@ -91,10 +94,10 @@ public void testRoundTrip() throws IOException { FORMAT_VERSION_V4, "s3://bucket/data/file.parquet", FileFormat.PARQUET, - partition(7), RECORD_COUNT, FILE_SIZE_IN_BYTES, - 0, + SPEC.specId(), + partition(7), null, SORT_ORDER_ID, dv, @@ -103,37 +106,30 @@ public void testRoundTrip() throws IOException { ImmutableList.of(50L, 100L), null); - InputFile manifest = writeManifest(PARTITION_TYPE, ImmutableList.of(file)); + InputFile manifest = writeManifest(format, PARTITION_TYPE, ImmutableList.of(file)); List read = read(manifest, PARTITIONED_SPECS); assertThat(read).hasSize(1); TrackedFile actual = read.get(0); - assertThat(actual.contentType()).isEqualTo(FileContent.DATA); - assertThat(actual.formatVersion()).isEqualTo(FORMAT_VERSION_V4); - assertThat(actual.location()).isEqualTo("s3://bucket/data/file.parquet"); - assertThat(actual.fileFormat()).isEqualTo(FileFormat.PARQUET); - assertThat(actual.recordCount()).isEqualTo(RECORD_COUNT); - assertThat(actual.fileSizeInBytes()).isEqualTo(FILE_SIZE_IN_BYTES); - assertThat(actual.specId()).isEqualTo(0); - assertThat(actual.sortOrderId()).isEqualTo(SORT_ORDER_ID); - assertThat(actual.keyMetadata()).isEqualTo(ByteBuffer.wrap(new byte[] {1, 2, 3})); - assertThat(actual.splitOffsets()).containsExactly(50L, 100L); - assertThat(actual.partition().get(0, Integer.class)).isEqualTo(7); - - assertThat(actual.tracking()).isNotNull(); - assertThat(actual.tracking().status()).isEqualTo(EntryStatus.ADDED); - assertThat(actual.tracking().snapshotId()).isEqualTo(SNAPSHOT_ID); - - assertThat(actual.deletionVector()).isNotNull(); - assertThat(actual.deletionVector().location()).isEqualTo(DV_LOCATION); - assertThat(actual.deletionVector().offset()).isEqualTo(DV_OFFSET); - assertThat(actual.deletionVector().sizeInBytes()).isEqualTo(DV_SIZE_IN_BYTES); - assertThat(actual.deletionVector().cardinality()).isEqualTo(DV_CARDINALITY); - } - - @TestTemplate - public void testEqualityDeleteRoundTrip() throws IOException { + // the reader fills row_position (manifestPos) and manifestLocation, which the written file + // does not have; mirror them on the expected file before comparing + TrackingStruct expectedTracking = (TrackingStruct) ((TrackedFileStruct) file).tracking(); + expectedTracking.set(MANIFEST_POS_ORDINAL, 0L); + expectedTracking.setManifestLocation(manifest.location()); + + Types.StructType readType = + TypeUtil.replaceFieldTypes( + TrackedFile.schema(PARTITION_TYPE, Types.StructType.of()), + ImmutableMap.of(TrackedFile.TRACKING.fieldId(), TrackingStruct.BASE_TYPE)) + .asStruct(); + assertThat(Comparators.forType(readType).compare((StructLike) file, (StructLike) actual)) + .isEqualTo(0); + } + + @ParameterizedTest + @FieldSource("FORMATS") + public void testEqualityDeleteRoundTrip(FileFormat format) throws IOException { TrackedFile delete = new TrackedFileStruct( addedTracking(), @@ -141,10 +137,10 @@ public void testEqualityDeleteRoundTrip() throws IOException { FORMAT_VERSION_V4, "s3://bucket/eq-delete.parquet", FileFormat.PARQUET, - EMPTY_PARTITION_DATA, RECORD_COUNT, FILE_SIZE_IN_BYTES, 0, + EMPTY_PARTITION_DATA, null, null, null, @@ -153,15 +149,16 @@ public void testEqualityDeleteRoundTrip() throws IOException { null, ImmutableList.of(1, 2)); - InputFile manifest = writeManifest(EMPTY_PARTITION, ImmutableList.of(delete)); + InputFile manifest = writeManifest(format, EMPTY_PARTITION, ImmutableList.of(delete)); TrackedFile actual = read(manifest, UNPARTITIONED_SPECS).get(0); assertThat(actual.contentType()).isEqualTo(FileContent.EQUALITY_DELETES); assertThat(actual.equalityIds()).containsExactly(1, 2); } - @TestTemplate - public void testLiveFilesExcludesDeletedAndReplaced() throws IOException { + @ParameterizedTest + @FieldSource("FORMATS") + public void testStatusFiltering(FileFormat format) throws IOException { List files = ImmutableList.of( fileWithStatus(EntryStatus.ADDED, "s3://bucket/added.parquet"), @@ -170,15 +167,17 @@ public void testLiveFilesExcludesDeletedAndReplaced() throws IOException { fileWithStatus(EntryStatus.DELETED, "s3://bucket/deleted.parquet"), fileWithStatus(EntryStatus.REPLACED, "s3://bucket/replaced.parquet")); - InputFile manifest = writeManifest(EMPTY_PARTITION, files); + InputFile manifest = writeManifest(format, EMPTY_PARTITION, files); - try (V4ManifestReader reader = newReader(manifest, UNPARTITIONED_SPECS).build()) { + try (V4ManifestReader reader = + V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS).build()) { assertThat(reader) .extracting(file -> file.tracking().status()) .containsExactly(EntryStatus.ADDED, EntryStatus.EXISTING, EntryStatus.MODIFIED); } - try (V4ManifestReader reader = newReader(manifest, UNPARTITIONED_SPECS).includeAll().build()) { + try (V4ManifestReader reader = + V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS).includeAll().build()) { assertThat(reader) .extracting(file -> file.tracking().status()) .containsExactly( @@ -190,15 +189,16 @@ public void testLiveFilesExcludesDeletedAndReplaced() throws IOException { } } - @TestTemplate - public void testManifestLocationAndPosition() throws IOException { + @ParameterizedTest + @FieldSource("FORMATS") + public void testManifestLocationAndPosition(FileFormat format) throws IOException { List files = ImmutableList.of( dataFile("s3://bucket/a.parquet", EMPTY_PARTITION_DATA), dataFile("s3://bucket/b.parquet", EMPTY_PARTITION_DATA), dataFile("s3://bucket/c.parquet", EMPTY_PARTITION_DATA)); - InputFile manifest = writeManifest(EMPTY_PARTITION, files); + InputFile manifest = writeManifest(format, EMPTY_PARTITION, files); List read = read(manifest, UNPARTITIONED_SPECS); assertThat(read) @@ -207,8 +207,9 @@ public void testManifestLocationAndPosition() throws IOException { assertThat(read).extracting(file -> file.tracking().manifestPos()).containsExactly(0L, 1L, 2L); } - @TestTemplate - public void testProjectionRestrictsFields() throws IOException { + @ParameterizedTest + @FieldSource("FORMATS") + public void testProjectionRestrictsFields(FileFormat format) throws IOException { TrackedFile file = new TrackedFileStruct( addedTracking(), @@ -216,10 +217,10 @@ public void testProjectionRestrictsFields() throws IOException { FORMAT_VERSION_V4, "s3://bucket/file.parquet", FileFormat.PARQUET, - EMPTY_PARTITION_DATA, RECORD_COUNT, FILE_SIZE_IN_BYTES, 0, + EMPTY_PARTITION_DATA, null, SORT_ORDER_ID, null, @@ -228,11 +229,11 @@ public void testProjectionRestrictsFields() throws IOException { null, null); - InputFile manifest = writeManifest(EMPTY_PARTITION, ImmutableList.of(file)); + InputFile manifest = writeManifest(format, EMPTY_PARTITION, ImmutableList.of(file)); Schema projection = new Schema(TrackedFile.LOCATION); try (V4ManifestReader reader = - newReader(manifest, UNPARTITIONED_SPECS).project(projection).build()) { + V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS).project(projection).build()) { TrackedFile actual = Lists.newArrayList(reader).get(0); assertThat(actual.location()).isEqualTo(file.location()); // tracking and content_type are always projected, even though the caller omitted them @@ -248,19 +249,20 @@ public void testProjectionRestrictsFields() throws IOException { } } - @TestTemplate - public void testRowFilterForcesRecordCount() throws IOException { + @ParameterizedTest + @FieldSource("FORMATS") + public void testRowFilterForcesRecordCount(FileFormat format) throws IOException { TrackedFile file = dataFile("s3://bucket/file.parquet", EMPTY_PARTITION_DATA); - InputFile manifest = writeManifest(EMPTY_PARTITION, ImmutableList.of(file)); + InputFile manifest = writeManifest(format, EMPTY_PARTITION, ImmutableList.of(file)); // record_count is read when evaluating a row filter against file metrics, so it is projected // even though the caller selected only location Schema projection = new Schema(TrackedFile.LOCATION); try (V4ManifestReader reader = - newReader(manifest, UNPARTITIONED_SPECS) + V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS) .project(projection) - .filterRows(Expressions.equal("id", 1)) + .filter(Expressions.equal("id", 1)) .build()) { TrackedFile actual = Lists.newArrayList(reader).get(0); assertThat(actual.location()).isEqualTo(file.location()); @@ -268,14 +270,17 @@ public void testRowFilterForcesRecordCount() throws IOException { } } - @TestTemplate - public void testSelectRestrictsFields() throws IOException { + @ParameterizedTest + @FieldSource("FORMATS") + public void testSelectRestrictsFields(FileFormat format) throws IOException { TrackedFile file = dataFile("s3://bucket/file.parquet", EMPTY_PARTITION_DATA); - InputFile manifest = writeManifest(EMPTY_PARTITION, ImmutableList.of(file)); + InputFile manifest = writeManifest(format, EMPTY_PARTITION, ImmutableList.of(file)); try (V4ManifestReader reader = - newReader(manifest, UNPARTITIONED_SPECS).select(ImmutableList.of("location")).build()) { + V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS) + .select(ImmutableList.of("location")) + .build()) { TrackedFile actual = Lists.newArrayList(reader).get(0); assertThat(actual.location()).isEqualTo(file.location()); // tracking status and content_type are joined in even though not selected @@ -290,14 +295,15 @@ public void testSelectRestrictsFields() throws IOException { } } - @TestTemplate - public void testCaseInsensitiveSelect() throws IOException { + @ParameterizedTest + @FieldSource("FORMATS") + public void testCaseInsensitiveSelect(FileFormat format) throws IOException { TrackedFile file = dataFile("s3://bucket/file.parquet", EMPTY_PARTITION_DATA); - InputFile manifest = writeManifest(EMPTY_PARTITION, ImmutableList.of(file)); + InputFile manifest = writeManifest(format, EMPTY_PARTITION, ImmutableList.of(file)); try (V4ManifestReader reader = - newReader(manifest, UNPARTITIONED_SPECS) + V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS) .select(ImmutableList.of("LOCATION")) .caseSensitive(false) .build()) { @@ -307,13 +313,13 @@ public void testCaseInsensitiveSelect() throws IOException { } } - @TestTemplate + @Test public void testProjectionModesAreMutuallyExclusive() { InputFile manifest = fileIO.newInputFile(tempDir.resolve("manifest.avro").toString()); assertThatThrownBy( () -> - newReader(manifest, UNPARTITIONED_SPECS) + V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS) .select(ImmutableList.of("location")) .project(new Schema(TrackedFile.LOCATION))) .isInstanceOf(IllegalStateException.class) @@ -322,7 +328,7 @@ public void testProjectionModesAreMutuallyExclusive() { assertThatThrownBy( () -> - newReader(manifest, UNPARTITIONED_SPECS) + V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS) .project(new Schema(TrackedFile.LOCATION)) .select(ImmutableList.of("location"))) .isInstanceOf(IllegalStateException.class) @@ -331,7 +337,7 @@ public void testProjectionModesAreMutuallyExclusive() { assertThatThrownBy( () -> - newReader(manifest, UNPARTITIONED_SPECS) + V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS) .forScanPlanning() .select(ImmutableList.of("location"))) .isInstanceOf(IllegalStateException.class) @@ -339,7 +345,7 @@ public void testProjectionModesAreMutuallyExclusive() { assertThatThrownBy( () -> - newReader(manifest, UNPARTITIONED_SPECS) + V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS) .select(ImmutableList.of("location")) .forScanPlanning()) .isInstanceOf(IllegalStateException.class) @@ -348,7 +354,7 @@ public void testProjectionModesAreMutuallyExclusive() { assertThatThrownBy( () -> - newReader(manifest, UNPARTITIONED_SPECS) + V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS) .forScanPlanning() .project(new Schema(TrackedFile.LOCATION))) .isInstanceOf(IllegalStateException.class) @@ -356,7 +362,7 @@ public void testProjectionModesAreMutuallyExclusive() { assertThatThrownBy( () -> - newReader(manifest, UNPARTITIONED_SPECS) + V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS) .project(new Schema(TrackedFile.LOCATION)) .forScanPlanning()) .isInstanceOf(IllegalStateException.class) @@ -364,9 +370,12 @@ public void testProjectionModesAreMutuallyExclusive() { "Cannot use forScanPlanning() with select(Collection) or project(Schema)"); } - @TestTemplate - public void testProjectionPreservesNarrowTrackingProjection() throws IOException { - InputFile manifest = writeManifest(EMPTY_PARTITION, ImmutableList.of(fileWithFullTracking())); + @ParameterizedTest + @FieldSource("FORMATS") + public void testProjectionPreservesNarrowTrackingProjection(FileFormat format) + throws IOException { + InputFile manifest = + writeManifest(format, EMPTY_PARTITION, ImmutableList.of(fileWithFullTracking())); Schema projection = new Schema( @@ -374,7 +383,7 @@ public void testProjectionPreservesNarrowTrackingProjection() throws IOException TrackedFile.TRACKING.fieldId(), "tracking", Types.StructType.of(Tracking.STATUS))); try (V4ManifestReader reader = - newReader(manifest, UNPARTITIONED_SPECS).project(projection).build()) { + V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS).project(projection).build()) { Tracking actual = Lists.newArrayList(reader).get(0).tracking(); assertThat(actual.status()).isEqualTo(EntryStatus.ADDED); // the narrow tracking projection is not widened to the full tracking type @@ -385,8 +394,9 @@ public void testProjectionPreservesNarrowTrackingProjection() throws IOException } } - @TestTemplate - public void testSelectListColumn() throws IOException { + @ParameterizedTest + @FieldSource("FORMATS") + public void testSelectListColumn(FileFormat format) throws IOException { TrackedFile file = new TrackedFileStruct( addedTracking(), @@ -394,10 +404,10 @@ public void testSelectListColumn() throws IOException { FORMAT_VERSION_V4, "s3://bucket/file.parquet", FileFormat.PARQUET, - EMPTY_PARTITION_DATA, RECORD_COUNT, FILE_SIZE_IN_BYTES, 0, + EMPTY_PARTITION_DATA, null, null, null, @@ -406,10 +416,10 @@ public void testSelectListColumn() throws IOException { ImmutableList.of(50L, 100L), null); - InputFile manifest = writeManifest(EMPTY_PARTITION, ImmutableList.of(file)); + InputFile manifest = writeManifest(format, EMPTY_PARTITION, ImmutableList.of(file)); try (V4ManifestReader reader = - newReader(manifest, UNPARTITIONED_SPECS) + V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS) .select(ImmutableList.of("split_offsets")) .build()) { TrackedFile actual = Lists.newArrayList(reader).get(0); @@ -418,30 +428,34 @@ public void testSelectListColumn() throws IOException { } } - @TestTemplate - public void testSelectWithPartitionFilterProjectsFilterFields() throws IOException { + @ParameterizedTest + @FieldSource("FORMATS") + public void testSelectWithPartitionFilterProjectsFilterFields(FileFormat format) + throws IOException { TrackedFile keep = dataFile("keep.parquet", partition(1)); TrackedFile prune = dataFile("prune.parquet", partition(2)); - InputFile manifest = writeManifest(PARTITION_TYPE, ImmutableList.of(keep, prune)); + InputFile manifest = writeManifest(format, PARTITION_TYPE, ImmutableList.of(keep, prune)); // the caller selects only location; the reader must still project spec_id and partition // for the partition filter or every row would be pruned try (V4ManifestReader reader = - newReader(manifest, PARTITIONED_SPECS) + V4ManifestReader.builder(manifest, PARTITIONED_SPECS) .select(ImmutableList.of("location")) - .filterRows(Expressions.equal("id", 1)) + .filter(Expressions.equal("id", 1)) .build()) { assertThat(reader).extracting(TrackedFile::location).containsExactly(keep.location()); } } - @TestTemplate - public void testForScanPlanningOmitsChangeTrackingFields() throws IOException { - InputFile manifest = writeManifest(EMPTY_PARTITION, ImmutableList.of(fileWithFullTracking())); + @ParameterizedTest + @FieldSource("FORMATS") + public void testForScanPlanningOmitsChangeTrackingFields(FileFormat format) throws IOException { + InputFile manifest = + writeManifest(format, EMPTY_PARTITION, ImmutableList.of(fileWithFullTracking())); try (V4ManifestReader reader = - newReader(manifest, UNPARTITIONED_SPECS).forScanPlanning().build()) { + V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS).forScanPlanning().build()) { Tracking actual = Lists.newArrayList(reader).get(0).tracking(); // scan-relevant tracking fields are projected assertThat(actual.status()).isEqualTo(EntryStatus.ADDED); @@ -456,9 +470,11 @@ public void testForScanPlanningOmitsChangeTrackingFields() throws IOException { } } - @TestTemplate - public void testDefaultReadsFullTracking() throws IOException { - InputFile manifest = writeManifest(EMPTY_PARTITION, ImmutableList.of(fileWithFullTracking())); + @ParameterizedTest + @FieldSource("FORMATS") + public void testDefaultReadsFullTracking(FileFormat format) throws IOException { + InputFile manifest = + writeManifest(format, EMPTY_PARTITION, ImmutableList.of(fileWithFullTracking())); // without a projection, the reader returns the full schema for copying to other manifests, // including the change-tracking fields @@ -473,47 +489,50 @@ public void testDefaultReadsFullTracking() throws IOException { assertThat(actual.replacedPositions()).isEqualTo(ByteBuffer.wrap(new byte[] {3, 4})); } - @TestTemplate - public void testPartitionFilterForceProjectsFilterFields() throws IOException { + @ParameterizedTest + @FieldSource("FORMATS") + public void testPartitionFilterForceProjectsFilterFields(FileFormat format) throws IOException { TrackedFile keep = dataFile("keep.parquet", partition(1)); TrackedFile prune = dataFile("prune.parquet", partition(2)); - InputFile manifest = writeManifest(PARTITION_TYPE, ImmutableList.of(keep, prune)); + InputFile manifest = writeManifest(format, PARTITION_TYPE, ImmutableList.of(keep, prune)); // the caller projects only location; the reader must still project the fields the partition // filter reads (content_type, spec_id, partition) or every row would be pruned Schema projection = new Schema(TrackedFile.LOCATION); try (V4ManifestReader reader = - newReader(manifest, PARTITIONED_SPECS) + V4ManifestReader.builder(manifest, PARTITIONED_SPECS) .project(projection) - .filterRows(Expressions.equal("id", 1)) + .filter(Expressions.equal("id", 1)) .build()) { assertThat(reader).extracting(TrackedFile::location).containsExactly(keep.location()); } } - @TestTemplate - public void testUnpartitioned() throws IOException { + @ParameterizedTest + @FieldSource("FORMATS") + public void testUnpartitioned(FileFormat format) throws IOException { TrackedFile file = dataFile("s3://bucket/file.parquet", EMPTY_PARTITION_DATA); - InputFile manifest = writeManifest(EMPTY_PARTITION, ImmutableList.of(file)); + InputFile manifest = writeManifest(format, EMPTY_PARTITION, ImmutableList.of(file)); TrackedFile actual = read(manifest, UNPARTITIONED_SPECS).get(0); // unpartitioned manifests omit the partition field, which is read as null assertThat(actual.partition()).isNull(); } - @TestTemplate - public void testPartitionFilterPrunesNonMatchingFiles() throws IOException { + @ParameterizedTest + @FieldSource("FORMATS") + public void testPartitionFilterPrunesNonMatchingFiles(FileFormat format) throws IOException { TrackedFile keep = dataFile("keep.parquet", partition(1)); TrackedFile prune = dataFile("prune.parquet", partition(2)); - InputFile manifest = writeManifest(PARTITION_TYPE, ImmutableList.of(keep, prune)); + InputFile manifest = writeManifest(format, PARTITION_TYPE, ImmutableList.of(keep, prune)); ScanMetrics metrics = ScanMetrics.of(new DefaultMetricsContext()); try (V4ManifestReader reader = - newReader(manifest, PARTITIONED_SPECS) - .filterRows(Expressions.equal("id", 1)) + V4ManifestReader.builder(manifest, PARTITIONED_SPECS) + .filter(Expressions.equal("id", 1)) .scanMetrics(metrics) .build()) { assertThat(reader).extracting(TrackedFile::location).containsExactly(keep.location()); @@ -522,8 +541,9 @@ public void testPartitionFilterPrunesNonMatchingFiles() throws IOException { assertThat(metrics.skippedDataFiles().value()).isEqualTo(1L); } - @TestTemplate - public void testPartitionFilterCountsSkippedDeleteFiles() throws IOException { + @ParameterizedTest + @FieldSource("FORMATS") + public void testPartitionFilterCountsSkippedDeleteFiles(FileFormat format) throws IOException { TrackedFile delete = new TrackedFileStruct( addedTracking(), @@ -531,10 +551,10 @@ public void testPartitionFilterCountsSkippedDeleteFiles() throws IOException { FORMAT_VERSION_V4, "delete.parquet", FileFormat.PARQUET, - partition(2), RECORD_COUNT, FILE_SIZE_IN_BYTES, 0, + partition(2), null, null, null, @@ -543,12 +563,12 @@ public void testPartitionFilterCountsSkippedDeleteFiles() throws IOException { null, ImmutableList.of(1)); - InputFile manifest = writeManifest(PARTITION_TYPE, ImmutableList.of(delete)); + InputFile manifest = writeManifest(format, PARTITION_TYPE, ImmutableList.of(delete)); ScanMetrics metrics = ScanMetrics.of(new DefaultMetricsContext()); try (V4ManifestReader reader = - newReader(manifest, PARTITIONED_SPECS) - .filterRows(Expressions.equal("id", 1)) + V4ManifestReader.builder(manifest, PARTITIONED_SPECS) + .filter(Expressions.equal("id", 1)) .scanMetrics(metrics) .build()) { assertThat(reader).isEmpty(); @@ -558,8 +578,9 @@ public void testPartitionFilterCountsSkippedDeleteFiles() throws IOException { assertThat(metrics.skippedDataFiles().value()).isEqualTo(0L); } - @TestTemplate - public void testPartitionFilterKeepsManifestReferences() throws IOException { + @ParameterizedTest + @FieldSource("FORMATS") + public void testPartitionFilterKeepsManifestReferences(FileFormat format) throws IOException { TrackedFile keep = dataFile("data-1.parquet", partition(1)); TrackedFile prune = dataFile("data-2.parquet", partition(2)); // a real manifest reference has a null spec_id and no partition tuple; these refs carry a @@ -570,12 +591,14 @@ public void testPartitionFilterKeepsManifestReferences() throws IOException { InputFile manifest = writeManifest( - PARTITION_TYPE, ImmutableList.of(keep, prune, dataManifestRef, deleteManifestRef)); + format, + PARTITION_TYPE, + ImmutableList.of(keep, prune, dataManifestRef, deleteManifestRef)); ScanMetrics metrics = ScanMetrics.of(new DefaultMetricsContext()); try (V4ManifestReader reader = - newReader(manifest, PARTITIONED_SPECS) - .filterRows(Expressions.equal("id", 1)) + V4ManifestReader.builder(manifest, PARTITIONED_SPECS) + .filter(Expressions.equal("id", 1)) .scanMetrics(metrics) .build()) { assertThat(reader) @@ -590,17 +613,18 @@ public void testPartitionFilterKeepsManifestReferences() throws IOException { assertThat(metrics.skippedDeleteManifests().value()).isEqualTo(0L); } - @TestTemplate - public void testRowFilterOnUnpartitionedTableKeepsAllFiles() throws IOException { + @ParameterizedTest + @FieldSource("FORMATS") + public void testRowFilterOnUnpartitionedTableKeepsAllFiles(FileFormat format) throws IOException { TrackedFile file1 = dataFile("s3://bucket/a.parquet", EMPTY_PARTITION_DATA); TrackedFile file2 = dataFile("s3://bucket/b.parquet", EMPTY_PARTITION_DATA); - InputFile manifest = writeManifest(EMPTY_PARTITION, ImmutableList.of(file1, file2)); + InputFile manifest = writeManifest(format, EMPTY_PARTITION, ImmutableList.of(file1, file2)); ScanMetrics metrics = ScanMetrics.of(new DefaultMetricsContext()); try (V4ManifestReader reader = - newReader(manifest, UNPARTITIONED_SPECS) - .filterRows(Expressions.equal("id", 1)) + V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS) + .filter(Expressions.equal("id", 1)) .scanMetrics(metrics) .build()) { assertThat(reader) @@ -612,41 +636,44 @@ public void testRowFilterOnUnpartitionedTableKeepsAllFiles() throws IOException assertThat(metrics.skippedDeleteFiles().value()).isEqualTo(0L); } - @TestTemplate + @Test public void testInvalidBuilderArguments() { InputFile manifest = fileIO.newInputFile(tempDir.resolve("manifest.avro").toString()); - assertThatThrownBy(() -> newReader(manifest, UNPARTITIONED_SPECS).filterRows(null)) + assertThatThrownBy(() -> V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS).filter(null)) .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Invalid row filter: null"); + .hasMessage("Invalid filter: null"); - assertThatThrownBy(() -> newReader(manifest, UNPARTITIONED_SPECS).scanMetrics(null)) + assertThatThrownBy( + () -> V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS).scanMetrics(null)) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Invalid scan metrics: null"); - assertThatThrownBy(() -> newReader(manifest, UNPARTITIONED_SPECS).select(null)) + assertThatThrownBy(() -> V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS).select(null)) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Invalid columns: null"); } - @TestTemplate - public void testCaseInsensitivePartitionFilter() throws IOException { + @ParameterizedTest + @FieldSource("FORMATS") + public void testCaseInsensitivePartitionFilter(FileFormat format) throws IOException { TrackedFile keep = dataFile("keep.parquet", partition(1)); TrackedFile prune = dataFile("prune.parquet", partition(2)); - InputFile manifest = writeManifest(PARTITION_TYPE, ImmutableList.of(keep, prune)); + InputFile manifest = writeManifest(format, PARTITION_TYPE, ImmutableList.of(keep, prune)); try (V4ManifestReader reader = - newReader(manifest, PARTITIONED_SPECS) - .filterRows(Expressions.equal("ID", 1)) + V4ManifestReader.builder(manifest, PARTITIONED_SPECS) + .filter(Expressions.equal("ID", 1)) .caseSensitive(false) .build()) { assertThat(reader).extracting(TrackedFile::location).containsExactly(keep.location()); } } - @TestTemplate - public void testMultiSpecPartitionPruning() throws IOException { + @ParameterizedTest + @FieldSource("FORMATS") + public void testMultiSpecPartitionPruning(FileFormat format) throws IOException { PartitionSpec spec0 = PartitionSpec.builderFor(TABLE_SCHEMA).withSpecId(0).identity("id").build(); PartitionSpec spec1 = @@ -663,12 +690,10 @@ public void testMultiSpecPartitionPruning() throws IOException { dataFile("spec1-data.parquet", unionPartition(unionType, null, "x"), 1); InputFile manifest = - writeManifest(unionType, ImmutableList.of(keepById, prunedById, keptOtherSpec)); + writeManifest(format, unionType, ImmutableList.of(keepById, prunedById, keptOtherSpec)); try (V4ManifestReader reader = - V4ManifestReader.builder(manifest, specsById) - .filterRows(Expressions.equal("id", 1)) - .build()) { + V4ManifestReader.builder(manifest, specsById).filter(Expressions.equal("id", 1)).build()) { // spec0 entries are pruned by id; the spec1 entry is not partitioned by id so it survives assertThat(reader) .extracting(TrackedFile::location) @@ -676,17 +701,19 @@ public void testMultiSpecPartitionPruning() throws IOException { } } - @TestTemplate - public void testIteratorReturnsLiveCopies() throws IOException { + @ParameterizedTest + @FieldSource("FORMATS") + public void testIteratorReturnsLiveCopies(FileFormat format) throws IOException { TrackedFile added1 = dataFile("s3://bucket/added-1.parquet", EMPTY_PARTITION_DATA); TrackedFile added2 = dataFile("s3://bucket/added-2.parquet", EMPTY_PARTITION_DATA); List files = ImmutableList.of( added1, added2, fileWithStatus(EntryStatus.DELETED, "s3://bucket/deleted.parquet")); - InputFile manifest = writeManifest(EMPTY_PARTITION, files); + InputFile manifest = writeManifest(format, EMPTY_PARTITION, files); - try (V4ManifestReader reader = newReader(manifest, UNPARTITIONED_SPECS).build()) { + try (V4ManifestReader reader = + V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS).build()) { List read = Lists.newArrayList(reader); assertThat(read) .hasSize(2) @@ -698,40 +725,47 @@ public void testIteratorReturnsLiveCopies() throws IOException { } } - @TestTemplate - public void testUnknownManifestFormatThrows() throws IOException { + @ParameterizedTest + @FieldSource("FORMATS") + public void testUnknownManifestFormatThrows(FileFormat format) throws IOException { InputFile badFile = fileIO.newInputFile(tempDir.resolve("manifest-" + System.nanoTime() + ".txt").toString()); - try (V4ManifestReader reader = newReader(badFile, UNPARTITIONED_SPECS).build()) { + try (V4ManifestReader reader = V4ManifestReader.builder(badFile, UNPARTITIONED_SPECS).build()) { assertThatThrownBy(reader::iterator) .isInstanceOf(IllegalArgumentException.class) .hasMessageContaining("Cannot determine format of manifest"); } } - @TestTemplate - public void testPartitionFilterKeepsFileWithUnknownSpec() throws IOException { + @ParameterizedTest + @FieldSource("FORMATS") + public void testPartitionFilterKeepsFileWithUnknownSpec(FileFormat format) throws IOException { // spec ID 5 is not in PARTITIONED_SPECS, so no partition filter applies to this file TrackedFile file = dataFile("orphan.parquet", partition(1), 5); - InputFile manifest = writeManifest(PARTITION_TYPE, ImmutableList.of(file)); + InputFile manifest = writeManifest(format, PARTITION_TYPE, ImmutableList.of(file)); // the filter would prune partition id=1 under spec 0, but cannot be applied to spec 5 try (V4ManifestReader reader = - newReader(manifest, PARTITIONED_SPECS).filterRows(Expressions.equal("id", 2)).build()) { + V4ManifestReader.builder(manifest, PARTITIONED_SPECS) + .filter(Expressions.equal("id", 2)) + .build()) { assertThat(reader).extracting(TrackedFile::location).containsExactly(file.location()); } } - @TestTemplate - public void testPartitionFilterKeepsFileWithNullSpecId() throws IOException { + @ParameterizedTest + @FieldSource("FORMATS") + public void testPartitionFilterKeepsFileWithNullSpecId(FileFormat format) throws IOException { TrackedFile file = dataFile("no-spec.parquet", null, null); - InputFile manifest = writeManifest(PARTITION_TYPE, ImmutableList.of(file)); + InputFile manifest = writeManifest(format, PARTITION_TYPE, ImmutableList.of(file)); try (V4ManifestReader reader = - newReader(manifest, PARTITIONED_SPECS).filterRows(Expressions.equal("id", 2)).build()) { + V4ManifestReader.builder(manifest, PARTITIONED_SPECS) + .filter(Expressions.equal("id", 2)) + .build()) { assertThat(reader).extracting(TrackedFile::location).containsExactly(file.location()); } } @@ -747,10 +781,10 @@ private static TrackedFile dataFile(String location, PartitionData partition, In FORMAT_VERSION_V4, location, FileFormat.PARQUET, - partition, RECORD_COUNT, FILE_SIZE_IN_BYTES, specId, + partition, null, null, null, @@ -777,10 +811,10 @@ private static TrackedFile fileWithFullTracking() { FORMAT_VERSION_V4, "s3://bucket/file.parquet", FileFormat.PARQUET, - EMPTY_PARTITION_DATA, RECORD_COUNT, FILE_SIZE_IN_BYTES, 0, + EMPTY_PARTITION_DATA, null, null, null, @@ -798,10 +832,10 @@ private static TrackedFile manifestRef(FileContent content, String location) { FORMAT_VERSION_V4, location, FileFormat.PARQUET, - partition(2), RECORD_COUNT, FILE_SIZE_IN_BYTES, 0, + partition(2), null, null, null, @@ -812,17 +846,26 @@ private static TrackedFile manifestRef(FileContent content, String location) { } private static TrackedFile fileWithStatus(EntryStatus status, String location) { - Tracking tracking = new TrackingStruct(status, SNAPSHOT_ID, 3L, 3L, null, null, null, null); + Tracking tracking = + new TrackingStruct( + status, + SNAPSHOT_ID, + 3L, // data sequence number + 3L, // file sequence number + null, // dv snapshot id + null, // first row id + null, // deleted positions + null); // replaced positions return new TrackedFileStruct( tracking, FileContent.DATA, FORMAT_VERSION_V4, location, FileFormat.PARQUET, - EMPTY_PARTITION_DATA, RECORD_COUNT, FILE_SIZE_IN_BYTES, 0, + EMPTY_PARTITION_DATA, null, null, null, @@ -838,12 +881,12 @@ private static Tracking addedTracking() { private static DeletionVector deletionVector( String location, long offset, long sizeInBytes, long cardinality) { - DeletionVectorStruct dv = new DeletionVectorStruct(DeletionVector.schema()); - dv.set(0, location); - dv.set(1, offset); - dv.set(2, sizeInBytes); - dv.set(3, cardinality); - return dv; + return DeletionVectorStruct.builder() + .location(location) + .offset(offset) + .sizeInBytes(sizeInBytes) + .cardinality(cardinality) + .build(); } private static PartitionData partition(int id) { @@ -859,16 +902,11 @@ private static PartitionData unionPartition(Types.StructType unionType, Integer return partition; } - private InputFile writeManifest(Types.StructType partitionType, Iterable files) + private InputFile writeManifest( + FileFormat format, Types.StructType partitionType, Iterable files) throws IOException { - Schema writeSchema = - new Schema(TrackedFile.schema(partitionType, Types.StructType.of()).fields()); - OutputFile out = - fileIO.newOutputFile( - tempDir - .resolve( - "manifest-" + System.nanoTime() + "." + format.name().toLowerCase(Locale.ROOT)) - .toString()); + Schema writeSchema = TrackedFile.schema(partitionType, Types.StructType.of()); + OutputFile out = new InMemoryOutputFile("manifest." + format.name().toLowerCase(Locale.ROOT)); try (FileAppender appender = InternalData.write(format, out).schema(writeSchema).named("tracked_file").build()) { for (TrackedFile file : files) { @@ -876,17 +914,12 @@ private InputFile writeManifest(Types.StructType partitionType, Iterable specsById) { - return V4ManifestReader.builder(manifest, specsById); + return out.toInputFile(); } private List read(InputFile manifest, Map specsById) throws IOException { - try (V4ManifestReader reader = newReader(manifest, specsById).build()) { + try (V4ManifestReader reader = V4ManifestReader.builder(manifest, specsById).build()) { return Lists.newArrayList(reader); } } From f6bf7ef9b56d5f5257ed6f60bdc2331a388e6748 Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Fri, 24 Jul 2026 13:46:12 -0700 Subject: [PATCH 21/26] Use the helper in test rather than constructor --- .../apache/iceberg/TestV4ManifestReader.java | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java index f304fe43a926..34795435434a 100644 --- a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java +++ b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java @@ -210,24 +210,7 @@ public void testManifestLocationAndPosition(FileFormat format) throws IOExceptio @ParameterizedTest @FieldSource("FORMATS") public void testProjectionRestrictsFields(FileFormat format) throws IOException { - TrackedFile file = - new TrackedFileStruct( - addedTracking(), - FileContent.DATA, - FORMAT_VERSION_V4, - "s3://bucket/file.parquet", - FileFormat.PARQUET, - RECORD_COUNT, - FILE_SIZE_IN_BYTES, - 0, - EMPTY_PARTITION_DATA, - null, - SORT_ORDER_ID, - null, - null, - null, - null, - null); + TrackedFile file = dataFile("s3://bucket/file.parquet", EMPTY_PARTITION_DATA); InputFile manifest = writeManifest(format, EMPTY_PARTITION, ImmutableList.of(file)); From c5d2018fd3f970ba1fabc19193c19f17d03a0de6 Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Fri, 24 Jul 2026 18:05:43 -0700 Subject: [PATCH 22/26] Incorporate test feedback from Ryan --- .../org/apache/iceberg/V4ManifestReader.java | 6 + .../apache/iceberg/TestTrackedFileStruct.java | 4 +- .../apache/iceberg/TestV4ManifestReader.java | 549 ++++++++---------- 3 files changed, 241 insertions(+), 318 deletions(-) diff --git a/core/src/main/java/org/apache/iceberg/V4ManifestReader.java b/core/src/main/java/org/apache/iceberg/V4ManifestReader.java index 808cd4f82412..6d2c3358259b 100644 --- a/core/src/main/java/org/apache/iceberg/V4ManifestReader.java +++ b/core/src/main/java/org/apache/iceberg/V4ManifestReader.java @@ -18,6 +18,7 @@ */ package org.apache.iceberg; +import java.util.Arrays; import java.util.Collection; import java.util.Map; import java.util.Set; @@ -210,6 +211,11 @@ Builder forScanPlanning() { return this; } + /** Selects columns to read by name; fields needed by the reader are always read. */ + Builder select(String... newColumns) { + return select(Arrays.asList(newColumns)); + } + /** Selects columns to read by name; fields needed by the reader are always read. */ Builder select(Collection newColumns) { Preconditions.checkArgument(newColumns != null, "Invalid columns: null"); diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java b/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java index 010695e43c08..d7e838266093 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java @@ -355,11 +355,11 @@ void serializationRoundTrip(RoundTripSerializer serializer) t FileContent.DATA, FORMAT_VERSION_V4, "s3://bucket/data/file.parquet", - FileFormat.PARQUET, // PartitionData has its own serialization tests + FileFormat.PARQUET, 100L, 1024L, 7, - null, + null, // PartitionData has its own serialization tests null, 1, null, // DeletionVector has its own serialization tests diff --git a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java index 34795435434a..b2a93974f68c 100644 --- a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java +++ b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java @@ -25,9 +25,12 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.file.Path; +import java.util.Collection; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.function.Consumer; +import java.util.stream.Stream; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.inmemory.InMemoryOutputFile; import org.apache.iceberg.io.FileAppender; @@ -38,6 +41,7 @@ import org.apache.iceberg.metrics.ScanMetrics; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.relocated.com.google.common.collect.Iterables; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.transforms.Transforms; import org.apache.iceberg.types.Comparators; @@ -46,7 +50,9 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.FieldSource; +import org.junit.jupiter.params.provider.MethodSource; public class TestV4ManifestReader { private static final long SNAPSHOT_ID = 42L; @@ -58,25 +64,57 @@ public class TestV4ManifestReader { private static final long DV_OFFSET = 100L; private static final long DV_SIZE_IN_BYTES = 50L; private static final long DV_CARDINALITY = 5L; + private static final DeletionVector DV = + DeletionVectorStruct.builder() + .location(DV_LOCATION) + .offset(DV_OFFSET) + .sizeInBytes(DV_SIZE_IN_BYTES) + .cardinality(DV_CARDINALITY) + .build(); private static final Schema TABLE_SCHEMA = new Schema( optional(1, "id", Types.IntegerType.get()), optional(2, "data", Types.StringType.get())); - private static final PartitionSpec SPEC = + private static final PartitionSpec ID_PARTITIONING = PartitionSpec.builderFor(TABLE_SCHEMA).identity("id").build(); - private static final Types.StructType PARTITION_TYPE = SPEC.partitionType(); + private static final Types.StructType PARTITION_TYPE = ID_PARTITIONING.partitionType(); private static final Types.StructType EMPTY_PARTITION = Types.StructType.of(); private static final PartitionData EMPTY_PARTITION_DATA = new PartitionData(EMPTY_PARTITION); private static final Map PARTITIONED_SPECS = - ImmutableMap.of(SPEC.specId(), SPEC); + ImmutableMap.of(ID_PARTITIONING.specId(), ID_PARTITIONING); private static final Map UNPARTITIONED_SPECS = ImmutableMap.of(PartitionSpec.unpartitioned().specId(), PartitionSpec.unpartitioned()); private static final List FORMATS = ImmutableList.of(FileFormat.AVRO, FileFormat.PARQUET); - // row_position is appended after the tracking schema fields by the reader - private static final int MANIFEST_POS_ORDINAL = Tracking.schema().fields().size(); + // a data file whose tracking carries every inheritable and change-tracking value set + private static final TrackedFile FILE_WITH_FULL_TRACKING = + new TrackedFileStruct( + new TrackingStruct( + EntryStatus.ADDED, + SNAPSHOT_ID, + 5L, // data sequence number + 6L, // file sequence number + 7L, // dv snapshot id + 8L, // first row id + new byte[] {1, 2}, // deleted positions + new byte[] {3, 4}), // replaced positions + FileContent.DATA, + FORMAT_VERSION_V4, + "s3://bucket/file.parquet", + FileFormat.PARQUET, + RECORD_COUNT, + FILE_SIZE_IN_BYTES, + 0, + EMPTY_PARTITION_DATA, + null, + null, + null, + null, + null, + null, + null); @TempDir private Path tempDir; @@ -84,9 +122,7 @@ public class TestV4ManifestReader { @ParameterizedTest @FieldSource("FORMATS") - public void testRoundTrip(FileFormat format) throws IOException { - DeletionVector dv = deletionVector(DV_LOCATION, DV_OFFSET, DV_SIZE_IN_BYTES, DV_CARDINALITY); - + public void testReadsWrittenFile(FileFormat format) throws IOException { TrackedFile file = new TrackedFileStruct( addedTracking(), @@ -96,11 +132,11 @@ public void testRoundTrip(FileFormat format) throws IOException { FileFormat.PARQUET, RECORD_COUNT, FILE_SIZE_IN_BYTES, - SPEC.specId(), + ID_PARTITIONING.specId(), partition(7), null, SORT_ORDER_ID, - dv, + DV, null, ByteBuffer.wrap(new byte[] {1, 2, 3}), ImmutableList.of(50L, 100L), @@ -108,23 +144,19 @@ public void testRoundTrip(FileFormat format) throws IOException { InputFile manifest = writeManifest(format, PARTITION_TYPE, ImmutableList.of(file)); - List read = read(manifest, PARTITIONED_SPECS); - assertThat(read).hasSize(1); - TrackedFile actual = read.get(0); - - // the reader fills row_position (manifestPos) and manifestLocation, which the written file - // does not have; mirror them on the expected file before comparing - TrackingStruct expectedTracking = (TrackingStruct) ((TrackedFileStruct) file).tracking(); - expectedTracking.set(MANIFEST_POS_ORDINAL, 0L); - expectedTracking.setManifestLocation(manifest.location()); + TrackedFile actual = Iterables.getOnlyElement(read(manifest, PARTITIONED_SPECS)); - Types.StructType readType = + // compare with tracking reduced to status: the reader fills status-independent tracking + // fields (row position, sequence numbers via inheritance) that the written file does not have + Types.StructType comparisonType = TypeUtil.replaceFieldTypes( TrackedFile.schema(PARTITION_TYPE, Types.StructType.of()), - ImmutableMap.of(TrackedFile.TRACKING.fieldId(), TrackingStruct.BASE_TYPE)) + ImmutableMap.of( + TrackedFile.TRACKING.fieldId(), Types.StructType.of(Tracking.STATUS))) .asStruct(); - assertThat(Comparators.forType(readType).compare((StructLike) file, (StructLike) actual)) - .isEqualTo(0); + assertThat((StructLike) actual) + .usingComparator(Comparators.forType(comparisonType)) + .isEqualTo((StructLike) file); } @ParameterizedTest @@ -151,7 +183,7 @@ public void testEqualityDeleteRoundTrip(FileFormat format) throws IOException { InputFile manifest = writeManifest(format, EMPTY_PARTITION, ImmutableList.of(delete)); - TrackedFile actual = read(manifest, UNPARTITIONED_SPECS).get(0); + TrackedFile actual = Iterables.getOnlyElement(read(manifest, UNPARTITIONED_SPECS)); assertThat(actual.contentType()).isEqualTo(FileContent.EQUALITY_DELETES); assertThat(actual.equalityIds()).containsExactly(1, 2); } @@ -207,31 +239,49 @@ public void testManifestLocationAndPosition(FileFormat format) throws IOExceptio assertThat(read).extracting(file -> file.tracking().manifestPos()).containsExactly(0L, 1L, 2L); } - @ParameterizedTest - @FieldSource("FORMATS") - public void testProjectionRestrictsFields(FileFormat format) throws IOException { - TrackedFile file = dataFile("s3://bucket/file.parquet", EMPTY_PARTITION_DATA); + @ParameterizedTest(name = "{0} / {2}") + @MethodSource("restrictedReadModes") + public void testRestrictedReadReturnsOnlyRequestedFields( + FileFormat format, Consumer configureRead, String description) + throws IOException { + List files = + ImmutableList.of( + dataFile("s3://bucket/live.parquet", EMPTY_PARTITION_DATA), + fileWithStatus(EntryStatus.DELETED, "s3://bucket/deleted.parquet"), + fileWithStatus(EntryStatus.REPLACED, "s3://bucket/replaced.parquet")); - InputFile manifest = writeManifest(format, EMPTY_PARTITION, ImmutableList.of(file)); + InputFile manifest = writeManifest(format, EMPTY_PARTITION, files); - Schema projection = new Schema(TrackedFile.LOCATION); - try (V4ManifestReader reader = - V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS).project(projection).build()) { - TrackedFile actual = Lists.newArrayList(reader).get(0); - assertThat(actual.location()).isEqualTo(file.location()); - // tracking and content_type are always projected, even though the caller omitted them - assertThat(actual.tracking()).isNotNull(); - assertThat(actual.tracking().status()).isEqualTo(EntryStatus.ADDED); - assertThat(actual.contentType()).isEqualTo(FileContent.DATA); - // sort_order_id, file_format, spec_id, and record_count are null because they were not - // projected and no row filter forces them - assertThat(actual.sortOrderId()).isNull(); + V4ManifestReader.Builder builder = V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS); + configureRead.accept(builder); + try (V4ManifestReader reader = builder.build()) { + // content_type and status are projected for liveness filtering, so only the live entry + // survives even though the caller requested only location + TrackedFile actual = Iterables.getOnlyElement(reader); + assertThat(actual.location()).isEqualTo("s3://bucket/live.parquet"); + // fields the caller did not request are not read assertThat(actual.fileFormat()).isNull(); assertThat(actual.specId()).isNull(); - assertThat(actual.recordCount()).isEqualTo(-1L); + assertThat(actual.sortOrderId()).isNull(); } } + private static Stream restrictedReadModes() { + Map> modes = + ImmutableMap.of( + "project", + builder -> builder.project(new Schema(TrackedFile.LOCATION)), + "select", + builder -> builder.select("location"), + "case-insensitive select", + builder -> builder.select("LOCATION").caseSensitive(false)); + return FORMATS.stream() + .flatMap( + format -> + modes.entrySet().stream() + .map(mode -> Arguments.of(format, mode.getValue(), mode.getKey()))); + } + @ParameterizedTest @FieldSource("FORMATS") public void testRowFilterForcesRecordCount(FileFormat format) throws IOException { @@ -247,55 +297,12 @@ public void testRowFilterForcesRecordCount(FileFormat format) throws IOException .project(projection) .filter(Expressions.equal("id", 1)) .build()) { - TrackedFile actual = Lists.newArrayList(reader).get(0); + TrackedFile actual = Iterables.getOnlyElement(reader); assertThat(actual.location()).isEqualTo(file.location()); assertThat(actual.recordCount()).isEqualTo(RECORD_COUNT); } } - @ParameterizedTest - @FieldSource("FORMATS") - public void testSelectRestrictsFields(FileFormat format) throws IOException { - TrackedFile file = dataFile("s3://bucket/file.parquet", EMPTY_PARTITION_DATA); - - InputFile manifest = writeManifest(format, EMPTY_PARTITION, ImmutableList.of(file)); - - try (V4ManifestReader reader = - V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS) - .select(ImmutableList.of("location")) - .build()) { - TrackedFile actual = Lists.newArrayList(reader).get(0); - assertThat(actual.location()).isEqualTo(file.location()); - // tracking status and content_type are joined in even though not selected - assertThat(actual.tracking()).isNotNull(); - assertThat(actual.tracking().status()).isEqualTo(EntryStatus.ADDED); - assertThat(actual.contentType()).isEqualTo(FileContent.DATA); - // only status is joined from tracking, not the other tracking fields - assertThat(actual.tracking().snapshotId()).isNull(); - // file_format and spec_id are null because they were not selected - assertThat(actual.fileFormat()).isNull(); - assertThat(actual.specId()).isNull(); - } - } - - @ParameterizedTest - @FieldSource("FORMATS") - public void testCaseInsensitiveSelect(FileFormat format) throws IOException { - TrackedFile file = dataFile("s3://bucket/file.parquet", EMPTY_PARTITION_DATA); - - InputFile manifest = writeManifest(format, EMPTY_PARTITION, ImmutableList.of(file)); - - try (V4ManifestReader reader = - V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS) - .select(ImmutableList.of("LOCATION")) - .caseSensitive(false) - .build()) { - TrackedFile actual = Lists.newArrayList(reader).get(0); - assertThat(actual.location()).isEqualTo(file.location()); - assertThat(actual.fileFormat()).isNull(); - } - } - @Test public void testProjectionModesAreMutuallyExclusive() { InputFile manifest = fileIO.newInputFile(tempDir.resolve("manifest.avro").toString()); @@ -303,7 +310,7 @@ public void testProjectionModesAreMutuallyExclusive() { assertThatThrownBy( () -> V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS) - .select(ImmutableList.of("location")) + .select("location") .project(new Schema(TrackedFile.LOCATION))) .isInstanceOf(IllegalStateException.class) .hasMessage( @@ -313,7 +320,7 @@ public void testProjectionModesAreMutuallyExclusive() { () -> V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS) .project(new Schema(TrackedFile.LOCATION)) - .select(ImmutableList.of("location"))) + .select("location")) .isInstanceOf(IllegalStateException.class) .hasMessage( "Cannot select columns using both select(Collection) and project(Schema)"); @@ -322,14 +329,14 @@ public void testProjectionModesAreMutuallyExclusive() { () -> V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS) .forScanPlanning() - .select(ImmutableList.of("location"))) + .select("location")) .isInstanceOf(IllegalStateException.class) .hasMessage("Cannot use select(Collection) with forScanPlanning()"); assertThatThrownBy( () -> V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS) - .select(ImmutableList.of("location")) + .select("location") .forScanPlanning()) .isInstanceOf(IllegalStateException.class) .hasMessage( @@ -358,16 +365,11 @@ public void testProjectionModesAreMutuallyExclusive() { public void testProjectionPreservesNarrowTrackingProjection(FileFormat format) throws IOException { InputFile manifest = - writeManifest(format, EMPTY_PARTITION, ImmutableList.of(fileWithFullTracking())); - - Schema projection = - new Schema( - Types.NestedField.required( - TrackedFile.TRACKING.fieldId(), "tracking", Types.StructType.of(Tracking.STATUS))); + writeManifest(format, EMPTY_PARTITION, ImmutableList.of(FILE_WITH_FULL_TRACKING)); try (V4ManifestReader reader = - V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS).project(projection).build()) { - Tracking actual = Lists.newArrayList(reader).get(0).tracking(); + V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS).select("tracking.status").build()) { + Tracking actual = Iterables.getOnlyElement(reader).tracking(); assertThat(actual.status()).isEqualTo(EntryStatus.ADDED); // the narrow tracking projection is not widened to the full tracking type assertThat(actual.snapshotId()).isNull(); @@ -377,69 +379,15 @@ public void testProjectionPreservesNarrowTrackingProjection(FileFormat format) } } - @ParameterizedTest - @FieldSource("FORMATS") - public void testSelectListColumn(FileFormat format) throws IOException { - TrackedFile file = - new TrackedFileStruct( - addedTracking(), - FileContent.DATA, - FORMAT_VERSION_V4, - "s3://bucket/file.parquet", - FileFormat.PARQUET, - RECORD_COUNT, - FILE_SIZE_IN_BYTES, - 0, - EMPTY_PARTITION_DATA, - null, - null, - null, - null, - null, - ImmutableList.of(50L, 100L), - null); - - InputFile manifest = writeManifest(format, EMPTY_PARTITION, ImmutableList.of(file)); - - try (V4ManifestReader reader = - V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS) - .select(ImmutableList.of("split_offsets")) - .build()) { - TrackedFile actual = Lists.newArrayList(reader).get(0); - assertThat(actual.splitOffsets()).containsExactly(50L, 100L); - assertThat(actual.location()).isNull(); - } - } - - @ParameterizedTest - @FieldSource("FORMATS") - public void testSelectWithPartitionFilterProjectsFilterFields(FileFormat format) - throws IOException { - TrackedFile keep = dataFile("keep.parquet", partition(1)); - TrackedFile prune = dataFile("prune.parquet", partition(2)); - - InputFile manifest = writeManifest(format, PARTITION_TYPE, ImmutableList.of(keep, prune)); - - // the caller selects only location; the reader must still project spec_id and partition - // for the partition filter or every row would be pruned - try (V4ManifestReader reader = - V4ManifestReader.builder(manifest, PARTITIONED_SPECS) - .select(ImmutableList.of("location")) - .filter(Expressions.equal("id", 1)) - .build()) { - assertThat(reader).extracting(TrackedFile::location).containsExactly(keep.location()); - } - } - @ParameterizedTest @FieldSource("FORMATS") public void testForScanPlanningOmitsChangeTrackingFields(FileFormat format) throws IOException { InputFile manifest = - writeManifest(format, EMPTY_PARTITION, ImmutableList.of(fileWithFullTracking())); + writeManifest(format, EMPTY_PARTITION, ImmutableList.of(FILE_WITH_FULL_TRACKING)); try (V4ManifestReader reader = V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS).forScanPlanning().build()) { - Tracking actual = Lists.newArrayList(reader).get(0).tracking(); + Tracking actual = Iterables.getOnlyElement(reader).tracking(); // scan-relevant tracking fields are projected assertThat(actual.status()).isEqualTo(EntryStatus.ADDED); assertThat(actual.snapshotId()).isEqualTo(SNAPSHOT_ID); @@ -457,19 +405,22 @@ public void testForScanPlanningOmitsChangeTrackingFields(FileFormat format) thro @FieldSource("FORMATS") public void testDefaultReadsFullTracking(FileFormat format) throws IOException { InputFile manifest = - writeManifest(format, EMPTY_PARTITION, ImmutableList.of(fileWithFullTracking())); + writeManifest(format, EMPTY_PARTITION, ImmutableList.of(FILE_WITH_FULL_TRACKING)); - // without a projection, the reader returns the full schema for copying to other manifests, - // including the change-tracking fields - Tracking actual = read(manifest, UNPARTITIONED_SPECS).get(0).tracking(); - assertThat(actual.status()).isEqualTo(EntryStatus.ADDED); - assertThat(actual.snapshotId()).isEqualTo(SNAPSHOT_ID); - assertThat(actual.dataSequenceNumber()).isEqualTo(5L); - assertThat(actual.fileSequenceNumber()).isEqualTo(6L); - assertThat(actual.firstRowId()).isEqualTo(8L); - assertThat(actual.dvSnapshotId()).isEqualTo(7L); - assertThat(actual.deletedPositions()).isEqualTo(ByteBuffer.wrap(new byte[] {1, 2})); - assertThat(actual.replacedPositions()).isEqualTo(ByteBuffer.wrap(new byte[] {3, 4})); + // without scanPlanning, select, or project, the reader returns the full schema for copying to + // other manifests, including the change-tracking fields + try (V4ManifestReader reader = + V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS).build()) { + Tracking actual = Iterables.getOnlyElement(reader).tracking(); + assertThat(actual.status()).isEqualTo(EntryStatus.ADDED); + assertThat(actual.snapshotId()).isEqualTo(SNAPSHOT_ID); + assertThat(actual.dataSequenceNumber()).isEqualTo(5L); + assertThat(actual.fileSequenceNumber()).isEqualTo(6L); + assertThat(actual.firstRowId()).isEqualTo(8L); + assertThat(actual.dvSnapshotId()).isEqualTo(7L); + assertThat(actual.deletedPositions()).isEqualTo(ByteBuffer.wrap(new byte[] {1, 2})); + assertThat(actual.replacedPositions()).isEqualTo(ByteBuffer.wrap(new byte[] {3, 4})); + } } @ParameterizedTest @@ -494,81 +445,45 @@ public void testPartitionFilterForceProjectsFilterFields(FileFormat format) thro @ParameterizedTest @FieldSource("FORMATS") - public void testUnpartitioned(FileFormat format) throws IOException { - TrackedFile file = dataFile("s3://bucket/file.parquet", EMPTY_PARTITION_DATA); - - InputFile manifest = writeManifest(format, EMPTY_PARTITION, ImmutableList.of(file)); - - TrackedFile actual = read(manifest, UNPARTITIONED_SPECS).get(0); - // unpartitioned manifests omit the partition field, which is read as null - assertThat(actual.partition()).isNull(); - } - - @ParameterizedTest - @FieldSource("FORMATS") - public void testPartitionFilterPrunesNonMatchingFiles(FileFormat format) throws IOException { + public void testSelectWithPartitionFilterProjectsFilterFields(FileFormat format) + throws IOException { TrackedFile keep = dataFile("keep.parquet", partition(1)); TrackedFile prune = dataFile("prune.parquet", partition(2)); InputFile manifest = writeManifest(format, PARTITION_TYPE, ImmutableList.of(keep, prune)); - ScanMetrics metrics = ScanMetrics.of(new DefaultMetricsContext()); + // the caller selects only location; the reader must still project spec_id and partition + // for the partition filter or every row would be pruned try (V4ManifestReader reader = V4ManifestReader.builder(manifest, PARTITIONED_SPECS) + .select("location") .filter(Expressions.equal("id", 1)) - .scanMetrics(metrics) .build()) { assertThat(reader).extracting(TrackedFile::location).containsExactly(keep.location()); } - - assertThat(metrics.skippedDataFiles().value()).isEqualTo(1L); } @ParameterizedTest @FieldSource("FORMATS") - public void testPartitionFilterCountsSkippedDeleteFiles(FileFormat format) throws IOException { - TrackedFile delete = - new TrackedFileStruct( - addedTracking(), - FileContent.EQUALITY_DELETES, - FORMAT_VERSION_V4, - "delete.parquet", - FileFormat.PARQUET, - RECORD_COUNT, - FILE_SIZE_IN_BYTES, - 0, - partition(2), - null, - null, - null, - null, - null, - null, - ImmutableList.of(1)); - - InputFile manifest = writeManifest(format, PARTITION_TYPE, ImmutableList.of(delete)); + public void testUnpartitioned(FileFormat format) throws IOException { + TrackedFile file = dataFile("s3://bucket/file.parquet", EMPTY_PARTITION_DATA); - ScanMetrics metrics = ScanMetrics.of(new DefaultMetricsContext()); - try (V4ManifestReader reader = - V4ManifestReader.builder(manifest, PARTITIONED_SPECS) - .filter(Expressions.equal("id", 1)) - .scanMetrics(metrics) - .build()) { - assertThat(reader).isEmpty(); - } + InputFile manifest = writeManifest(format, EMPTY_PARTITION, ImmutableList.of(file)); - assertThat(metrics.skippedDeleteFiles().value()).isEqualTo(1L); - assertThat(metrics.skippedDataFiles().value()).isEqualTo(0L); + TrackedFile actual = Iterables.getOnlyElement(read(manifest, UNPARTITIONED_SPECS)); + // unpartitioned manifests omit the partition field, which is read as null + assertThat(actual.partition()).isNull(); } @ParameterizedTest @FieldSource("FORMATS") - public void testPartitionFilterKeepsManifestReferences(FileFormat format) throws IOException { - TrackedFile keep = dataFile("data-1.parquet", partition(1)); - TrackedFile prune = dataFile("data-2.parquet", partition(2)); - // a real manifest reference has a null spec_id and no partition tuple; these refs carry a - // spec and a tuple that fails the filter so that pruning would be detected if the manifest - // passthrough broke + public void testPartitionFilterPrunesFilesAndCountsSkips(FileFormat format) throws IOException { + // one data file and one delete file match the filter; their counterparts are pruned; manifest + // references have no partition and are always kept + TrackedFile keepData = dataFile("keep-data.parquet", partition(1)); + TrackedFile pruneData = dataFile("prune-data.parquet", partition(2)); + TrackedFile keepDelete = deleteFile("keep-delete.parquet", partition(1)); + TrackedFile pruneDelete = deleteFile("prune-delete.parquet", partition(2)); TrackedFile dataManifestRef = manifestRef(FileContent.DATA_MANIFEST, "data-leaf.parquet"); TrackedFile deleteManifestRef = manifestRef(FileContent.DELETE_MANIFEST, "delete-leaf.parquet"); @@ -576,7 +491,8 @@ public void testPartitionFilterKeepsManifestReferences(FileFormat format) throws writeManifest( format, PARTITION_TYPE, - ImmutableList.of(keep, prune, dataManifestRef, deleteManifestRef)); + ImmutableList.of( + keepData, pruneData, keepDelete, pruneDelete, dataManifestRef, deleteManifestRef)); ScanMetrics metrics = ScanMetrics.of(new DefaultMetricsContext()); try (V4ManifestReader reader = @@ -587,18 +503,30 @@ public void testPartitionFilterKeepsManifestReferences(FileFormat format) throws assertThat(reader) .extracting(TrackedFile::location) .containsExactlyInAnyOrder( - keep.location(), dataManifestRef.location(), deleteManifestRef.location()); + keepData.location(), + keepDelete.location(), + dataManifestRef.location(), + deleteManifestRef.location()); } - // the manifest references bypass the filter instead of being evaluated and skipped - assertThat(metrics.skippedDataFiles().value()).isEqualTo(1L); - assertThat(metrics.skippedDataManifests().value()).isEqualTo(0L); - assertThat(metrics.skippedDeleteManifests().value()).isEqualTo(0L); + assertThat(metrics.skippedDataFiles().value()) + .as("one data file is pruned by the partition filter") + .isEqualTo(1L); + assertThat(metrics.skippedDeleteFiles().value()) + .as("one delete file is pruned by the partition filter") + .isEqualTo(1L); + assertThat(metrics.skippedDataManifests().value()) + .as("manifests have no partition and are not pruned") + .isEqualTo(0L); + assertThat(metrics.skippedDeleteManifests().value()) + .as("manifests have no partition and are not pruned") + .isEqualTo(0L); } @ParameterizedTest @FieldSource("FORMATS") - public void testRowFilterOnUnpartitionedTableKeepsAllFiles(FileFormat format) throws IOException { + public void testRowFilterKeepsFilesWithoutStats(FileFormat format) throws IOException { + // with no content stats to evaluate, the row filter cannot prune any file TrackedFile file1 = dataFile("s3://bucket/a.parquet", EMPTY_PARTITION_DATA); TrackedFile file2 = dataFile("s3://bucket/b.parquet", EMPTY_PARTITION_DATA); @@ -619,24 +547,6 @@ public void testRowFilterOnUnpartitionedTableKeepsAllFiles(FileFormat format) th assertThat(metrics.skippedDeleteFiles().value()).isEqualTo(0L); } - @Test - public void testInvalidBuilderArguments() { - InputFile manifest = fileIO.newInputFile(tempDir.resolve("manifest.avro").toString()); - - assertThatThrownBy(() -> V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS).filter(null)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Invalid filter: null"); - - assertThatThrownBy( - () -> V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS).scanMetrics(null)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Invalid scan metrics: null"); - - assertThatThrownBy(() -> V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS).select(null)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("Invalid columns: null"); - } - @ParameterizedTest @FieldSource("FORMATS") public void testCaseInsensitivePartitionFilter(FileFormat format) throws IOException { @@ -658,19 +568,25 @@ public void testCaseInsensitivePartitionFilter(FileFormat format) throws IOExcep @FieldSource("FORMATS") public void testMultiSpecPartitionPruning(FileFormat format) throws IOException { PartitionSpec spec0 = - PartitionSpec.builderFor(TABLE_SCHEMA).withSpecId(0).identity("id").build(); + PartitionSpec.builderFor(TABLE_SCHEMA) + .withSpecId(0) + .add(1, 1000, "id", Transforms.identity()) + .build(); PartitionSpec spec1 = PartitionSpec.builderFor(TABLE_SCHEMA) .withSpecId(1) .add(2, 1001, "data", Transforms.identity()) .build(); - Map specsById = ImmutableMap.of(0, spec0, 1, spec1); + Map specsById = + ImmutableMap.of(spec0.specId(), spec0, spec1.specId(), spec1); Types.StructType unionType = Partitioning.unionPartitionTypes(specsById.values()); - TrackedFile keepById = dataFile("spec0-id1.parquet", unionPartition(unionType, 1, null), 0); - TrackedFile prunedById = dataFile("spec0-id2.parquet", unionPartition(unionType, 2, null), 0); + TrackedFile keepById = + dataFile("spec0-id1.parquet", spec0.specId(), unionPartition(unionType, 1, null)); + TrackedFile prunedById = + dataFile("spec0-id2.parquet", spec0.specId(), unionPartition(unionType, 2, null)); TrackedFile keptOtherSpec = - dataFile("spec1-data.parquet", unionPartition(unionType, null, "x"), 1); + dataFile("spec1-data.parquet", spec1.specId(), unionPartition(unionType, null, "x")); InputFile manifest = writeManifest(format, unionType, ImmutableList.of(keepById, prunedById, keptOtherSpec)); @@ -684,6 +600,38 @@ public void testMultiSpecPartitionPruning(FileFormat format) throws IOException } } + @ParameterizedTest + @FieldSource("FORMATS") + public void testPartitionFilterKeepsFileWithUnknownSpec(FileFormat format) throws IOException { + // spec ID 5 is not in PARTITIONED_SPECS, so no partition filter applies to this file + TrackedFile file = dataFile("orphan.parquet", 5, partition(1)); + + InputFile manifest = writeManifest(format, PARTITION_TYPE, ImmutableList.of(file)); + + // the filter would prune partition id=1 under spec 0, but cannot be applied to spec 5 + try (V4ManifestReader reader = + V4ManifestReader.builder(manifest, PARTITIONED_SPECS) + .filter(Expressions.equal("id", 2)) + .build()) { + assertThat(reader).extracting(TrackedFile::location).containsExactly(file.location()); + } + } + + @ParameterizedTest + @FieldSource("FORMATS") + public void testPartitionFilterKeepsFileWithNullSpecId(FileFormat format) throws IOException { + TrackedFile file = dataFile("no-spec.parquet", null, null); + + InputFile manifest = writeManifest(format, PARTITION_TYPE, ImmutableList.of(file)); + + try (V4ManifestReader reader = + V4ManifestReader.builder(manifest, PARTITIONED_SPECS) + .filter(Expressions.equal("id", 2)) + .build()) { + assertThat(reader).extracting(TrackedFile::location).containsExactly(file.location()); + } + } + @ParameterizedTest @FieldSource("FORMATS") public void testIteratorReturnsLiveCopies(FileFormat format) throws IOException { @@ -702,9 +650,9 @@ public void testIteratorReturnsLiveCopies(FileFormat format) throws IOException .hasSize(2) .extracting(TrackedFile::location) .containsExactly(added1.location(), added2.location()); - // iterator() copies each entry, so the collected instances are independent of the reused - // container (they would be the same object if iterator() did not copy) - assertThat(read.get(0)).isNotSameAs(read.get(1)); + assertThat(read.get(0)) + .as("iterator() should copy each entry rather than yield one reused container") + .isNotSameAs(read.get(1)); } } @@ -721,43 +669,32 @@ public void testUnknownManifestFormatThrows(FileFormat format) throws IOExceptio } } - @ParameterizedTest - @FieldSource("FORMATS") - public void testPartitionFilterKeepsFileWithUnknownSpec(FileFormat format) throws IOException { - // spec ID 5 is not in PARTITIONED_SPECS, so no partition filter applies to this file - TrackedFile file = dataFile("orphan.parquet", partition(1), 5); - - InputFile manifest = writeManifest(format, PARTITION_TYPE, ImmutableList.of(file)); - - // the filter would prune partition id=1 under spec 0, but cannot be applied to spec 5 - try (V4ManifestReader reader = - V4ManifestReader.builder(manifest, PARTITIONED_SPECS) - .filter(Expressions.equal("id", 2)) - .build()) { - assertThat(reader).extracting(TrackedFile::location).containsExactly(file.location()); - } - } + @Test + public void testInvalidBuilderArguments() { + InputFile manifest = fileIO.newInputFile(tempDir.resolve("manifest.avro").toString()); - @ParameterizedTest - @FieldSource("FORMATS") - public void testPartitionFilterKeepsFileWithNullSpecId(FileFormat format) throws IOException { - TrackedFile file = dataFile("no-spec.parquet", null, null); + assertThatThrownBy(() -> V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS).filter(null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid filter: null"); - InputFile manifest = writeManifest(format, PARTITION_TYPE, ImmutableList.of(file)); + assertThatThrownBy( + () -> V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS).scanMetrics(null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid scan metrics: null"); - try (V4ManifestReader reader = - V4ManifestReader.builder(manifest, PARTITIONED_SPECS) - .filter(Expressions.equal("id", 2)) - .build()) { - assertThat(reader).extracting(TrackedFile::location).containsExactly(file.location()); - } + assertThatThrownBy( + () -> + V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS) + .select((Collection) null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid columns: null"); } private static TrackedFile dataFile(String location, PartitionData partition) { - return dataFile(location, partition, 0); + return dataFile(location, 0, partition); } - private static TrackedFile dataFile(String location, PartitionData partition, Integer specId) { + private static TrackedFile dataFile(String location, Integer specId, PartitionData partition) { return new TrackedFileStruct( addedTracking(), FileContent.DATA, @@ -777,34 +714,24 @@ private static TrackedFile dataFile(String location, PartitionData partition, In null); } - private static TrackedFile fileWithFullTracking() { - Tracking tracking = - new TrackingStruct( - EntryStatus.ADDED, - SNAPSHOT_ID, - 5L, // data sequence number - 6L, // file sequence number - 7L, // dv snapshot id - 8L, // first row id - new byte[] {1, 2}, // deleted positions - new byte[] {3, 4}); // replaced positions + private static TrackedFile deleteFile(String location, PartitionData partition) { return new TrackedFileStruct( - tracking, - FileContent.DATA, + addedTracking(), + FileContent.EQUALITY_DELETES, FORMAT_VERSION_V4, - "s3://bucket/file.parquet", + location, FileFormat.PARQUET, RECORD_COUNT, FILE_SIZE_IN_BYTES, 0, - EMPTY_PARTITION_DATA, + partition, null, null, null, null, null, null, - null); + ImmutableList.of(1)); } private static TrackedFile manifestRef(FileContent content, String location) { @@ -817,15 +744,15 @@ private static TrackedFile manifestRef(FileContent content, String location) { FileFormat.PARQUET, RECORD_COUNT, FILE_SIZE_IN_BYTES, - 0, - partition(2), - null, - null, - null, + null, // spec_id: a manifest reference has no spec + null, // partition: a manifest reference has no partition tuple + null, // content_stats + null, // sort_order_id + null, // deletion_vector info, - null, - null, - null); + null, // key_metadata + null, // split_offsets + null); // equality_ids } private static TrackedFile fileWithStatus(EntryStatus status, String location) { @@ -862,16 +789,6 @@ private static Tracking addedTracking() { return new TrackingStruct(EntryStatus.ADDED, SNAPSHOT_ID, null, null, null, null, null, null); } - private static DeletionVector deletionVector( - String location, long offset, long sizeInBytes, long cardinality) { - return DeletionVectorStruct.builder() - .location(location) - .offset(offset) - .sizeInBytes(sizeInBytes) - .cardinality(cardinality) - .build(); - } - private static PartitionData partition(int id) { PartitionData partition = new PartitionData(PARTITION_TYPE); partition.set(0, id); From 1d2b21f21c611d93d4dd838a32c82e7f56df43ff Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Sun, 26 Jul 2026 08:25:40 -0700 Subject: [PATCH 23/26] Standardize constants --- .../apache/iceberg/TestV4ManifestReader.java | 70 +++++++++---------- 1 file changed, 32 insertions(+), 38 deletions(-) diff --git a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java index b2a93974f68c..c33f0ddb4781 100644 --- a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java +++ b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java @@ -59,17 +59,12 @@ public class TestV4ManifestReader { private static final int FORMAT_VERSION_V4 = 4; private static final long RECORD_COUNT = 100L; private static final long FILE_SIZE_IN_BYTES = 1024L; - private static final int SORT_ORDER_ID = 1; - private static final String DV_LOCATION = "s3://bucket/dv.puffin"; - private static final long DV_OFFSET = 100L; - private static final long DV_SIZE_IN_BYTES = 50L; - private static final long DV_CARDINALITY = 5L; private static final DeletionVector DV = DeletionVectorStruct.builder() - .location(DV_LOCATION) - .offset(DV_OFFSET) - .sizeInBytes(DV_SIZE_IN_BYTES) - .cardinality(DV_CARDINALITY) + .location("s3://bucket/dv.puffin") + .offset(100L) + .sizeInBytes(50L) + .cardinality(5L) .build(); private static final Schema TABLE_SCHEMA = @@ -116,6 +111,16 @@ public class TestV4ManifestReader { null, null); + // shared data files: FILE_A is in partition id=1, FILE_B in partition id=2 + private static final TrackedFile FILE_A = dataFile("data-a.parquet", partition(1)); + private static final TrackedFile FILE_B = dataFile("data-b.parquet", partition(2)); + private static final TrackedFile EQ_DELETES_A = deleteFile("eq-deletes-a.parquet", partition(1)); + private static final TrackedFile EQ_DELETES_B = deleteFile("eq-deletes-b.parquet", partition(2)); + private static final TrackedFile DATA_MANIFEST_REF = + manifestRef(FileContent.DATA_MANIFEST, "data-leaf.parquet"); + private static final TrackedFile DELETE_MANIFEST_REF = + manifestRef(FileContent.DELETE_MANIFEST, "delete-leaf.parquet"); + @TempDir private Path tempDir; private final FileIO fileIO = new TestTables.LocalFileIO(); @@ -135,7 +140,7 @@ public void testReadsWrittenFile(FileFormat format) throws IOException { ID_PARTITIONING.specId(), partition(7), null, - SORT_ORDER_ID, + 1, // sort order id DV, null, ByteBuffer.wrap(new byte[] {1, 2, 3}), @@ -426,10 +431,7 @@ public void testDefaultReadsFullTracking(FileFormat format) throws IOException { @ParameterizedTest @FieldSource("FORMATS") public void testPartitionFilterForceProjectsFilterFields(FileFormat format) throws IOException { - TrackedFile keep = dataFile("keep.parquet", partition(1)); - TrackedFile prune = dataFile("prune.parquet", partition(2)); - - InputFile manifest = writeManifest(format, PARTITION_TYPE, ImmutableList.of(keep, prune)); + InputFile manifest = writeManifest(format, PARTITION_TYPE, ImmutableList.of(FILE_A, FILE_B)); // the caller projects only location; the reader must still project the fields the partition // filter reads (content_type, spec_id, partition) or every row would be pruned @@ -439,7 +441,7 @@ public void testPartitionFilterForceProjectsFilterFields(FileFormat format) thro .project(projection) .filter(Expressions.equal("id", 1)) .build()) { - assertThat(reader).extracting(TrackedFile::location).containsExactly(keep.location()); + assertThat(reader).extracting(TrackedFile::location).containsExactly(FILE_A.location()); } } @@ -447,10 +449,7 @@ public void testPartitionFilterForceProjectsFilterFields(FileFormat format) thro @FieldSource("FORMATS") public void testSelectWithPartitionFilterProjectsFilterFields(FileFormat format) throws IOException { - TrackedFile keep = dataFile("keep.parquet", partition(1)); - TrackedFile prune = dataFile("prune.parquet", partition(2)); - - InputFile manifest = writeManifest(format, PARTITION_TYPE, ImmutableList.of(keep, prune)); + InputFile manifest = writeManifest(format, PARTITION_TYPE, ImmutableList.of(FILE_A, FILE_B)); // the caller selects only location; the reader must still project spec_id and partition // for the partition filter or every row would be pruned @@ -459,7 +458,7 @@ public void testSelectWithPartitionFilterProjectsFilterFields(FileFormat format) .select("location") .filter(Expressions.equal("id", 1)) .build()) { - assertThat(reader).extracting(TrackedFile::location).containsExactly(keep.location()); + assertThat(reader).extracting(TrackedFile::location).containsExactly(FILE_A.location()); } } @@ -478,21 +477,19 @@ public void testUnpartitioned(FileFormat format) throws IOException { @ParameterizedTest @FieldSource("FORMATS") public void testPartitionFilterPrunesFilesAndCountsSkips(FileFormat format) throws IOException { - // one data file and one delete file match the filter; their counterparts are pruned; manifest + // FILE_A and EQ_DELETES_A match the filter; FILE_B and EQ_DELETES_B are pruned; manifest // references have no partition and are always kept - TrackedFile keepData = dataFile("keep-data.parquet", partition(1)); - TrackedFile pruneData = dataFile("prune-data.parquet", partition(2)); - TrackedFile keepDelete = deleteFile("keep-delete.parquet", partition(1)); - TrackedFile pruneDelete = deleteFile("prune-delete.parquet", partition(2)); - TrackedFile dataManifestRef = manifestRef(FileContent.DATA_MANIFEST, "data-leaf.parquet"); - TrackedFile deleteManifestRef = manifestRef(FileContent.DELETE_MANIFEST, "delete-leaf.parquet"); - InputFile manifest = writeManifest( format, PARTITION_TYPE, ImmutableList.of( - keepData, pruneData, keepDelete, pruneDelete, dataManifestRef, deleteManifestRef)); + FILE_A, + FILE_B, + EQ_DELETES_A, + EQ_DELETES_B, + DATA_MANIFEST_REF, + DELETE_MANIFEST_REF)); ScanMetrics metrics = ScanMetrics.of(new DefaultMetricsContext()); try (V4ManifestReader reader = @@ -503,10 +500,10 @@ public void testPartitionFilterPrunesFilesAndCountsSkips(FileFormat format) thro assertThat(reader) .extracting(TrackedFile::location) .containsExactlyInAnyOrder( - keepData.location(), - keepDelete.location(), - dataManifestRef.location(), - deleteManifestRef.location()); + FILE_A.location(), + EQ_DELETES_A.location(), + DATA_MANIFEST_REF.location(), + DELETE_MANIFEST_REF.location()); } assertThat(metrics.skippedDataFiles().value()) @@ -550,17 +547,14 @@ public void testRowFilterKeepsFilesWithoutStats(FileFormat format) throws IOExce @ParameterizedTest @FieldSource("FORMATS") public void testCaseInsensitivePartitionFilter(FileFormat format) throws IOException { - TrackedFile keep = dataFile("keep.parquet", partition(1)); - TrackedFile prune = dataFile("prune.parquet", partition(2)); - - InputFile manifest = writeManifest(format, PARTITION_TYPE, ImmutableList.of(keep, prune)); + InputFile manifest = writeManifest(format, PARTITION_TYPE, ImmutableList.of(FILE_A, FILE_B)); try (V4ManifestReader reader = V4ManifestReader.builder(manifest, PARTITIONED_SPECS) .filter(Expressions.equal("ID", 1)) .caseSensitive(false) .build()) { - assertThat(reader).extracting(TrackedFile::location).containsExactly(keep.location()); + assertThat(reader).extracting(TrackedFile::location).containsExactly(FILE_A.location()); } } From 47ecda8ba31f2b0b2cb0dba9f72ce315f2992229 Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Tue, 28 Jul 2026 17:54:23 -0700 Subject: [PATCH 24/26] PR feedback from Dan and Eduard --- .../apache/iceberg/types/TestTypeUtil.java | 102 +++++++++- .../org/apache/iceberg/V4ManifestReader.java | 24 +-- .../apache/iceberg/TestTrackedFileStruct.java | 8 +- .../apache/iceberg/TestV4ManifestReader.java | 174 +++++++++++------- 4 files changed, 219 insertions(+), 89 deletions(-) diff --git a/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java b/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java index 3c25e930a3fb..d540d239614e 100644 --- a/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java +++ b/api/src/test/java/org/apache/iceberg/types/TestTypeUtil.java @@ -1047,12 +1047,23 @@ public void testReplaceFieldTypes() { required(1, "id", IntegerType.get()), required(2, "s", Types.StructType.of(required(3, "a", Types.LongType.get())))); - Schema result = TypeUtil.replaceFieldTypes(schema, ImmutableMap.of(2, (Type) replacement)); + Schema result = TypeUtil.replaceFieldTypes(schema, ImmutableMap.of(2, replacement)); assertThat(result.findField(1).type()).isEqualTo(IntegerType.get()); assertThat(result.findField(2).type()).isEqualTo(replacement); } + @Test + public void testReplaceFieldTypesPrimitive() { + Schema schema = + new Schema(required(1, "id", IntegerType.get()), required(2, "count", IntegerType.get())); + + Schema result = TypeUtil.replaceFieldTypes(schema, ImmutableMap.of(2, Types.LongType.get())); + + assertThat(result.findField(1).type()).isEqualTo(IntegerType.get()); + assertThat(result.findField(2).type()).isEqualTo(Types.LongType.get()); + } + @Test public void testReplaceFieldTypesListElement() { Schema schema = @@ -1060,19 +1071,100 @@ public void testReplaceFieldTypesListElement() { Schema result = TypeUtil.replaceFieldTypes( - schema, - ImmutableMap.of(2, (Type) Types.StructType.of(required(3, "x", IntegerType.get())))); + schema, ImmutableMap.of(2, Types.StructType.of(required(3, "x", IntegerType.get())))); Types.ListType list = (Types.ListType) result.findField(1).type(); assertThat(list.elementType().asStructType().field(3).name()).isEqualTo("x"); assertThat(list.isElementRequired()).isTrue(); } + @Test + public void testReplaceFieldTypesMapKeyAndValue() { + Schema schema = + new Schema( + required(1, "m", Types.MapType.ofRequired(2, 3, IntegerType.get(), IntegerType.get()))); + + Schema result = + TypeUtil.replaceFieldTypes( + schema, ImmutableMap.of(2, Types.LongType.get(), 3, Types.StringType.get())); + + Types.MapType map = (Types.MapType) result.findField(1).type(); + assertThat(map.keyType()).isEqualTo(Types.LongType.get()); + assertThat(map.valueType()).isEqualTo(Types.StringType.get()); + assertThat(map.isValueRequired()).isTrue(); + } + + @Test + public void testReplaceFieldTypesPreservesDefaultsOnCompatibleType() { + Types.NestedField field = + Types.NestedField.optional("count") + .withId(1) + .ofType(IntegerType.get()) + .withInitialDefault(Literal.of(5)) + .withWriteDefault(Literal.of(7)) + .build(); + Schema schema = new Schema(field); + + Schema result = TypeUtil.replaceFieldTypes(schema, ImmutableMap.of(1, Types.LongType.get())); + + Types.NestedField replaced = result.findField(1); + assertThat(replaced.type()).isEqualTo(Types.LongType.get()); + assertThat(replaced.initialDefault()).isEqualTo(5L); + assertThat(replaced.writeDefault()).isEqualTo(7L); + } + + @Test + public void testReplaceFieldTypesRejectsDefaultIncompatibleWithType() { + Types.NestedField field = + Types.NestedField.optional("count") + .withId(1) + .ofType(IntegerType.get()) + .withInitialDefault(Literal.of(5)) + .build(); + Schema schema = new Schema(field); + + assertThatThrownBy( + () -> TypeUtil.replaceFieldTypes(schema, ImmutableMap.of(1, Types.StringType.get()))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Cannot cast default value to string"); + } + + @Test + public void testReplaceFieldTypesPreservesIdentifierField() { + Schema schema = + new Schema( + Lists.newArrayList( + required(1, "id", IntegerType.get()), required(2, "data", Types.StringType.get())), + Sets.newHashSet(1)); + + Schema result = TypeUtil.replaceFieldTypes(schema, ImmutableMap.of(1, Types.LongType.get())); + + assertThat(result.findField(1).type()).isEqualTo(Types.LongType.get()); + assertThat(result.identifierFieldIds()).containsExactly(1); + } + + @Test + public void testReplaceFieldTypesRejectsNonPrimitiveIdentifierField() { + Schema schema = + new Schema( + Lists.newArrayList( + required(1, "id", IntegerType.get()), required(2, "data", Types.StringType.get())), + Sets.newHashSet(1)); + + // an identifier field must be primitive, so replacing it with a struct is rejected + assertThatThrownBy( + () -> + TypeUtil.replaceFieldTypes( + schema, + ImmutableMap.of(1, Types.StructType.of(required(3, "x", IntegerType.get()))))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("not a primitive type field"); + } + @Test public void testReplaceFieldTypesNoMatchReturnsSameSchema() { Schema schema = new Schema(required(1, "id", IntegerType.get())); - Schema result = - TypeUtil.replaceFieldTypes(schema, ImmutableMap.of(99, (Type) Types.LongType.get())); + Schema result = TypeUtil.replaceFieldTypes(schema, ImmutableMap.of(99, Types.LongType.get())); assertThat(result).isSameAs(schema); } } diff --git a/core/src/main/java/org/apache/iceberg/V4ManifestReader.java b/core/src/main/java/org/apache/iceberg/V4ManifestReader.java index 6d2c3358259b..a823454845dd 100644 --- a/core/src/main/java/org/apache/iceberg/V4ManifestReader.java +++ b/core/src/main/java/org/apache/iceberg/V4ManifestReader.java @@ -109,20 +109,11 @@ private boolean matchesPartition(TrackedFile trackedFile) { private void incrementSkipCount(FileContent content) { switch (content) { - case DATA: - scanMetrics.skippedDataFiles().increment(); - break; - case EQUALITY_DELETES: - scanMetrics.skippedDeleteFiles().increment(); - break; - case DATA_MANIFEST: - scanMetrics.skippedDataManifests().increment(); - break; - case DELETE_MANIFEST: - scanMetrics.skippedDeleteManifests().increment(); - break; - default: - throw new UnsupportedOperationException("Unsupported content type: " + content); + case DATA -> scanMetrics.skippedDataFiles().increment(); + case EQUALITY_DELETES -> scanMetrics.skippedDeleteFiles().increment(); + case DATA_MANIFEST -> scanMetrics.skippedDataManifests().increment(); + case DELETE_MANIFEST -> scanMetrics.skippedDeleteManifests().increment(); + default -> throw new UnsupportedOperationException("Unsupported content type: " + content); } } @@ -223,7 +214,7 @@ Builder select(Collection newColumns) { !scanPlanning, "Cannot use select(Collection) with forScanPlanning()"); Preconditions.checkState( requestedProjection == null, - "Cannot select columns using both select(Collection) and project(Schema)"); + "Cannot use select(Collection) with project(Schema)"); this.columns = newColumns; return this; } @@ -232,8 +223,7 @@ Builder select(Collection newColumns) { Builder project(Schema newProjection) { Preconditions.checkState(!scanPlanning, "Cannot use project(Schema) with forScanPlanning()"); Preconditions.checkState( - columns == null, - "Cannot select columns using both select(Collection) and project(Schema)"); + columns == null, "Cannot use project(Schema) with select(Collection)"); this.requestedProjection = newProjection; return this; } diff --git a/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java b/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java index d7e838266093..14265c91f692 100644 --- a/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java +++ b/core/src/test/java/org/apache/iceberg/TestTrackedFileStruct.java @@ -34,7 +34,7 @@ class TestTrackedFileStruct { private static final int FORMAT_VERSION_V4 = 4; - private static final List FIELDS = + private static final List DEFAULT_FIELDS = TrackedFile.schema(Types.StructType.of(), Types.StructType.of()).asStruct().fields(); private static final Tracking TRACKING = Mockito.mock(Tracking.class); @@ -343,7 +343,7 @@ void projectedStructLike() { @Test void structLikeSize() { TrackedFileStruct file = new TrackedFileStruct(); - assertThat(file.size()).isEqualTo(FIELDS.size()); + assertThat(file.size()).isEqualTo(DEFAULT_FIELDS.size()); } @ParameterizedTest @@ -388,8 +388,8 @@ void serializationRoundTrip(RoundTripSerializer serializer) t } private static int pos(String fieldName) { - for (int i = 0; i < FIELDS.size(); i += 1) { - if (FIELDS.get(i).name().equals(fieldName)) { + for (int i = 0; i < DEFAULT_FIELDS.size(); i += 1) { + if (DEFAULT_FIELDS.get(i).name().equals(fieldName)) { return i; } } diff --git a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java index c33f0ddb4781..069b307f6a12 100644 --- a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java +++ b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java @@ -31,6 +31,7 @@ import java.util.Map; import java.util.function.Consumer; import java.util.stream.Stream; +import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.inmemory.InMemoryOutputFile; import org.apache.iceberg.io.FileAppender; @@ -47,6 +48,7 @@ import org.apache.iceberg.types.Comparators; import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.types.Types; +import org.junit.jupiter.api.Named; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; @@ -54,7 +56,7 @@ import org.junit.jupiter.params.provider.FieldSource; import org.junit.jupiter.params.provider.MethodSource; -public class TestV4ManifestReader { +class TestV4ManifestReader { private static final long SNAPSHOT_ID = 42L; private static final int FORMAT_VERSION_V4 = 4; private static final long RECORD_COUNT = 100L; @@ -72,15 +74,15 @@ public class TestV4ManifestReader { optional(1, "id", Types.IntegerType.get()), optional(2, "data", Types.StringType.get())); private static final PartitionSpec ID_PARTITIONING = PartitionSpec.builderFor(TABLE_SCHEMA).identity("id").build(); - private static final Types.StructType PARTITION_TYPE = ID_PARTITIONING.partitionType(); + private static final Types.StructType ID_PARTITION_TYPE = ID_PARTITIONING.partitionType(); private static final Types.StructType EMPTY_PARTITION = Types.StructType.of(); private static final PartitionData EMPTY_PARTITION_DATA = new PartitionData(EMPTY_PARTITION); - private static final Map PARTITIONED_SPECS = + private static final Map ID_PARTITIONING_SPECS = ImmutableMap.of(ID_PARTITIONING.specId(), ID_PARTITIONING); private static final Map UNPARTITIONED_SPECS = ImmutableMap.of(PartitionSpec.unpartitioned().specId(), PartitionSpec.unpartitioned()); - private static final List FORMATS = + private static final List MANIFEST_FORMATS = ImmutableList.of(FileFormat.AVRO, FileFormat.PARQUET); // a data file whose tracking carries every inheritable and change-tracking value set @@ -126,7 +128,7 @@ public class TestV4ManifestReader { private final FileIO fileIO = new TestTables.LocalFileIO(); @ParameterizedTest - @FieldSource("FORMATS") + @FieldSource("MANIFEST_FORMATS") public void testReadsWrittenFile(FileFormat format) throws IOException { TrackedFile file = new TrackedFileStruct( @@ -147,25 +149,25 @@ public void testReadsWrittenFile(FileFormat format) throws IOException { ImmutableList.of(50L, 100L), null); - InputFile manifest = writeManifest(format, PARTITION_TYPE, ImmutableList.of(file)); + InputFile manifest = writeManifest(format, ID_PARTITION_TYPE, ImmutableList.of(file)); - TrackedFile actual = Iterables.getOnlyElement(read(manifest, PARTITIONED_SPECS)); + TrackedFile actual = Iterables.getOnlyElement(read(manifest, ID_PARTITIONING_SPECS)); // compare with tracking reduced to status: the reader fills status-independent tracking // fields (row position, sequence numbers via inheritance) that the written file does not have Types.StructType comparisonType = TypeUtil.replaceFieldTypes( - TrackedFile.schema(PARTITION_TYPE, Types.StructType.of()), + TrackedFile.schema(ID_PARTITION_TYPE, Types.StructType.of()), ImmutableMap.of( TrackedFile.TRACKING.fieldId(), Types.StructType.of(Tracking.STATUS))) .asStruct(); assertThat((StructLike) actual) .usingComparator(Comparators.forType(comparisonType)) - .isEqualTo((StructLike) file); + .isEqualTo(file); } @ParameterizedTest - @FieldSource("FORMATS") + @FieldSource("MANIFEST_FORMATS") public void testEqualityDeleteRoundTrip(FileFormat format) throws IOException { TrackedFile delete = new TrackedFileStruct( @@ -194,7 +196,7 @@ public void testEqualityDeleteRoundTrip(FileFormat format) throws IOException { } @ParameterizedTest - @FieldSource("FORMATS") + @FieldSource("MANIFEST_FORMATS") public void testStatusFiltering(FileFormat format) throws IOException { List files = ImmutableList.of( @@ -227,7 +229,7 @@ public void testStatusFiltering(FileFormat format) throws IOException { } @ParameterizedTest - @FieldSource("FORMATS") + @FieldSource("MANIFEST_FORMATS") public void testManifestLocationAndPosition(FileFormat format) throws IOException { List files = ImmutableList.of( @@ -244,11 +246,10 @@ public void testManifestLocationAndPosition(FileFormat format) throws IOExceptio assertThat(read).extracting(file -> file.tracking().manifestPos()).containsExactly(0L, 1L, 2L); } - @ParameterizedTest(name = "{0} / {2}") - @MethodSource("restrictedReadModes") - public void testRestrictedReadReturnsOnlyRequestedFields( - FileFormat format, Consumer configureRead, String description) - throws IOException { + @ParameterizedTest(name = "{0} / {1}") + @MethodSource("selectiveReadModes") + void testSelectiveReadReturnsOnlyRequestedFields( + FileFormat format, Consumer configureRead) throws IOException { List files = ImmutableList.of( dataFile("s3://bucket/live.parquet", EMPTY_PARTITION_DATA), @@ -260,18 +261,35 @@ public void testRestrictedReadReturnsOnlyRequestedFields( V4ManifestReader.Builder builder = V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS); configureRead.accept(builder); try (V4ManifestReader reader = builder.build()) { - // content_type and status are projected for liveness filtering, so only the live entry - // survives even though the caller requested only location TrackedFile actual = Iterables.getOnlyElement(reader); + + // the requested field is read assertThat(actual.location()).isEqualTo("s3://bucket/live.parquet"); - // fields the caller did not request are not read + + // the reader always projects the fields it consumes internally, even though the caller + // selected only location: content type and status (liveness filtering keeps only the live + // entry), and manifest position (from row_position) + assertThat(actual.contentType()).isEqualTo(FileContent.DATA); + assertThat(actual.tracking().status()).isEqualTo(EntryStatus.ADDED); + assertThat(actual.tracking().manifestPos()).isEqualTo(0L); + + // every field the caller did not request and the reader does not require is omitted; + // content stats in particular (the largest projection) is not read + assertThat(actual.contentStats()).isNull(); assertThat(actual.fileFormat()).isNull(); + assertThat(actual.recordCount()).isEqualTo(-1L); + assertThat(actual.fileSizeInBytes()).isEqualTo(-1L); assertThat(actual.specId()).isNull(); + assertThat(actual.partition()).isNull(); assertThat(actual.sortOrderId()).isNull(); + assertThat(actual.deletionVector()).isNull(); + assertThat(actual.keyMetadata()).isNull(); + assertThat(actual.splitOffsets()).isNull(); + assertThat(actual.equalityIds()).isNull(); } } - private static Stream restrictedReadModes() { + private static Stream selectiveReadModes() { Map> modes = ImmutableMap.of( "project", @@ -280,15 +298,15 @@ private static Stream restrictedReadModes() { builder -> builder.select("location"), "case-insensitive select", builder -> builder.select("LOCATION").caseSensitive(false)); - return FORMATS.stream() + return MANIFEST_FORMATS.stream() .flatMap( format -> modes.entrySet().stream() - .map(mode -> Arguments.of(format, mode.getValue(), mode.getKey()))); + .map(mode -> Arguments.of(format, Named.of(mode.getKey(), mode.getValue())))); } @ParameterizedTest - @FieldSource("FORMATS") + @FieldSource("MANIFEST_FORMATS") public void testRowFilterForcesRecordCount(FileFormat format) throws IOException { TrackedFile file = dataFile("s3://bucket/file.parquet", EMPTY_PARTITION_DATA); @@ -318,8 +336,7 @@ public void testProjectionModesAreMutuallyExclusive() { .select("location") .project(new Schema(TrackedFile.LOCATION))) .isInstanceOf(IllegalStateException.class) - .hasMessage( - "Cannot select columns using both select(Collection) and project(Schema)"); + .hasMessage("Cannot use project(Schema) with select(Collection)"); assertThatThrownBy( () -> @@ -327,8 +344,7 @@ public void testProjectionModesAreMutuallyExclusive() { .project(new Schema(TrackedFile.LOCATION)) .select("location")) .isInstanceOf(IllegalStateException.class) - .hasMessage( - "Cannot select columns using both select(Collection) and project(Schema)"); + .hasMessage("Cannot use select(Collection) with project(Schema)"); assertThatThrownBy( () -> @@ -366,7 +382,7 @@ public void testProjectionModesAreMutuallyExclusive() { } @ParameterizedTest - @FieldSource("FORMATS") + @FieldSource("MANIFEST_FORMATS") public void testProjectionPreservesNarrowTrackingProjection(FileFormat format) throws IOException { InputFile manifest = @@ -385,7 +401,7 @@ public void testProjectionPreservesNarrowTrackingProjection(FileFormat format) } @ParameterizedTest - @FieldSource("FORMATS") + @FieldSource("MANIFEST_FORMATS") public void testForScanPlanningOmitsChangeTrackingFields(FileFormat format) throws IOException { InputFile manifest = writeManifest(format, EMPTY_PARTITION, ImmutableList.of(FILE_WITH_FULL_TRACKING)); @@ -407,7 +423,7 @@ public void testForScanPlanningOmitsChangeTrackingFields(FileFormat format) thro } @ParameterizedTest - @FieldSource("FORMATS") + @FieldSource("MANIFEST_FORMATS") public void testDefaultReadsFullTracking(FileFormat format) throws IOException { InputFile manifest = writeManifest(format, EMPTY_PARTITION, ImmutableList.of(FILE_WITH_FULL_TRACKING)); @@ -429,42 +445,65 @@ public void testDefaultReadsFullTracking(FileFormat format) throws IOException { } @ParameterizedTest - @FieldSource("FORMATS") + @FieldSource("MANIFEST_FORMATS") + public void testProjectNullReadsFullSchema(FileFormat format) throws IOException { + InputFile manifest = + writeManifest(format, EMPTY_PARTITION, ImmutableList.of(FILE_WITH_FULL_TRACKING)); + + // project(null) clears the projection and reads the full schema, like no projection at all + try (V4ManifestReader reader = + V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS).project(null).build()) { + TrackedFile actual = Iterables.getOnlyElement(reader); + assertThat(actual.location()).isEqualTo("s3://bucket/file.parquet"); + assertThat(actual.fileFormat()).isEqualTo(FileFormat.PARQUET); + assertThat(actual.recordCount()).isEqualTo(RECORD_COUNT); + assertThat(actual.fileSizeInBytes()).isEqualTo(FILE_SIZE_IN_BYTES); + } + } + + @ParameterizedTest + @FieldSource("MANIFEST_FORMATS") public void testPartitionFilterForceProjectsFilterFields(FileFormat format) throws IOException { - InputFile manifest = writeManifest(format, PARTITION_TYPE, ImmutableList.of(FILE_A, FILE_B)); + InputFile manifest = writeManifest(format, ID_PARTITION_TYPE, ImmutableList.of(FILE_A, FILE_B)); // the caller projects only location; the reader must still project the fields the partition - // filter reads (content_type, spec_id, partition) or every row would be pruned + // filter reads (spec_id, partition) or every row would be pruned Schema projection = new Schema(TrackedFile.LOCATION); try (V4ManifestReader reader = - V4ManifestReader.builder(manifest, PARTITIONED_SPECS) + V4ManifestReader.builder(manifest, ID_PARTITIONING_SPECS) .project(projection) .filter(Expressions.equal("id", 1)) .build()) { - assertThat(reader).extracting(TrackedFile::location).containsExactly(FILE_A.location()); + TrackedFile actual = Iterables.getOnlyElement(reader); + assertThat(actual.location()).isEqualTo(FILE_A.location()); + assertThat(actual.specId()).isEqualTo(ID_PARTITIONING.specId()); + assertThat(actual.partition().get(0, Integer.class)).isEqualTo(1); } } @ParameterizedTest - @FieldSource("FORMATS") + @FieldSource("MANIFEST_FORMATS") public void testSelectWithPartitionFilterProjectsFilterFields(FileFormat format) throws IOException { - InputFile manifest = writeManifest(format, PARTITION_TYPE, ImmutableList.of(FILE_A, FILE_B)); + InputFile manifest = writeManifest(format, ID_PARTITION_TYPE, ImmutableList.of(FILE_A, FILE_B)); // the caller selects only location; the reader must still project spec_id and partition // for the partition filter or every row would be pruned try (V4ManifestReader reader = - V4ManifestReader.builder(manifest, PARTITIONED_SPECS) + V4ManifestReader.builder(manifest, ID_PARTITIONING_SPECS) .select("location") .filter(Expressions.equal("id", 1)) .build()) { - assertThat(reader).extracting(TrackedFile::location).containsExactly(FILE_A.location()); + TrackedFile actual = Iterables.getOnlyElement(reader); + assertThat(actual.location()).isEqualTo(FILE_A.location()); + assertThat(actual.specId()).isEqualTo(ID_PARTITIONING.specId()); + assertThat(actual.partition().get(0, Integer.class)).isEqualTo(1); } } @ParameterizedTest - @FieldSource("FORMATS") - public void testUnpartitioned(FileFormat format) throws IOException { + @FieldSource("MANIFEST_FORMATS") + public void testUnpartitionedProducesNullPartitionValue(FileFormat format) throws IOException { TrackedFile file = dataFile("s3://bucket/file.parquet", EMPTY_PARTITION_DATA); InputFile manifest = writeManifest(format, EMPTY_PARTITION, ImmutableList.of(file)); @@ -475,14 +514,14 @@ public void testUnpartitioned(FileFormat format) throws IOException { } @ParameterizedTest - @FieldSource("FORMATS") + @FieldSource("MANIFEST_FORMATS") public void testPartitionFilterPrunesFilesAndCountsSkips(FileFormat format) throws IOException { // FILE_A and EQ_DELETES_A match the filter; FILE_B and EQ_DELETES_B are pruned; manifest // references have no partition and are always kept InputFile manifest = writeManifest( format, - PARTITION_TYPE, + ID_PARTITION_TYPE, ImmutableList.of( FILE_A, FILE_B, @@ -493,7 +532,7 @@ public void testPartitionFilterPrunesFilesAndCountsSkips(FileFormat format) thro ScanMetrics metrics = ScanMetrics.of(new DefaultMetricsContext()); try (V4ManifestReader reader = - V4ManifestReader.builder(manifest, PARTITIONED_SPECS) + V4ManifestReader.builder(manifest, ID_PARTITIONING_SPECS) .filter(Expressions.equal("id", 1)) .scanMetrics(metrics) .build()) { @@ -521,7 +560,7 @@ public void testPartitionFilterPrunesFilesAndCountsSkips(FileFormat format) thro } @ParameterizedTest - @FieldSource("FORMATS") + @FieldSource("MANIFEST_FORMATS") public void testRowFilterKeepsFilesWithoutStats(FileFormat format) throws IOException { // with no content stats to evaluate, the row filter cannot prune any file TrackedFile file1 = dataFile("s3://bucket/a.parquet", EMPTY_PARTITION_DATA); @@ -545,22 +584,32 @@ public void testRowFilterKeepsFilesWithoutStats(FileFormat format) throws IOExce } @ParameterizedTest - @FieldSource("FORMATS") + @FieldSource("MANIFEST_FORMATS") public void testCaseInsensitivePartitionFilter(FileFormat format) throws IOException { - InputFile manifest = writeManifest(format, PARTITION_TYPE, ImmutableList.of(FILE_A, FILE_B)); + InputFile manifest = writeManifest(format, ID_PARTITION_TYPE, ImmutableList.of(FILE_A, FILE_B)); + // a case-insensitive filter binds the mismatched-case "ID" reference and prunes FILE_B try (V4ManifestReader reader = - V4ManifestReader.builder(manifest, PARTITIONED_SPECS) + V4ManifestReader.builder(manifest, ID_PARTITIONING_SPECS) .filter(Expressions.equal("ID", 1)) .caseSensitive(false) .build()) { assertThat(reader).extracting(TrackedFile::location).containsExactly(FILE_A.location()); } + + // the same filter is case-sensitive by default, so "ID" fails to bind to the "id" field + assertThatThrownBy( + () -> + V4ManifestReader.builder(manifest, ID_PARTITIONING_SPECS) + .filter(Expressions.equal("ID", 1)) + .build()) + .isInstanceOf(ValidationException.class) + .hasMessageContaining("Cannot find field 'ID'"); } @ParameterizedTest - @FieldSource("FORMATS") - public void testMultiSpecPartitionPruning(FileFormat format) throws IOException { + @FieldSource("MANIFEST_FORMATS") + public void testFilterMatchesFilesAcrossDisjointSpecs(FileFormat format) throws IOException { PartitionSpec spec0 = PartitionSpec.builderFor(TABLE_SCHEMA) .withSpecId(0) @@ -595,16 +644,16 @@ public void testMultiSpecPartitionPruning(FileFormat format) throws IOException } @ParameterizedTest - @FieldSource("FORMATS") + @FieldSource("MANIFEST_FORMATS") public void testPartitionFilterKeepsFileWithUnknownSpec(FileFormat format) throws IOException { - // spec ID 5 is not in PARTITIONED_SPECS, so no partition filter applies to this file + // spec ID 5 is not in ID_PARTITIONING_SPECS, so no partition filter applies to this file TrackedFile file = dataFile("orphan.parquet", 5, partition(1)); - InputFile manifest = writeManifest(format, PARTITION_TYPE, ImmutableList.of(file)); + InputFile manifest = writeManifest(format, ID_PARTITION_TYPE, ImmutableList.of(file)); // the filter would prune partition id=1 under spec 0, but cannot be applied to spec 5 try (V4ManifestReader reader = - V4ManifestReader.builder(manifest, PARTITIONED_SPECS) + V4ManifestReader.builder(manifest, ID_PARTITIONING_SPECS) .filter(Expressions.equal("id", 2)) .build()) { assertThat(reader).extracting(TrackedFile::location).containsExactly(file.location()); @@ -612,14 +661,14 @@ public void testPartitionFilterKeepsFileWithUnknownSpec(FileFormat format) throw } @ParameterizedTest - @FieldSource("FORMATS") + @FieldSource("MANIFEST_FORMATS") public void testPartitionFilterKeepsFileWithNullSpecId(FileFormat format) throws IOException { TrackedFile file = dataFile("no-spec.parquet", null, null); - InputFile manifest = writeManifest(format, PARTITION_TYPE, ImmutableList.of(file)); + InputFile manifest = writeManifest(format, ID_PARTITION_TYPE, ImmutableList.of(file)); try (V4ManifestReader reader = - V4ManifestReader.builder(manifest, PARTITIONED_SPECS) + V4ManifestReader.builder(manifest, ID_PARTITIONING_SPECS) .filter(Expressions.equal("id", 2)) .build()) { assertThat(reader).extracting(TrackedFile::location).containsExactly(file.location()); @@ -627,7 +676,7 @@ public void testPartitionFilterKeepsFileWithNullSpecId(FileFormat format) throws } @ParameterizedTest - @FieldSource("FORMATS") + @FieldSource("MANIFEST_FORMATS") public void testIteratorReturnsLiveCopies(FileFormat format) throws IOException { TrackedFile added1 = dataFile("s3://bucket/added-1.parquet", EMPTY_PARTITION_DATA); TrackedFile added2 = dataFile("s3://bucket/added-2.parquet", EMPTY_PARTITION_DATA); @@ -650,9 +699,8 @@ public void testIteratorReturnsLiveCopies(FileFormat format) throws IOException } } - @ParameterizedTest - @FieldSource("FORMATS") - public void testUnknownManifestFormatThrows(FileFormat format) throws IOException { + @Test + void testUnknownManifestFormatThrows() throws IOException { InputFile badFile = fileIO.newInputFile(tempDir.resolve("manifest-" + System.nanoTime() + ".txt").toString()); @@ -784,7 +832,7 @@ private static Tracking addedTracking() { } private static PartitionData partition(int id) { - PartitionData partition = new PartitionData(PARTITION_TYPE); + PartitionData partition = new PartitionData(ID_PARTITION_TYPE); partition.set(0, id); return partition; } From d244e63d092bb006e230b52f595b69e14075dcc3 Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Tue, 28 Jul 2026 18:01:04 -0700 Subject: [PATCH 25/26] add testPartialFilterStillPrunesOnCompatibleField --- .../apache/iceberg/TestV4ManifestReader.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java index 069b307f6a12..c6878d56aeec 100644 --- a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java +++ b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java @@ -643,6 +643,27 @@ public void testFilterMatchesFilesAcrossDisjointSpecs(FileFormat format) throws } } + @ParameterizedTest + @FieldSource("MANIFEST_FORMATS") + public void testPartialFilterStillPrunesOnCompatibleField(FileFormat format) throws IOException { + // the spec partitions on id only; a filter of id = 1 AND data = 'z' should still prune by id + // even though data is not a partition source + TrackedFile keep = dataFile("id1.parquet", partition(1)); + TrackedFile prune = dataFile("id2.parquet", partition(2)); + + InputFile manifest = writeManifest(format, ID_PARTITION_TYPE, ImmutableList.of(keep, prune)); + + try (V4ManifestReader reader = + V4ManifestReader.builder(manifest, ID_PARTITIONING_SPECS) + .filter(Expressions.and(Expressions.equal("id", 1), Expressions.equal("data", "z"))) + .build()) { + assertThat(reader) + .extracting(TrackedFile::location) + .as("the id predicate prunes even though data is not a partition field") + .containsExactly(keep.location()); + } + } + @ParameterizedTest @FieldSource("MANIFEST_FORMATS") public void testPartitionFilterKeepsFileWithUnknownSpec(FileFormat format) throws IOException { From ae84803ab6b7a7f69308d1b7b2f7909853feb3ed Mon Sep 17 00:00:00 2001 From: Anoop Johnson Date: Wed, 29 Jul 2026 00:19:10 -0700 Subject: [PATCH 26/26] drop the test prefix --- .../apache/iceberg/TestV4ManifestReader.java | 50 +++++++++---------- 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java index c6878d56aeec..c8a5cfd61a31 100644 --- a/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java +++ b/core/src/test/java/org/apache/iceberg/TestV4ManifestReader.java @@ -129,7 +129,7 @@ class TestV4ManifestReader { @ParameterizedTest @FieldSource("MANIFEST_FORMATS") - public void testReadsWrittenFile(FileFormat format) throws IOException { + public void readsWrittenFile(FileFormat format) throws IOException { TrackedFile file = new TrackedFileStruct( addedTracking(), @@ -168,7 +168,7 @@ public void testReadsWrittenFile(FileFormat format) throws IOException { @ParameterizedTest @FieldSource("MANIFEST_FORMATS") - public void testEqualityDeleteRoundTrip(FileFormat format) throws IOException { + public void equalityDeleteRoundTrip(FileFormat format) throws IOException { TrackedFile delete = new TrackedFileStruct( addedTracking(), @@ -197,7 +197,7 @@ public void testEqualityDeleteRoundTrip(FileFormat format) throws IOException { @ParameterizedTest @FieldSource("MANIFEST_FORMATS") - public void testStatusFiltering(FileFormat format) throws IOException { + public void statusFiltering(FileFormat format) throws IOException { List files = ImmutableList.of( fileWithStatus(EntryStatus.ADDED, "s3://bucket/added.parquet"), @@ -230,7 +230,7 @@ public void testStatusFiltering(FileFormat format) throws IOException { @ParameterizedTest @FieldSource("MANIFEST_FORMATS") - public void testManifestLocationAndPosition(FileFormat format) throws IOException { + public void manifestLocationAndPosition(FileFormat format) throws IOException { List files = ImmutableList.of( dataFile("s3://bucket/a.parquet", EMPTY_PARTITION_DATA), @@ -248,7 +248,7 @@ public void testManifestLocationAndPosition(FileFormat format) throws IOExceptio @ParameterizedTest(name = "{0} / {1}") @MethodSource("selectiveReadModes") - void testSelectiveReadReturnsOnlyRequestedFields( + public void selectiveReadReturnsOnlyRequestedFields( FileFormat format, Consumer configureRead) throws IOException { List files = ImmutableList.of( @@ -307,7 +307,7 @@ private static Stream selectiveReadModes() { @ParameterizedTest @FieldSource("MANIFEST_FORMATS") - public void testRowFilterForcesRecordCount(FileFormat format) throws IOException { + public void rowFilterForcesRecordCount(FileFormat format) throws IOException { TrackedFile file = dataFile("s3://bucket/file.parquet", EMPTY_PARTITION_DATA); InputFile manifest = writeManifest(format, EMPTY_PARTITION, ImmutableList.of(file)); @@ -327,7 +327,7 @@ public void testRowFilterForcesRecordCount(FileFormat format) throws IOException } @Test - public void testProjectionModesAreMutuallyExclusive() { + public void projectionModesAreMutuallyExclusive() { InputFile manifest = fileIO.newInputFile(tempDir.resolve("manifest.avro").toString()); assertThatThrownBy( @@ -383,8 +383,7 @@ public void testProjectionModesAreMutuallyExclusive() { @ParameterizedTest @FieldSource("MANIFEST_FORMATS") - public void testProjectionPreservesNarrowTrackingProjection(FileFormat format) - throws IOException { + public void projectionPreservesNarrowTrackingProjection(FileFormat format) throws IOException { InputFile manifest = writeManifest(format, EMPTY_PARTITION, ImmutableList.of(FILE_WITH_FULL_TRACKING)); @@ -402,7 +401,7 @@ public void testProjectionPreservesNarrowTrackingProjection(FileFormat format) @ParameterizedTest @FieldSource("MANIFEST_FORMATS") - public void testForScanPlanningOmitsChangeTrackingFields(FileFormat format) throws IOException { + public void forScanPlanningOmitsChangeTrackingFields(FileFormat format) throws IOException { InputFile manifest = writeManifest(format, EMPTY_PARTITION, ImmutableList.of(FILE_WITH_FULL_TRACKING)); @@ -424,7 +423,7 @@ public void testForScanPlanningOmitsChangeTrackingFields(FileFormat format) thro @ParameterizedTest @FieldSource("MANIFEST_FORMATS") - public void testDefaultReadsFullTracking(FileFormat format) throws IOException { + public void defaultReadsFullTracking(FileFormat format) throws IOException { InputFile manifest = writeManifest(format, EMPTY_PARTITION, ImmutableList.of(FILE_WITH_FULL_TRACKING)); @@ -446,7 +445,7 @@ public void testDefaultReadsFullTracking(FileFormat format) throws IOException { @ParameterizedTest @FieldSource("MANIFEST_FORMATS") - public void testProjectNullReadsFullSchema(FileFormat format) throws IOException { + public void projectNullReadsFullSchema(FileFormat format) throws IOException { InputFile manifest = writeManifest(format, EMPTY_PARTITION, ImmutableList.of(FILE_WITH_FULL_TRACKING)); @@ -463,7 +462,7 @@ public void testProjectNullReadsFullSchema(FileFormat format) throws IOException @ParameterizedTest @FieldSource("MANIFEST_FORMATS") - public void testPartitionFilterForceProjectsFilterFields(FileFormat format) throws IOException { + public void partitionFilterForceProjectsFilterFields(FileFormat format) throws IOException { InputFile manifest = writeManifest(format, ID_PARTITION_TYPE, ImmutableList.of(FILE_A, FILE_B)); // the caller projects only location; the reader must still project the fields the partition @@ -483,8 +482,7 @@ public void testPartitionFilterForceProjectsFilterFields(FileFormat format) thro @ParameterizedTest @FieldSource("MANIFEST_FORMATS") - public void testSelectWithPartitionFilterProjectsFilterFields(FileFormat format) - throws IOException { + public void selectWithPartitionFilterProjectsFilterFields(FileFormat format) throws IOException { InputFile manifest = writeManifest(format, ID_PARTITION_TYPE, ImmutableList.of(FILE_A, FILE_B)); // the caller selects only location; the reader must still project spec_id and partition @@ -503,7 +501,7 @@ public void testSelectWithPartitionFilterProjectsFilterFields(FileFormat format) @ParameterizedTest @FieldSource("MANIFEST_FORMATS") - public void testUnpartitionedProducesNullPartitionValue(FileFormat format) throws IOException { + public void unpartitionedProducesNullPartitionValue(FileFormat format) throws IOException { TrackedFile file = dataFile("s3://bucket/file.parquet", EMPTY_PARTITION_DATA); InputFile manifest = writeManifest(format, EMPTY_PARTITION, ImmutableList.of(file)); @@ -515,7 +513,7 @@ public void testUnpartitionedProducesNullPartitionValue(FileFormat format) throw @ParameterizedTest @FieldSource("MANIFEST_FORMATS") - public void testPartitionFilterPrunesFilesAndCountsSkips(FileFormat format) throws IOException { + public void partitionFilterPrunesFilesAndCountsSkips(FileFormat format) throws IOException { // FILE_A and EQ_DELETES_A match the filter; FILE_B and EQ_DELETES_B are pruned; manifest // references have no partition and are always kept InputFile manifest = @@ -561,7 +559,7 @@ public void testPartitionFilterPrunesFilesAndCountsSkips(FileFormat format) thro @ParameterizedTest @FieldSource("MANIFEST_FORMATS") - public void testRowFilterKeepsFilesWithoutStats(FileFormat format) throws IOException { + public void rowFilterKeepsFilesWithoutStats(FileFormat format) throws IOException { // with no content stats to evaluate, the row filter cannot prune any file TrackedFile file1 = dataFile("s3://bucket/a.parquet", EMPTY_PARTITION_DATA); TrackedFile file2 = dataFile("s3://bucket/b.parquet", EMPTY_PARTITION_DATA); @@ -585,7 +583,7 @@ public void testRowFilterKeepsFilesWithoutStats(FileFormat format) throws IOExce @ParameterizedTest @FieldSource("MANIFEST_FORMATS") - public void testCaseInsensitivePartitionFilter(FileFormat format) throws IOException { + public void caseInsensitivePartitionFilter(FileFormat format) throws IOException { InputFile manifest = writeManifest(format, ID_PARTITION_TYPE, ImmutableList.of(FILE_A, FILE_B)); // a case-insensitive filter binds the mismatched-case "ID" reference and prunes FILE_B @@ -609,7 +607,7 @@ public void testCaseInsensitivePartitionFilter(FileFormat format) throws IOExcep @ParameterizedTest @FieldSource("MANIFEST_FORMATS") - public void testFilterMatchesFilesAcrossDisjointSpecs(FileFormat format) throws IOException { + public void filterMatchesFilesAcrossDisjointSpecs(FileFormat format) throws IOException { PartitionSpec spec0 = PartitionSpec.builderFor(TABLE_SCHEMA) .withSpecId(0) @@ -645,7 +643,7 @@ public void testFilterMatchesFilesAcrossDisjointSpecs(FileFormat format) throws @ParameterizedTest @FieldSource("MANIFEST_FORMATS") - public void testPartialFilterStillPrunesOnCompatibleField(FileFormat format) throws IOException { + public void partialFilterStillPrunesOnCompatibleField(FileFormat format) throws IOException { // the spec partitions on id only; a filter of id = 1 AND data = 'z' should still prune by id // even though data is not a partition source TrackedFile keep = dataFile("id1.parquet", partition(1)); @@ -666,7 +664,7 @@ public void testPartialFilterStillPrunesOnCompatibleField(FileFormat format) thr @ParameterizedTest @FieldSource("MANIFEST_FORMATS") - public void testPartitionFilterKeepsFileWithUnknownSpec(FileFormat format) throws IOException { + public void partitionFilterKeepsFileWithUnknownSpec(FileFormat format) throws IOException { // spec ID 5 is not in ID_PARTITIONING_SPECS, so no partition filter applies to this file TrackedFile file = dataFile("orphan.parquet", 5, partition(1)); @@ -683,7 +681,7 @@ public void testPartitionFilterKeepsFileWithUnknownSpec(FileFormat format) throw @ParameterizedTest @FieldSource("MANIFEST_FORMATS") - public void testPartitionFilterKeepsFileWithNullSpecId(FileFormat format) throws IOException { + public void partitionFilterKeepsFileWithNullSpecId(FileFormat format) throws IOException { TrackedFile file = dataFile("no-spec.parquet", null, null); InputFile manifest = writeManifest(format, ID_PARTITION_TYPE, ImmutableList.of(file)); @@ -698,7 +696,7 @@ public void testPartitionFilterKeepsFileWithNullSpecId(FileFormat format) throws @ParameterizedTest @FieldSource("MANIFEST_FORMATS") - public void testIteratorReturnsLiveCopies(FileFormat format) throws IOException { + public void iteratorReturnsLiveCopies(FileFormat format) throws IOException { TrackedFile added1 = dataFile("s3://bucket/added-1.parquet", EMPTY_PARTITION_DATA); TrackedFile added2 = dataFile("s3://bucket/added-2.parquet", EMPTY_PARTITION_DATA); List files = @@ -721,7 +719,7 @@ public void testIteratorReturnsLiveCopies(FileFormat format) throws IOException } @Test - void testUnknownManifestFormatThrows() throws IOException { + public void unknownManifestFormatThrows() throws IOException { InputFile badFile = fileIO.newInputFile(tempDir.resolve("manifest-" + System.nanoTime() + ".txt").toString()); @@ -733,7 +731,7 @@ void testUnknownManifestFormatThrows() throws IOException { } @Test - public void testInvalidBuilderArguments() { + public void invalidBuilderArguments() { InputFile manifest = fileIO.newInputFile(tempDir.resolve("manifest.avro").toString()); assertThatThrownBy(() -> V4ManifestReader.builder(manifest, UNPARTITIONED_SPECS).filter(null))