diff --git a/.palantir/revapi.yml b/.palantir/revapi.yml index 80fa8f15f168..048ca8660fdd 100644 --- a/.palantir/revapi.yml +++ b/.palantir/revapi.yml @@ -573,6 +573,232 @@ acceptedBreaks: \ org.apache.iceberg.PartitionStatsHandler::readPartitionStatsFile(org.apache.iceberg.Schema,\ \ org.apache.iceberg.io.InputFile)" justification: "Removed deprecated functionality for partition stats" + "1.12.0": + org.apache.iceberg:iceberg-core: + - code: "java.class.removed" + old: "class org.apache.iceberg.SystemProperties" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.class.removed" + old: "class org.apache.iceberg.data.avro.DataReader" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.field.removed" + old: "field org.apache.iceberg.SystemConfigs.NETFLIX_UNSAFE_PARQUET_ID_FALLBACK_ENABLED" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.field.removed" + old: "field org.apache.iceberg.rest.RESTUtil.NAMESPACE_JOINER" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.field.removed" + old: "field org.apache.iceberg.rest.RESTUtil.NAMESPACE_SPLITTER" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.field.removedWithConstant" + old: "field org.apache.iceberg.TableProperties.MANIFEST_LISTS_ENABLED" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.field.removedWithConstant" + old: "field org.apache.iceberg.TableProperties.MANIFEST_LISTS_ENABLED_DEFAULT" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.field.removedWithConstant" + old: "field org.apache.iceberg.rest.RESTSessionCatalog.REST_PAGE_SIZE" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.noLongerDefault" + old: "method org.apache.iceberg.deletes.PositionDeleteWriter\ + \ org.apache.iceberg.RewriteTablePathUtil.PositionDeleteReaderWriter::writer(org.apache.iceberg.io.OutputFile,\ + \ org.apache.iceberg.FileFormat, org.apache.iceberg.PartitionSpec, org.apache.iceberg.StructLike)\ + \ throws java.io.IOException" + new: "method org.apache.iceberg.deletes.PositionDeleteWriter\ + \ org.apache.iceberg.RewriteTablePathUtil.PositionDeleteReaderWriter::writer(org.apache.iceberg.io.OutputFile,\ + \ org.apache.iceberg.FileFormat, org.apache.iceberg.PartitionSpec, org.apache.iceberg.StructLike)\ + \ throws java.io.IOException" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.nowAbstract" + old: "method org.apache.iceberg.deletes.PositionDeleteWriter\ + \ org.apache.iceberg.RewriteTablePathUtil.PositionDeleteReaderWriter::writer(org.apache.iceberg.io.OutputFile,\ + \ org.apache.iceberg.FileFormat, org.apache.iceberg.PartitionSpec, org.apache.iceberg.StructLike)\ + \ throws java.io.IOException" + new: "method org.apache.iceberg.deletes.PositionDeleteWriter\ + \ org.apache.iceberg.RewriteTablePathUtil.PositionDeleteReaderWriter::writer(org.apache.iceberg.io.OutputFile,\ + \ org.apache.iceberg.FileFormat, org.apache.iceberg.PartitionSpec, org.apache.iceberg.StructLike)\ + \ throws java.io.IOException" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.removed" + old: "method B org.apache.iceberg.rest.responses.BaseScanTaskResponse.Builder, R>, R\ + \ extends org.apache.iceberg.rest.responses.BaseScanTaskResponse>::withDeleteFiles(java.util.List)" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.removed" + old: "method boolean org.apache.iceberg.util.TableScanUtil::hasDeletes(org.apache.iceberg.CombinedScanTask)" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.removed" + old: "method boolean org.apache.iceberg.util.TableScanUtil::hasEqDeletes(org.apache.iceberg.CombinedScanTask)" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.removed" + old: "method java.lang.String org.apache.iceberg.rest.RESTUtil::encodeNamespace(org.apache.iceberg.catalog.Namespace)" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.removed" + old: "method java.nio.ByteBuffer org.apache.iceberg.encryption.StandardEncryptionManager::unwrapKey(java.nio.ByteBuffer)" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.removed" + old: "method java.nio.ByteBuffer org.apache.iceberg.encryption.StandardEncryptionManager::wrapKey(java.nio.ByteBuffer)" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.removed" + old: "method java.util.List org.apache.iceberg.util.SnapshotUtil::newFiles(java.lang.Long,\ + \ long, java.util.function.Function,\ + \ org.apache.iceberg.io.FileIO)" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.removed" + old: "method java.util.concurrent.ExecutorService org.apache.iceberg.util.ThreadPools::newWorkerPool(java.lang.String)" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.removed" + old: "method java.util.concurrent.ExecutorService org.apache.iceberg.util.ThreadPools::newWorkerPool(java.lang.String,\ + \ int)" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.removed" + old: "method org.apache.avro.Schema org.apache.iceberg.avro.AvroSchemaUtil::pruneColumns(org.apache.avro.Schema,\ + \ java.util.Set, org.apache.iceberg.mapping.NameMapping)" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.ManifestReader\ + \ org.apache.iceberg.ManifestFiles::read(org.apache.iceberg.ManifestFile,\ + \ org.apache.iceberg.io.FileIO)" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.MetricsConfig org.apache.iceberg.MetricsConfig::forPositionDelete(org.apache.iceberg.Table)" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.catalog.Namespace org.apache.iceberg.rest.RESTUtil::decodeNamespace(java.lang.String)" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.deletes.PositionDeleteWriter\ + \ org.apache.iceberg.RewriteTablePathUtil.PositionDeleteReaderWriter::writer(org.apache.iceberg.io.OutputFile,\ + \ org.apache.iceberg.FileFormat, org.apache.iceberg.PartitionSpec, org.apache.iceberg.StructLike,\ + \ org.apache.iceberg.Schema) throws java.io.IOException" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.io.CloseableIterable org.apache.iceberg.ManifestFiles::readPaths(org.apache.iceberg.ManifestFile,\ + \ org.apache.iceberg.io.FileIO)" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.io.CloseableIterable\ + \ org.apache.iceberg.util.SnapshotUtil::newFilesBetween(java.lang.Long, long,\ + \ java.util.function.Function,\ + \ org.apache.iceberg.io.FileIO)" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.io.FileIO org.apache.iceberg.BaseScan>::io() @ org.apache.iceberg.AllDataFilesTable.AllDataFilesTableScan" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.io.FileIO org.apache.iceberg.BaseScan>::io() @ org.apache.iceberg.AllDeleteFilesTable.AllDeleteFilesTableScan" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.io.FileIO org.apache.iceberg.BaseScan>::io() @ org.apache.iceberg.AllFilesTable.AllFilesTableScan" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.io.FileIO org.apache.iceberg.BaseScan>::io() @ org.apache.iceberg.AllManifestsTable.AllManifestsTableScan" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.io.FileIO org.apache.iceberg.BaseScan>::io() @ org.apache.iceberg.DataFilesTable.DataFilesTableScan" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.io.FileIO org.apache.iceberg.BaseScan>::io() @ org.apache.iceberg.DataTableScan" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.io.FileIO org.apache.iceberg.BaseScan>::io() @ org.apache.iceberg.DeleteFilesTable.DeleteFilesTableScan" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.io.FileIO org.apache.iceberg.BaseScan>::io() @ org.apache.iceberg.FilesTable.FilesTableScan" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.io.FileIO org.apache.iceberg.BaseScan>::io() @ org.apache.iceberg.PositionDeletesTable.PositionDeletesBatchScan" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.io.FileIO org.apache.iceberg.BaseScan>::io() @ org.apache.iceberg.SnapshotScan>" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.rest.responses.LoadTableResponse org.apache.iceberg.rest.CatalogHandlers::loadTable(org.apache.iceberg.catalog.Catalog,\ + \ org.apache.iceberg.catalog.TableIdentifier)" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.removed" + old: "method void org.apache.iceberg.data.avro.RawDecoder::(org.apache.iceberg.Schema,\ + \ java.util.function.Function>,\ + \ org.apache.avro.Schema)" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.removed" + old: "method void org.apache.iceberg.encryption.StandardEncryptionManager::(java.lang.String,\ + \ int, org.apache.iceberg.encryption.KeyManagementClient)" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.removed" + old: "method void org.apache.iceberg.hadoop.HadoopFileIO::(org.apache.iceberg.util.SerializableSupplier)" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.removed" + old: "method void org.apache.iceberg.io.ContentCache::invalidateAll()" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + org.apache.iceberg:iceberg-data: + - code: "java.class.removed" + old: "class org.apache.iceberg.data.BaseFileWriterFactory" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.class.removed" + old: "class org.apache.iceberg.data.GenericAppenderFactory" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.removed" + old: "method void org.apache.iceberg.data.GenericFileWriterFactory::configureDataWrite(org.apache.iceberg.avro.Avro.DataWriteBuilder)" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.removed" + old: "method void org.apache.iceberg.data.GenericFileWriterFactory::configureDataWrite(org.apache.iceberg.orc.ORC.DataWriteBuilder)" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.removed" + old: "method void org.apache.iceberg.data.GenericFileWriterFactory::configureDataWrite(org.apache.iceberg.parquet.Parquet.DataWriteBuilder)" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.removed" + old: "method void org.apache.iceberg.data.GenericFileWriterFactory::configureEqualityDelete(org.apache.iceberg.avro.Avro.DeleteWriteBuilder)" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.removed" + old: "method void org.apache.iceberg.data.GenericFileWriterFactory::configureEqualityDelete(org.apache.iceberg.orc.ORC.DeleteWriteBuilder)" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.removed" + old: "method void org.apache.iceberg.data.GenericFileWriterFactory::configureEqualityDelete(org.apache.iceberg.parquet.Parquet.DeleteWriteBuilder)" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.removed" + old: "method void org.apache.iceberg.data.GenericFileWriterFactory::configurePositionDelete(org.apache.iceberg.avro.Avro.DeleteWriteBuilder)" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.removed" + old: "method void org.apache.iceberg.data.GenericFileWriterFactory::configurePositionDelete(org.apache.iceberg.orc.ORC.DeleteWriteBuilder)" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.removed" + old: "method void org.apache.iceberg.data.GenericFileWriterFactory::configurePositionDelete(org.apache.iceberg.parquet.Parquet.DeleteWriteBuilder)" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + org.apache.iceberg:iceberg-parquet: + - code: "java.method.removed" + old: "method org.apache.iceberg.parquet.ParquetValueReader org.apache.iceberg.data.parquet.BaseParquetReaders::createStructReader(java.util.List>,\ + \ org.apache.iceberg.types.Types.StructType) @ org.apache.iceberg.data.parquet.GenericParquetReaders" + justification: "Removing deprecated API scheduled for removal in 1.12.0" + - code: "java.method.removed" + old: "method org.apache.iceberg.parquet.ParquetValueReader org.apache.iceberg.data.parquet.BaseParquetReaders::createStructReader(java.util.List>,\ + \ org.apache.iceberg.types.Types.StructType) @ org.apache.iceberg.data.parquet.InternalReader" + justification: "Removing deprecated API scheduled for removal in 1.12.0" "1.2.0": org.apache.iceberg:iceberg-api: - code: "java.field.constantValueChanged" diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3ObjectMapper.java b/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3ObjectMapper.java deleted file mode 100644 index 7f1d6c3cc848..000000000000 --- a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3ObjectMapper.java +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iceberg.aws.s3.signer; - -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.PropertyAccessor; -import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.JsonDeserializer; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.JsonSerializer; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.PropertyNamingStrategies; -import com.fasterxml.jackson.databind.SerializerProvider; -import com.fasterxml.jackson.databind.module.SimpleModule; -import java.io.IOException; -import org.apache.iceberg.rest.RESTSerializers.ErrorResponseDeserializer; -import org.apache.iceberg.rest.RESTSerializers.ErrorResponseSerializer; -import org.apache.iceberg.rest.RESTSerializers.OAuthTokenResponseDeserializer; -import org.apache.iceberg.rest.RESTSerializers.OAuthTokenResponseSerializer; -import org.apache.iceberg.rest.responses.ErrorResponse; -import org.apache.iceberg.rest.responses.OAuthTokenResponse; - -/** - * @deprecated since 1.11.0, will be removed in 1.12.0; use {@code RESTObjectMapper} instead. - */ -@Deprecated -public class S3ObjectMapper { - - private static final JsonFactory FACTORY = new JsonFactory(); - private static final ObjectMapper MAPPER = new ObjectMapper(FACTORY); - private static volatile boolean isInitialized = false; - - private S3ObjectMapper() {} - - static ObjectMapper mapper() { - if (!isInitialized) { - synchronized (S3ObjectMapper.class) { - if (!isInitialized) { - MAPPER.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY); - MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - // even though using new PropertyNamingStrategy.KebabCaseStrategy() is deprecated - // and PropertyNamingStrategies.KebabCaseStrategy.INSTANCE (introduced in jackson 2.14) is - // recommended, we can't use it because Spark still relies on jackson 2.13.x stuff - MAPPER.setPropertyNamingStrategy(new PropertyNamingStrategies.KebabCaseStrategy()); - MAPPER.registerModule(initModule()); - isInitialized = true; - } - } - } - - return MAPPER; - } - - public static SimpleModule initModule() { - return new SimpleModule() - .addSerializer(ErrorResponse.class, new ErrorResponseSerializer()) - .addDeserializer(ErrorResponse.class, new ErrorResponseDeserializer()) - .addSerializer(OAuthTokenResponse.class, new OAuthTokenResponseSerializer()) - .addDeserializer(OAuthTokenResponse.class, new OAuthTokenResponseDeserializer()) - .addSerializer(S3SignRequest.class, new S3SignRequestSerializer<>()) - .addSerializer(ImmutableS3SignRequest.class, new S3SignRequestSerializer<>()) - .addDeserializer(S3SignRequest.class, new S3SignRequestDeserializer<>()) - .addDeserializer(ImmutableS3SignRequest.class, new S3SignRequestDeserializer<>()) - .addSerializer(S3SignResponse.class, new S3SignResponseSerializer<>()) - .addSerializer(ImmutableS3SignResponse.class, new S3SignResponseSerializer<>()) - .addDeserializer(S3SignResponse.class, new S3SignResponseDeserializer<>()) - .addDeserializer(ImmutableS3SignResponse.class, new S3SignResponseDeserializer<>()); - } - - public static class S3SignRequestSerializer extends JsonSerializer { - @Override - public void serialize(T request, JsonGenerator gen, SerializerProvider serializers) - throws IOException { - S3SignRequestParser.toJson(request, gen); - } - } - - public static class S3SignRequestDeserializer - extends JsonDeserializer { - @Override - public T deserialize(JsonParser p, DeserializationContext context) throws IOException { - JsonNode jsonNode = p.getCodec().readTree(p); - return (T) S3SignRequestParser.fromJson(jsonNode); - } - } - - public static class S3SignResponseSerializer extends JsonSerializer { - @Override - public void serialize(T request, JsonGenerator gen, SerializerProvider serializers) - throws IOException { - S3SignResponseParser.toJson(request, gen); - } - } - - public static class S3SignResponseDeserializer - extends JsonDeserializer { - @Override - public T deserialize(JsonParser p, DeserializationContext context) throws IOException { - JsonNode jsonNode = p.getCodec().readTree(p); - return (T) S3SignResponseParser.fromJson(jsonNode); - } - } -} diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignRequest.java b/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignRequest.java deleted file mode 100644 index 995f6e7e4860..000000000000 --- a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignRequest.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iceberg.aws.s3.signer; - -import org.apache.iceberg.rest.requests.RemoteSignRequest; -import org.immutables.value.Value; - -/** - * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link RemoteSignRequest} instead. - */ -@Deprecated -@Value.Immutable -@SuppressWarnings("immutables:subtype") -public interface S3SignRequest extends RemoteSignRequest {} diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignRequestParser.java b/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignRequestParser.java deleted file mode 100644 index 5d2a7d684460..000000000000 --- a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignRequestParser.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iceberg.aws.s3.signer; - -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.JsonNode; -import java.io.IOException; -import java.util.List; -import java.util.Map; -import org.apache.iceberg.rest.requests.RemoteSignRequest; -import org.apache.iceberg.rest.requests.RemoteSignRequestParser; - -/** - * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link RemoteSignRequestParser} instead. - */ -@Deprecated -public class S3SignRequestParser { - - private S3SignRequestParser() {} - - public static String toJson(S3SignRequest request) { - return RemoteSignRequestParser.toJson(request, false); - } - - public static String toJson(S3SignRequest request, boolean pretty) { - return RemoteSignRequestParser.toJson(request, pretty); - } - - public static void toJson(S3SignRequest request, JsonGenerator gen) throws IOException { - RemoteSignRequestParser.toJson(request, gen); - } - - public static S3SignRequest fromJson(String json) { - RemoteSignRequest request = RemoteSignRequestParser.fromJson(json); - return ImmutableS3SignRequest.builder().from(request).build(); - } - - public static S3SignRequest fromJson(JsonNode json) { - RemoteSignRequest request = RemoteSignRequestParser.fromJson(json); - return ImmutableS3SignRequest.builder().from(request).build(); - } - - static void headersToJson(String property, Map> headers, JsonGenerator gen) - throws IOException { - RemoteSignRequestParser.headersToJson(property, headers, gen); - } - - static Map> headersFromJson(String property, JsonNode json) { - return RemoteSignRequestParser.headersFromJson(property, json); - } -} diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignResponse.java b/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignResponse.java deleted file mode 100644 index 6fbaa90fe7af..000000000000 --- a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignResponse.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iceberg.aws.s3.signer; - -import org.apache.iceberg.rest.responses.RemoteSignResponse; -import org.immutables.value.Value; - -/** - * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link RemoteSignResponse} instead. - */ -@Deprecated -@Value.Immutable -@SuppressWarnings("immutables:subtype") -public interface S3SignResponse extends RemoteSignResponse {} diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignResponseParser.java b/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignResponseParser.java deleted file mode 100644 index be63a51b38fb..000000000000 --- a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3SignResponseParser.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iceberg.aws.s3.signer; - -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.JsonNode; -import java.io.IOException; -import org.apache.iceberg.rest.responses.RemoteSignResponse; -import org.apache.iceberg.rest.responses.RemoteSignResponseParser; - -/** - * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link RemoteSignResponseParser} - * instead. - */ -@Deprecated -public class S3SignResponseParser { - - private S3SignResponseParser() {} - - public static String toJson(S3SignResponse response) { - return RemoteSignResponseParser.toJson(response, false); - } - - public static String toJson(S3SignResponse response, boolean pretty) { - return RemoteSignResponseParser.toJson(response, pretty); - } - - public static void toJson(S3SignResponse response, JsonGenerator gen) throws IOException { - RemoteSignResponseParser.toJson(response, gen); - } - - public static S3SignResponse fromJson(String json) { - RemoteSignResponse result = RemoteSignResponseParser.fromJson(json); - return ImmutableS3SignResponse.builder().from(result).build(); - } - - public static S3SignResponse fromJson(JsonNode json) { - RemoteSignResponse result = RemoteSignResponseParser.fromJson(json); - return ImmutableS3SignResponse.builder().from(result).build(); - } -} diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3V4RestSignerClient.java b/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3V4RestSignerClient.java index 7a463abd3d2d..20e12406cb5e 100644 --- a/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3V4RestSignerClient.java +++ b/aws/src/main/java/org/apache/iceberg/aws/s3/signer/S3V4RestSignerClient.java @@ -71,23 +71,6 @@ public abstract class S3V4RestSignerClient public static final String S3_PROVIDER = "s3"; - /** - * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link - * RESTCatalogProperties#SIGNER_URI} instead. - */ - @Deprecated public static final String S3_SIGNER_URI = "s3.signer.uri"; - - /** - * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link - * RESTCatalogProperties#SIGNER_URI} instead. - */ - @Deprecated public static final String S3_SIGNER_ENDPOINT = "s3.signer.endpoint"; - - /** - * @deprecated since 1.11.0, will be removed in 1.12.0; there is no replacement. - */ - @Deprecated static final String S3_SIGNER_DEFAULT_ENDPOINT = "v1/aws/s3/sign"; - @VisibleForTesting static final String UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD"; private static final String CACHE_CONTROL = "Cache-Control"; @@ -115,27 +98,17 @@ public Supplier> requestPropertiesSupplier() { @Value.Lazy public String baseSignerUri() { - // TODO remove in 1.12.0 - if (properties().containsKey(S3_SIGNER_URI)) { - return properties().get(S3_SIGNER_URI); - } - return properties() .getOrDefault(RESTCatalogProperties.SIGNER_URI, properties().get(CatalogProperties.URI)); } @Value.Lazy public String endpoint() { - // TODO remove in 1.12.0 - String endpointPath; - if (properties().containsKey(S3_SIGNER_ENDPOINT)) { - endpointPath = properties().get(S3_SIGNER_ENDPOINT); - } else { - endpointPath = - properties() - .getOrDefault(RESTCatalogProperties.SIGNER_ENDPOINT, S3_SIGNER_DEFAULT_ENDPOINT); - } - + String endpointPath = properties().get(RESTCatalogProperties.SIGNER_ENDPOINT); + Preconditions.checkArgument( + endpointPath != null, + "S3 signer endpoint (%s) is required", + RESTCatalogProperties.SIGNER_ENDPOINT); return RESTUtil.resolveEndpoint(baseSignerUri(), endpointPath); } @@ -232,36 +205,13 @@ private boolean credentialProvided() { @Value.Check protected void check() { Preconditions.checkArgument( - properties().containsKey(S3_SIGNER_URI) - || properties().containsKey(RESTCatalogProperties.SIGNER_URI) + properties().containsKey(RESTCatalogProperties.SIGNER_URI) || properties().containsKey(CatalogProperties.URI), "S3 signer service URI is required"); - - if (properties().containsKey(S3_SIGNER_URI) - && !properties().containsKey(RESTCatalogProperties.SIGNER_URI)) { - LOG.warn( - "S3 signer URI is configured via deprecated property {}, this won't be supported in future releases. " - + "Please use {} instead.", - S3_SIGNER_URI, - RESTCatalogProperties.SIGNER_URI); - } - - if (properties().containsKey(S3_SIGNER_ENDPOINT) - && !properties().containsKey(RESTCatalogProperties.SIGNER_ENDPOINT)) { - LOG.warn( - "Signer endpoint is configured via deprecated property {}, this won't be supported in future releases. " - + "Please use {} instead.", - S3_SIGNER_ENDPOINT, - RESTCatalogProperties.SIGNER_ENDPOINT); - } - - // TODO change to required in 1.12.0 - if (!properties().containsKey(S3_SIGNER_ENDPOINT) - && !properties().containsKey(RESTCatalogProperties.SIGNER_ENDPOINT)) { - LOG.warn( - "Signer endpoint is not set, this won't be supported in future releases. Using deprecated default: {}", - S3_SIGNER_DEFAULT_ENDPOINT); - } + Preconditions.checkArgument( + properties().containsKey(RESTCatalogProperties.SIGNER_ENDPOINT), + "S3 signer endpoint (%s) is required", + RESTCatalogProperties.SIGNER_ENDPOINT); } @Override diff --git a/aws/src/test/java/org/apache/iceberg/aws/TestS3FileIOProperties.java b/aws/src/test/java/org/apache/iceberg/aws/TestS3FileIOProperties.java index f374a18c0411..e0932e216d94 100644 --- a/aws/src/test/java/org/apache/iceberg/aws/TestS3FileIOProperties.java +++ b/aws/src/test/java/org/apache/iceberg/aws/TestS3FileIOProperties.java @@ -28,6 +28,7 @@ import org.apache.iceberg.aws.s3.signer.S3V4RestSignerClient; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.rest.RESTCatalogProperties; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; import org.mockito.Mockito; @@ -225,7 +226,12 @@ public void testS3RemoteSigningEnabled() { String uri = "http://localhost:12345"; Map properties = ImmutableMap.of( - S3FileIOProperties.REMOTE_SIGNING_ENABLED, "true", CatalogProperties.URI, uri); + S3FileIOProperties.REMOTE_SIGNING_ENABLED, + "true", + CatalogProperties.URI, + uri, + RESTCatalogProperties.SIGNER_ENDPOINT, + "v1/aws/s3/sign"); S3FileIOProperties s3Properties = new S3FileIOProperties(properties); S3ClientBuilder builder = S3Client.builder(); @@ -244,7 +250,12 @@ public void s3RemoteSigningEnabledWithUserAgentAndRetryPolicy() { String uri = "http://localhost:12345"; Map properties = ImmutableMap.of( - S3FileIOProperties.REMOTE_SIGNING_ENABLED, "true", CatalogProperties.URI, uri); + S3FileIOProperties.REMOTE_SIGNING_ENABLED, + "true", + CatalogProperties.URI, + uri, + RESTCatalogProperties.SIGNER_ENDPOINT, + "v1/aws/s3/sign"); S3FileIOProperties s3Properties = new S3FileIOProperties(properties); S3ClientBuilder builder = S3Client.builder(); diff --git a/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3FileIOProperties.java b/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3FileIOProperties.java index 953f73d45d4a..5318ac6c2e03 100644 --- a/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3FileIOProperties.java +++ b/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3FileIOProperties.java @@ -32,6 +32,7 @@ import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.relocated.com.google.common.collect.Sets; +import org.apache.iceberg.rest.RESTCatalogProperties; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; import org.mockito.Mockito; @@ -504,7 +505,9 @@ public void testApplySignerConfiguration() { S3FileIOProperties.REMOTE_SIGNING_ENABLED, "true", CatalogProperties.URI, - "http://localhost:12345"); + "http://localhost:12345", + RESTCatalogProperties.SIGNER_ENDPOINT, + "v1/aws/s3/sign"); S3FileIOProperties s3FileIOProperties = new S3FileIOProperties(properties); S3ClientBuilder mockS3ClientBuilder = Mockito.mock(S3ClientBuilder.class); s3FileIOProperties.applySignerConfiguration(mockS3ClientBuilder); diff --git a/aws/src/test/java/org/apache/iceberg/aws/s3/signer/TestS3V4RestSignerClient.java b/aws/src/test/java/org/apache/iceberg/aws/s3/signer/TestS3V4RestSignerClient.java index aadbf036b567..6ffc99e85a86 100644 --- a/aws/src/test/java/org/apache/iceberg/aws/s3/signer/TestS3V4RestSignerClient.java +++ b/aws/src/test/java/org/apache/iceberg/aws/s3/signer/TestS3V4RestSignerClient.java @@ -198,21 +198,9 @@ void legacySignerProperties( } } - @SuppressWarnings("deprecation") public static Stream legacySignerProperties() { return Stream.of( - // Only legacy properties - Arguments.of( - Map.of( - CatalogProperties.URI, - "https://catalog.com", - S3V4RestSignerClient.S3_SIGNER_URI, - "https://legacy-signer.com", - S3V4RestSignerClient.S3_SIGNER_ENDPOINT, - "v1/legacy/sign"), - "https://legacy-signer.com", - "https://legacy-signer.com/v1/legacy/sign"), - // Only new properties + // Signer URI + endpoint Arguments.of( Map.of( CatalogProperties.URI, @@ -223,25 +211,14 @@ public static Stream legacySignerProperties() { "v1/new/sign"), "https://new-signer.com", "https://new-signer.com/v1/new/sign"), - // Mixed properties: legacy properties take precedence + // No signer URI: the catalog URI is used as base Arguments.of( Map.of( CatalogProperties.URI, "https://catalog.com", - RESTCatalogProperties.SIGNER_URI, - "https://new-signer.com", RESTCatalogProperties.SIGNER_ENDPOINT, - "v1/new/sign", - S3V4RestSignerClient.S3_SIGNER_URI, - "https://legacy-signer.com", - S3V4RestSignerClient.S3_SIGNER_ENDPOINT, - "v1/legacy/sign"), - "https://legacy-signer.com", - "https://legacy-signer.com/v1/legacy/sign"), - // No signer properties: the catalog URI and the deprecated default endpoint are used - Arguments.of( - Map.of(CatalogProperties.URI, "https://catalog.com"), + "v1/tables/t/sign"), "https://catalog.com", - "https://catalog.com/" + S3V4RestSignerClient.S3_SIGNER_DEFAULT_ENDPOINT)); + "https://catalog.com/v1/tables/t/sign")); } } diff --git a/bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java b/bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java index dd01246cb01f..df81d5b8ac84 100644 --- a/bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java +++ b/bigquery/src/main/java/org/apache/iceberg/gcp/bigquery/BigQueryMetastoreCatalog.java @@ -57,24 +57,6 @@ public class BigQueryMetastoreCatalog extends BaseMetastoreCatalog implements SupportsNamespaces, Configurable { - /** - * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link BigQueryProperties#PROJECT_ID} - * instead. - */ - @Deprecated public static final String PROJECT_ID = "gcp.bigquery.project-id"; - - /** - * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link - * BigQueryProperties#GCP_LOCATION} instead. - */ - @Deprecated public static final String GCP_LOCATION = "gcp.bigquery.location"; - - /** - * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link - * BigQueryProperties#LIST_ALL_TABLES} instead. - */ - @Deprecated public static final String LIST_ALL_TABLES = "gcp.bigquery.list-all-tables"; - private static final Logger LOG = LoggerFactory.getLogger(BigQueryMetastoreCatalog.class); private String catalogName; diff --git a/core/src/main/java/org/apache/iceberg/BaseDistributedDataScan.java b/core/src/main/java/org/apache/iceberg/BaseDistributedDataScan.java index e951ae830737..36f3e4d867d3 100644 --- a/core/src/main/java/org/apache/iceberg/BaseDistributedDataScan.java +++ b/core/src/main/java/org/apache/iceberg/BaseDistributedDataScan.java @@ -194,7 +194,7 @@ public CloseableIterable> planTasks() { } private List findMatchingDataManifests(Snapshot snapshot) { - List dataManifests = snapshot.dataManifests(io()); + List dataManifests = snapshot.dataManifests(table().io()); scanMetrics().totalDataManifests().increment(dataManifests.size()); List matchingDataManifests = filterManifests(dataManifests); @@ -205,7 +205,7 @@ private List findMatchingDataManifests(Snapshot snapshot) { } private List findMatchingDeleteManifests(Snapshot snapshot) { - List deleteManifests = snapshot.deleteManifests(io()); + List deleteManifests = snapshot.deleteManifests(table().io()); scanMetrics().totalDeleteManifests().increment(deleteManifests.size()); List matchingDeleteManifests = filterManifests(deleteManifests); @@ -293,7 +293,7 @@ private CompletableFuture newDeletesFuture( } private DeleteFileIndex planDeletesLocally(List deleteManifests) { - DeleteFileIndex.Builder builder = DeleteFileIndex.builderFor(io(), deleteManifests); + DeleteFileIndex.Builder builder = DeleteFileIndex.builderFor(table().io(), deleteManifests); if (shouldPlanWithExecutor() && deleteManifests.size() > 1) { builder.planWith(planExecutor()); diff --git a/core/src/main/java/org/apache/iceberg/BaseScan.java b/core/src/main/java/org/apache/iceberg/BaseScan.java index 242a5aaacc09..3c5692c50d2b 100644 --- a/core/src/main/java/org/apache/iceberg/BaseScan.java +++ b/core/src/main/java/org/apache/iceberg/BaseScan.java @@ -103,14 +103,6 @@ public Table table() { return table; } - /** - * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link BaseScan#fileIO()} instead. - */ - @Deprecated - protected FileIO io() { - return table.io(); - } - @Override public Supplier fileIO() { return table::io; diff --git a/core/src/main/java/org/apache/iceberg/DataScan.java b/core/src/main/java/org/apache/iceberg/DataScan.java index 336824e4cc91..7921591abf95 100644 --- a/core/src/main/java/org/apache/iceberg/DataScan.java +++ b/core/src/main/java/org/apache/iceberg/DataScan.java @@ -49,7 +49,7 @@ protected ManifestGroup newManifestGroup( boolean withColumnStats) { ManifestGroup manifestGroup = - new ManifestGroup(io(), dataManifests, deleteManifests) + new ManifestGroup(table().io(), dataManifests, deleteManifests) .caseSensitive(isCaseSensitive()) .select(withColumnStats ? SCAN_WITH_STATS_COLUMNS : SCAN_COLUMNS) .filterData(filter()) diff --git a/core/src/main/java/org/apache/iceberg/ManifestFiles.java b/core/src/main/java/org/apache/iceberg/ManifestFiles.java index 5ac55f0cf41f..008f60caac62 100644 --- a/core/src/main/java/org/apache/iceberg/ManifestFiles.java +++ b/core/src/main/java/org/apache/iceberg/ManifestFiles.java @@ -108,39 +108,6 @@ public static CloseableIterable readPaths( entry -> entry.file().location()); } - /** - * Returns a {@link CloseableIterable} of file paths in the {@link ManifestFile}. - * - * @param manifest a ManifestFile - * @param io a FileIO - * @return a manifest reader - * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link #readPaths(ManifestFile, - * FileIO, Map)} instead. - */ - @Deprecated - public static CloseableIterable readPaths(ManifestFile manifest, FileIO io) { - return readPaths(manifest, io, null); - } - - /** - * Returns a new {@link ManifestReader} for a {@link ManifestFile}. - * - *

Note: Callers should use {@link ManifestFiles#read(ManifestFile, FileIO, Map)} to - * ensure the schema used by filters is the latest table schema. This should be used only when - * reading a manifest without filters. - * - * @param manifest a ManifestFile - * @param io a FileIO - * @return a manifest reader - * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link #read(ManifestFile, FileIO, - * Map)} instead. Reading partition specs from manifest file metadata will not be supported - * for non-Avro manifest formats. - */ - @Deprecated - public static ManifestReader read(ManifestFile manifest, FileIO io) { - return read(manifest, io, null); - } - /** * Returns a new {@link ManifestReader} for a {@link ManifestFile}. * @@ -437,15 +404,6 @@ public static ManifestFile decode(byte[] manifestData) throws IOException { return AvroEncoderUtil.decode(manifestData); } - /** - * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link #open(ManifestFile, FileIO, - * Map)} instead. - */ - @Deprecated - static ManifestReader open(ManifestFile manifest, FileIO io) { - return open(manifest, io, null); - } - static ManifestReader open( ManifestFile manifest, FileIO io, Map specsById) { switch (manifest.content()) { diff --git a/core/src/main/java/org/apache/iceberg/MetricsConfig.java b/core/src/main/java/org/apache/iceberg/MetricsConfig.java index 2b55bcbeab22..0de04596c29a 100644 --- a/core/src/main/java/org/apache/iceberg/MetricsConfig.java +++ b/core/src/main/java/org/apache/iceberg/MetricsConfig.java @@ -32,7 +32,6 @@ import javax.annotation.concurrent.Immutable; import org.apache.iceberg.MetricsModes.MetricsMode; import org.apache.iceberg.exceptions.ValidationException; -import org.apache.iceberg.relocated.com.google.common.base.Joiner; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.relocated.com.google.common.collect.Sets; @@ -49,7 +48,6 @@ public final class MetricsConfig implements Serializable { private static final Logger LOG = LoggerFactory.getLogger(MetricsConfig.class); - private static final Joiner DOT = Joiner.on('.'); // Disable metrics by default for wide tables to prevent excessive metadata private static final MetricsMode DEFAULT_MODE = @@ -100,34 +98,6 @@ public static MetricsConfig forTable(Table table) { return from(table.properties(), table.schema(), table.sortOrder()); } - /** - * Creates a metrics config for a position delete file. - * - * @param table an Iceberg table - * @deprecated This method is deprecated as of version 1.11.0 and will be removed in 1.12.0. - * Position deletes that include row data are no longer supported. Use {@link - * #forPositionDelete()} instead. - */ - @Deprecated - public static MetricsConfig forPositionDelete(Table table) { - ImmutableMap.Builder columnModes = ImmutableMap.builder(); - - columnModes.put(MetadataColumns.DELETE_FILE_PATH.name(), MetricsModes.Full.get()); - columnModes.put(MetadataColumns.DELETE_FILE_POS.name(), MetricsModes.Full.get()); - - MetricsConfig tableConfig = forTable(table); - - MetricsMode defaultMode = tableConfig.defaultMode; - tableConfig.columnModes.forEach( - (columnAlias, mode) -> { - String positionDeleteColumnAlias = - DOT.join(MetadataColumns.DELETE_FILE_ROW_FIELD_NAME, columnAlias); - columnModes.put(positionDeleteColumnAlias, mode); - }); - - return new MetricsConfig(columnModes.build(), defaultMode); - } - static Set limitFieldIds(Schema schema, int limit) { return TypeUtil.visit( schema, diff --git a/core/src/main/java/org/apache/iceberg/SystemConfigs.java b/core/src/main/java/org/apache/iceberg/SystemConfigs.java index be59424992ee..66b122003c3b 100644 --- a/core/src/main/java/org/apache/iceberg/SystemConfigs.java +++ b/core/src/main/java/org/apache/iceberg/SystemConfigs.java @@ -80,21 +80,6 @@ private SystemConfigs() {} 8, Integer::parseUnsignedInt); - /** - * @deprecated will be removed in 1.12.0; use name mapping instead - */ - @Deprecated - public static final ConfigEntry NETFLIX_UNSAFE_PARQUET_ID_FALLBACK_ENABLED = - new ConfigEntry<>( - "iceberg.netflix.unsafe-parquet-id-fallback.enabled", - "ICEBERG_NETFLIX_UNSAFE_PARQUET_ID_FALLBACK_ENABLED", - true, - s -> { - LOG.warn( - "Fallback ID assignment in Parquet is UNSAFE and will be removed in 1.12.0. Use name mapping instead."); - return Boolean.parseBoolean(s); - }); - public static class ConfigEntry { private final String propertyKey; private final String envKey; diff --git a/core/src/main/java/org/apache/iceberg/SystemProperties.java b/core/src/main/java/org/apache/iceberg/SystemProperties.java deleted file mode 100644 index 484879bb21b1..000000000000 --- a/core/src/main/java/org/apache/iceberg/SystemProperties.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iceberg; - -/** - * Configuration properties that are controlled by Java system properties. - * - * @deprecated Use {@link SystemConfigs} instead; will be removed in 1.12.0 - */ -@Deprecated -public class SystemProperties { - - private SystemProperties() {} - - /** - * Sets the size of the worker pool. The worker pool limits the number of tasks concurrently - * processing manifests in the base table implementation across all concurrent planning or commit - * operations. - */ - public static final String WORKER_THREAD_POOL_SIZE_PROP = "iceberg.worker.num-threads"; - - /** Whether to use the shared worker pool when planning table scans. */ - public static final String SCAN_THREAD_POOL_ENABLED = "iceberg.scan.plan-in-worker-pool"; - - /** - * Maximum number of distinct {@link org.apache.iceberg.io.FileIO} that is allowed to have - * associated {@link org.apache.iceberg.io.ContentCache} in memory at a time. - */ - public static final String IO_MANIFEST_CACHE_MAX_FILEIO = "iceberg.io.manifest.cache.fileio-max"; - - public static final int IO_MANIFEST_CACHE_MAX_FILEIO_DEFAULT = 8; - - static boolean getBoolean(String systemProperty, boolean defaultValue) { - String value = System.getProperty(systemProperty); - if (value != null) { - return Boolean.parseBoolean(value); - } - - return defaultValue; - } -} diff --git a/core/src/main/java/org/apache/iceberg/TableProperties.java b/core/src/main/java/org/apache/iceberg/TableProperties.java index 021ef95d9122..dbd9295a064e 100644 --- a/core/src/main/java/org/apache/iceberg/TableProperties.java +++ b/core/src/main/java/org/apache/iceberg/TableProperties.java @@ -309,16 +309,6 @@ private TableProperties() {} public static final String WRITE_PARTITION_SUMMARY_LIMIT = "write.summary.partition-limit"; public static final int WRITE_PARTITION_SUMMARY_LIMIT_DEFAULT = 0; - /** - * @deprecated will be removed in 1.12.0, writing manifest lists is always enabled - */ - @Deprecated public static final String MANIFEST_LISTS_ENABLED = "write.manifest-lists.enabled"; - - /** - * @deprecated will be removed in 1.12.0, writing manifest lists is always enabled - */ - @Deprecated public static final boolean MANIFEST_LISTS_ENABLED_DEFAULT = true; - public static final String METADATA_COMPRESSION = "write.metadata.compression-codec"; public static final String METADATA_COMPRESSION_DEFAULT = "none"; diff --git a/core/src/main/java/org/apache/iceberg/avro/AvroSchemaUtil.java b/core/src/main/java/org/apache/iceberg/avro/AvroSchemaUtil.java index c67a3089a6bf..e0e940622342 100644 --- a/core/src/main/java/org/apache/iceberg/avro/AvroSchemaUtil.java +++ b/core/src/main/java/org/apache/iceberg/avro/AvroSchemaUtil.java @@ -128,16 +128,6 @@ public static Schema pruneColumns(Schema schema, Set selectedIds) { return new PruneColumns(selectedIds, null).rootSchema(schema); } - /** - * @deprecated will be removed in 1.12.0; use applyNameMapping and pruneColumns(Schema, Set) - * instead. - */ - @Deprecated - public static Schema pruneColumns( - Schema schema, Set selectedIds, NameMapping nameMapping) { - return new PruneColumns(selectedIds, nameMapping).rootSchema(schema); - } - public static Schema buildAvroProjection( Schema schema, org.apache.iceberg.Schema expected, Map renames) { return AvroCustomOrderSchemaVisitor.visit(schema, new BuildAvroProjection(expected, renames)); diff --git a/core/src/main/java/org/apache/iceberg/data/avro/DataReader.java b/core/src/main/java/org/apache/iceberg/data/avro/DataReader.java deleted file mode 100644 index 5f813f8db576..000000000000 --- a/core/src/main/java/org/apache/iceberg/data/avro/DataReader.java +++ /dev/null @@ -1,193 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iceberg.data.avro; - -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.function.Supplier; -import org.apache.avro.LogicalType; -import org.apache.avro.LogicalTypes; -import org.apache.avro.Schema; -import org.apache.avro.io.DatumReader; -import org.apache.avro.io.Decoder; -import org.apache.iceberg.avro.AvroSchemaUtil; -import org.apache.iceberg.avro.AvroSchemaWithTypeVisitor; -import org.apache.iceberg.avro.SupportsRowPosition; -import org.apache.iceberg.avro.ValueReader; -import org.apache.iceberg.avro.ValueReaders; -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; -import org.apache.iceberg.types.Type; -import org.apache.iceberg.types.Types; - -/** - * @deprecated will be removed in 1.12.0; use {@link PlannedDataReader} instead. - */ -@Deprecated -public class DataReader implements DatumReader, SupportsRowPosition { - - public static DataReader create( - org.apache.iceberg.Schema expectedSchema, Schema readSchema) { - return create(expectedSchema, readSchema, ImmutableMap.of()); - } - - public static DataReader create( - org.apache.iceberg.Schema expectedSchema, Schema readSchema, Map idToConstant) { - return new DataReader<>(expectedSchema, readSchema, idToConstant); - } - - private final Schema readSchema; - private final ValueReader reader; - private Schema fileSchema = null; - - @SuppressWarnings("unchecked") - protected DataReader( - org.apache.iceberg.Schema expectedSchema, Schema readSchema, Map idToConstant) { - this.readSchema = readSchema; - this.reader = - (ValueReader) - AvroSchemaWithTypeVisitor.visit( - expectedSchema, readSchema, new ReadBuilder(idToConstant)); - } - - @Override - public void setSchema(Schema newFileSchema) { - this.fileSchema = Schema.applyAliases(newFileSchema, readSchema); - } - - @Override - public T read(T reuse, Decoder decoder) throws IOException { - return DecoderResolver.resolveAndRead(decoder, readSchema, fileSchema, reader, reuse); - } - - @Override - public void setRowPositionSupplier(Supplier posSupplier) { - if (reader instanceof SupportsRowPosition) { - ((SupportsRowPosition) reader).setRowPositionSupplier(posSupplier); - } - } - - protected ValueReader createStructReader( - Types.StructType struct, List> fields, Map idToConstant) { - return GenericReaders.struct(struct, fields, idToConstant); - } - - private class ReadBuilder extends AvroSchemaWithTypeVisitor> { - private final Map idToConstant; - - private ReadBuilder(Map idToConstant) { - this.idToConstant = idToConstant; - } - - @Override - public ValueReader record( - Types.StructType struct, Schema record, List names, List> fields) { - return createStructReader(struct, fields, idToConstant); - } - - @Override - public ValueReader union(Type ignored, Schema union, List> options) { - return ValueReaders.union(options); - } - - @Override - public ValueReader array( - Types.ListType ignored, Schema array, ValueReader elementReader) { - return ValueReaders.array(elementReader); - } - - @Override - public ValueReader map( - Types.MapType iMap, Schema map, ValueReader keyReader, ValueReader valueReader) { - return ValueReaders.arrayMap(keyReader, valueReader); - } - - @Override - public ValueReader map(Types.MapType ignored, Schema map, ValueReader valueReader) { - return ValueReaders.map(ValueReaders.strings(), valueReader); - } - - @Override - public ValueReader primitive(Type.PrimitiveType ignored, Schema primitive) { - LogicalType logicalType = primitive.getLogicalType(); - if (logicalType != null) { - switch (logicalType.getName()) { - case "date": - return GenericReaders.dates(); - - case "time-micros": - return GenericReaders.times(); - - case "timestamp-micros": - if (AvroSchemaUtil.isTimestamptz(primitive)) { - return GenericReaders.timestamptz(); - } - return GenericReaders.timestamps(); - - case "timestamp-nanos": - if (AvroSchemaUtil.isTimestamptz(primitive)) { - return GenericReaders.timestamptzNanos(); - } - return GenericReaders.timestampNanos(); - - case "timestamp-millis": - if (AvroSchemaUtil.isTimestamptz(primitive)) { - return GenericReaders.timestamptzMillis(); - } - return GenericReaders.timestampMillis(); - - case "decimal": - return ValueReaders.decimal( - ValueReaders.decimalBytesReader(primitive), - ((LogicalTypes.Decimal) logicalType).getScale()); - - case "uuid": - return ValueReaders.uuids(); - - default: - throw new IllegalArgumentException("Unknown logical type: " + logicalType); - } - } - - switch (primitive.getType()) { - case NULL: - return ValueReaders.nulls(); - case BOOLEAN: - return ValueReaders.booleans(); - case INT: - return ValueReaders.ints(); - case LONG: - return ValueReaders.longs(); - case FLOAT: - return ValueReaders.floats(); - case DOUBLE: - return ValueReaders.doubles(); - case STRING: - // might want to use a binary-backed container like Utf8 - return ValueReaders.strings(); - case FIXED: - return ValueReaders.fixed(primitive.getFixedSize()); - case BYTES: - return ValueReaders.byteBuffers(); - default: - throw new IllegalArgumentException("Unsupported type: " + primitive); - } - } - } -} diff --git a/core/src/main/java/org/apache/iceberg/data/avro/RawDecoder.java b/core/src/main/java/org/apache/iceberg/data/avro/RawDecoder.java index 9f3cdfdad5f4..eba06489d9a4 100644 --- a/core/src/main/java/org/apache/iceberg/data/avro/RawDecoder.java +++ b/core/src/main/java/org/apache/iceberg/data/avro/RawDecoder.java @@ -27,8 +27,6 @@ import org.apache.avro.io.DatumReader; import org.apache.avro.io.DecoderFactory; import org.apache.avro.message.MessageDecoder; -import org.apache.iceberg.avro.ProjectionDatumReader; -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; public class RawDecoder extends MessageDecoder.BaseDecoder { private static final ThreadLocal DECODER = new ThreadLocal<>(); @@ -56,26 +54,6 @@ public static RawDecoder create( private final DatumReader reader; - /** - * Creates a new {@link MessageDecoder} that constructs datum instances described by the {@link - * Schema readSchema}. - * - *

The {@code readSchema} is used for the expected schema and the {@code writeSchema} is the - * schema used to decode buffers. The {@code writeSchema} must be the schema that was used to - * encode all buffers decoded by this class. - * - * @deprecated will be removed in 1.12.0; use {@link #create(org.apache.iceberg.Schema, Function, - * Schema)} instead - */ - @Deprecated - public RawDecoder( - org.apache.iceberg.Schema readSchema, - Function> readerFunction, - Schema writeSchema) { - this.reader = new ProjectionDatumReader<>(readerFunction, readSchema, ImmutableMap.of(), null); - this.reader.setSchema(writeSchema); - } - /** * Creates a new {@link MessageDecoder} that constructs datum instances using the {@code reader}. */ diff --git a/core/src/main/java/org/apache/iceberg/encryption/StandardEncryptionManager.java b/core/src/main/java/org/apache/iceberg/encryption/StandardEncryptionManager.java index bb5126d23e3f..f1b19fa30489 100644 --- a/core/src/main/java/org/apache/iceberg/encryption/StandardEncryptionManager.java +++ b/core/src/main/java/org/apache/iceberg/encryption/StandardEncryptionManager.java @@ -53,15 +53,6 @@ public class StandardEncryptionManager implements EncryptionManager { private transient volatile LoadingCache unwrappedKeyCache; private transient volatile SecureRandom lazyRNG = null; - /** - * @deprecated will be removed in 1.12.0. - */ - @Deprecated - public StandardEncryptionManager( - String tableKeyId, int dataKeyLength, KeyManagementClient kmsClient) { - this(List.of(), tableKeyId, dataKeyLength, kmsClient); - } - /** * @param keys encryption keys from table metadata * @param tableKeyId table encryption key id @@ -137,22 +128,6 @@ private SecureRandom workerRNG() { return lazyRNG; } - /** - * @deprecated will be removed in 1.12.0. - */ - @Deprecated - public ByteBuffer wrapKey(ByteBuffer secretKey) { - return kmsClient.wrapKey(secretKey, tableKeyId); - } - - /** - * @deprecated will be removed in 1.12.0. - */ - @Deprecated - public ByteBuffer unwrapKey(ByteBuffer wrappedSecretKey) { - return kmsClient.unwrapKey(wrappedSecretKey, tableKeyId); - } - Map encryptionKeys() { return encryptionKeys; } diff --git a/core/src/main/java/org/apache/iceberg/hadoop/HadoopFileIO.java b/core/src/main/java/org/apache/iceberg/hadoop/HadoopFileIO.java index 877290f48e3f..fbb50abb85a7 100644 --- a/core/src/main/java/org/apache/iceberg/hadoop/HadoopFileIO.java +++ b/core/src/main/java/org/apache/iceberg/hadoop/HadoopFileIO.java @@ -66,16 +66,7 @@ public class HadoopFileIO implements HadoopConfigurable, DelegateFileIO { public HadoopFileIO() {} public HadoopFileIO(Configuration hadoopConf) { - this(new SerializableConfiguration(hadoopConf)); - } - - /** - * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link - * HadoopFileIO#HadoopFileIO(Configuration)} instead. - */ - @Deprecated - public HadoopFileIO(SerializableSupplier hadoopConf) { - this.hadoopConf = hadoopConf; + this.hadoopConf = new SerializableConfiguration(hadoopConf); } public Configuration conf() { @@ -138,10 +129,6 @@ public Configuration getConf() { return hadoopConf.get(); } - /** - * @deprecated since 1.11.0, will be removed in 1.12.0. - */ - @Deprecated @Override public void serializeConfWith( Function> confSerializer) { diff --git a/core/src/main/java/org/apache/iceberg/io/ContentCache.java b/core/src/main/java/org/apache/iceberg/io/ContentCache.java index 484306690c67..22bb8f1e455b 100644 --- a/core/src/main/java/org/apache/iceberg/io/ContentCache.java +++ b/core/src/main/java/org/apache/iceberg/io/ContentCache.java @@ -139,17 +139,6 @@ public void invalidate(String key) { cache.invalidate(key); } - /** - * @deprecated since 1.7.0, will be removed in 1.12.0; This method does only best-effort - * invalidation and is susceptible to a race condition. If the caller changed the state that - * could be cached (perhaps files on the storage) and calls this method, there is no guarantee - * that the cache will not contain stale entries some time after this method returns. - */ - @Deprecated - public void invalidateAll() { - cache.invalidateAll(); - } - public void cleanUp() { cache.cleanUp(); } diff --git a/core/src/main/java/org/apache/iceberg/rest/CatalogHandlers.java b/core/src/main/java/org/apache/iceberg/rest/CatalogHandlers.java index 3a1e62260aae..226ea67d6a21 100644 --- a/core/src/main/java/org/apache/iceberg/rest/CatalogHandlers.java +++ b/core/src/main/java/org/apache/iceberg/rest/CatalogHandlers.java @@ -502,15 +502,6 @@ public static void tableExists(Catalog catalog, TableIdentifier ident) { } } - /** - * @deprecated since 1.11.0, will be removed in 1.12.0. Use {@link #loadTable(Catalog, - * TableIdentifier, SnapshotMode)} instead. - */ - @Deprecated - public static LoadTableResponse loadTable(Catalog catalog, TableIdentifier ident) { - return loadTable(catalog, ident, SnapshotMode.ALL); - } - public static LoadTableResponse loadTable( Catalog catalog, TableIdentifier ident, SnapshotMode mode) { Table table = catalog.loadTable(ident); diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java b/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java index 9effb875e05f..57f93829e9bb 100644 --- a/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java +++ b/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java @@ -116,12 +116,6 @@ public class RESTSessionCatalog extends BaseViewSessionCatalog private static final Logger LOG = LoggerFactory.getLogger(RESTSessionCatalog.class); private static final String DEFAULT_FILE_IO_IMPL = "org.apache.iceberg.io.ResolvingFileIO"; - /** - * @deprecated will be removed in 1.12.0. Use {@link - * org.apache.iceberg.rest.RESTCatalogProperties#PAGE_SIZE} instead. - */ - @Deprecated public static final String REST_PAGE_SIZE = "rest-page-size"; - // these default endpoints must not be updated in order to maintain backwards compatibility with // legacy servers private static final Set DEFAULT_ENDPOINTS = diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTUtil.java b/core/src/main/java/org/apache/iceberg/rest/RESTUtil.java index f4fdf3af26e7..3686fae0b237 100644 --- a/core/src/main/java/org/apache/iceberg/rest/RESTUtil.java +++ b/core/src/main/java/org/apache/iceberg/rest/RESTUtil.java @@ -40,20 +40,6 @@ public class RESTUtil { /** The namespace separator as url encoded UTF-8 character */ static final String NAMESPACE_SEPARATOR_URLENCODED_UTF_8 = "%1F"; - /** - * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link - * RESTUtil#namespaceToQueryParam(Namespace)}} instead. - */ - @Deprecated - public static final Joiner NAMESPACE_JOINER = Joiner.on(NAMESPACE_SEPARATOR_AS_UNICODE); - - /** - * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link - * RESTUtil#namespaceFromQueryParam(String)} instead. - */ - @Deprecated - public static final Splitter NAMESPACE_SPLITTER = Splitter.on(NAMESPACE_SEPARATOR_AS_UNICODE); - public static final String IDEMPOTENCY_KEY_HEADER = "Idempotency-Key"; private RESTUtil() {} @@ -171,9 +157,8 @@ public static String decodeString(String encoded) { /** * This converts the given namespace to a string and separates each part in a multipart namespace - * using the unicode character '\u001f'. Note that this method is different from {@link - * RESTUtil#encodeNamespace(Namespace)}, which uses the UTF-8 escaped version of '\u001f', which - * is '0x1F'. + * using the unicode character '\u001f'. Note that this method uses the raw unicode separator, + * unlike the percent-encoded form used by the REST catalog path encoding. * *

{@link #namespaceFromQueryParam(String)} should be used to convert the namespace string back * to a {@link Namespace} instance. @@ -188,8 +173,8 @@ public static String namespaceToQueryParam(Namespace namespace) { /** * This converts the given namespace to a string and separates each part in a multipart namespace - * using the provided unicode separator. Note that this method is different from {@link - * RESTUtil#encodeNamespace(Namespace)}, which uses a UTF-8 escaped separator. + * using the provided unicode separator. Note that this method uses the raw unicode separator, + * unlike the percent-encoded form used by the REST catalog path encoding. * *

{@link #namespaceFromQueryParam(String, String)} should be used to convert the namespace * string back to a {@link Namespace} instance. @@ -253,24 +238,6 @@ public static Namespace namespaceFromQueryParam( return Namespace.of(splitter.splitToStream(namespace).toArray(String[]::new)); } - /** - * Returns a String representation of a namespace that is suitable for use in a URL / URI. - * - *

This function needs to be called when a namespace is used as a path variable (or query - * parameter etc.), to format the namespace per the spec. - * - *

{@link #decodeNamespace} should be used to parse the namespace from a URL parameter. - * - * @param ns namespace to encode - * @return UTF-8 encoded string representing the namespace, suitable for use as a URL parameter - * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link - * RESTUtil#encodeNamespace(Namespace, String)} instead. - */ - @Deprecated - public static String encodeNamespace(Namespace ns) { - return encodeNamespace(ns, NAMESPACE_SEPARATOR_URLENCODED_UTF_8); - } - /** * Returns a String representation of a namespace that is suitable for use in a URL / URI. * @@ -299,22 +266,6 @@ public static String encodeNamespace(Namespace namespace, String separator) { return Joiner.on(separator).join(encodedLevels); } - /** - * Takes in a string representation of a namespace as used for a URL parameter and returns the - * corresponding namespace. - * - *

See also {@link #encodeNamespace} for generating correctly formatted URLs. - * - * @param encodedNs a namespace to decode - * @return a namespace - * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link - * RESTUtil#decodeNamespace(String, String)} instead. - */ - @Deprecated - public static Namespace decodeNamespace(String encodedNs) { - return decodeNamespace(encodedNs, NAMESPACE_SEPARATOR_URLENCODED_UTF_8); - } - /** * Takes in a string representation of a namespace as used for a URL parameter and returns the * corresponding namespace. diff --git a/core/src/main/java/org/apache/iceberg/rest/responses/BaseScanTaskResponse.java b/core/src/main/java/org/apache/iceberg/rest/responses/BaseScanTaskResponse.java index e83cacc48459..3d42ffce4726 100644 --- a/core/src/main/java/org/apache/iceberg/rest/responses/BaseScanTaskResponse.java +++ b/core/src/main/java/org/apache/iceberg/rest/responses/BaseScanTaskResponse.java @@ -93,15 +93,6 @@ public B withFileScanTasks(List tasks) { return self(); } - /** - * @deprecated since 1.11.0, will be removed in 1.12.0. - */ - @Deprecated - public B withDeleteFiles(List deleteFilesList) { - this.deleteFiles = DeleteFileSet.of(deleteFilesList); - return self(); - } - /** * @deprecated since 1.11.0, visibility will be reduced in 1.12.0. */ diff --git a/core/src/main/java/org/apache/iceberg/util/SnapshotUtil.java b/core/src/main/java/org/apache/iceberg/util/SnapshotUtil.java index 370bbfed336e..e34cd34dd8b8 100644 --- a/core/src/main/java/org/apache/iceberg/util/SnapshotUtil.java +++ b/core/src/main/java/org/apache/iceberg/util/SnapshotUtil.java @@ -21,20 +21,14 @@ import java.util.Iterator; import java.util.List; import java.util.NoSuchElementException; -import java.util.Objects; import java.util.function.Function; import org.apache.iceberg.BaseMetadataTable; -import org.apache.iceberg.DataFile; import org.apache.iceberg.HistoryEntry; import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; -import org.apache.iceberg.SnapshotChanges; import org.apache.iceberg.SnapshotRef; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; -import org.apache.iceberg.exceptions.ValidationException; -import org.apache.iceberg.io.CloseableIterable; -import org.apache.iceberg.io.FileIO; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.Iterables; @@ -284,66 +278,6 @@ private static Iterable toIds(Iterable snapshots) { return Iterables.transform(snapshots, Snapshot::snapshotId); } - /** - * @deprecated will be removed in 1.12.0, use {@link SnapshotChanges} with {@link - * #ancestorsBetween(long, Long, Function)} instead. - */ - @Deprecated - public static List newFiles( - Long baseSnapshotId, long latestSnapshotId, Function lookup, FileIO io) { - List newFiles = Lists.newArrayList(); - Snapshot lastSnapshot = null; - for (Snapshot currentSnapshot : ancestorsOf(latestSnapshotId, lookup)) { - lastSnapshot = currentSnapshot; - if (Objects.equals(currentSnapshot.snapshotId(), baseSnapshotId)) { - return newFiles; - } - - Iterables.addAll(newFiles, currentSnapshot.addedDataFiles(io)); - } - - ValidationException.check( - Objects.equals(lastSnapshot.parentId(), baseSnapshotId), - "Cannot determine history between read snapshot %s and the last known ancestor %s", - baseSnapshotId, - lastSnapshot.snapshotId()); - - return newFiles; - } - - /** - * @deprecated will be removed in 1.12.0, use {@link SnapshotChanges} with {@link - * #ancestorsBetween(long, Long, Function)} instead. - */ - @Deprecated - public static CloseableIterable newFilesBetween( - Long startSnapshotId, long endSnapshotId, Function lookup, FileIO io) { - - List snapshots = Lists.newArrayList(); - Snapshot lastSnapshot = null; - for (Snapshot currentSnapshot : ancestorsOf(endSnapshotId, lookup)) { - lastSnapshot = currentSnapshot; - if (Objects.equals(currentSnapshot.snapshotId(), startSnapshotId)) { - break; - } - - snapshots.add(currentSnapshot); - } - - if (lastSnapshot != null) { - ValidationException.check( - Objects.equals(lastSnapshot.snapshotId(), startSnapshotId) - || Objects.equals(lastSnapshot.parentId(), startSnapshotId), - "Cannot determine history between read snapshot %s and the last known ancestor %s", - startSnapshotId, - lastSnapshot.snapshotId()); - } - - return new ParallelIterable<>( - Iterables.transform(snapshots, snapshot -> snapshot.addedDataFiles(io)), - ThreadPools.getWorkerPool()); - } - /** * Traverses the history of the table's current snapshot and finds the snapshot with the given * snapshot id as its parent. diff --git a/core/src/main/java/org/apache/iceberg/util/TableScanUtil.java b/core/src/main/java/org/apache/iceberg/util/TableScanUtil.java index 6291acbf9c73..f143a300b87c 100644 --- a/core/src/main/java/org/apache/iceberg/util/TableScanUtil.java +++ b/core/src/main/java/org/apache/iceberg/util/TableScanUtil.java @@ -25,7 +25,6 @@ import org.apache.iceberg.BaseCombinedScanTask; import org.apache.iceberg.BaseScanTaskGroup; import org.apache.iceberg.CombinedScanTask; -import org.apache.iceberg.FileContent; import org.apache.iceberg.FileScanTask; import org.apache.iceberg.MergeableScanTask; import org.apache.iceberg.PartitionData; @@ -51,30 +50,6 @@ public class TableScanUtil { private TableScanUtil() {} - /** - * @deprecated since 1.11.0 and will be removed in 1.12.0 - */ - @Deprecated - public static boolean hasDeletes(CombinedScanTask task) { - return task.files().stream().anyMatch(TableScanUtil::hasDeletes); - } - - /** - * This is temporarily introduced since we plan to support pos-delete vectorized read first, then - * get to the equality-delete support. We will remove this method once both are supported. - * - * @deprecated since 1.11.0 and will be removed in 1.12.0 - */ - @Deprecated - public static boolean hasEqDeletes(CombinedScanTask task) { - return task.files().stream() - .anyMatch( - t -> - t.deletes().stream() - .anyMatch( - deleteFile -> deleteFile.content().equals(FileContent.EQUALITY_DELETES))); - } - public static boolean hasDeletes(FileScanTask task) { return !task.deletes().isEmpty(); } diff --git a/core/src/main/java/org/apache/iceberg/util/ThreadPools.java b/core/src/main/java/org/apache/iceberg/util/ThreadPools.java index bb508295ecfa..7167d14e7d7e 100644 --- a/core/src/main/java/org/apache/iceberg/util/ThreadPools.java +++ b/core/src/main/java/org/apache/iceberg/util/ThreadPools.java @@ -101,48 +101,6 @@ private static class AuthRefreshPoolHolder { "auth-session-refresh", AUTH_REFRESH_THREAD_POOL_SIZE, Duration.ZERO); } - /** - * Creates a fixed-size thread pool that uses daemon threads. The pool is wrapped with {@link - * MoreExecutors#getExitingExecutorService(ThreadPoolExecutor)}, which registers a shutdown hook - * to ensure the pool terminates when the JVM exits. Important: Even if the pool is - * explicitly shut down using {@link ExecutorService#shutdown()}, the shutdown hook is not - * removed. This can lead to accumulation of shutdown hooks if this method is used repeatedly for - * short-lived thread pools. - * - *

For clarity and to avoid potential issues with shutdown hook accumulation, prefer using - * either {@link #newExitingWorkerPool(String, int)} or {@link #newFixedThreadPool(String, int)}, - * depending on the intended lifecycle of the thread pool. - * - * @deprecated will be removed in 1.12.0. Use {@link #newExitingWorkerPool(String, int)} for - * long-lived thread pools that require a shutdown hook, or {@link #newFixedThreadPool(String, - * int)} for short-lived thread pools where you manage the lifecycle. - */ - @Deprecated - public static ExecutorService newWorkerPool(String namePrefix) { - return newExitingWorkerPool(namePrefix, WORKER_THREAD_POOL_SIZE); - } - - /** - * Creates a fixed-size thread pool that uses daemon threads. The pool is wrapped with {@link - * MoreExecutors#getExitingExecutorService(ThreadPoolExecutor)}, which registers a shutdown hook - * to ensure the pool terminates when the JVM exits. Important: Even if the pool is - * explicitly shut down using {@link ExecutorService#shutdown()}, the shutdown hook is not - * removed. This can lead to accumulation of shutdown hooks if this method is used repeatedly for - * short-lived thread pools. - * - *

For clarity and to avoid potential issues with shutdown hook accumulation, prefer using - * either {@link #newExitingWorkerPool(String, int)} or {@link #newFixedThreadPool(String, int)}, - * depending on the intended lifecycle of the thread pool. - * - * @deprecated will be removed in 1.12.0. Use {@link #newExitingWorkerPool(String, int)} for - * long-lived thread pools that require a shutdown hook, or {@link #newFixedThreadPool(String, - * int)} for short-lived thread pools where you manage the lifecycle. - */ - @Deprecated - public static ExecutorService newWorkerPool(String namePrefix, int poolSize) { - return newExitingWorkerPool(namePrefix, poolSize); - } - /** * Creates a fixed-size thread pool that uses daemon threads and registers a shutdown hook to * ensure the pool terminates when the JVM exits. This is suitable for long-lived thread pools diff --git a/core/src/test/java/org/apache/iceberg/TestManifestReader.java b/core/src/test/java/org/apache/iceberg/TestManifestReader.java index de2b7fd859e6..75551bafce94 100644 --- a/core/src/test/java/org/apache/iceberg/TestManifestReader.java +++ b/core/src/test/java/org/apache/iceberg/TestManifestReader.java @@ -270,15 +270,14 @@ public void testDataFileSplitOffsetsNullWhenInvalid() throws IOException { } } - @SuppressWarnings("deprecation") @TestTemplate - public void testDeprecatedReadWithoutSpecsById() throws IOException { + public void testReadWithoutSpecsById() throws IOException { assumeThat(formatVersion) .as("Deprecated read without specsById requires Avro metadata; V4 uses Parquet") .isLessThan(TableMetadata.MIN_FORMAT_VERSION_PARQUET_MANIFESTS); ManifestFile manifest = writeManifest(1000L, manifestEntry(Status.EXISTING, 1000L, FILE_A)); - try (ManifestReader reader = ManifestFiles.read(manifest, FILE_IO)) { + try (ManifestReader reader = ManifestFiles.read(manifest, FILE_IO, null)) { ManifestEntry entry = Iterables.getOnlyElement(reader.entries()); assertThat(entry.status()).isEqualTo(Status.EXISTING); assertThat(entry.file().location()).isEqualTo(FILE_A.location()); diff --git a/core/src/test/java/org/apache/iceberg/data/avro/TestDataReader.java b/core/src/test/java/org/apache/iceberg/data/avro/TestDataReader.java deleted file mode 100644 index 966f01267d67..000000000000 --- a/core/src/test/java/org/apache/iceberg/data/avro/TestDataReader.java +++ /dev/null @@ -1,208 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iceberg.data.avro; - -import static org.apache.iceberg.avro.AvroSchemaUtil.ADJUST_TO_UTC_PROP; -import static org.assertj.core.api.Assertions.assertThat; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.time.LocalDateTime; -import java.time.OffsetDateTime; -import java.time.ZoneOffset; -import org.apache.avro.LogicalTypes; -import org.apache.avro.Schema; -import org.apache.avro.SchemaBuilder; -import org.apache.avro.generic.GenericData; -import org.apache.avro.generic.GenericDatumWriter; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.io.BinaryEncoder; -import org.apache.avro.io.DecoderFactory; -import org.apache.avro.io.EncoderFactory; -import org.apache.iceberg.data.Record; -import org.apache.iceberg.types.Types; -import org.apache.iceberg.util.DateTimeUtil; -import org.junit.jupiter.api.Test; - -class TestDataReader { - - @Test - public void timestampDataReader() throws IOException { - org.apache.iceberg.Schema icebergSchema = - new org.apache.iceberg.Schema( - Types.NestedField.required(1, "timestamp_nanos", Types.TimestampType.withoutZone()), - Types.NestedField.required(2, "timestamp_micros", Types.TimestampType.withoutZone()), - Types.NestedField.required(3, "timestamp_millis", Types.TimestampType.withoutZone())); - - Schema avroSchema = - SchemaBuilder.record("test_programmatic") - .fields() - .name("timestamp_nanos") - .type(LogicalTypes.timestampNanos().addToSchema(Schema.create(Schema.Type.LONG))) - .noDefault() - .name("timestamp_micros") - .type(LogicalTypes.timestampMicros().addToSchema(Schema.create(Schema.Type.LONG))) - .noDefault() - .name("timestamp_millis") - .type(LogicalTypes.timestampMillis().addToSchema(Schema.create(Schema.Type.LONG))) - .noDefault() - .endRecord(); - - avroSchema.getField("timestamp_nanos").addProp("field-id", 1); - avroSchema.getField("timestamp_micros").addProp("field-id", 2); - avroSchema.getField("timestamp_millis").addProp("field-id", 3); - - DataReader reader = DataReader.create(icebergSchema, avroSchema); - reader.setSchema(avroSchema); - - // post-epoch timestamps - GenericRecord avroRecord = new GenericData.Record(avroSchema); - LocalDateTime timestampNanos = LocalDateTime.of(2023, 10, 15, 14, 30, 45, 123456789); - LocalDateTime timestampMicros = LocalDateTime.of(2023, 10, 15, 14, 30, 45, 123456000); - LocalDateTime timestampMillis = LocalDateTime.of(2023, 10, 15, 14, 30, 45, 123000000); - - avroRecord.put("timestamp_nanos", DateTimeUtil.nanosFromTimestamp(timestampNanos)); - avroRecord.put("timestamp_micros", DateTimeUtil.microsFromTimestamp(timestampMicros)); - avroRecord.put("timestamp_millis", DateTimeUtil.millisFromTimestamp(timestampMillis)); - - Record result = readRecord(reader, avroSchema, avroRecord); - - assertThat(result.getField("timestamp_nanos")).isEqualTo(timestampNanos); - assertThat(result.getField("timestamp_micros")).isEqualTo(timestampMicros); - assertThat(result.getField("timestamp_millis")).isEqualTo(timestampMillis); - - // pre-epoch timestamps - GenericRecord preEpochRecord = new GenericData.Record(avroSchema); - LocalDateTime preEpochNanos = LocalDateTime.of(1969, 1, 1, 10, 11, 12, 123456789); - LocalDateTime preEpochMicros = LocalDateTime.of(1968, 1, 1, 10, 11, 12, 123456000); - LocalDateTime preEpochMillis = LocalDateTime.of(1967, 1, 1, 10, 11, 12, 123000000); - - preEpochRecord.put("timestamp_nanos", DateTimeUtil.nanosFromTimestamp(preEpochNanos)); - preEpochRecord.put("timestamp_micros", DateTimeUtil.microsFromTimestamp(preEpochMicros)); - preEpochRecord.put("timestamp_millis", DateTimeUtil.millisFromTimestamp(preEpochMillis)); - - Record preEpochResult = readRecord(reader, avroSchema, preEpochRecord); - - assertThat(preEpochResult.getField("timestamp_nanos")).isEqualTo(preEpochNanos); - assertThat(preEpochResult.getField("timestamp_micros")).isEqualTo(preEpochMicros); - assertThat(preEpochResult.getField("timestamp_millis")).isEqualTo(preEpochMillis); - } - - @Test - public void timestampTzDataReader() throws IOException { - org.apache.iceberg.Schema icebergSchema = - new org.apache.iceberg.Schema( - Types.NestedField.required(1, "timestamp_nanos_tz", Types.TimestampType.withZone()), - Types.NestedField.required(2, "timestamp_micros_tz", Types.TimestampType.withZone()), - Types.NestedField.required(3, "timestamp_millis_tz", Types.TimestampType.withZone())); - - Schema avroSchema = - SchemaBuilder.record("test_tz") - .fields() - .name("timestamp_nanos_tz") - .type(LogicalTypes.timestampNanos().addToSchema(utcAdjustedLongSchema())) - .noDefault() - .name("timestamp_micros_tz") - .type(LogicalTypes.timestampMicros().addToSchema(utcAdjustedLongSchema())) - .noDefault() - .name("timestamp_millis_tz") - .type(LogicalTypes.timestampMillis().addToSchema(utcAdjustedLongSchema())) - .noDefault() - .endRecord(); - - avroSchema.getField("timestamp_nanos_tz").addProp("field-id", 1); - avroSchema.getField("timestamp_micros_tz").addProp("field-id", 2); - avroSchema.getField("timestamp_millis_tz").addProp("field-id", 3); - - DataReader reader = DataReader.create(icebergSchema, avroSchema); - reader.setSchema(avroSchema); - - // post-epoch timestamps - GenericRecord avroRecord = new GenericData.Record(avroSchema); - - OffsetDateTime offsetTimestampNanos = - OffsetDateTime.of(2023, 10, 15, 14, 30, 45, 123456789, ZoneOffset.ofHours(-8)); - OffsetDateTime offsetTimestampMicros = - OffsetDateTime.of(2023, 10, 15, 14, 30, 45, 123456000, ZoneOffset.ofHours(5)); - OffsetDateTime offsetTimestampMillis = - OffsetDateTime.of(2023, 10, 15, 14, 30, 45, 123000000, ZoneOffset.ofHours(-3)); - - avroRecord.put("timestamp_nanos_tz", DateTimeUtil.nanosFromTimestamptz(offsetTimestampNanos)); - avroRecord.put( - "timestamp_micros_tz", DateTimeUtil.microsFromTimestamptz(offsetTimestampMicros)); - avroRecord.put( - "timestamp_millis_tz", DateTimeUtil.millisFromTimestamptz(offsetTimestampMillis)); - - Record result = readRecord(reader, avroSchema, avroRecord); - - assertThat(result.getField("timestamp_nanos_tz")) - .isEqualTo(offsetTimestampNanos.withOffsetSameInstant(ZoneOffset.UTC)); - assertThat(result.getField("timestamp_micros_tz")) - .isEqualTo(offsetTimestampMicros.withOffsetSameInstant(ZoneOffset.UTC)); - assertThat(result.getField("timestamp_millis_tz")) - .isEqualTo(offsetTimestampMillis.withOffsetSameInstant(ZoneOffset.UTC)); - - // pre-epoch timestamps - GenericRecord preEpochRecord = new GenericData.Record(avroSchema); - - OffsetDateTime preEpochTimestampNanos = - OffsetDateTime.of(1969, 1, 1, 10, 11, 12, 123456789, ZoneOffset.ofHours(-8)); - OffsetDateTime preEpochTimestampMicros = - OffsetDateTime.of(1968, 1, 1, 10, 11, 12, 123456000, ZoneOffset.ofHours(5)); - OffsetDateTime preEpochTimestampMillis = - OffsetDateTime.of(1967, 1, 1, 10, 11, 12, 123000000, ZoneOffset.ofHours(-3)); - - preEpochRecord.put( - "timestamp_nanos_tz", DateTimeUtil.nanosFromTimestamptz(preEpochTimestampNanos)); - preEpochRecord.put( - "timestamp_micros_tz", DateTimeUtil.microsFromTimestamptz(preEpochTimestampMicros)); - preEpochRecord.put( - "timestamp_millis_tz", DateTimeUtil.millisFromTimestamptz(preEpochTimestampMillis)); - - Record preEpochResult = readRecord(reader, avroSchema, preEpochRecord); - - assertThat(preEpochResult.getField("timestamp_nanos_tz")) - .isEqualTo(preEpochTimestampNanos.withOffsetSameInstant(ZoneOffset.UTC)); - assertThat(preEpochResult.getField("timestamp_micros_tz")) - .isEqualTo(preEpochTimestampMicros.withOffsetSameInstant(ZoneOffset.UTC)); - assertThat(preEpochResult.getField("timestamp_millis_tz")) - .isEqualTo(preEpochTimestampMillis.withOffsetSameInstant(ZoneOffset.UTC)); - } - - private Record readRecord(DataReader reader, Schema avroSchema, GenericRecord avroRecord) - throws IOException { - try (ByteArrayOutputStream out = new ByteArrayOutputStream()) { - BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(out, null); - GenericDatumWriter writer = new GenericDatumWriter<>(avroSchema); - writer.write(avroRecord, encoder); - encoder.flush(); - - try (ByteArrayInputStream in = new ByteArrayInputStream(out.toByteArray())) { - return reader.read(null, DecoderFactory.get().binaryDecoder(in, null)); - } - } - } - - private Schema utcAdjustedLongSchema() { - Schema schema = Schema.create(Schema.Type.LONG); - schema.addProp(ADJUST_TO_UTC_PROP, "true"); - return schema; - } -} diff --git a/core/src/test/java/org/apache/iceberg/rest/TestRESTUtil.java b/core/src/test/java/org/apache/iceberg/rest/TestRESTUtil.java index 1ed732ebc91a..253c719c0ac0 100644 --- a/core/src/test/java/org/apache/iceberg/rest/TestRESTUtil.java +++ b/core/src/test/java/org/apache/iceberg/rest/TestRESTUtil.java @@ -108,8 +108,8 @@ public void testRoundTripUrlEncodeDecodeNamespace(String namespaceSeparator) { @Test public void encodeAsOldClientAndDecodeAsNewServer() { Namespace namespace = Namespace.of("first", "second", "third"); - // old client would call encodeNamespace without specifying a separator - String encodedNamespace = RESTUtil.encodeNamespace(namespace); + // old client would call encodeNamespace with the legacy separator + String encodedNamespace = RESTUtil.encodeNamespace(namespace, "%1F"); assertThat(encodedNamespace).contains(RESTUtil.NAMESPACE_SEPARATOR_URLENCODED_UTF_8); // old client would also call namespaceToQueryParam without specifying a separator @@ -130,11 +130,11 @@ public void encodeAsOldClientAndDecodeAsNewServer() { @Test public void testNamespaceUrlEncodeDecodeDoesNotAllowNull() { assertThatExceptionOfType(IllegalArgumentException.class) - .isThrownBy(() -> RESTUtil.encodeNamespace(null)) + .isThrownBy(() -> RESTUtil.encodeNamespace(null, "%1F")) .withMessage("Invalid namespace: null"); assertThatExceptionOfType(IllegalArgumentException.class) - .isThrownBy(() -> RESTUtil.decodeNamespace(null)) + .isThrownBy(() -> RESTUtil.decodeNamespace(null, "%1F")) .withMessage("Invalid namespace: null"); } diff --git a/core/src/test/java/org/apache/iceberg/rest/TestResourcePaths.java b/core/src/test/java/org/apache/iceberg/rest/TestResourcePaths.java index a742b89a7627..823c85912a6b 100644 --- a/core/src/test/java/org/apache/iceberg/rest/TestResourcePaths.java +++ b/core/src/test/java/org/apache/iceberg/rest/TestResourcePaths.java @@ -119,14 +119,14 @@ public void nestedNamespaceWithLegacySeparator() { // legacy separator is always used by default, so no need to configure it ResourcePaths pathsWithLegacySeparator = ResourcePaths.forCatalogProperties(ImmutableMap.of()); - // Encode namespace using legacy separator. No need to provide the separator to encodeNamespace - String legacyEncodedNamespace = RESTUtil.encodeNamespace(namespace); + // Encode namespace using legacy separator. + String legacyEncodedNamespace = RESTUtil.encodeNamespace(namespace, "%1F"); assertThat(pathsWithLegacySeparator.namespace(namespace)) .contains(legacyEncodedNamespace) .contains(legacySeparator); - // Decode the namespace containing legacy separator without providing the separator - assertThat(RESTUtil.decodeNamespace(legacyEncodedNamespace)).isEqualTo(namespace); + // Decode the namespace containing legacy separator + assertThat(RESTUtil.decodeNamespace(legacyEncodedNamespace, "%1F")).isEqualTo(namespace); // Decode the namespace containing legacy separator with providing the new separator assertThat(RESTUtil.decodeNamespace(legacyEncodedNamespace, newSeparator)).isEqualTo(namespace); diff --git a/core/src/test/java/org/apache/iceberg/rest/responses/TestFetchPlanningResultResponseParser.java b/core/src/test/java/org/apache/iceberg/rest/responses/TestFetchPlanningResultResponseParser.java index 841083f88baf..f0eb1cfdf284 100644 --- a/core/src/test/java/org/apache/iceberg/rest/responses/TestFetchPlanningResultResponseParser.java +++ b/core/src/test/java/org/apache/iceberg/rest/responses/TestFetchPlanningResultResponseParser.java @@ -144,18 +144,6 @@ public void roundTripSerdeWithInvalidPlanStatusSubmittedWithTasksPresent() { @Test public void roundTripSerdeWithInvalidPlanStatusSubmittedWithDeleteFilesNoFileScanTasksPresent() { - PlanStatus planStatus = PlanStatus.fromName("submitted"); - assertThatThrownBy( - () -> { - FetchPlanningResultResponse.builder() - .withPlanStatus(planStatus) - .withDeleteFiles(List.of(FILE_A_DELETES)) - .build(); - }) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage( - "Invalid response: deleteFiles should only be returned with fileScanTasks that reference them"); - String invalidJson = "{\"status\":\"submitted\"," + "\"delete-files\":[{\"spec-id\":0,\"content\":\"position-deletes\"," diff --git a/core/src/test/java/org/apache/iceberg/rest/responses/TestFetchScanTasksResponseParser.java b/core/src/test/java/org/apache/iceberg/rest/responses/TestFetchScanTasksResponseParser.java index 7b44b3533a5b..b3845bf42398 100644 --- a/core/src/test/java/org/apache/iceberg/rest/responses/TestFetchScanTasksResponseParser.java +++ b/core/src/test/java/org/apache/iceberg/rest/responses/TestFetchScanTasksResponseParser.java @@ -83,16 +83,6 @@ public void roundTripSerdeWithPlanTasks() { @Test public void roundTripSerdeWithDeleteFilesNoFileScanTasksPresent() { - assertThatThrownBy( - () -> - FetchScanTasksResponse.builder() - .withPlanTasks(List.of("task1", "task2")) - .withDeleteFiles(List.of(FILE_A_DELETES)) - .build()) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage( - "Invalid response: deleteFiles should only be returned with fileScanTasks that reference them"); - String invalidJson = "{\"plan-tasks\":[\"task1\",\"task2\"]," + "\"delete-files\":[{\"spec-id\":0,\"content\":\"position-deletes\"," diff --git a/core/src/test/java/org/apache/iceberg/rest/responses/TestPlanTableScanResponseParser.java b/core/src/test/java/org/apache/iceberg/rest/responses/TestPlanTableScanResponseParser.java index 6354e7bf246f..c248392e3934 100644 --- a/core/src/test/java/org/apache/iceberg/rest/responses/TestPlanTableScanResponseParser.java +++ b/core/src/test/java/org/apache/iceberg/rest/responses/TestPlanTableScanResponseParser.java @@ -196,17 +196,6 @@ public void roundTripSerdeWithInvalidPlanIdWithIncorrectStatus() { @Test public void roundTripSerdeWithInvalidPlanStatusSubmittedWithDeleteFilesNoFileScanTasksPresent() { - assertThatThrownBy( - () -> - PlanTableScanResponse.builder() - .withPlanStatus(PlanStatus.SUBMITTED) - .withPlanId("somePlanId") - .withDeleteFiles(List.of(FILE_A_DELETES)) - .build()) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage( - "Invalid response: deleteFiles should only be returned with fileScanTasks that reference them"); - String invalidJson = "{\"status\":\"submitted\"," + "\"plan-id\":\"somePlanId\"," @@ -264,7 +253,6 @@ public void roundTripSerdeWithValidStatusAndFileScanTasks() { .withPlanStatus(fromResponse.planStatus()) .withPlanId(fromResponse.planId()) .withPlanTasks(fromResponse.planTasks()) - .withDeleteFiles(fromResponse.deleteFiles()) .withFileScanTasks(fromResponse.fileScanTasks()) .withSpecsById(PARTITION_SPECS_BY_ID) .build(); diff --git a/data/src/main/java/org/apache/iceberg/data/BaseFileWriterFactory.java b/data/src/main/java/org/apache/iceberg/data/BaseFileWriterFactory.java deleted file mode 100644 index 444c0d0226bd..000000000000 --- a/data/src/main/java/org/apache/iceberg/data/BaseFileWriterFactory.java +++ /dev/null @@ -1,375 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iceberg.data; - -import java.io.IOException; -import java.io.Serializable; -import java.io.UncheckedIOException; -import java.util.Map; -import org.apache.iceberg.FileFormat; -import org.apache.iceberg.MetricsConfig; -import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.Schema; -import org.apache.iceberg.SortOrder; -import org.apache.iceberg.StructLike; -import org.apache.iceberg.Table; -import org.apache.iceberg.avro.Avro; -import org.apache.iceberg.deletes.EqualityDeleteWriter; -import org.apache.iceberg.deletes.PositionDeleteWriter; -import org.apache.iceberg.encryption.EncryptedOutputFile; -import org.apache.iceberg.encryption.EncryptionKeyMetadata; -import org.apache.iceberg.io.DataWriter; -import org.apache.iceberg.io.FileWriterFactory; -import org.apache.iceberg.orc.ORC; -import org.apache.iceberg.parquet.Parquet; -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; - -/** - * A base writer factory to be extended by query engine integrations. - * - * @deprecated since version 1.11.0 and will be removed in 1.12.0. Use {@link - * RegistryBasedFileWriterFactory} - */ -@Deprecated -public abstract class BaseFileWriterFactory implements FileWriterFactory, Serializable { - private final Table table; - private final FileFormat dataFileFormat; - private final Schema dataSchema; - private final SortOrder dataSortOrder; - private final FileFormat deleteFileFormat; - private final int[] equalityFieldIds; - private final Schema equalityDeleteRowSchema; - private final SortOrder equalityDeleteSortOrder; - private final Schema positionDeleteRowSchema; - private final Map writerProperties; - - protected BaseFileWriterFactory( - Table table, - FileFormat dataFileFormat, - Schema dataSchema, - SortOrder dataSortOrder, - FileFormat deleteFileFormat, - int[] equalityFieldIds, - Schema equalityDeleteRowSchema, - SortOrder equalityDeleteSortOrder, - Map writerProperties) { - this.table = table; - this.dataFileFormat = dataFileFormat; - this.dataSchema = dataSchema; - this.dataSortOrder = dataSortOrder; - this.deleteFileFormat = deleteFileFormat; - this.equalityFieldIds = equalityFieldIds; - this.equalityDeleteRowSchema = equalityDeleteRowSchema; - this.equalityDeleteSortOrder = equalityDeleteSortOrder; - this.writerProperties = writerProperties; - this.positionDeleteRowSchema = null; - } - - protected BaseFileWriterFactory( - Table table, - FileFormat dataFileFormat, - Schema dataSchema, - SortOrder dataSortOrder, - FileFormat deleteFileFormat, - int[] equalityFieldIds, - Schema equalityDeleteRowSchema, - SortOrder equalityDeleteSortOrder, - Schema positionDeleteRowSchema, - Map writerProperties) { - this.table = table; - this.dataFileFormat = dataFileFormat; - this.dataSchema = dataSchema; - this.dataSortOrder = dataSortOrder; - this.deleteFileFormat = deleteFileFormat; - this.equalityFieldIds = equalityFieldIds; - this.equalityDeleteRowSchema = equalityDeleteRowSchema; - this.equalityDeleteSortOrder = equalityDeleteSortOrder; - this.positionDeleteRowSchema = positionDeleteRowSchema; - this.writerProperties = writerProperties; - } - - @Deprecated - protected BaseFileWriterFactory( - Table table, - FileFormat dataFileFormat, - Schema dataSchema, - SortOrder dataSortOrder, - FileFormat deleteFileFormat, - int[] equalityFieldIds, - Schema equalityDeleteRowSchema, - SortOrder equalityDeleteSortOrder, - Schema positionDeleteRowSchema) { - this.table = table; - this.dataFileFormat = dataFileFormat; - this.dataSchema = dataSchema; - this.dataSortOrder = dataSortOrder; - this.deleteFileFormat = deleteFileFormat; - this.equalityFieldIds = equalityFieldIds; - this.equalityDeleteRowSchema = equalityDeleteRowSchema; - this.equalityDeleteSortOrder = equalityDeleteSortOrder; - this.positionDeleteRowSchema = positionDeleteRowSchema; - this.writerProperties = ImmutableMap.of(); - } - - protected abstract void configureDataWrite(Avro.DataWriteBuilder builder); - - protected abstract void configureEqualityDelete(Avro.DeleteWriteBuilder builder); - - protected abstract void configurePositionDelete(Avro.DeleteWriteBuilder builder); - - protected abstract void configureDataWrite(Parquet.DataWriteBuilder builder); - - protected abstract void configureEqualityDelete(Parquet.DeleteWriteBuilder builder); - - protected abstract void configurePositionDelete(Parquet.DeleteWriteBuilder builder); - - protected abstract void configureDataWrite(ORC.DataWriteBuilder builder); - - protected abstract void configureEqualityDelete(ORC.DeleteWriteBuilder builder); - - protected abstract void configurePositionDelete(ORC.DeleteWriteBuilder builder); - - @Override - public DataWriter newDataWriter( - EncryptedOutputFile file, PartitionSpec spec, StructLike partition) { - EncryptionKeyMetadata keyMetadata = file.keyMetadata(); - Map properties = table == null ? ImmutableMap.of() : table.properties(); - MetricsConfig metricsConfig = - table == null ? MetricsConfig.getDefault() : MetricsConfig.forTable(table); - - try { - switch (dataFileFormat) { - case AVRO: - Avro.DataWriteBuilder avroBuilder = - Avro.writeData(file) - .schema(dataSchema) - .setAll(properties) - .setAll(writerProperties) - .metricsConfig(metricsConfig) - .withSpec(spec) - .withPartition(partition) - .withKeyMetadata(keyMetadata) - .withSortOrder(dataSortOrder) - .overwrite(); - - configureDataWrite(avroBuilder); - - return avroBuilder.build(); - - case PARQUET: - Parquet.DataWriteBuilder parquetBuilder = - Parquet.writeData(file) - .schema(dataSchema) - .setAll(properties) - .setAll(writerProperties) - .metricsConfig(metricsConfig) - .withSpec(spec) - .withPartition(partition) - .withKeyMetadata(keyMetadata) - .withSortOrder(dataSortOrder) - .overwrite(); - - configureDataWrite(parquetBuilder); - - return parquetBuilder.build(); - - case ORC: - ORC.DataWriteBuilder orcBuilder = - ORC.writeData(file) - .schema(dataSchema) - .setAll(properties) - .setAll(writerProperties) - .metricsConfig(metricsConfig) - .withSpec(spec) - .withPartition(partition) - .withKeyMetadata(keyMetadata) - .withSortOrder(dataSortOrder) - .overwrite(); - - configureDataWrite(orcBuilder); - - return orcBuilder.build(); - - default: - throw new UnsupportedOperationException( - "Unsupported data file format: " + dataFileFormat); - } - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - @Override - public EqualityDeleteWriter newEqualityDeleteWriter( - EncryptedOutputFile file, PartitionSpec spec, StructLike partition) { - EncryptionKeyMetadata keyMetadata = file.keyMetadata(); - Map properties = table == null ? ImmutableMap.of() : table.properties(); - MetricsConfig metricsConfig = - table == null ? MetricsConfig.getDefault() : MetricsConfig.forTable(table); - - try { - switch (deleteFileFormat) { - case AVRO: - Avro.DeleteWriteBuilder avroBuilder = - Avro.writeDeletes(file) - .setAll(properties) - .setAll(writerProperties) - .metricsConfig(metricsConfig) - .rowSchema(equalityDeleteRowSchema) - .equalityFieldIds(equalityFieldIds) - .withSpec(spec) - .withPartition(partition) - .withKeyMetadata(keyMetadata) - .withSortOrder(equalityDeleteSortOrder) - .overwrite(); - - configureEqualityDelete(avroBuilder); - - return avroBuilder.buildEqualityWriter(); - - case PARQUET: - Parquet.DeleteWriteBuilder parquetBuilder = - Parquet.writeDeletes(file) - .setAll(properties) - .setAll(writerProperties) - .metricsConfig(metricsConfig) - .rowSchema(equalityDeleteRowSchema) - .equalityFieldIds(equalityFieldIds) - .withSpec(spec) - .withPartition(partition) - .withKeyMetadata(keyMetadata) - .withSortOrder(equalityDeleteSortOrder) - .overwrite(); - - configureEqualityDelete(parquetBuilder); - - return parquetBuilder.buildEqualityWriter(); - - case ORC: - ORC.DeleteWriteBuilder orcBuilder = - ORC.writeDeletes(file) - .setAll(properties) - .setAll(writerProperties) - .metricsConfig(metricsConfig) - .rowSchema(equalityDeleteRowSchema) - .equalityFieldIds(equalityFieldIds) - .withSpec(spec) - .withPartition(partition) - .withKeyMetadata(keyMetadata) - .withSortOrder(equalityDeleteSortOrder) - .overwrite(); - - configureEqualityDelete(orcBuilder); - - return orcBuilder.buildEqualityWriter(); - - default: - throw new UnsupportedOperationException( - "Unsupported format for equality deletes: " + deleteFileFormat); - } - } catch (IOException e) { - throw new UncheckedIOException("Failed to create new equality delete writer", e); - } - } - - @Override - public PositionDeleteWriter newPositionDeleteWriter( - EncryptedOutputFile file, PartitionSpec spec, StructLike partition) { - EncryptionKeyMetadata keyMetadata = file.keyMetadata(); - Map properties = table == null ? ImmutableMap.of() : table.properties(); - MetricsConfig metricsConfig = - table == null ? MetricsConfig.forPositionDelete() : MetricsConfig.forPositionDelete(table); - - try { - switch (deleteFileFormat) { - case AVRO: - Avro.DeleteWriteBuilder avroBuilder = - Avro.writeDeletes(file) - .setAll(properties) - .setAll(writerProperties) - .metricsConfig(metricsConfig) - .rowSchema(positionDeleteRowSchema) - .withSpec(spec) - .withPartition(partition) - .withKeyMetadata(keyMetadata) - .overwrite(); - - configurePositionDelete(avroBuilder); - - return avroBuilder.buildPositionWriter(); - - case PARQUET: - Parquet.DeleteWriteBuilder parquetBuilder = - Parquet.writeDeletes(file) - .setAll(properties) - .setAll(writerProperties) - .metricsConfig(metricsConfig) - .rowSchema(positionDeleteRowSchema) - .withSpec(spec) - .withPartition(partition) - .withKeyMetadata(keyMetadata) - .overwrite(); - - configurePositionDelete(parquetBuilder); - - return parquetBuilder.buildPositionWriter(); - - case ORC: - ORC.DeleteWriteBuilder orcBuilder = - ORC.writeDeletes(file) - .setAll(properties) - .setAll(writerProperties) - .metricsConfig(metricsConfig) - .rowSchema(positionDeleteRowSchema) - .withSpec(spec) - .withPartition(partition) - .withKeyMetadata(keyMetadata) - .overwrite(); - - configurePositionDelete(orcBuilder); - - return orcBuilder.buildPositionWriter(); - - default: - throw new UnsupportedOperationException( - "Unsupported format for position deletes: " + deleteFileFormat); - } - - } catch (IOException e) { - throw new UncheckedIOException("Failed to create new position delete writer", e); - } - } - - protected Schema dataSchema() { - return dataSchema; - } - - protected Schema equalityDeleteRowSchema() { - return equalityDeleteRowSchema; - } - - /** - * @deprecated This method is deprecated as of version 1.11.0 and will be removed in 1.12.0. - * Position deletes that include row data are no longer supported. - */ - @Deprecated - protected Schema positionDeleteRowSchema() { - return positionDeleteRowSchema; - } -} diff --git a/data/src/main/java/org/apache/iceberg/data/GenericAppenderFactory.java b/data/src/main/java/org/apache/iceberg/data/GenericAppenderFactory.java deleted file mode 100644 index dd78b48858a2..000000000000 --- a/data/src/main/java/org/apache/iceberg/data/GenericAppenderFactory.java +++ /dev/null @@ -1,350 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iceberg.data; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.util.Map; -import org.apache.iceberg.FileFormat; -import org.apache.iceberg.MetricsConfig; -import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.Schema; -import org.apache.iceberg.StructLike; -import org.apache.iceberg.Table; -import org.apache.iceberg.avro.Avro; -import org.apache.iceberg.data.avro.DataWriter; -import org.apache.iceberg.data.orc.GenericOrcWriter; -import org.apache.iceberg.data.parquet.GenericParquetWriter; -import org.apache.iceberg.deletes.EqualityDeleteWriter; -import org.apache.iceberg.deletes.PositionDeleteWriter; -import org.apache.iceberg.encryption.EncryptedOutputFile; -import org.apache.iceberg.encryption.EncryptionUtil; -import org.apache.iceberg.io.FileAppender; -import org.apache.iceberg.io.FileAppenderFactory; -import org.apache.iceberg.io.OutputFile; -import org.apache.iceberg.orc.ORC; -import org.apache.iceberg.parquet.Parquet; -import org.apache.iceberg.relocated.com.google.common.base.Preconditions; -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; -import org.apache.iceberg.relocated.com.google.common.collect.Maps; - -/** - * Factory to create a new {@link FileAppender} to write {@link Record}s. - * - * @deprecated will be removed in 1.12.0; use {@link GenericFileWriterFactory} instead. - */ -@Deprecated -public class GenericAppenderFactory implements FileAppenderFactory { - private final Table table; - private final Schema schema; - private final PartitionSpec spec; - private final int[] equalityFieldIds; - private final Schema eqDeleteRowSchema; - private final Schema posDeleteRowSchema; - private final Map config; - - public GenericAppenderFactory(Schema schema) { - this(schema, PartitionSpec.unpartitioned()); - } - - public GenericAppenderFactory(Schema schema, PartitionSpec spec) { - this(schema, spec, null, null, null); - } - - public GenericAppenderFactory( - Schema schema, - PartitionSpec spec, - int[] equalityFieldIds, - Schema eqDeleteRowSchema, - Schema posDeleteRowSchema) { - this(null, schema, spec, null, equalityFieldIds, eqDeleteRowSchema, posDeleteRowSchema); - } - - /** - * Constructor for GenericAppenderFactory. - * - * @param schema the schema of the records to write - * @param spec the partition spec of the records - * @param equalityFieldIds the field ids for equality delete - * @param eqDeleteRowSchema the schema for equality delete rows - */ - public GenericAppenderFactory( - Schema schema, PartitionSpec spec, int[] equalityFieldIds, Schema eqDeleteRowSchema) { - this(null, schema, spec, null, equalityFieldIds, eqDeleteRowSchema, null); - } - - /** - * Constructor for GenericAppenderFactory. - * - * @param table iceberg table - * @param schema the schema of the records to write - * @param spec the partition spec of the records - * @param config the configuration for the writer - * @param equalityFieldIds the field ids for equality delete - * @param eqDeleteRowSchema the schema for equality delete rows - */ - public GenericAppenderFactory( - Table table, - Schema schema, - PartitionSpec spec, - Map config, - int[] equalityFieldIds, - Schema eqDeleteRowSchema) { - this(table, schema, spec, config, equalityFieldIds, eqDeleteRowSchema, null); - } - - /** - * Constructor for GenericAppenderFactory. - * - * @param table iceberg table - * @param schema the schema of the records to write - * @param spec the partition spec of the records - * @param config the configuration for the writer - * @param equalityFieldIds the field ids for equality delete - * @param eqDeleteRowSchema the schema for equality delete rows - * @param posDeleteRowSchema the schema for position delete rows - * @deprecated This constructor is deprecated as of version 1.11.0 and will be removed in 1.12.0. - * Position deletes that include row data are no longer supported. Use {@link - * #GenericAppenderFactory(Table, Schema, PartitionSpec, Map, int[], Schema)} instead. - */ - @Deprecated - public GenericAppenderFactory( - Table table, - Schema schema, - PartitionSpec spec, - Map config, - int[] equalityFieldIds, - Schema eqDeleteRowSchema, - Schema posDeleteRowSchema) { - this.table = table; - this.config = config == null ? Maps.newHashMap() : config; - - if (table != null) { - // If the table is provided and schema and spec are not provided, derive them from the table - this.schema = schema == null ? table.schema() : schema; - this.spec = spec == null ? table.spec() : spec; - validateMetricsConfig(this.config); - } else { - this.schema = schema; - this.spec = spec; - } - - this.equalityFieldIds = equalityFieldIds; - this.eqDeleteRowSchema = eqDeleteRowSchema; - this.posDeleteRowSchema = posDeleteRowSchema; - } - - public GenericAppenderFactory set(String property, String value) { - validateMetricsConfig(ImmutableMap.of(property, value)); - config.put(property, value); - return this; - } - - public GenericAppenderFactory setAll(Map properties) { - validateMetricsConfig(properties); - config.putAll(properties); - return this; - } - - @Override - public FileAppender newAppender(OutputFile outputFile, FileFormat fileFormat) { - return newAppender(EncryptionUtil.plainAsEncryptedOutput(outputFile), fileFormat); - } - - @Override - public FileAppender newAppender( - EncryptedOutputFile encryptedOutputFile, FileFormat fileFormat) { - MetricsConfig metricsConfig = - table != null ? MetricsConfig.forTable(table) : MetricsConfig.fromProperties(config); - - try { - switch (fileFormat) { - case AVRO: - return Avro.write(encryptedOutputFile) - .schema(schema) - .createWriterFunc(DataWriter::create) - .metricsConfig(metricsConfig) - .setAll(config) - .overwrite() - .build(); - - case PARQUET: - return Parquet.write(encryptedOutputFile) - .schema(schema) - .createWriterFunc(GenericParquetWriter::create) - .setAll(config) - .metricsConfig(metricsConfig) - .overwrite() - .build(); - - case ORC: - return ORC.write(encryptedOutputFile) - .schema(schema) - .createWriterFunc(GenericOrcWriter::buildWriter) - .setAll(config) - .metricsConfig(metricsConfig) - .overwrite() - .build(); - - default: - throw new UnsupportedOperationException( - "Cannot write unknown file format: " + fileFormat); - } - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - @Override - public org.apache.iceberg.io.DataWriter newDataWriter( - EncryptedOutputFile file, FileFormat format, StructLike partition) { - return new org.apache.iceberg.io.DataWriter<>( - newAppender(file, format), - format, - file.encryptingOutputFile().location(), - spec, - partition, - file.keyMetadata()); - } - - @Override - public EqualityDeleteWriter newEqDeleteWriter( - EncryptedOutputFile file, FileFormat format, StructLike partition) { - Preconditions.checkState( - equalityFieldIds != null && equalityFieldIds.length > 0, - "Equality field ids shouldn't be null or empty when creating equality-delete writer"); - Preconditions.checkNotNull( - eqDeleteRowSchema, - "Equality delete row schema shouldn't be null when creating equality-delete writer"); - MetricsConfig metricsConfig = - table != null ? MetricsConfig.forTable(table) : MetricsConfig.fromProperties(config); - - try { - switch (format) { - case AVRO: - return Avro.writeDeletes(file) - .createWriterFunc(DataWriter::create) - .withPartition(partition) - .overwrite() - .setAll(config) - .rowSchema(eqDeleteRowSchema) - .withSpec(spec) - .withKeyMetadata(file.keyMetadata()) - .equalityFieldIds(equalityFieldIds) - .buildEqualityWriter(); - - case ORC: - return ORC.writeDeletes(file) - .createWriterFunc(GenericOrcWriter::buildWriter) - .withPartition(partition) - .overwrite() - .setAll(config) - .metricsConfig(metricsConfig) - .rowSchema(eqDeleteRowSchema) - .withSpec(spec) - .withKeyMetadata(file.keyMetadata()) - .equalityFieldIds(equalityFieldIds) - .buildEqualityWriter(); - - case PARQUET: - return Parquet.writeDeletes(file) - .createWriterFunc(GenericParquetWriter::create) - .withPartition(partition) - .overwrite() - .setAll(config) - .metricsConfig(metricsConfig) - .rowSchema(eqDeleteRowSchema) - .withSpec(spec) - .withKeyMetadata(file.keyMetadata()) - .equalityFieldIds(equalityFieldIds) - .buildEqualityWriter(); - - default: - throw new UnsupportedOperationException( - "Cannot write equality-deletes for unsupported file format: " + format); - } - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - @Override - public PositionDeleteWriter newPosDeleteWriter( - EncryptedOutputFile file, FileFormat format, StructLike partition) { - MetricsConfig metricsConfig = - table != null - ? MetricsConfig.forPositionDelete(table) - : MetricsConfig.fromProperties(config); - - try { - switch (format) { - case AVRO: - return Avro.writeDeletes(file) - .createWriterFunc(DataWriter::create) - .withPartition(partition) - .overwrite() - .setAll(config) - .rowSchema(posDeleteRowSchema) - .withSpec(spec) - .withKeyMetadata(file.keyMetadata()) - .buildPositionWriter(); - - case ORC: - return ORC.writeDeletes(file) - .createWriterFunc(GenericOrcWriter::buildWriter) - .withPartition(partition) - .overwrite() - .setAll(config) - .rowSchema(posDeleteRowSchema) - .withSpec(spec) - .withKeyMetadata(file.keyMetadata()) - .buildPositionWriter(); - - case PARQUET: - return Parquet.writeDeletes(file) - .createWriterFunc(GenericParquetWriter::create) - .withPartition(partition) - .overwrite() - .setAll(config) - .metricsConfig(metricsConfig) - .rowSchema(posDeleteRowSchema) - .withSpec(spec) - .withKeyMetadata(file.keyMetadata()) - .buildPositionWriter(); - - default: - throw new UnsupportedOperationException( - "Cannot write pos-deletes for unsupported file format: " + format); - } - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - private void validateMetricsConfig(Map writeConfig) { - if (table == null) { - return; - } - - if (writeConfig.keySet().stream().anyMatch(k -> k.startsWith("write.metadata.metrics."))) { - throw new IllegalArgumentException( - "Cannot set metrics properties when the table is provided, use table properties instead"); - } - } -} diff --git a/data/src/main/java/org/apache/iceberg/data/GenericFileWriterFactory.java b/data/src/main/java/org/apache/iceberg/data/GenericFileWriterFactory.java index 1e75b9eda961..914dc3672b77 100644 --- a/data/src/main/java/org/apache/iceberg/data/GenericFileWriterFactory.java +++ b/data/src/main/java/org/apache/iceberg/data/GenericFileWriterFactory.java @@ -241,10 +241,7 @@ public PositionDeleteWriter newPositionDeleteWriter( LOG.warn( "Deprecated feature used. Position delete row schema is used to create the position delete writer."); Map properties = table == null ? ImmutableMap.of() : table.properties(); - MetricsConfig metricsConfig = - table == null - ? MetricsConfig.forPositionDelete() - : MetricsConfig.forPositionDelete(table); + MetricsConfig metricsConfig = MetricsConfig.forPositionDelete(); try { return switch (format) { diff --git a/data/src/main/java/org/apache/iceberg/data/RegistryBasedFileWriterFactory.java b/data/src/main/java/org/apache/iceberg/data/RegistryBasedFileWriterFactory.java index 868b41f5840b..da2be7702909 100644 --- a/data/src/main/java/org/apache/iceberg/data/RegistryBasedFileWriterFactory.java +++ b/data/src/main/java/org/apache/iceberg/data/RegistryBasedFileWriterFactory.java @@ -160,8 +160,7 @@ public PositionDeleteWriter newPositionDeleteWriter( EncryptedOutputFile file, PartitionSpec spec, StructLike partition) { EncryptionKeyMetadata keyMetadata = file.keyMetadata(); Map properties = table != null ? table.properties() : ImmutableMap.of(); - MetricsConfig metricsConfig = - table != null ? MetricsConfig.forPositionDelete(table) : MetricsConfig.forPositionDelete(); + MetricsConfig metricsConfig = MetricsConfig.forPositionDelete(); try { FileWriterBuilder, ?> builder = diff --git a/data/src/test/java/org/apache/iceberg/TestGenericAppenderFactory.java b/data/src/test/java/org/apache/iceberg/TestGenericAppenderFactory.java deleted file mode 100644 index 5d940adaec58..000000000000 --- a/data/src/test/java/org/apache/iceberg/TestGenericAppenderFactory.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.iceberg; - -import static org.assertj.core.api.Assertions.assertThatNoException; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -import java.util.List; -import java.util.Map; -import org.apache.iceberg.data.GenericAppenderFactory; -import org.apache.iceberg.data.GenericRecord; -import org.apache.iceberg.data.Record; -import org.apache.iceberg.io.FileAppenderFactory; -import org.apache.iceberg.io.TestAppenderFactory; -import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; -import org.apache.iceberg.relocated.com.google.common.collect.Maps; -import org.apache.iceberg.util.ArrayUtil; -import org.apache.iceberg.util.StructLikeSet; -import org.junit.jupiter.api.TestTemplate; - -public class TestGenericAppenderFactory extends TestAppenderFactory { - - private final GenericRecord gRecord = GenericRecord.create(SCHEMA); - - @Override - protected FileAppenderFactory createAppenderFactory( - List equalityFieldIds, Schema eqDeleteSchema, Schema posDeleteRowSchema) { - return new GenericAppenderFactory( - table, - table.schema(), - table.spec(), - Maps.newHashMap(), - ArrayUtil.toIntArray(equalityFieldIds), - eqDeleteSchema, - posDeleteRowSchema); - } - - @Override - protected Record createRow(Integer id, String data) { - return gRecord.copy(ImmutableMap.of("id", id, "data", data)); - } - - @Override - protected StructLikeSet expectedRowSet(Iterable records) { - StructLikeSet set = StructLikeSet.create(table.schema().asStruct()); - records.forEach(set::add); - return set; - } - - @TestTemplate - void illegalSetConfig() { - GenericAppenderFactory appenderFactory = - (GenericAppenderFactory) createAppenderFactory(null, null, null); - - assertThatThrownBy( - () -> - appenderFactory.set( - TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS, - MetricsModes.None.get().toString())) - .as("Should not allow setting metrics property if the table was provided") - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining( - "Cannot set metrics properties when the table is provided, use table properties instead"); - } - - @TestTemplate - void illegalSetAllConfigs() { - GenericAppenderFactory appenderFactory = - (GenericAppenderFactory) createAppenderFactory(null, null, null); - - Map properties = - ImmutableMap.of( - TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS, - "10", - TableProperties.METRICS_MODE_COLUMN_CONF_PREFIX + "id", - MetricsModes.Full.get().toString()); - - assertThatThrownBy(() -> appenderFactory.setAll(properties)) - .as("Should not allow setting metrics property if the table was provided") - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining( - "Cannot set metrics properties when the table is provided, use table properties instead"); - } - - @TestTemplate - void setConfigExcludeMetrics() { - GenericAppenderFactory appenderFactory = - (GenericAppenderFactory) createAppenderFactory(null, null, null); - assertThatNoException().isThrownBy(() -> appenderFactory.set("key1", "value1")); - assertThatNoException() - .isThrownBy(() -> appenderFactory.setAll(ImmutableMap.of("key2", "value2"))); - } - - @TestTemplate - void setConfigWithoutTable() { - GenericAppenderFactory appenderFactory = new GenericAppenderFactory(SCHEMA); - assertThatNoException() - .isThrownBy( - () -> appenderFactory.set(TableProperties.METRICS_MAX_INFERRED_COLUMN_DEFAULTS, "10")); - assertThatNoException() - .isThrownBy( - () -> - appenderFactory.setAll( - ImmutableMap.of(TableProperties.DEFAULT_WRITE_METRICS_MODE, "full"))); - } - - @TestTemplate - void createFactoryWithConflictConfig() { - table - .updateProperties() - .set(TableProperties.DEFAULT_WRITE_METRICS_MODE, MetricsModes.Full.get().toString()) - .commit(); - Map config = - ImmutableMap.of( - TableProperties.DEFAULT_WRITE_METRICS_MODE, MetricsModes.None.get().toString()); - - assertThatThrownBy( - () -> new GenericAppenderFactory(table, SCHEMA, SPEC, config, null, null, null)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessageContaining( - "Cannot set metrics properties when the table is provided, use table properties instead"); - } -} diff --git a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/RewriteDataFiles.java b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/RewriteDataFiles.java index be77fda23d29..8d21dc0b6bcd 100644 --- a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/RewriteDataFiles.java +++ b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/RewriteDataFiles.java @@ -209,20 +209,6 @@ public Builder maxFilesToRewrite(int maxFilesToRewrite) { return this; } - /** - * A user provided filter for determining which files will be considered by the rewrite - * strategy. - * - * @param newFilter the filter expression to apply - * @return this for method chaining - * @deprecated will be removed in 1.12.0. Use {@link #filter(SerializableSupplier)} instead - */ - @Deprecated - public Builder filter(Expression newFilter) { - this.filterSupplier = () -> newFilter; - return this; - } - /** * A user-provided supplier of a filter expression that determines which files are considered by * the rewrite strategy. diff --git a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/sink/FlinkAppenderFactory.java b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/sink/FlinkAppenderFactory.java index 07a068391ce6..85dcc3133bd6 100644 --- a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/sink/FlinkAppenderFactory.java +++ b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/sink/FlinkAppenderFactory.java @@ -238,7 +238,7 @@ public EqualityDeleteWriter newEqDeleteWriter( @Override public PositionDeleteWriter newPosDeleteWriter( EncryptedOutputFile outputFile, FileFormat format, StructLike partition) { - MetricsConfig metricsConfig = MetricsConfig.forPositionDelete(table); + MetricsConfig metricsConfig = MetricsConfig.forPositionDelete(); try { switch (format) { case AVRO: diff --git a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestRewriteDataFiles.java b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestRewriteDataFiles.java index c27f6081af5a..a59ca80d99fd 100644 --- a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestRewriteDataFiles.java +++ b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestRewriteDataFiles.java @@ -521,7 +521,7 @@ void testRewriteWithFilter() throws Exception { .minFileSizeBytes(500_000L) .minInputFiles(2) // Only rewrite data files where id is 1 or 2 for testing rewrite - .filter(Expressions.in("id", 1, 2)) + .filter(() -> Expressions.in("id", 1, 2)) .partialProgressEnabled(true) .partialProgressMaxCommits(1) .maxRewriteBytes(100_000L) diff --git a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/RewriteDataFiles.java b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/RewriteDataFiles.java index be77fda23d29..8d21dc0b6bcd 100644 --- a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/RewriteDataFiles.java +++ b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/RewriteDataFiles.java @@ -209,20 +209,6 @@ public Builder maxFilesToRewrite(int maxFilesToRewrite) { return this; } - /** - * A user provided filter for determining which files will be considered by the rewrite - * strategy. - * - * @param newFilter the filter expression to apply - * @return this for method chaining - * @deprecated will be removed in 1.12.0. Use {@link #filter(SerializableSupplier)} instead - */ - @Deprecated - public Builder filter(Expression newFilter) { - this.filterSupplier = () -> newFilter; - return this; - } - /** * A user-provided supplier of a filter expression that determines which files are considered by * the rewrite strategy. diff --git a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/sink/FlinkAppenderFactory.java b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/sink/FlinkAppenderFactory.java index 07a068391ce6..85dcc3133bd6 100644 --- a/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/sink/FlinkAppenderFactory.java +++ b/flink/v2.0/flink/src/main/java/org/apache/iceberg/flink/sink/FlinkAppenderFactory.java @@ -238,7 +238,7 @@ public EqualityDeleteWriter newEqDeleteWriter( @Override public PositionDeleteWriter newPosDeleteWriter( EncryptedOutputFile outputFile, FileFormat format, StructLike partition) { - MetricsConfig metricsConfig = MetricsConfig.forPositionDelete(table); + MetricsConfig metricsConfig = MetricsConfig.forPositionDelete(); try { switch (format) { case AVRO: diff --git a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestRewriteDataFiles.java b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestRewriteDataFiles.java index c27f6081af5a..a59ca80d99fd 100644 --- a/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestRewriteDataFiles.java +++ b/flink/v2.0/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestRewriteDataFiles.java @@ -521,7 +521,7 @@ void testRewriteWithFilter() throws Exception { .minFileSizeBytes(500_000L) .minInputFiles(2) // Only rewrite data files where id is 1 or 2 for testing rewrite - .filter(Expressions.in("id", 1, 2)) + .filter(() -> Expressions.in("id", 1, 2)) .partialProgressEnabled(true) .partialProgressMaxCommits(1) .maxRewriteBytes(100_000L) diff --git a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/RewriteDataFiles.java b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/RewriteDataFiles.java index be77fda23d29..8d21dc0b6bcd 100644 --- a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/RewriteDataFiles.java +++ b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/maintenance/api/RewriteDataFiles.java @@ -209,20 +209,6 @@ public Builder maxFilesToRewrite(int maxFilesToRewrite) { return this; } - /** - * A user provided filter for determining which files will be considered by the rewrite - * strategy. - * - * @param newFilter the filter expression to apply - * @return this for method chaining - * @deprecated will be removed in 1.12.0. Use {@link #filter(SerializableSupplier)} instead - */ - @Deprecated - public Builder filter(Expression newFilter) { - this.filterSupplier = () -> newFilter; - return this; - } - /** * A user-provided supplier of a filter expression that determines which files are considered by * the rewrite strategy. diff --git a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/sink/FlinkAppenderFactory.java b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/sink/FlinkAppenderFactory.java index 07a068391ce6..85dcc3133bd6 100644 --- a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/sink/FlinkAppenderFactory.java +++ b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/sink/FlinkAppenderFactory.java @@ -238,7 +238,7 @@ public EqualityDeleteWriter newEqDeleteWriter( @Override public PositionDeleteWriter newPosDeleteWriter( EncryptedOutputFile outputFile, FileFormat format, StructLike partition) { - MetricsConfig metricsConfig = MetricsConfig.forPositionDelete(table); + MetricsConfig metricsConfig = MetricsConfig.forPositionDelete(); try { switch (format) { case AVRO: diff --git a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestRewriteDataFiles.java b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestRewriteDataFiles.java index c27f6081af5a..a59ca80d99fd 100644 --- a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestRewriteDataFiles.java +++ b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/maintenance/api/TestRewriteDataFiles.java @@ -521,7 +521,7 @@ void testRewriteWithFilter() throws Exception { .minFileSizeBytes(500_000L) .minInputFiles(2) // Only rewrite data files where id is 1 or 2 for testing rewrite - .filter(Expressions.in("id", 1, 2)) + .filter(() -> Expressions.in("id", 1, 2)) .partialProgressEnabled(true) .partialProgressMaxCommits(1) .maxRewriteBytes(100_000L) diff --git a/kafka-connect/kafka-connect-events/src/main/java/org/apache/iceberg/connect/events/TableReference.java b/kafka-connect/kafka-connect-events/src/main/java/org/apache/iceberg/connect/events/TableReference.java index cd1341822c58..4deaf61361fc 100644 --- a/kafka-connect/kafka-connect-events/src/main/java/org/apache/iceberg/connect/events/TableReference.java +++ b/kafka-connect/kafka-connect-events/src/main/java/org/apache/iceberg/connect/events/TableReference.java @@ -58,16 +58,6 @@ public class TableReference implements IndexedRecord { NestedField.optional(TABLE_UUID, "table_uuid", UUIDType.get())); private static final Schema AVRO_SCHEMA = AvroUtil.convert(ICEBERG_SCHEMA, TableReference.class); - /** - * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link TableReference#of(String, - * TableIdentifier, UUID)} - */ - @Deprecated - public static TableReference of(String catalog, TableIdentifier tableIdentifier) { - return new TableReference( - catalog, Arrays.asList(tableIdentifier.namespace().levels()), tableIdentifier.name(), null); - } - public static TableReference of(String catalog, TableIdentifier tableIdentifier, UUID tableUuid) { return new TableReference( catalog, @@ -81,21 +71,6 @@ public TableReference(Schema avroSchema) { this.avroSchema = avroSchema; } - /** - * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link TableReference#of(String, - * TableIdentifier, UUID)}. - */ - @Deprecated - public TableReference(String catalog, List namespace, String name) { - Preconditions.checkNotNull(catalog, "Catalog cannot be null"); - Preconditions.checkNotNull(namespace, "Namespace cannot be null"); - Preconditions.checkNotNull(name, "Name cannot be null"); - this.catalog = catalog; - this.namespace = namespace; - this.name = name; - this.avroSchema = AVRO_SCHEMA; - } - private TableReference(String catalog, List namespace, String name, UUID uuid) { Preconditions.checkNotNull(catalog, "Catalog cannot be null"); Preconditions.checkNotNull(namespace, "Namespace cannot be null"); diff --git a/kafka-connect/kafka-connect/src/main/java/org/apache/iceberg/connect/data/IcebergWriterResult.java b/kafka-connect/kafka-connect/src/main/java/org/apache/iceberg/connect/data/IcebergWriterResult.java index 5667399cd74e..2903f5507d47 100644 --- a/kafka-connect/kafka-connect/src/main/java/org/apache/iceberg/connect/data/IcebergWriterResult.java +++ b/kafka-connect/kafka-connect/src/main/java/org/apache/iceberg/connect/data/IcebergWriterResult.java @@ -21,7 +21,6 @@ import java.util.List; import org.apache.iceberg.DataFile; import org.apache.iceberg.DeleteFile; -import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.connect.events.TableReference; import org.apache.iceberg.types.Types.StructType; @@ -43,35 +42,10 @@ public IcebergWriterResult( this.partitionStruct = partitionStruct; } - /** - * @deprecated since 1.11.0, will be removed in 1.12.0; use {@link - * IcebergWriterResult#IcebergWriterResult(TableReference, List, List, StructType)} instead - */ - @Deprecated - public IcebergWriterResult( - TableIdentifier tableIdentifier, - List dataFiles, - List deleteFiles, - StructType partitionStruct) { - this.tableReference = TableReference.of("unknown", tableIdentifier); - this.dataFiles = dataFiles; - this.deleteFiles = deleteFiles; - this.partitionStruct = partitionStruct; - } - public TableReference tableReference() { return tableReference; } - /** - * @deprecated since 1.11.0, will be removed in 1.12.0; use {@code tableReference().identifier()} - * instead - */ - @Deprecated - public TableIdentifier tableIdentifier() { - return tableReference.identifier(); - } - public List dataFiles() { return dataFiles; } diff --git a/kafka-connect/kafka-connect/src/test/java/org/apache/iceberg/connect/channel/TestWorker.java b/kafka-connect/kafka-connect/src/test/java/org/apache/iceberg/connect/channel/TestWorker.java index 6cd5c0c86eab..641b812177bc 100644 --- a/kafka-connect/kafka-connect/src/test/java/org/apache/iceberg/connect/channel/TestWorker.java +++ b/kafka-connect/kafka-connect/src/test/java/org/apache/iceberg/connect/channel/TestWorker.java @@ -37,6 +37,7 @@ import org.apache.iceberg.connect.events.Event; import org.apache.iceberg.connect.events.PayloadType; import org.apache.iceberg.connect.events.StartCommit; +import org.apache.iceberg.connect.events.TableReference; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; @@ -67,7 +68,7 @@ public void testSave() { IcebergWriterResult writeResult = new IcebergWriterResult( - TableIdentifier.parse(TABLE_NAME), + TableReference.of("unknown", TableIdentifier.parse(TABLE_NAME), null), ImmutableList.of(EventTestUtil.createDataFile()), ImmutableList.of(), StructType.of()); diff --git a/kafka-connect/kafka-connect/src/test/java/org/apache/iceberg/connect/data/TestSinkWriter.java b/kafka-connect/kafka-connect/src/test/java/org/apache/iceberg/connect/data/TestSinkWriter.java index 09f7a373d5f2..008d9b5a4bed 100644 --- a/kafka-connect/kafka-connect/src/test/java/org/apache/iceberg/connect/data/TestSinkWriter.java +++ b/kafka-connect/kafka-connect/src/test/java/org/apache/iceberg/connect/data/TestSinkWriter.java @@ -38,6 +38,7 @@ import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.connect.IcebergSinkConfig; import org.apache.iceberg.connect.TableSinkConfig; +import org.apache.iceberg.connect.events.TableReference; import org.apache.iceberg.inmemory.InMemoryCatalog; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; @@ -163,7 +164,7 @@ public void testOffsetTrackedByOriginalTopicPartition() { IcebergWriterResult writeResult = new IcebergWriterResult( - TableIdentifier.parse(TABLE_NAME), + TableReference.of("unknown", TableIdentifier.parse(TABLE_NAME), null), ImmutableList.of(mock(DataFile.class)), ImmutableList.of(), Types.StructType.of()); @@ -239,7 +240,7 @@ private List sinkWriterTest( Map value, IcebergSinkConfig config) { IcebergWriterResult writeResult = new IcebergWriterResult( - TableIdentifier.parse(TABLE_NAME), + TableReference.of("unknown", TableIdentifier.parse(TABLE_NAME), null), ImmutableList.of(mock(DataFile.class)), ImmutableList.of(), Types.StructType.of()); diff --git a/parquet/src/main/java/org/apache/iceberg/data/parquet/BaseParquetReaders.java b/parquet/src/main/java/org/apache/iceberg/data/parquet/BaseParquetReaders.java index b1fd8f43a578..eae620e38e0a 100644 --- a/parquet/src/main/java/org/apache/iceberg/data/parquet/BaseParquetReaders.java +++ b/parquet/src/main/java/org/apache/iceberg/data/parquet/BaseParquetReaders.java @@ -78,26 +78,8 @@ protected ParquetValueReader createReader( } } - /** - * @deprecated will be removed in 1.12.0. Subclasses should override {@link - * #createStructReader(List, Types.StructType, Integer)} instead - */ - @Deprecated - protected ParquetValueReader createStructReader( - List> fieldReaders, Types.StructType structType) { - throw new UnsupportedOperationException( - "Deprecated method is not used in this implementation, only createStructReader(list, Types.Struct, Integer) should be used"); - } - - /** - * This method can be overridden to provide a custom implementation which also uses the fieldId of - * the Schema when creating the struct reader - */ - protected ParquetValueReader createStructReader( - List> fieldReaders, Types.StructType structType, Integer fieldId) { - // Fallback to the signature without fieldId if not overridden - return createStructReader(fieldReaders, structType); - } + protected abstract ParquetValueReader createStructReader( + List> fieldReaders, Types.StructType structType, Integer fieldId); protected abstract ParquetValueReader fixedReader(ColumnDescriptor desc); diff --git a/parquet/src/main/java/org/apache/iceberg/parquet/Parquet.java b/parquet/src/main/java/org/apache/iceberg/parquet/Parquet.java index f02974d6e79c..e45db652256c 100644 --- a/parquet/src/main/java/org/apache/iceberg/parquet/Parquet.java +++ b/parquet/src/main/java/org/apache/iceberg/parquet/Parquet.java @@ -75,7 +75,6 @@ import org.apache.iceberg.SchemaParser; import org.apache.iceberg.SortOrder; import org.apache.iceberg.StructLike; -import org.apache.iceberg.SystemConfigs; import org.apache.iceberg.Table; import org.apache.iceberg.avro.AvroSchemaUtil; import org.apache.iceberg.data.parquet.GenericParquetWriter; @@ -1489,10 +1488,8 @@ public CloseableIterable build() { NameMapping mapping; if (nameMapping != null) { mapping = nameMapping; - } else if (SystemConfigs.NETFLIX_UNSAFE_PARQUET_ID_FALLBACK_ENABLED.value()) { - mapping = null; } else { - mapping = NameMapping.empty(); + mapping = null; } Function> batchedFunc = diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkFileWriterFactory.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkFileWriterFactory.java index 39110f0b0597..5f13b8aac45b 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkFileWriterFactory.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkFileWriterFactory.java @@ -166,10 +166,7 @@ public PositionDeleteWriter newPositionDeleteWriter( } else { LOG.warn("Position deletes with deleted rows are deprecated and will be removed in 1.12.0."); Map properties = table == null ? ImmutableMap.of() : table.properties(); - MetricsConfig metricsConfig = - table == null - ? MetricsConfig.forPositionDelete() - : MetricsConfig.forPositionDelete(table); + MetricsConfig metricsConfig = MetricsConfig.forPositionDelete(); try { return switch (deleteFormat) { diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDVWriters.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDVWriters.java index dfc693d3094d..ac6e62524768 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDVWriters.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDVWriters.java @@ -44,7 +44,6 @@ protected FileWriterFactory newWriterFactory( .deleteFileFormat(dataFormat()) .equalityFieldIds(ArrayUtil.toIntArray(equalityFieldIds)) .equalityDeleteRowSchema(equalityDeleteRowSchema) - .positionDeleteRowSchema(positionDeleteRowSchema) .build(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkPartitioningWriters.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkPartitioningWriters.java index 979abd21e7f7..e47fc53ac8bf 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkPartitioningWriters.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkPartitioningWriters.java @@ -44,7 +44,6 @@ protected FileWriterFactory newWriterFactory( .deleteFileFormat(format()) .equalityFieldIds(ArrayUtil.toIntArray(equalityFieldIds)) .equalityDeleteRowSchema(equalityDeleteRowSchema) - .positionDeleteRowSchema(positionDeleteRowSchema) .build(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkRollingFileWriters.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkRollingFileWriters.java index 5ebeafcb8cef..ed2158727d86 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkRollingFileWriters.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkRollingFileWriters.java @@ -41,7 +41,6 @@ protected FileWriterFactory newWriterFactory( .deleteFileFormat(format()) .equalityFieldIds(ArrayUtil.toIntArray(equalityFieldIds)) .equalityDeleteRowSchema(equalityDeleteRowSchema) - .positionDeleteRowSchema(positionDeleteRowSchema) .build(); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkWriterMetrics.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkWriterMetrics.java index 06ecc20c2fc3..c6d7418a9061 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkWriterMetrics.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkWriterMetrics.java @@ -18,6 +18,10 @@ */ package org.apache.iceberg.spark.source; +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.ByteBuffer; +import java.util.Map; import org.apache.iceberg.FileFormat; import org.apache.iceberg.Table; import org.apache.iceberg.io.FileWriterFactory; @@ -38,7 +42,6 @@ protected FileWriterFactory newWriterFactory(Table sourceTable) { .dataSchema(sourceTable.schema()) .dataFileFormat(fileFormat) .deleteFileFormat(fileFormat) - .positionDeleteRowSchema(sourceTable.schema()) .build(); } @@ -64,4 +67,14 @@ protected InternalRow toGenericRow(int value, int repeated) { } return row; } + + @Override + protected void checkRowStatistics(Map bounds) { + assertThat(bounds).hasSize(2); + } + + @Override + protected void checkNotExistingRowStatistics(Map bounds) { + assertThat(bounds).isNull(); + } } diff --git a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkSchemaUtil.java b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkSchemaUtil.java index fcf5fbeb2acb..1338e712fda3 100644 --- a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkSchemaUtil.java +++ b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkSchemaUtil.java @@ -187,25 +187,6 @@ public static Schema convert(Schema baseSchema, StructType sparkType, boolean ca return SparkFixupTypes.fixup(schema, baseSchema); } - /** - * Convert a Spark {@link StructType struct} to a {@link Schema} based on the given schema. - * - *

This conversion will assign new ids for fields that are not found in the base schema. - * - *

Data types, field order, and nullability will match the spark type. This conversion may - * return a schema that is not compatible with base schema. - * - * @param baseSchema a Schema on which conversion is based - * @param sparkType a Spark StructType - * @return the equivalent Schema - * @throws IllegalArgumentException if the type cannot be converted or there are missing ids - * @deprecated since 1.11.0, will be removed in 1.12.0 - */ - @Deprecated - public static Schema convertWithFreshIds(Schema baseSchema, StructType sparkType) { - return convertWithFreshIds(baseSchema, sparkType, true); - } - /** * Convert a Spark {@link StructType struct} to a {@link Schema} based on the given schema. * @@ -251,32 +232,6 @@ public static Schema prune(Schema schema, StructType requestedType) { .fields()); } - /** - * Prune columns from a {@link Schema} using a {@link StructType Spark type} projection. - * - *

This requires that the Spark type is a projection of the Schema. Nullability and types must - * match. - * - *

The filters list of {@link Expression} is used to ensure that columns referenced by filters - * are projected. - * - * @param schema a Schema - * @param requestedType a projection of the Spark representation of the Schema - * @param filters a list of filters - * @return a Schema corresponding to the Spark projection - * @throws IllegalArgumentException if the Spark type does not match the Schema - * @deprecated since 1.11.0, will be removed in 1.12.0 - */ - @Deprecated - public static Schema prune(Schema schema, StructType requestedType, List filters) { - Set filterRefs = Binder.boundReferences(schema.asStruct(), filters, true); - return new Schema( - TypeUtil.visit(schema, new PruneColumnsWithoutReordering(requestedType, filterRefs)) - .asNestedType() - .asStructType() - .fields()); - } - /** * Prune columns from a {@link Schema} using a {@link StructType Spark type} projection. * diff --git a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkTableUtil.java b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkTableUtil.java index 0e9edac3fbd5..0b74edd67412 100644 --- a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkTableUtil.java +++ b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/SparkTableUtil.java @@ -95,19 +95,13 @@ import org.apache.spark.sql.catalyst.TableIdentifier; import org.apache.spark.sql.catalyst.analysis.NoSuchDatabaseException; import org.apache.spark.sql.catalyst.analysis.NoSuchTableException; -import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute; import org.apache.spark.sql.catalyst.catalog.CatalogTable; import org.apache.spark.sql.catalyst.catalog.CatalogTablePartition; import org.apache.spark.sql.catalyst.catalog.SessionCatalog; -import org.apache.spark.sql.catalyst.expressions.Expression; -import org.apache.spark.sql.catalyst.expressions.NamedExpression; -import org.apache.spark.sql.catalyst.parser.ParseException; -import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan; import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation; import org.apache.spark.sql.util.CaseInsensitiveStringMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import scala.Function2; import scala.Option; import scala.Some; import scala.Tuple2; @@ -115,7 +109,6 @@ import scala.collection.immutable.Map$; import scala.collection.immutable.Seq; import scala.collection.mutable.Builder; -import scala.runtime.AbstractPartialFunction; /** * Java version of the original SparkTableUtil.scala @@ -132,62 +125,6 @@ public class SparkTableUtil { private SparkTableUtil() {} - /** - * Returns a DataFrame with a row for each partition in the table. - * - *

The DataFrame has 3 columns, partition key (a=1/b=2), partition location, and format (avro - * or parquet). - * - * @param spark a Spark session - * @param table a table name and (optional) database - * @return a DataFrame of the table's partitions - * @deprecated since 1.11.0, will be removed in 1.12.0 - */ - @Deprecated - public static Dataset partitionDF(SparkSession spark, String table) { - List partitions = getPartitions(spark, table); - return spark - .createDataFrame(partitions, SparkPartition.class) - .toDF("partition", "uri", "format"); - } - - /** - * Returns a DataFrame with a row for each partition that matches the specified 'expression'. - * - * @param spark a Spark session. - * @param table name of the table. - * @param expression The expression whose matching partitions are returned. - * @return a DataFrame of the table partitions. - * @deprecated since 1.11.0, will be removed in 1.12.0 - */ - @Deprecated - public static Dataset partitionDFByFilter( - SparkSession spark, String table, String expression) { - List partitions = getPartitionsByFilter(spark, table, expression); - return spark - .createDataFrame(partitions, SparkPartition.class) - .toDF("partition", "uri", "format"); - } - - /** - * Returns all partitions in the table. - * - * @param spark a Spark session - * @param table a table name and (optional) database - * @return all table's partitions - * @deprecated since 1.11.0, will be removed in 1.12.0 - */ - @Deprecated - public static List getPartitions(SparkSession spark, String table) { - try { - TableIdentifier tableIdent = spark.sessionState().sqlParser().parseTableIdentifier(table); - return getPartitions(spark, tableIdent, null); - } catch (ParseException e) { - throw SparkExceptionUtil.toUncheckedException( - e, "Unable to parse table identifier: %s", table); - } - } - /** * Returns all partitions in the table. * @@ -225,80 +162,6 @@ public static List getPartitions( } } - /** - * Returns partitions that match the specified 'predicate'. - * - * @param spark a Spark session - * @param table a table name and (optional) database - * @param predicate a predicate on partition columns - * @return matching table's partitions - * @deprecated since 1.11.0, will be removed in 1.12.0 - */ - @Deprecated - public static List getPartitionsByFilter( - SparkSession spark, String table, String predicate) { - TableIdentifier tableIdent; - try { - tableIdent = spark.sessionState().sqlParser().parseTableIdentifier(table); - } catch (ParseException e) { - throw SparkExceptionUtil.toUncheckedException( - e, "Unable to parse the table identifier: %s", table); - } - - Expression unresolvedPredicateExpr; - try { - unresolvedPredicateExpr = spark.sessionState().sqlParser().parseExpression(predicate); - } catch (ParseException e) { - throw SparkExceptionUtil.toUncheckedException( - e, "Unable to parse the predicate expression: %s", predicate); - } - - Expression resolvedPredicateExpr = resolveAttrs(spark, table, unresolvedPredicateExpr); - return getPartitionsByFilter(spark, tableIdent, resolvedPredicateExpr); - } - - /** - * Returns partitions that match the specified 'predicate'. - * - * @param spark a Spark session - * @param tableIdent a table identifier - * @param predicateExpr a predicate expression on partition columns - * @return matching table's partitions - * @deprecated since 1.11.0, will be removed in 1.12.0 - */ - @Deprecated - public static List getPartitionsByFilter( - SparkSession spark, TableIdentifier tableIdent, Expression predicateExpr) { - try { - SessionCatalog catalog = spark.sessionState().catalog(); - CatalogTable catalogTable = catalog.getTableMetadata(tableIdent); - - Expression resolvedPredicateExpr; - if (!predicateExpr.resolved()) { - resolvedPredicateExpr = resolveAttrs(spark, tableIdent.quotedString(), predicateExpr); - } else { - resolvedPredicateExpr = predicateExpr; - } - Seq predicates = - JavaConverters.collectionAsScalaIterableConverter(ImmutableList.of(resolvedPredicateExpr)) - .asScala() - .toIndexedSeq(); - - Seq partitions = - catalog.listPartitionsByFilter(tableIdent, predicates).toIndexedSeq(); - - return JavaConverters.seqAsJavaListConverter(partitions).asJava().stream() - .map(catalogPartition -> toSparkPartition(catalogPartition, catalogTable)) - .collect(Collectors.toList()); - } catch (NoSuchDatabaseException e) { - throw SparkExceptionUtil.toUncheckedException( - e, "Unknown table: %s. Database not found in catalog.", tableIdent); - } catch (NoSuchTableException e) { - throw SparkExceptionUtil.toUncheckedException( - e, "Unknown table: %s. Table not found in catalog.", tableIdent); - } - } - private static List listPartition( SparkPartition partition, PartitionSpec spec, @@ -344,31 +207,6 @@ private static SparkPartition toSparkPartition( return new SparkPartition(partitionSpec, uri, format); } - private static Expression resolveAttrs(SparkSession spark, String table, Expression expr) { - Function2 resolver = spark.sessionState().analyzer().resolver(); - LogicalPlan plan = spark.table(table).queryExecution().analyzed(); - return expr.transform( - new AbstractPartialFunction() { - @Override - public Expression apply(Expression attr) { - UnresolvedAttribute unresolvedAttribute = (UnresolvedAttribute) attr; - Option namedExpressionOption = - plan.resolve(unresolvedAttribute.nameParts(), resolver); - if (namedExpressionOption.isDefined()) { - return (Expression) namedExpressionOption.get(); - } else { - throw new IllegalArgumentException( - String.format("Could not resolve %s using columns: %s", attr, plan.output())); - } - } - - @Override - public boolean isDefinedAt(Expression attr) { - return attr instanceof UnresolvedAttribute; - } - }); - } - private static Iterator buildManifest( int formatVersion, Long snapshotId, @@ -406,61 +244,6 @@ private static Iterator buildManifest( } } - /** - * Import files from an existing Spark table to an Iceberg table. - * - *

The import uses the Spark session to get table metadata. It assumes no operation is going on - * the original and target table and thus is not thread-safe. - * - * @param spark a Spark session - * @param sourceTableIdent an identifier of the source Spark table - * @param targetTable an Iceberg table where to import the data - * @param stagingDir a staging directory to store temporary manifest files - * @param partitionFilter only import partitions whose values match those in the map, can be - * partially defined - * @param checkDuplicateFiles if true, throw exception if import results in a duplicate data file - * @deprecated since 1.11.0, will be removed in 1.12.0 - */ - @Deprecated - public static void importSparkTable( - SparkSession spark, - TableIdentifier sourceTableIdent, - Table targetTable, - String stagingDir, - Map partitionFilter, - boolean checkDuplicateFiles) { - importSparkTable( - spark, sourceTableIdent, targetTable, stagingDir, partitionFilter, checkDuplicateFiles, 1); - } - - /** - * Import files from an existing Spark table to an Iceberg table. - * - *

The import uses the Spark session to get table metadata. It assumes no operation is going on - * the original and target table and thus is not thread-safe. - * - * @param spark a Spark session - * @param sourceTableIdent an identifier of the source Spark table - * @param targetTable an Iceberg table where to import the data - * @param stagingDir a staging directory to store temporary manifest files - * @param parallelism number of threads to use for file reading - * @deprecated since 1.11.0, will be removed in 1.12.0 - */ - @Deprecated - public static void importSparkTable( - SparkSession spark, - TableIdentifier sourceTableIdent, - Table targetTable, - String stagingDir, - int parallelism) { - importSparkTable( - spark, - sourceTableIdent, - targetTable, - stagingDir, - TableMigrationUtil.migrationService(parallelism)); - } - /** * Import files from an existing Spark table to an Iceberg table. * @@ -628,36 +411,6 @@ public static void importSparkTable( } } - /** - * Import files from an existing Spark table to an Iceberg table. - * - *

The import uses the Spark session to get table metadata. It assumes no operation is going on - * the original and target table and thus is not thread-safe. - * - * @param spark a Spark session - * @param sourceTableIdent an identifier of the source Spark table - * @param targetTable an Iceberg table where to import the data - * @param stagingDir a staging directory to store temporary manifest files - * @param checkDuplicateFiles if true, throw exception if import results in a duplicate data file - * @deprecated since 1.11.0, will be removed in 1.12.0 - */ - @Deprecated - public static void importSparkTable( - SparkSession spark, - TableIdentifier sourceTableIdent, - Table targetTable, - String stagingDir, - boolean checkDuplicateFiles) { - importSparkTable( - spark, - sourceTableIdent, - targetTable, - stagingDir, - Collections.emptyMap(), - checkDuplicateFiles, - 1); - } - /** * Import files from an existing Spark table to an Iceberg table. * @@ -733,28 +486,6 @@ private static void importUnpartitionedSparkTable( } } - /** - * Import files from given partitions to an Iceberg table. - * - * @param spark a Spark session - * @param partitions partitions to import - * @param targetTable an Iceberg table where to import the data - * @param spec a partition spec - * @param stagingDir a staging directory to store temporary manifest files - * @param checkDuplicateFiles if true, throw exception if import results in a duplicate data file - * @deprecated since 1.11.0, will be removed in 1.12.0 - */ - @Deprecated - public static void importSparkPartitions( - SparkSession spark, - List partitions, - Table targetTable, - PartitionSpec spec, - String stagingDir, - boolean checkDuplicateFiles) { - importSparkPartitions(spark, partitions, targetTable, spec, stagingDir, checkDuplicateFiles, 1); - } - /** * Import files from given partitions to an Iceberg table. * @@ -934,41 +665,6 @@ public static void importSparkPartitions( } } - /** - * Import files from given partitions to an Iceberg table. - * - * @param spark a Spark session - * @param partitions partitions to import - * @param targetTable an Iceberg table where to import the data - * @param spec a partition spec - * @param stagingDir a staging directory to store temporary manifest files - * @deprecated since 1.11.0, will be removed in 1.12.0 - */ - @Deprecated - public static void importSparkPartitions( - SparkSession spark, - List partitions, - Table targetTable, - PartitionSpec spec, - String stagingDir) { - importSparkPartitions(spark, partitions, targetTable, spec, stagingDir, false, 1); - } - - /** - * @deprecated since 1.11.0, will be removed in 1.12.0 - */ - @Deprecated - public static List filterPartitions( - List partitions, Map partitionFilter) { - if (partitionFilter.isEmpty()) { - return partitions; - } else { - return partitions.stream() - .filter(p -> p.getValues().entrySet().containsAll(partitionFilter.entrySet())) - .collect(Collectors.toList()); - } - } - private static void deleteManifests(FileIO io, List manifests) { CatalogUtil.deleteFiles(io, Lists.transform(manifests, ManifestFile::path), "manifests"); } diff --git a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkFileWriterFactory.java b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkFileWriterFactory.java index 39110f0b0597..5f13b8aac45b 100644 --- a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkFileWriterFactory.java +++ b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/source/SparkFileWriterFactory.java @@ -166,10 +166,7 @@ public PositionDeleteWriter newPositionDeleteWriter( } else { LOG.warn("Position deletes with deleted rows are deprecated and will be removed in 1.12.0."); Map properties = table == null ? ImmutableMap.of() : table.properties(); - MetricsConfig metricsConfig = - table == null - ? MetricsConfig.forPositionDelete() - : MetricsConfig.forPositionDelete(table); + MetricsConfig metricsConfig = MetricsConfig.forPositionDelete(); try { return switch (deleteFormat) { diff --git a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDVWriters.java b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDVWriters.java index dfc693d3094d..ac6e62524768 100644 --- a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDVWriters.java +++ b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDVWriters.java @@ -44,7 +44,6 @@ protected FileWriterFactory newWriterFactory( .deleteFileFormat(dataFormat()) .equalityFieldIds(ArrayUtil.toIntArray(equalityFieldIds)) .equalityDeleteRowSchema(equalityDeleteRowSchema) - .positionDeleteRowSchema(positionDeleteRowSchema) .build(); } diff --git a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkPartitioningWriters.java b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkPartitioningWriters.java index 979abd21e7f7..e47fc53ac8bf 100644 --- a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkPartitioningWriters.java +++ b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkPartitioningWriters.java @@ -44,7 +44,6 @@ protected FileWriterFactory newWriterFactory( .deleteFileFormat(format()) .equalityFieldIds(ArrayUtil.toIntArray(equalityFieldIds)) .equalityDeleteRowSchema(equalityDeleteRowSchema) - .positionDeleteRowSchema(positionDeleteRowSchema) .build(); } diff --git a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkRollingFileWriters.java b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkRollingFileWriters.java index 5ebeafcb8cef..ed2158727d86 100644 --- a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkRollingFileWriters.java +++ b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkRollingFileWriters.java @@ -41,7 +41,6 @@ protected FileWriterFactory newWriterFactory( .deleteFileFormat(format()) .equalityFieldIds(ArrayUtil.toIntArray(equalityFieldIds)) .equalityDeleteRowSchema(equalityDeleteRowSchema) - .positionDeleteRowSchema(positionDeleteRowSchema) .build(); } diff --git a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkWriterMetrics.java b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkWriterMetrics.java index 06ecc20c2fc3..c6d7418a9061 100644 --- a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkWriterMetrics.java +++ b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkWriterMetrics.java @@ -18,6 +18,10 @@ */ package org.apache.iceberg.spark.source; +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.ByteBuffer; +import java.util.Map; import org.apache.iceberg.FileFormat; import org.apache.iceberg.Table; import org.apache.iceberg.io.FileWriterFactory; @@ -38,7 +42,6 @@ protected FileWriterFactory newWriterFactory(Table sourceTable) { .dataSchema(sourceTable.schema()) .dataFileFormat(fileFormat) .deleteFileFormat(fileFormat) - .positionDeleteRowSchema(sourceTable.schema()) .build(); } @@ -64,4 +67,14 @@ protected InternalRow toGenericRow(int value, int repeated) { } return row; } + + @Override + protected void checkRowStatistics(Map bounds) { + assertThat(bounds).hasSize(2); + } + + @Override + protected void checkNotExistingRowStatistics(Map bounds) { + assertThat(bounds).isNull(); + } } diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkReadConf.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkReadConf.java index 8128babfa340..096061660843 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkReadConf.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkReadConf.java @@ -64,16 +64,6 @@ public SparkReadConf(SparkSession spark, Table table) { } public SparkReadConf(SparkSession spark, Table table, CaseInsensitiveStringMap options) { - this(spark, table, null, options); - } - - /** - * @deprecated since 1.11.0, will be removed in 1.12.0. Use {@link #SparkReadConf(SparkSession, - * Table, CaseInsensitiveStringMap)} instead. - */ - @Deprecated - public SparkReadConf( - SparkSession spark, Table table, String branch, CaseInsensitiveStringMap options) { this.spark = spark; this.table = table; this.confParser = new SparkConfParser(spark, table, options); diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkSchemaUtil.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkSchemaUtil.java index 4c3713d3fff3..9f299cb276ae 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkSchemaUtil.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkSchemaUtil.java @@ -188,25 +188,6 @@ public static Schema convert(Schema baseSchema, StructType sparkType, boolean ca return SparkFixupTypes.fixup(schema, baseSchema); } - /** - * Convert a Spark {@link StructType struct} to a {@link Schema} based on the given schema. - * - *

This conversion will assign new ids for fields that are not found in the base schema. - * - *

Data types, field order, and nullability will match the spark type. This conversion may - * return a schema that is not compatible with base schema. - * - * @param baseSchema a Schema on which conversion is based - * @param sparkType a Spark StructType - * @return the equivalent Schema - * @throws IllegalArgumentException if the type cannot be converted or there are missing ids - * @deprecated since 1.11.0, will be removed in 1.12.0 - */ - @Deprecated - public static Schema convertWithFreshIds(Schema baseSchema, StructType sparkType) { - return convertWithFreshIds(baseSchema, sparkType, true); - } - /** * Convert a Spark {@link StructType struct} to a {@link Schema} based on the given schema. * @@ -252,32 +233,6 @@ public static Schema prune(Schema schema, StructType requestedType) { .fields()); } - /** - * Prune columns from a {@link Schema} using a {@link StructType Spark type} projection. - * - *

This requires that the Spark type is a projection of the Schema. Nullability and types must - * match. - * - *

The filters list of {@link Expression} is used to ensure that columns referenced by filters - * are projected. - * - * @param schema a Schema - * @param requestedType a projection of the Spark representation of the Schema - * @param filters a list of filters - * @return a Schema corresponding to the Spark projection - * @throws IllegalArgumentException if the Spark type does not match the Schema - * @deprecated since 1.11.0, will be removed in 1.12.0 - */ - @Deprecated - public static Schema prune(Schema schema, StructType requestedType, List filters) { - Set filterRefs = Binder.boundReferences(schema.asStruct(), filters, true); - return new Schema( - TypeUtil.visit(schema, new PruneColumnsWithoutReordering(requestedType, filterRefs)) - .asNestedType() - .asStructType() - .fields()); - } - /** * Prune columns from a {@link Schema} using a {@link StructType Spark type} projection. * diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkTableUtil.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkTableUtil.java index 96499184cab3..d56ebd0b1df8 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkTableUtil.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkTableUtil.java @@ -94,19 +94,13 @@ import org.apache.spark.sql.catalyst.TableIdentifier; import org.apache.spark.sql.catalyst.analysis.NoSuchDatabaseException; import org.apache.spark.sql.catalyst.analysis.NoSuchTableException; -import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute; import org.apache.spark.sql.catalyst.catalog.CatalogTable; import org.apache.spark.sql.catalyst.catalog.CatalogTablePartition; import org.apache.spark.sql.catalyst.catalog.SessionCatalog; -import org.apache.spark.sql.catalyst.expressions.Expression; -import org.apache.spark.sql.catalyst.expressions.NamedExpression; -import org.apache.spark.sql.catalyst.parser.ParseException; -import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan; import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation; import org.apache.spark.sql.util.CaseInsensitiveStringMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import scala.Function2; import scala.Option; import scala.Some; import scala.Tuple2; @@ -114,7 +108,6 @@ import scala.collection.immutable.Map$; import scala.collection.immutable.Seq; import scala.collection.mutable.Builder; -import scala.runtime.AbstractPartialFunction; /** * Java version of the original SparkTableUtil.scala @@ -131,62 +124,6 @@ public class SparkTableUtil { private SparkTableUtil() {} - /** - * Returns a DataFrame with a row for each partition in the table. - * - *

The DataFrame has 3 columns, partition key (a=1/b=2), partition location, and format (avro - * or parquet). - * - * @param spark a Spark session - * @param table a table name and (optional) database - * @return a DataFrame of the table's partitions - * @deprecated since 1.11.0, will be removed in 1.12.0 - */ - @Deprecated - public static Dataset partitionDF(SparkSession spark, String table) { - List partitions = getPartitions(spark, table); - return spark - .createDataFrame(partitions, SparkPartition.class) - .toDF("partition", "uri", "format"); - } - - /** - * Returns a DataFrame with a row for each partition that matches the specified 'expression'. - * - * @param spark a Spark session. - * @param table name of the table. - * @param expression The expression whose matching partitions are returned. - * @return a DataFrame of the table partitions. - * @deprecated since 1.11.0, will be removed in 1.12.0 - */ - @Deprecated - public static Dataset partitionDFByFilter( - SparkSession spark, String table, String expression) { - List partitions = getPartitionsByFilter(spark, table, expression); - return spark - .createDataFrame(partitions, SparkPartition.class) - .toDF("partition", "uri", "format"); - } - - /** - * Returns all partitions in the table. - * - * @param spark a Spark session - * @param table a table name and (optional) database - * @return all table's partitions - * @deprecated since 1.11.0, will be removed in 1.12.0 - */ - @Deprecated - public static List getPartitions(SparkSession spark, String table) { - try { - TableIdentifier tableIdent = spark.sessionState().sqlParser().parseTableIdentifier(table); - return getPartitions(spark, tableIdent, null); - } catch (ParseException e) { - throw SparkExceptionUtil.toUncheckedException( - e, "Unable to parse table identifier: %s", table); - } - } - /** * Returns all partitions in the table. * @@ -224,80 +161,6 @@ public static List getPartitions( } } - /** - * Returns partitions that match the specified 'predicate'. - * - * @param spark a Spark session - * @param table a table name and (optional) database - * @param predicate a predicate on partition columns - * @return matching table's partitions - * @deprecated since 1.11.0, will be removed in 1.12.0 - */ - @Deprecated - public static List getPartitionsByFilter( - SparkSession spark, String table, String predicate) { - TableIdentifier tableIdent; - try { - tableIdent = spark.sessionState().sqlParser().parseTableIdentifier(table); - } catch (ParseException e) { - throw SparkExceptionUtil.toUncheckedException( - e, "Unable to parse the table identifier: %s", table); - } - - Expression unresolvedPredicateExpr; - try { - unresolvedPredicateExpr = spark.sessionState().sqlParser().parseExpression(predicate); - } catch (ParseException e) { - throw SparkExceptionUtil.toUncheckedException( - e, "Unable to parse the predicate expression: %s", predicate); - } - - Expression resolvedPredicateExpr = resolveAttrs(spark, table, unresolvedPredicateExpr); - return getPartitionsByFilter(spark, tableIdent, resolvedPredicateExpr); - } - - /** - * Returns partitions that match the specified 'predicate'. - * - * @param spark a Spark session - * @param tableIdent a table identifier - * @param predicateExpr a predicate expression on partition columns - * @return matching table's partitions - * @deprecated since 1.11.0, will be removed in 1.12.0 - */ - @Deprecated - public static List getPartitionsByFilter( - SparkSession spark, TableIdentifier tableIdent, Expression predicateExpr) { - try { - SessionCatalog catalog = spark.sessionState().catalog(); - CatalogTable catalogTable = catalog.getTableMetadata(tableIdent); - - Expression resolvedPredicateExpr; - if (!predicateExpr.resolved()) { - resolvedPredicateExpr = resolveAttrs(spark, tableIdent.quotedString(), predicateExpr); - } else { - resolvedPredicateExpr = predicateExpr; - } - Seq predicates = - JavaConverters.collectionAsScalaIterableConverter(ImmutableList.of(resolvedPredicateExpr)) - .asScala() - .toIndexedSeq(); - - Seq partitions = - catalog.listPartitionsByFilter(tableIdent, predicates).toIndexedSeq(); - - return JavaConverters.seqAsJavaListConverter(partitions).asJava().stream() - .map(catalogPartition -> toSparkPartition(catalogPartition, catalogTable)) - .collect(Collectors.toList()); - } catch (NoSuchDatabaseException e) { - throw SparkExceptionUtil.toUncheckedException( - e, "Unknown table: %s. Database not found in catalog.", tableIdent); - } catch (NoSuchTableException e) { - throw SparkExceptionUtil.toUncheckedException( - e, "Unknown table: %s. Table not found in catalog.", tableIdent); - } - } - private static List listPartition( SparkPartition partition, PartitionSpec spec, @@ -343,31 +206,6 @@ private static SparkPartition toSparkPartition( return new SparkPartition(partitionSpec, uri, format); } - private static Expression resolveAttrs(SparkSession spark, String table, Expression expr) { - Function2 resolver = spark.sessionState().analyzer().resolver(); - LogicalPlan plan = spark.table(table).queryExecution().analyzed(); - return expr.transform( - new AbstractPartialFunction() { - @Override - public Expression apply(Expression attr) { - UnresolvedAttribute unresolvedAttribute = (UnresolvedAttribute) attr; - Option namedExpressionOption = - plan.resolve(unresolvedAttribute.nameParts(), resolver); - if (namedExpressionOption.isDefined()) { - return (Expression) namedExpressionOption.get(); - } else { - throw new IllegalArgumentException( - String.format("Could not resolve %s using columns: %s", attr, plan.output())); - } - } - - @Override - public boolean isDefinedAt(Expression attr) { - return attr instanceof UnresolvedAttribute; - } - }); - } - private static Iterator buildManifest( int formatVersion, Long snapshotId, @@ -405,61 +243,6 @@ private static Iterator buildManifest( } } - /** - * Import files from an existing Spark table to an Iceberg table. - * - *

The import uses the Spark session to get table metadata. It assumes no operation is going on - * the original and target table and thus is not thread-safe. - * - * @param spark a Spark session - * @param sourceTableIdent an identifier of the source Spark table - * @param targetTable an Iceberg table where to import the data - * @param stagingDir a staging directory to store temporary manifest files - * @param partitionFilter only import partitions whose values match those in the map, can be - * partially defined - * @param checkDuplicateFiles if true, throw exception if import results in a duplicate data file - * @deprecated since 1.11.0, will be removed in 1.12.0 - */ - @Deprecated - public static void importSparkTable( - SparkSession spark, - TableIdentifier sourceTableIdent, - Table targetTable, - String stagingDir, - Map partitionFilter, - boolean checkDuplicateFiles) { - importSparkTable( - spark, sourceTableIdent, targetTable, stagingDir, partitionFilter, checkDuplicateFiles, 1); - } - - /** - * Import files from an existing Spark table to an Iceberg table. - * - *

The import uses the Spark session to get table metadata. It assumes no operation is going on - * the original and target table and thus is not thread-safe. - * - * @param spark a Spark session - * @param sourceTableIdent an identifier of the source Spark table - * @param targetTable an Iceberg table where to import the data - * @param stagingDir a staging directory to store temporary manifest files - * @param parallelism number of threads to use for file reading - * @deprecated since 1.11.0, will be removed in 1.12.0 - */ - @Deprecated - public static void importSparkTable( - SparkSession spark, - TableIdentifier sourceTableIdent, - Table targetTable, - String stagingDir, - int parallelism) { - importSparkTable( - spark, - sourceTableIdent, - targetTable, - stagingDir, - TableMigrationUtil.migrationService(parallelism)); - } - /** * Import files from an existing Spark table to an Iceberg table. * @@ -627,36 +410,6 @@ public static void importSparkTable( } } - /** - * Import files from an existing Spark table to an Iceberg table. - * - *

The import uses the Spark session to get table metadata. It assumes no operation is going on - * the original and target table and thus is not thread-safe. - * - * @param spark a Spark session - * @param sourceTableIdent an identifier of the source Spark table - * @param targetTable an Iceberg table where to import the data - * @param stagingDir a staging directory to store temporary manifest files - * @param checkDuplicateFiles if true, throw exception if import results in a duplicate data file - * @deprecated since 1.11.0, will be removed in 1.12.0 - */ - @Deprecated - public static void importSparkTable( - SparkSession spark, - TableIdentifier sourceTableIdent, - Table targetTable, - String stagingDir, - boolean checkDuplicateFiles) { - importSparkTable( - spark, - sourceTableIdent, - targetTable, - stagingDir, - Collections.emptyMap(), - checkDuplicateFiles, - 1); - } - /** * Import files from an existing Spark table to an Iceberg table. * @@ -732,28 +485,6 @@ private static void importUnpartitionedSparkTable( } } - /** - * Import files from given partitions to an Iceberg table. - * - * @param spark a Spark session - * @param partitions partitions to import - * @param targetTable an Iceberg table where to import the data - * @param spec a partition spec - * @param stagingDir a staging directory to store temporary manifest files - * @param checkDuplicateFiles if true, throw exception if import results in a duplicate data file - * @deprecated since 1.11.0, will be removed in 1.12.0 - */ - @Deprecated - public static void importSparkPartitions( - SparkSession spark, - List partitions, - Table targetTable, - PartitionSpec spec, - String stagingDir, - boolean checkDuplicateFiles) { - importSparkPartitions(spark, partitions, targetTable, spec, stagingDir, checkDuplicateFiles, 1); - } - /** * Import files from given partitions to an Iceberg table. * @@ -933,41 +664,6 @@ public static void importSparkPartitions( } } - /** - * Import files from given partitions to an Iceberg table. - * - * @param spark a Spark session - * @param partitions partitions to import - * @param targetTable an Iceberg table where to import the data - * @param spec a partition spec - * @param stagingDir a staging directory to store temporary manifest files - * @deprecated since 1.11.0, will be removed in 1.12.0 - */ - @Deprecated - public static void importSparkPartitions( - SparkSession spark, - List partitions, - Table targetTable, - PartitionSpec spec, - String stagingDir) { - importSparkPartitions(spark, partitions, targetTable, spec, stagingDir, false, 1); - } - - /** - * @deprecated since 1.11.0, will be removed in 1.12.0 - */ - @Deprecated - public static List filterPartitions( - List partitions, Map partitionFilter) { - if (partitionFilter.isEmpty()) { - return partitions; - } else { - return partitions.stream() - .filter(p -> p.getValues().entrySet().containsAll(partitionFilter.entrySet())) - .collect(Collectors.toList()); - } - } - private static void deleteManifests(FileIO io, List manifests) { CatalogUtil.deleteFiles(io, Lists.transform(manifests, ManifestFile::path), "manifests"); } diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java index 80f93427805a..373273e537e4 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java @@ -94,20 +94,10 @@ public class SparkWriteConf { private final SparkConfParser confParser; public SparkWriteConf(SparkSession spark, Table table) { - this(spark, table, null, CaseInsensitiveStringMap.empty()); + this(spark, table, CaseInsensitiveStringMap.empty()); } public SparkWriteConf(SparkSession spark, Table table, CaseInsensitiveStringMap options) { - this(spark, table, null, options); - } - - /** - * @deprecated since 1.11.0, will be removed in 1.12.0. Use {@link #SparkWriteConf(SparkSession, - * Table, CaseInsensitiveStringMap)} instead. - */ - @Deprecated - public SparkWriteConf( - SparkSession spark, Table table, String branch, CaseInsensitiveStringMap options) { this.spark = spark; this.table = table; this.sessionConf = spark.conf(); diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkFileWriterFactory.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkFileWriterFactory.java index 39110f0b0597..5f13b8aac45b 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkFileWriterFactory.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/SparkFileWriterFactory.java @@ -166,10 +166,7 @@ public PositionDeleteWriter newPositionDeleteWriter( } else { LOG.warn("Position deletes with deleted rows are deprecated and will be removed in 1.12.0."); Map properties = table == null ? ImmutableMap.of() : table.properties(); - MetricsConfig metricsConfig = - table == null - ? MetricsConfig.forPositionDelete() - : MetricsConfig.forPositionDelete(table); + MetricsConfig metricsConfig = MetricsConfig.forPositionDelete(); try { return switch (deleteFormat) { diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDVWriters.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDVWriters.java index dfc693d3094d..ac6e62524768 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDVWriters.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDVWriters.java @@ -44,7 +44,6 @@ protected FileWriterFactory newWriterFactory( .deleteFileFormat(dataFormat()) .equalityFieldIds(ArrayUtil.toIntArray(equalityFieldIds)) .equalityDeleteRowSchema(equalityDeleteRowSchema) - .positionDeleteRowSchema(positionDeleteRowSchema) .build(); } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkPartitioningWriters.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkPartitioningWriters.java index 979abd21e7f7..e47fc53ac8bf 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkPartitioningWriters.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkPartitioningWriters.java @@ -44,7 +44,6 @@ protected FileWriterFactory newWriterFactory( .deleteFileFormat(format()) .equalityFieldIds(ArrayUtil.toIntArray(equalityFieldIds)) .equalityDeleteRowSchema(equalityDeleteRowSchema) - .positionDeleteRowSchema(positionDeleteRowSchema) .build(); } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkRollingFileWriters.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkRollingFileWriters.java index 5ebeafcb8cef..ed2158727d86 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkRollingFileWriters.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkRollingFileWriters.java @@ -41,7 +41,6 @@ protected FileWriterFactory newWriterFactory( .deleteFileFormat(format()) .equalityFieldIds(ArrayUtil.toIntArray(equalityFieldIds)) .equalityDeleteRowSchema(equalityDeleteRowSchema) - .positionDeleteRowSchema(positionDeleteRowSchema) .build(); } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkWriterMetrics.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkWriterMetrics.java index 06ecc20c2fc3..c6d7418a9061 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkWriterMetrics.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkWriterMetrics.java @@ -18,6 +18,10 @@ */ package org.apache.iceberg.spark.source; +import static org.assertj.core.api.Assertions.assertThat; + +import java.nio.ByteBuffer; +import java.util.Map; import org.apache.iceberg.FileFormat; import org.apache.iceberg.Table; import org.apache.iceberg.io.FileWriterFactory; @@ -38,7 +42,6 @@ protected FileWriterFactory newWriterFactory(Table sourceTable) { .dataSchema(sourceTable.schema()) .dataFileFormat(fileFormat) .deleteFileFormat(fileFormat) - .positionDeleteRowSchema(sourceTable.schema()) .build(); } @@ -64,4 +67,14 @@ protected InternalRow toGenericRow(int value, int repeated) { } return row; } + + @Override + protected void checkRowStatistics(Map bounds) { + assertThat(bounds).hasSize(2); + } + + @Override + protected void checkNotExistingRowStatistics(Map bounds) { + assertThat(bounds).isNull(); + } }