diff --git a/open-api/src/testFixtures/java/org/apache/iceberg/rest/RESTCatalogServer.java b/open-api/src/testFixtures/java/org/apache/iceberg/rest/RESTCatalogServer.java index 2e4541b50b33..daed482c74a5 100644 --- a/open-api/src/testFixtures/java/org/apache/iceberg/rest/RESTCatalogServer.java +++ b/open-api/src/testFixtures/java/org/apache/iceberg/rest/RESTCatalogServer.java @@ -92,7 +92,7 @@ private CatalogContext initializeBackendCatalog() throws IOException { if (warehouseLocation == null) { File tmp = java.nio.file.Files.createTempDirectory("iceberg_warehouse").toFile(); tmp.deleteOnExit(); - warehouseLocation = new File(tmp, "iceberg_data").getAbsolutePath(); + warehouseLocation = new File(tmp, "iceberg_data").toURI().toString(); catalogProperties.put(CatalogProperties.WAREHOUSE_LOCATION, warehouseLocation); LOG.info("No warehouse location set. Defaulting to temp location: {}", warehouseLocation); diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java index 893f9931cfa2..72988ae0ed9e 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java @@ -46,11 +46,10 @@ import java.util.Arrays; import java.util.List; import java.util.Map; -import java.util.Random; import java.util.Set; import java.util.UUID; -import java.util.concurrent.ThreadLocalRandom; import java.util.stream.Collectors; +import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.DataFile; import org.apache.iceberg.FileFormat; import org.apache.iceberg.Files; @@ -83,8 +82,6 @@ @ExtendWith(ParameterizedTestExtension.class) public abstract class SparkRowLevelOperationsTestBase extends ExtensionsTestBase { - private static final Random RANDOM = ThreadLocalRandom.current(); - @Parameter(index = 3) protected FileFormat fileFormat; @@ -135,79 +132,22 @@ public static Object[][] parameters() { "default-namespace", "default"), FileFormat.PARQUET, true, - WRITE_DISTRIBUTION_MODE_NONE, - false, - "test", - DISTRIBUTED, - 2 - }, - { - "testhadoop", - SparkCatalog.class.getName(), - ImmutableMap.of("type", "hadoop"), - FileFormat.PARQUET, - RANDOM.nextBoolean(), WRITE_DISTRIBUTION_MODE_HASH, - true, - null, - LOCAL, - 2 - }, - { - "spark_catalog", - SparkSessionCatalog.class.getName(), - ImmutableMap.of( - "type", "hive", - "default-namespace", "default", - "clients", "1", - "parquet-enabled", "false", - "cache-enabled", - "false" // Spark will delete tables using v1, leaving the cache out of sync - ), - FileFormat.AVRO, false, - WRITE_DISTRIBUTION_MODE_RANGE, - false, - "test", + null, DISTRIBUTED, 2 }, - { - "testhadoop", - SparkCatalog.class.getName(), - ImmutableMap.of("type", "hadoop"), - FileFormat.PARQUET, - true, - WRITE_DISTRIBUTION_MODE_HASH, - true, - null, - LOCAL, - 3 - }, - { - "testhadoop", - SparkCatalog.class.getName(), - ImmutableMap.of("type", "hadoop"), - FileFormat.PARQUET, - false, - WRITE_DISTRIBUTION_MODE_HASH, - true, - null, - LOCAL, - 3 - }, { "spark_catalog", SparkSessionCatalog.class.getName(), ImmutableMap.of( "type", - "hive", + "rest", + CatalogProperties.URI, + restCatalog.properties().get(CatalogProperties.URI), "default-namespace", "default", - "clients", - "1", - "parquet-enabled", - "false", "cache-enabled", "false" // Spark will delete tables using v1, leaving the cache out of sync ), diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRewriteTablePathProcedure.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRewriteTablePathProcedure.java index ceb3077c5670..996fb2636ada 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRewriteTablePathProcedure.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRewriteTablePathProcedure.java @@ -35,7 +35,6 @@ import org.apache.iceberg.TableUtil; import org.apache.iceberg.data.FileHelpers; import org.apache.iceberg.relocated.com.google.common.collect.Lists; -import org.apache.iceberg.spark.SparkCatalogConfig; import org.apache.iceberg.util.Pair; import org.apache.spark.sql.AnalysisException; import org.junit.jupiter.api.AfterEach; @@ -222,12 +221,6 @@ public void testRewriteTablePathWithManifestAndDeleteCounts() throws IOException File file = new File(removePrefix(table.location()) + "/data/deletes.parquet"); String filePath = file.toURI().toString(); - if (SparkCatalogConfig.REST.catalogName().equals(catalogName)) { - // We applied this special handling because the base path for - // matching the RESTCATALOG's Hive BaseLocation is represented - // in the form of an AbsolutePath. - filePath = file.getAbsolutePath().toString(); - } DeleteFile positionDeletes = FileHelpers.writeDeleteFile(table, table.io().newOutputFile(filePath), rowsToDelete) diff --git a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java index b5d641576314..72988ae0ed9e 100644 --- a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java +++ b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java @@ -46,11 +46,10 @@ import java.util.Arrays; import java.util.List; import java.util.Map; -import java.util.Random; import java.util.Set; import java.util.UUID; -import java.util.concurrent.ThreadLocalRandom; import java.util.stream.Collectors; +import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.DataFile; import org.apache.iceberg.FileFormat; import org.apache.iceberg.Files; @@ -83,8 +82,6 @@ @ExtendWith(ParameterizedTestExtension.class) public abstract class SparkRowLevelOperationsTestBase extends ExtensionsTestBase { - private static final Random RANDOM = ThreadLocalRandom.current(); - @Parameter(index = 3) protected FileFormat fileFormat; @@ -135,67 +132,22 @@ public static Object[][] parameters() { "default-namespace", "default"), FileFormat.PARQUET, true, - WRITE_DISTRIBUTION_MODE_NONE, - false, - "test", - DISTRIBUTED, - 2 - }, - { - "testhadoop", - SparkCatalog.class.getName(), - ImmutableMap.of("type", "hadoop"), - FileFormat.PARQUET, - RANDOM.nextBoolean(), WRITE_DISTRIBUTION_MODE_HASH, - true, - null, - LOCAL, - 2 - }, - { - "spark_catalog", - SparkSessionCatalog.class.getName(), - ImmutableMap.of( - "type", "hive", - "default-namespace", "default", - "clients", "1", - "parquet-enabled", "false", - "cache-enabled", - "false" // Spark will delete tables using v1, leaving the cache out of sync - ), - FileFormat.AVRO, false, - WRITE_DISTRIBUTION_MODE_RANGE, - false, - "test", + null, DISTRIBUTED, 2 }, - { - "testhadoop", - SparkCatalog.class.getName(), - ImmutableMap.of("type", "hadoop"), - FileFormat.PARQUET, - RANDOM.nextBoolean(), - WRITE_DISTRIBUTION_MODE_HASH, - true, - null, - LOCAL, - 3 - }, { "spark_catalog", SparkSessionCatalog.class.getName(), ImmutableMap.of( "type", - "hive", + "rest", + CatalogProperties.URI, + restCatalog.properties().get(CatalogProperties.URI), "default-namespace", "default", - "clients", - "1", - "parquet-enabled", - "false", "cache-enabled", "false" // Spark will delete tables using v1, leaving the cache out of sync ), diff --git a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRewriteTablePathProcedure.java b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRewriteTablePathProcedure.java index 78f6b80ac948..cab33c8b005c 100644 --- a/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRewriteTablePathProcedure.java +++ b/spark/v4.0/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRewriteTablePathProcedure.java @@ -35,7 +35,6 @@ import org.apache.iceberg.TableUtil; import org.apache.iceberg.data.FileHelpers; import org.apache.iceberg.relocated.com.google.common.collect.Lists; -import org.apache.iceberg.spark.SparkCatalogConfig; import org.apache.iceberg.util.Pair; import org.apache.spark.sql.AnalysisException; import org.junit.jupiter.api.AfterEach; @@ -224,12 +223,6 @@ public void testRewriteTablePathWithManifestAndDeleteCounts() throws IOException File file = new File(removePrefix(table.location()) + "/data/deletes.parquet"); String filePath = file.toURI().toString(); - if (SparkCatalogConfig.REST.catalogName().equals(catalogName)) { - // We applied this special handling because the base path for - // matching the RESTCATALOG's Hive BaseLocation is represented - // in the form of an AbsolutePath. - filePath = file.getAbsolutePath().toString(); - } DeleteFile positionDeletes = FileHelpers.writeDeleteFile(table, table.io().newOutputFile(filePath), rowsToDelete) diff --git a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java index b5d641576314..72988ae0ed9e 100644 --- a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java +++ b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java @@ -46,11 +46,10 @@ import java.util.Arrays; import java.util.List; import java.util.Map; -import java.util.Random; import java.util.Set; import java.util.UUID; -import java.util.concurrent.ThreadLocalRandom; import java.util.stream.Collectors; +import org.apache.iceberg.CatalogProperties; import org.apache.iceberg.DataFile; import org.apache.iceberg.FileFormat; import org.apache.iceberg.Files; @@ -83,8 +82,6 @@ @ExtendWith(ParameterizedTestExtension.class) public abstract class SparkRowLevelOperationsTestBase extends ExtensionsTestBase { - private static final Random RANDOM = ThreadLocalRandom.current(); - @Parameter(index = 3) protected FileFormat fileFormat; @@ -135,67 +132,22 @@ public static Object[][] parameters() { "default-namespace", "default"), FileFormat.PARQUET, true, - WRITE_DISTRIBUTION_MODE_NONE, - false, - "test", - DISTRIBUTED, - 2 - }, - { - "testhadoop", - SparkCatalog.class.getName(), - ImmutableMap.of("type", "hadoop"), - FileFormat.PARQUET, - RANDOM.nextBoolean(), WRITE_DISTRIBUTION_MODE_HASH, - true, - null, - LOCAL, - 2 - }, - { - "spark_catalog", - SparkSessionCatalog.class.getName(), - ImmutableMap.of( - "type", "hive", - "default-namespace", "default", - "clients", "1", - "parquet-enabled", "false", - "cache-enabled", - "false" // Spark will delete tables using v1, leaving the cache out of sync - ), - FileFormat.AVRO, false, - WRITE_DISTRIBUTION_MODE_RANGE, - false, - "test", + null, DISTRIBUTED, 2 }, - { - "testhadoop", - SparkCatalog.class.getName(), - ImmutableMap.of("type", "hadoop"), - FileFormat.PARQUET, - RANDOM.nextBoolean(), - WRITE_DISTRIBUTION_MODE_HASH, - true, - null, - LOCAL, - 3 - }, { "spark_catalog", SparkSessionCatalog.class.getName(), ImmutableMap.of( "type", - "hive", + "rest", + CatalogProperties.URI, + restCatalog.properties().get(CatalogProperties.URI), "default-namespace", "default", - "clients", - "1", - "parquet-enabled", - "false", "cache-enabled", "false" // Spark will delete tables using v1, leaving the cache out of sync ), diff --git a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRewriteTablePathProcedure.java b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRewriteTablePathProcedure.java index 78f6b80ac948..cab33c8b005c 100644 --- a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRewriteTablePathProcedure.java +++ b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRewriteTablePathProcedure.java @@ -35,7 +35,6 @@ import org.apache.iceberg.TableUtil; import org.apache.iceberg.data.FileHelpers; import org.apache.iceberg.relocated.com.google.common.collect.Lists; -import org.apache.iceberg.spark.SparkCatalogConfig; import org.apache.iceberg.util.Pair; import org.apache.spark.sql.AnalysisException; import org.junit.jupiter.api.AfterEach; @@ -224,12 +223,6 @@ public void testRewriteTablePathWithManifestAndDeleteCounts() throws IOException File file = new File(removePrefix(table.location()) + "/data/deletes.parquet"); String filePath = file.toURI().toString(); - if (SparkCatalogConfig.REST.catalogName().equals(catalogName)) { - // We applied this special handling because the base path for - // matching the RESTCATALOG's Hive BaseLocation is represented - // in the form of an AbsolutePath. - filePath = file.getAbsolutePath().toString(); - } DeleteFile positionDeletes = FileHelpers.writeDeleteFile(table, table.io().newOutputFile(filePath), rowsToDelete)