Skip to content

Spark: Trim row-level test parameter rows from 6 to 3 #16549

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
stevenzwu wants to merge 5 commits into
base: main
Choose a base branch
from
Open

Spark: Trim row-level test parameter rows from 6 to 3 #16549

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ef563ee
Spark: Trim row-level test parameter rows from 6 to 3
stevenzwu May 24, 2026
be47c2e
Replace non-session REST row with second testhive row
stevenzwu May 24, 2026
5864e06
Spark: Fix TestBase.move() to handle locations without URI schemes
stevenzwu May 24, 2026
503d290
Build: Fix REST catalog test fixture warehouse path scheme
stevenzwu May 24, 2026
6f4ed58
Spark: Drop REST-only path workaround in TestRewriteTablePathProcedure
stevenzwu May 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion open-api/src/testFixtures/java/org/apache/iceberg/rest/RESTCatalogServer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ private CatalogContext initializeBackendCatalog() throws IOException {
if (warehouseLocation == null) {
File tmp = java.nio.file.Files.createTempDirectory("iceberg_warehouse").toFile();
tmp.deleteOnExit();
warehouseLocation = new File(tmp, "iceberg_data").getAbsolutePath();
warehouseLocation = new File(tmp, "iceberg_data").toURI().toString();
Copy link
Copy Markdown
Contributor Author

@stevenzwu stevenzwu May 24, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Either we fix the RESTCatalogServer and add the file:// scheme in in the warehouse location, or we have to handle the path with and without scheme in read.

catalogProperties.put(CatalogProperties.WAREHOUSE_LOCATION, warehouseLocation);

LOG.info("No warehouse location set. Defaulting to temp location: {}", warehouseLocation);
Expand Down
70 changes: 5 additions & 65 deletions ...ns/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,10 @@
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ThreadLocalRandom;
import java.util.stream.Collectors;
import org.apache.iceberg.CatalogProperties;
import org.apache.iceberg.DataFile;
import org.apache.iceberg.FileFormat;
import org.apache.iceberg.Files;
Expand Down Expand Up @@ -83,8 +82,6 @@
@ExtendWith(ParameterizedTestExtension.class)
public abstract class SparkRowLevelOperationsTestBase extends ExtensionsTestBase {

private static final Random RANDOM = ThreadLocalRandom.current();

@Parameter(index = 3)
protected FileFormat fileFormat;

Expand Down Expand Up @@ -135,79 +132,22 @@ public static Object[][] parameters() {
"default-namespace", "default"),
FileFormat.PARQUET,
true,
WRITE_DISTRIBUTION_MODE_NONE,
false,
"test",
DISTRIBUTED,
2
},
{
"testhadoop",
SparkCatalog.class.getName(),
ImmutableMap.of("type", "hadoop"),
FileFormat.PARQUET,
RANDOM.nextBoolean(),
WRITE_DISTRIBUTION_MODE_HASH,
true,
null,
LOCAL,
2
},
{
"spark_catalog",
SparkSessionCatalog.class.getName(),
ImmutableMap.of(
"type", "hive",
"default-namespace", "default",
"clients", "1",
"parquet-enabled", "false",
"cache-enabled",
"false" // Spark will delete tables using v1, leaving the cache out of sync
),
FileFormat.AVRO,
false,
WRITE_DISTRIBUTION_MODE_RANGE,
false,
"test",
null,
DISTRIBUTED,
2
},
{
"testhadoop",
SparkCatalog.class.getName(),
ImmutableMap.of("type", "hadoop"),
FileFormat.PARQUET,
true,
WRITE_DISTRIBUTION_MODE_HASH,
true,
null,
LOCAL,
3
},
{
"testhadoop",
SparkCatalog.class.getName(),
ImmutableMap.of("type", "hadoop"),
FileFormat.PARQUET,
false,
WRITE_DISTRIBUTION_MODE_HASH,
true,
null,
LOCAL,
3
},
{
"spark_catalog",
SparkSessionCatalog.class.getName(),
ImmutableMap.of(
"type",
"hive",
"rest",
CatalogProperties.URI,
restCatalog.properties().get(CatalogProperties.URI),
"default-namespace",
"default",
"clients",
"1",
"parquet-enabled",
"false",
"cache-enabled",
"false" // Spark will delete tables using v1, leaving the cache out of sync
),
Expand Down
7 changes: 0 additions & 7 deletions ...ions/src/test/java/org/apache/iceberg/spark/extensions/TestRewriteTablePathProcedure.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@
import org.apache.iceberg.TableUtil;
import org.apache.iceberg.data.FileHelpers;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.spark.SparkCatalogConfig;
import org.apache.iceberg.util.Pair;
import org.apache.spark.sql.AnalysisException;
import org.junit.jupiter.api.AfterEach;
Expand Down Expand Up @@ -222,12 +221,6 @@ public void testRewriteTablePathWithManifestAndDeleteCounts() throws IOException

File file = new File(removePrefix(table.location()) + "/data/deletes.parquet");
String filePath = file.toURI().toString();
if (SparkCatalogConfig.REST.catalogName().equals(catalogName)) {
// We applied this special handling because the base path for
// matching the RESTCATALOG's Hive BaseLocation is represented
// in the form of an AbsolutePath.
filePath = file.getAbsolutePath().toString();
}

DeleteFile positionDeletes =
FileHelpers.writeDeleteFile(table, table.io().newOutputFile(filePath), rowsToDelete)
Expand Down
58 changes: 5 additions & 53 deletions ...ns/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,10 @@
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ThreadLocalRandom;
import java.util.stream.Collectors;
import org.apache.iceberg.CatalogProperties;
import org.apache.iceberg.DataFile;
import org.apache.iceberg.FileFormat;
import org.apache.iceberg.Files;
Expand Down Expand Up @@ -83,8 +82,6 @@
@ExtendWith(ParameterizedTestExtension.class)
public abstract class SparkRowLevelOperationsTestBase extends ExtensionsTestBase {

private static final Random RANDOM = ThreadLocalRandom.current();

@Parameter(index = 3)
protected FileFormat fileFormat;

Expand Down Expand Up @@ -135,67 +132,22 @@ public static Object[][] parameters() {
"default-namespace", "default"),
FileFormat.PARQUET,
true,
WRITE_DISTRIBUTION_MODE_NONE,
false,
"test",
DISTRIBUTED,
2
},
{
"testhadoop",
SparkCatalog.class.getName(),
ImmutableMap.of("type", "hadoop"),
FileFormat.PARQUET,
RANDOM.nextBoolean(),
WRITE_DISTRIBUTION_MODE_HASH,
true,
null,
LOCAL,
2
},
{
"spark_catalog",
SparkSessionCatalog.class.getName(),
ImmutableMap.of(
"type", "hive",
"default-namespace", "default",
"clients", "1",
"parquet-enabled", "false",
"cache-enabled",
"false" // Spark will delete tables using v1, leaving the cache out of sync
),
FileFormat.AVRO,
false,
WRITE_DISTRIBUTION_MODE_RANGE,
false,
"test",
null,
DISTRIBUTED,
2
},
{
"testhadoop",
SparkCatalog.class.getName(),
ImmutableMap.of("type", "hadoop"),
FileFormat.PARQUET,
RANDOM.nextBoolean(),
WRITE_DISTRIBUTION_MODE_HASH,
true,
null,
LOCAL,
3
},
{
"spark_catalog",
SparkSessionCatalog.class.getName(),
ImmutableMap.of(
"type",
"hive",
"rest",
CatalogProperties.URI,
restCatalog.properties().get(CatalogProperties.URI),
"default-namespace",
"default",
"clients",
"1",
"parquet-enabled",
"false",
"cache-enabled",
"false" // Spark will delete tables using v1, leaving the cache out of sync
),
Expand Down
7 changes: 0 additions & 7 deletions ...ions/src/test/java/org/apache/iceberg/spark/extensions/TestRewriteTablePathProcedure.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@
import org.apache.iceberg.TableUtil;
import org.apache.iceberg.data.FileHelpers;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.spark.SparkCatalogConfig;
import org.apache.iceberg.util.Pair;
import org.apache.spark.sql.AnalysisException;
import org.junit.jupiter.api.AfterEach;
Expand Down Expand Up @@ -224,12 +223,6 @@ public void testRewriteTablePathWithManifestAndDeleteCounts() throws IOException

File file = new File(removePrefix(table.location()) + "/data/deletes.parquet");
String filePath = file.toURI().toString();
if (SparkCatalogConfig.REST.catalogName().equals(catalogName)) {
// We applied this special handling because the base path for
// matching the RESTCATALOG's Hive BaseLocation is represented
// in the form of an AbsolutePath.
filePath = file.getAbsolutePath().toString();
}

DeleteFile positionDeletes =
FileHelpers.writeDeleteFile(table, table.io().newOutputFile(filePath), rowsToDelete)
Expand Down
58 changes: 5 additions & 53 deletions ...ns/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,10 @@
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ThreadLocalRandom;
import java.util.stream.Collectors;
import org.apache.iceberg.CatalogProperties;
import org.apache.iceberg.DataFile;
import org.apache.iceberg.FileFormat;
import org.apache.iceberg.Files;
Expand Down Expand Up @@ -83,8 +82,6 @@
@ExtendWith(ParameterizedTestExtension.class)
public abstract class SparkRowLevelOperationsTestBase extends ExtensionsTestBase {

private static final Random RANDOM = ThreadLocalRandom.current();

@Parameter(index = 3)
protected FileFormat fileFormat;

Expand Down Expand Up @@ -135,67 +132,22 @@ public static Object[][] parameters() {
"default-namespace", "default"),
FileFormat.PARQUET,
true,
WRITE_DISTRIBUTION_MODE_NONE,
false,
"test",
DISTRIBUTED,
2
},
{
"testhadoop",
SparkCatalog.class.getName(),
ImmutableMap.of("type", "hadoop"),
FileFormat.PARQUET,
RANDOM.nextBoolean(),
WRITE_DISTRIBUTION_MODE_HASH,
true,
null,
LOCAL,
2
},
{
"spark_catalog",
SparkSessionCatalog.class.getName(),
ImmutableMap.of(
"type", "hive",
"default-namespace", "default",
"clients", "1",
"parquet-enabled", "false",
"cache-enabled",
"false" // Spark will delete tables using v1, leaving the cache out of sync
),
FileFormat.AVRO,
false,
WRITE_DISTRIBUTION_MODE_RANGE,
false,
"test",
null,
DISTRIBUTED,
2
},
{
"testhadoop",
SparkCatalog.class.getName(),
ImmutableMap.of("type", "hadoop"),
FileFormat.PARQUET,
RANDOM.nextBoolean(),
WRITE_DISTRIBUTION_MODE_HASH,
true,
null,
LOCAL,
3
},
{
"spark_catalog",
SparkSessionCatalog.class.getName(),
ImmutableMap.of(
"type",
"hive",
"rest",
CatalogProperties.URI,
restCatalog.properties().get(CatalogProperties.URI),
"default-namespace",
"default",
"clients",
"1",
"parquet-enabled",
"false",
"cache-enabled",
"false" // Spark will delete tables using v1, leaving the cache out of sync
),
Expand Down
7 changes: 0 additions & 7 deletions ...ions/src/test/java/org/apache/iceberg/spark/extensions/TestRewriteTablePathProcedure.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@
import org.apache.iceberg.TableUtil;
import org.apache.iceberg.data.FileHelpers;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.spark.SparkCatalogConfig;
import org.apache.iceberg.util.Pair;
import org.apache.spark.sql.AnalysisException;
import org.junit.jupiter.api.AfterEach;
Expand Down Expand Up @@ -224,12 +223,6 @@ public void testRewriteTablePathWithManifestAndDeleteCounts() throws IOException

File file = new File(removePrefix(table.location()) + "/data/deletes.parquet");
String filePath = file.toURI().toString();
if (SparkCatalogConfig.REST.catalogName().equals(catalogName)) {
Copy link
Copy Markdown
Contributor Author

@stevenzwu stevenzwu May 24, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we don't need this special handling anymore after adding the file:// scheme to the path in the RESTCatalogServer.

// We applied this special handling because the base path for
// matching the RESTCATALOG's Hive BaseLocation is represented
// in the form of an AbsolutePath.
filePath = file.getAbsolutePath().toString();
}

DeleteFile positionDeletes =
FileHelpers.writeDeleteFile(table, table.io().newOutputFile(filePath), rowsToDelete)
Expand Down