From 18fbcfe0b5c11dd143512295366b45d98c83aae1 Mon Sep 17 00:00:00 2001 From: ca61688 <206189192+ca61688@users.noreply.github.com> Date: Mon, 6 Jul 2026 09:00:01 +0000 Subject: [PATCH 01/50] 6593: Add CommandArguments to DeployNewInstance --- .../clients/deploy/DeployNewInstanceWrk.java | 245 ++++++++++++++++++ .../sleeper/clients/table/AddTableClient.java | 26 +- .../java/sleeper/clients/util/FileReader.java | 38 +++ 3 files changed, 288 insertions(+), 21 deletions(-) create mode 100644 java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstanceWrk.java create mode 100644 java/clients/src/main/java/sleeper/clients/util/FileReader.java diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstanceWrk.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstanceWrk.java new file mode 100644 index 00000000000..1711dbc74ab --- /dev/null +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstanceWrk.java @@ -0,0 +1,245 @@ +/* + * Copyright 2022-2026 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package sleeper.clients.deploy; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.regions.PartitionMetadata; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain; +import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.ecr.EcrClient; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.sts.StsClient; + +import sleeper.clients.table.AddTableClient; +import sleeper.clients.util.cdk.CdkCommand; +import sleeper.configuration.properties.S3InstanceProperties; +import sleeper.configuration.properties.S3TableProperties; +import sleeper.core.deploy.SleeperInstanceConfiguration; +import sleeper.core.properties.instance.InstanceProperties; +import sleeper.core.properties.model.SleeperInternalCdkApp; +import sleeper.core.properties.table.TableProperties; +import sleeper.core.util.cli.CommandArguments; +import sleeper.core.util.cli.CommandArgumentsException; +import sleeper.core.util.cli.CommandLineUsage; +import sleeper.core.util.cli.CommandOption; +import sleeper.statestore.StateStoreFactory; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; + +import static sleeper.core.properties.instance.CommonProperty.ID; +import static sleeper.core.properties.instance.CommonProperty.SUBNETS; +import static sleeper.core.properties.instance.CommonProperty.VPC_ID; + +public class DeployNewInstanceWrk { + private static final Logger LOGGER = LoggerFactory.getLogger(DeployNewInstance.class); + + private final DeployInstance deployInstance; + private final String accountName; + private final S3Client s3Client; + private final DynamoDbClient dynamoClient; + private final SleeperInstanceConfiguration deployInstanceConfiguration; + private final SleeperInternalCdkApp cdkApp; + private final boolean deployPaused; + + private DeployNewInstanceWrk(Builder builder) { + deployInstance = builder.deployInstance; + accountName = builder.accountName; + s3Client = builder.s3Client; + dynamoClient = builder.dynamoClient; + deployInstanceConfiguration = builder.deployInstanceConfiguration; + cdkApp = builder.cdkApp; + deployPaused = builder.deployPaused; + } + + public static Builder builder() { + return new Builder(); + } + + public static final CommandLineUsage USAGE = CommandLineUsage.builder() + .positionalArguments(List.of("instance-id")) + .positionalArguments(List.of("vpc")) + .positionalArguments(List.of("subnets")) + .options(List.of( + CommandOption.longOption("instance-properties"), + CommandOption.longOption("config-dir"), + CommandOption.longFlag("deployPaused"))) + .helpSummary("" + + "Deploys a new instance of Sleeper.\n" + + "Positional Argumemts:\n" + + "Instance ID, VPC, Subnets\n" + + "Optional Arguments\n" + + "--instance-properties \n" + + "Optional path to an instance properties file. If not set, default instance properties will be used.\n" + + "\n" + + "--config-dir \n" + + "Path to a directory containing instance.properties.") + .build(); + + public static Arguments readArguments(CommandArguments arguments) { + return new Arguments( + arguments.getString("instance-id"), + arguments.getString("vpc"), + arguments.getString("subnets"), + arguments.getOptionalString("instance-properties").map(Path::of).orElse(null), + arguments.getOptionalString("config-dir").map(Path::of).orElse(null), + arguments.isFlagSet("deployPaused")); + } + + public static void main(String[] rawArgs) throws IOException, InterruptedException { + Arguments args = CommandArguments.parseAndValidateOrExit(USAGE, rawArgs, a -> readArguments(a)); + + Path scriptsDirectory = Path.of(rawArgs[0]); + Path instancePropertiesFile = args.resolvePropertiesFile(); + boolean deployPaused = args.deployPaused(); + try (S3Client s3Client = S3Client.create(); + DynamoDbClient dynamoClient = DynamoDbClient.create(); + StsClient stsClient = StsClient.create(); + EcrClient ecrClient = EcrClient.create()) { + String accountName = stsClient.getCallerIdentity().account(); + Region region = DefaultAwsRegionProviderChain.builder().build().getRegion(); + PartitionMetadata partitionMetadata = PartitionMetadata.of(region); + + SleeperInstanceConfiguration config = SleeperInstanceConfiguration.fromLocalConfiguration(instancePropertiesFile); + + config.getInstanceProperties().set(ID, args.instanceId()); + config.getInstanceProperties().set(VPC_ID, args.vpcId()); + config.getInstanceProperties().set(SUBNETS, args.subnetIds()); + + builder() + .deployInstance(DeployInstance.fromScriptsDirectory(scriptsDirectory, accountName, region, partitionMetadata, s3Client, ecrClient)) + .accountName(accountName) + .s3Client(s3Client) + .dynamoClient(dynamoClient) + .deployInstanceConfiguration(config) + .deployPaused(deployPaused) + .cdkApp(SleeperInternalCdkApp.STANDARD) + .build().deploy(); + } + } + + public void deploy() throws IOException, InterruptedException { + deployInstanceConfiguration.validate(); + + deployInstance.deploy(DeployInstanceRequest.builder() + .instanceConfig(deployInstanceConfiguration) + .cdkCommand(deployPaused ? CdkCommand.deployNewPaused() : CdkCommand.deployNew()) + .cdkApp(cdkApp) + .build()); + + InstanceProperties instanceProperties = S3InstanceProperties.loadGivenAccountAndInstanceId(s3Client, accountName, deployInstanceConfiguration.getInstanceId()); + for (TableProperties tableProperties : deployInstanceConfiguration.getTableProperties()) { + LOGGER.info("Adding table " + tableProperties.getStatus()); + new AddTableClient(tableProperties, + S3TableProperties.createStore(instanceProperties, s3Client, dynamoClient), + StateStoreFactory.createProvider(instanceProperties, s3Client, dynamoClient)) + .run(); + } + LOGGER.info("Finished deployment of new instance"); + } + + public record Arguments( + String instanceId, + String vpcId, + String subnetIds, + Path propertiesFile, + Path configDir, + boolean deployPaused) { + + public Arguments { + if (instanceId == null) { + throw new CommandArgumentsException("instance-id must not be null"); + } + + if (vpcId == null) { + throw new CommandArgumentsException("vpcId must not be null"); + } + + if (subnetIds == null) { + throw new CommandArgumentsException("subnetIds must not be null"); + } + + if (propertiesFile == null && configDir == null) { + throw new CommandArgumentsException("Either --instance-properties or --config-dir must be provided"); + } + } + + public Path resolvePropertiesFile() { + return propertiesFile != null ? propertiesFile : configDir.resolve("instance.properties"); + } + } + + public static final class Builder { + private DeployInstance deployInstance; + private String accountName; + private S3Client s3Client; + private DynamoDbClient dynamoClient; + private SleeperInstanceConfiguration deployInstanceConfiguration; + private SleeperInternalCdkApp cdkApp; + private boolean deployPaused; + + private Builder() { + } + + public Builder deployInstance(DeployInstance deployInstance) { + this.deployInstance = deployInstance; + return this; + } + + public Builder accountName(String accountName) { + this.accountName = accountName; + return this; + } + + public Builder s3Client(S3Client s3Client) { + this.s3Client = s3Client; + return this; + } + + public Builder dynamoClient(DynamoDbClient dynamoClient) { + this.dynamoClient = dynamoClient; + return this; + } + + public Builder deployInstanceConfiguration(SleeperInstanceConfiguration deployInstanceConfiguration) { + this.deployInstanceConfiguration = deployInstanceConfiguration; + return this; + } + + public Builder cdkApp(SleeperInternalCdkApp cdkApp) { + this.cdkApp = cdkApp; + return this; + } + + public Builder deployPaused(boolean deployPaused) { + this.deployPaused = deployPaused; + return this; + } + + public DeployNewInstanceWrk build() { + return new DeployNewInstanceWrk(this); + } + + public void deployWithClients(S3Client s3Client, DynamoDbClient dynamoClient) throws IOException, InterruptedException { + s3Client(s3Client) + .dynamoClient(dynamoClient) + .build().deploy(); + } + } +} diff --git a/java/clients/src/main/java/sleeper/clients/table/AddTableClient.java b/java/clients/src/main/java/sleeper/clients/table/AddTableClient.java index 5df30181030..af2ba8e7ece 100644 --- a/java/clients/src/main/java/sleeper/clients/table/AddTableClient.java +++ b/java/clients/src/main/java/sleeper/clients/table/AddTableClient.java @@ -21,6 +21,7 @@ import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.sts.StsClient; +import sleeper.clients.util.FileReader; import sleeper.configuration.properties.S3InstanceProperties; import sleeper.configuration.properties.S3TableProperties; import sleeper.core.properties.PropertiesUtils; @@ -37,7 +38,6 @@ import sleeper.statestore.StateStoreFactory; import java.io.IOException; -import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; @@ -98,9 +98,9 @@ public static Arguments readArguments(CommandArguments arguments, FileReader fil Properties rawTableProperties; rawTableProperties = tablePropertiesFile.isPresent() - ? PropertiesUtils.loadProperties(readFile(files, tablePropertiesFile.get())) + ? PropertiesUtils.loadProperties(FileReader.readFile(files, tablePropertiesFile.get())) : configDir.isPresent() - ? PropertiesUtils.loadProperties(readFile(files, configDir.get().resolve("table.properties"))) + ? PropertiesUtils.loadProperties(FileReader.readFile(files, configDir.get().resolve("table.properties"))) : null; return new Arguments( @@ -132,9 +132,9 @@ public static void main(String[] rawArgs) throws IOException { } } - public static TableProperties createTablePropertiesWithLoaders(Arguments args, InstancePropertiesLoader instance, FileReader files) { + public static TableProperties createTablePropertiesWithLoaders(Arguments args, FileReader.InstancePropertiesLoader instance, FileReader files) { TableProperties tableProperties = createTableProperties(instance.load(args.instanceId()), args); - tableProperties.setSchema(new SchemaSerDe().fromJson(readFile(files, args.resolveSchemaFile()))); + tableProperties.setSchema(new SchemaSerDe().fromJson(FileReader.readFile(files, args.resolveSchemaFile()))); return tableProperties; } @@ -180,20 +180,4 @@ public Path resolveSchemaFile() { return schemaFile != null ? schemaFile : configDir.resolve("schema.json"); } } - - private static String readFile(FileReader reader, Path path) { - try { - return reader.readStringChecked(path); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - public interface InstancePropertiesLoader { - InstanceProperties load(String instanceId); - } - - public interface FileReader { - String readStringChecked(Path path) throws IOException; - } } diff --git a/java/clients/src/main/java/sleeper/clients/util/FileReader.java b/java/clients/src/main/java/sleeper/clients/util/FileReader.java new file mode 100644 index 00000000000..666cf726ad2 --- /dev/null +++ b/java/clients/src/main/java/sleeper/clients/util/FileReader.java @@ -0,0 +1,38 @@ +/* + * Copyright 2022-2026 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package sleeper.clients.util; + +import sleeper.core.properties.instance.InstanceProperties; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Path; + +public interface FileReader { + String readStringChecked(Path path) throws IOException; + + static String readFile(FileReader reader, Path path) { + try { + return reader.readStringChecked(path); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + interface InstancePropertiesLoader { + InstanceProperties load(String instanceId); + } +} From cc331739f1eddecd32891af62ab1d408b2339cdc Mon Sep 17 00:00:00 2001 From: ca61688 <206189192+ca61688@users.noreply.github.com> Date: Mon, 6 Jul 2026 09:41:37 +0000 Subject: [PATCH 02/50] 6593: Make deployNew script use new DeployNewInstance file --- .../clients/deploy/DeployNewInstanceWrk.java | 33 ++++++++++++------- scripts/deploy/deployNew.sh | 7 +--- 2 files changed, 22 insertions(+), 18 deletions(-) diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstanceWrk.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstanceWrk.java index 1711dbc74ab..19cdaea2ccb 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstanceWrk.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstanceWrk.java @@ -73,33 +73,37 @@ public static Builder builder() { } public static final CommandLineUsage USAGE = CommandLineUsage.builder() - .positionalArguments(List.of("instance-id")) - .positionalArguments(List.of("vpc")) - .positionalArguments(List.of("subnets")) + .systemArguments(List.of("scriptsDirectory")) + .positionalArguments(List.of("scriptsDirectory", "instance-id", "vpcId", "subnetIds")) .options(List.of( CommandOption.longOption("instance-properties"), CommandOption.longOption("config-dir"), - CommandOption.longFlag("deployPaused"))) + CommandOption.longFlag("paused"))) .helpSummary("" + "Deploys a new instance of Sleeper.\n" + - "Positional Argumemts:\n" + - "Instance ID, VPC, Subnets\n" + - "Optional Arguments\n" + + "\n" + "--instance-properties \n" + - "Optional path to an instance properties file. If not set, default instance properties will be used.\n" + + "Path to an instance properties file.\n" + + "One of --instance-properties and --config-dir must be set but not both.\n" + "\n" + "--config-dir \n" + - "Path to a directory containing instance.properties.") + "Path to a directory containing an instance.properties file.\n" + + "One of --instance-properties and --config-dir must be set but not both.\n" + + "\n" + + "--paused\n" + + "If set, the instance will be deployed paused. Periodic background processes will not run until " + + "the instance is manually resumed.") .build(); public static Arguments readArguments(CommandArguments arguments) { return new Arguments( + Path.of(arguments.getString("scriptsDirectory")), arguments.getString("instance-id"), - arguments.getString("vpc"), - arguments.getString("subnets"), + arguments.getString("vpcId"), + arguments.getString("subnetIds"), arguments.getOptionalString("instance-properties").map(Path::of).orElse(null), arguments.getOptionalString("config-dir").map(Path::of).orElse(null), - arguments.isFlagSet("deployPaused")); + arguments.isFlagSet("paused")); } public static void main(String[] rawArgs) throws IOException, InterruptedException { @@ -155,6 +159,7 @@ public void deploy() throws IOException, InterruptedException { } public record Arguments( + Path scriptsDirectory, String instanceId, String vpcId, String subnetIds, @@ -163,6 +168,10 @@ public record Arguments( boolean deployPaused) { public Arguments { + if (scriptsDirectory == null) { + throw new CommandArgumentsException("scriptsDirectory must not be null"); + } + if (instanceId == null) { throw new CommandArgumentsException("instance-id must not be null"); } diff --git a/scripts/deploy/deployNew.sh b/scripts/deploy/deployNew.sh index 990b14c3e8a..4597edbc4b2 100755 --- a/scripts/deploy/deployNew.sh +++ b/scripts/deploy/deployNew.sh @@ -16,12 +16,7 @@ set -e unset CDPATH -if [ "$#" -lt 3 ] || [ "$#" -gt 5 ]; then - echo "Usage: $0 " - exit 1 -fi - SCRIPTS_DIR=$(cd "$(dirname "$0")" && cd .. && pwd) VERSION=$(cat "${SCRIPTS_DIR}/templates/version.txt") -java -cp "${SCRIPTS_DIR}/jars/clients-${VERSION}-utility.jar" sleeper.clients.deploy.DeployNewInstance "${SCRIPTS_DIR}" "$@" +java -cp "${SCRIPTS_DIR}/jars/clients-${VERSION}-utility.jar" sleeper.clients.deploy.DeployNewInstanceWrk "${SCRIPTS_DIR}" "$@" From c74b7d4c459094b287ba9af1524c692581266b13 Mon Sep 17 00:00:00 2001 From: ca61688 <206189192+ca61688@users.noreply.github.com> Date: Mon, 6 Jul 2026 10:22:56 +0000 Subject: [PATCH 03/50] 6593: Move uses of DeployNewIsntance to new Class --- .../clients/deploy/DeployNewInstanceWrk.java | 91 +++---------------- .../drivers/cdk/DeployNewTestInstance.java | 14 +-- .../instance/AwsSleeperInstanceDriver.java | 12 +-- 3 files changed, 19 insertions(+), 98 deletions(-) diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstanceWrk.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstanceWrk.java index 19cdaea2ccb..5f15a43d45d 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstanceWrk.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstanceWrk.java @@ -58,18 +58,16 @@ public class DeployNewInstanceWrk { private final SleeperInternalCdkApp cdkApp; private final boolean deployPaused; - private DeployNewInstanceWrk(Builder builder) { - deployInstance = builder.deployInstance; - accountName = builder.accountName; - s3Client = builder.s3Client; - dynamoClient = builder.dynamoClient; - deployInstanceConfiguration = builder.deployInstanceConfiguration; - cdkApp = builder.cdkApp; - deployPaused = builder.deployPaused; - } - - public static Builder builder() { - return new Builder(); + public DeployNewInstanceWrk(DeployInstance deployInstance, String accountName, S3Client s3Client, + DynamoDbClient dynamoClient, SleeperInstanceConfiguration deployInstanceConfiguration, + SleeperInternalCdkApp cdkApp, boolean deployPaused) { + this.deployInstance = deployInstance; + this.accountName = accountName; + this.s3Client = s3Client; + this.dynamoClient = dynamoClient; + this.deployInstanceConfiguration = deployInstanceConfiguration; + this.cdkApp = cdkApp; + this.deployPaused = deployPaused; } public static final CommandLineUsage USAGE = CommandLineUsage.builder() @@ -126,15 +124,8 @@ public static void main(String[] rawArgs) throws IOException, InterruptedExcepti config.getInstanceProperties().set(VPC_ID, args.vpcId()); config.getInstanceProperties().set(SUBNETS, args.subnetIds()); - builder() - .deployInstance(DeployInstance.fromScriptsDirectory(scriptsDirectory, accountName, region, partitionMetadata, s3Client, ecrClient)) - .accountName(accountName) - .s3Client(s3Client) - .dynamoClient(dynamoClient) - .deployInstanceConfiguration(config) - .deployPaused(deployPaused) - .cdkApp(SleeperInternalCdkApp.STANDARD) - .build().deploy(); + new DeployNewInstanceWrk(DeployInstance.fromScriptsDirectory(scriptsDirectory, accountName, region, partitionMetadata, s3Client, ecrClient), + accountName, s3Client, dynamoClient, config, SleeperInternalCdkApp.STANDARD, deployPaused).deploy(); } } @@ -193,62 +184,4 @@ public Path resolvePropertiesFile() { return propertiesFile != null ? propertiesFile : configDir.resolve("instance.properties"); } } - - public static final class Builder { - private DeployInstance deployInstance; - private String accountName; - private S3Client s3Client; - private DynamoDbClient dynamoClient; - private SleeperInstanceConfiguration deployInstanceConfiguration; - private SleeperInternalCdkApp cdkApp; - private boolean deployPaused; - - private Builder() { - } - - public Builder deployInstance(DeployInstance deployInstance) { - this.deployInstance = deployInstance; - return this; - } - - public Builder accountName(String accountName) { - this.accountName = accountName; - return this; - } - - public Builder s3Client(S3Client s3Client) { - this.s3Client = s3Client; - return this; - } - - public Builder dynamoClient(DynamoDbClient dynamoClient) { - this.dynamoClient = dynamoClient; - return this; - } - - public Builder deployInstanceConfiguration(SleeperInstanceConfiguration deployInstanceConfiguration) { - this.deployInstanceConfiguration = deployInstanceConfiguration; - return this; - } - - public Builder cdkApp(SleeperInternalCdkApp cdkApp) { - this.cdkApp = cdkApp; - return this; - } - - public Builder deployPaused(boolean deployPaused) { - this.deployPaused = deployPaused; - return this; - } - - public DeployNewInstanceWrk build() { - return new DeployNewInstanceWrk(this); - } - - public void deployWithClients(S3Client s3Client, DynamoDbClient dynamoClient) throws IOException, InterruptedException { - s3Client(s3Client) - .dynamoClient(dynamoClient) - .build().deploy(); - } - } } diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java index ff6ba0519c8..2581fb5f53c 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java @@ -24,7 +24,7 @@ import software.amazon.awssdk.services.sts.StsClient; import sleeper.clients.deploy.DeployInstance; -import sleeper.clients.deploy.DeployNewInstance; +import sleeper.clients.deploy.DeployNewInstanceWrk; import sleeper.core.deploy.SleeperInstanceConfiguration; import sleeper.core.deploy.SleeperInstanceConfigurationFromTemplates; import sleeper.core.properties.model.SleeperInternalCdkApp; @@ -68,15 +68,9 @@ public static void main(String[] args) throws IOException, InterruptedException config.getInstanceProperties().set(ID, instanceId); config.getInstanceProperties().set(VPC_ID, vpcId); config.getInstanceProperties().set(SUBNETS, subnetIds); - DeployNewInstance.builder() - .deployInstance(DeployInstance.fromScriptsDirectory(scriptsDirectory, accountName, region, partitionMetadata, s3Client, ecrClient)) - .accountName(accountName) - .s3Client(s3Client) - .dynamoClient(dynamoClient) - .deployInstanceConfiguration(config) - .cdkApp(SleeperInternalCdkApp.DEMONSTRATION) - .deployPaused(deployPaused) - .build().deploy(); + new DeployNewInstanceWrk(DeployInstance.fromScriptsDirectory(scriptsDirectory, accountName, region, partitionMetadata, s3Client, ecrClient), + accountName, s3Client, dynamoClient, config, + SleeperInternalCdkApp.DEMONSTRATION, deployPaused).deploy(); } } diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java index fb85316ca03..8796b146fdc 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java @@ -26,7 +26,7 @@ import sleeper.clients.deploy.DeployExistingInstance; import sleeper.clients.deploy.DeployInstance; -import sleeper.clients.deploy.DeployNewInstance; +import sleeper.clients.deploy.DeployNewInstanceWrk; import sleeper.configuration.properties.S3InstanceProperties; import sleeper.core.deploy.SleeperInstanceConfiguration; import sleeper.core.properties.instance.InstanceProperties; @@ -84,14 +84,8 @@ public boolean deployInstanceIfNotPresent(String instanceId, SleeperInstanceConf deployConfig.getInstanceProperties().set(VPC_ID, parameters.getVpcId()); deployConfig.getInstanceProperties().set(SUBNETS, parameters.getSubnetIds()); try { - DeployNewInstance.builder() - .deployInstance(deployInstance) - .accountName(parameters.getAccount()) - .s3Client(s3) - .dynamoClient(dynamoDB) - .deployInstanceConfiguration(deployConfig) - .cdkApp(SleeperInternalCdkApp.STANDARD) - .build().deploy(); + new DeployNewInstanceWrk(deployInstance, parameters.getAccount(), s3, dynamoDB, deployConfig, + SleeperInternalCdkApp.STANDARD, false).deploy(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RuntimeException(e); From e6b6070d00b6da029e2077b3b2f5e904a3e4e525 Mon Sep 17 00:00:00 2001 From: ca61688 <206189192+ca61688@users.noreply.github.com> Date: Mon, 6 Jul 2026 12:26:50 +0000 Subject: [PATCH 04/50] 6593: Replace old deployNewInstance with new version --- .../clients/deploy/DeployNewInstance.java | 183 ++++++++--------- .../clients/deploy/DeployNewInstanceWrk.java | 187 ------------------ 2 files changed, 92 insertions(+), 278 deletions(-) delete mode 100644 java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstanceWrk.java diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java index ad4325258e3..d44931af496 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java @@ -33,12 +33,16 @@ import sleeper.core.properties.instance.InstanceProperties; import sleeper.core.properties.model.SleeperInternalCdkApp; import sleeper.core.properties.table.TableProperties; +import sleeper.core.util.cli.CommandArguments; +import sleeper.core.util.cli.CommandArgumentsException; +import sleeper.core.util.cli.CommandLineUsage; +import sleeper.core.util.cli.CommandOption; import sleeper.statestore.StateStoreFactory; import java.io.IOException; import java.nio.file.Path; +import java.util.List; -import static sleeper.clients.util.ClientUtils.optionalArgument; import static sleeper.core.properties.instance.CommonProperty.ID; import static sleeper.core.properties.instance.CommonProperty.SUBNETS; import static sleeper.core.properties.instance.CommonProperty.VPC_ID; @@ -54,31 +58,58 @@ public class DeployNewInstance { private final SleeperInternalCdkApp cdkApp; private final boolean deployPaused; - private DeployNewInstance(Builder builder) { - deployInstance = builder.deployInstance; - accountName = builder.accountName; - s3Client = builder.s3Client; - dynamoClient = builder.dynamoClient; - deployInstanceConfiguration = builder.deployInstanceConfiguration; - cdkApp = builder.cdkApp; - deployPaused = builder.deployPaused; + public DeployNewInstance(DeployInstance deployInstance, String accountName, S3Client s3Client, + DynamoDbClient dynamoClient, SleeperInstanceConfiguration deployInstanceConfiguration, + SleeperInternalCdkApp cdkApp, boolean deployPaused) { + this.deployInstance = deployInstance; + this.accountName = accountName; + this.s3Client = s3Client; + this.dynamoClient = dynamoClient; + this.deployInstanceConfiguration = deployInstanceConfiguration; + this.cdkApp = cdkApp; + this.deployPaused = deployPaused; } - public static Builder builder() { - return new Builder(); + public static final CommandLineUsage USAGE = CommandLineUsage.builder() + .systemArguments(List.of("scriptsDirectory")) + .positionalArguments(List.of("scriptsDirectory", "instance-id", "vpcId", "subnetIds")) + .options(List.of( + CommandOption.longOption("instance-properties"), + CommandOption.longOption("config-dir"), + CommandOption.longFlag("paused"))) + .helpSummary("" + + "Deploys a new instance of Sleeper.\n" + + "\n" + + "--instance-properties \n" + + "Path to an instance properties file.\n" + + "One of --instance-properties and --config-dir must be set but not both.\n" + + "\n" + + "--config-dir \n" + + "Path to a directory containing an instance.properties file.\n" + + "One of --instance-properties and --config-dir must be set but not both.\n" + + "\n" + + "--paused\n" + + "If set, the instance will be deployed paused. Periodic background processes will not run until " + + "the instance is manually resumed.") + .build(); + + public static Arguments readArguments(CommandArguments arguments) { + return new Arguments( + Path.of(arguments.getString("scriptsDirectory")), + arguments.getString("instance-id"), + arguments.getString("vpcId"), + arguments.getString("subnetIds"), + arguments.getOptionalString("instance-properties").map(Path::of).orElse(null), + arguments.getOptionalString("config-dir").map(Path::of).orElse(null), + arguments.isFlagSet("paused")); } - public static void main(String[] args) throws IOException, InterruptedException { - if (args.length < 4 || args.length > 6) { - throw new IllegalArgumentException("Usage: " + - " "); - } - Path scriptsDirectory = Path.of(args[0]); - String instanceId = args[1]; - String vpcId = args[2]; - String subnetIds = args[3]; - Path instancePropertiesFile = optionalArgument(args, 4).map(Path::of).orElse(null); - boolean deployPaused = "true".equalsIgnoreCase(optionalArgument(args, 5).orElse("false")); + public static void main(String[] rawArgs) throws IOException, InterruptedException { + Arguments args = CommandArguments.parseAndValidateOrExit(USAGE, rawArgs, a -> readArguments(a)); + + Path scriptsDirectory = Path.of(rawArgs[0]); + Path instancePropertiesFile = args.resolvePropertiesFile(); + boolean deployPaused = args.deployPaused(); try (S3Client s3Client = S3Client.create(); DynamoDbClient dynamoClient = DynamoDbClient.create(); StsClient stsClient = StsClient.create(); @@ -87,22 +118,14 @@ public static void main(String[] args) throws IOException, InterruptedException Region region = DefaultAwsRegionProviderChain.builder().build().getRegion(); PartitionMetadata partitionMetadata = PartitionMetadata.of(region); - SleeperInstanceConfiguration config = SleeperInstanceConfiguration.forNewInstanceDefaultingInstance( - instancePropertiesFile, scriptsDirectory.resolve("templates")); - - config.getInstanceProperties().set(ID, instanceId); - config.getInstanceProperties().set(VPC_ID, vpcId); - config.getInstanceProperties().set(SUBNETS, subnetIds); - - builder() - .deployInstance(DeployInstance.fromScriptsDirectory(scriptsDirectory, accountName, region, partitionMetadata, s3Client, ecrClient)) - .accountName(accountName) - .s3Client(s3Client) - .dynamoClient(dynamoClient) - .deployInstanceConfiguration(config) - .deployPaused(deployPaused) - .cdkApp(SleeperInternalCdkApp.STANDARD) - .build().deploy(); + SleeperInstanceConfiguration config = SleeperInstanceConfiguration.fromLocalConfiguration(instancePropertiesFile); + + config.getInstanceProperties().set(ID, args.instanceId()); + config.getInstanceProperties().set(VPC_ID, args.vpcId()); + config.getInstanceProperties().set(SUBNETS, args.subnetIds()); + + new DeployNewInstance(DeployInstance.fromScriptsDirectory(scriptsDirectory, accountName, region, partitionMetadata, s3Client, ecrClient), + accountName, s3Client, dynamoClient, config, SleeperInternalCdkApp.STANDARD, deployPaused).deploy(); } } @@ -126,61 +149,39 @@ public void deploy() throws IOException, InterruptedException { LOGGER.info("Finished deployment of new instance"); } - public static final class Builder { - private DeployInstance deployInstance; - private String accountName; - private S3Client s3Client; - private DynamoDbClient dynamoClient; - private SleeperInstanceConfiguration deployInstanceConfiguration; - private SleeperInternalCdkApp cdkApp; - private boolean deployPaused; - - private Builder() { - } - - public Builder deployInstance(DeployInstance deployInstance) { - this.deployInstance = deployInstance; - return this; - } - - public Builder accountName(String accountName) { - this.accountName = accountName; - return this; - } - - public Builder s3Client(S3Client s3Client) { - this.s3Client = s3Client; - return this; - } - - public Builder dynamoClient(DynamoDbClient dynamoClient) { - this.dynamoClient = dynamoClient; - return this; - } - - public Builder deployInstanceConfiguration(SleeperInstanceConfiguration deployInstanceConfiguration) { - this.deployInstanceConfiguration = deployInstanceConfiguration; - return this; - } - - public Builder cdkApp(SleeperInternalCdkApp cdkApp) { - this.cdkApp = cdkApp; - return this; - } - - public Builder deployPaused(boolean deployPaused) { - this.deployPaused = deployPaused; - return this; - } - - public DeployNewInstance build() { - return new DeployNewInstance(this); + public record Arguments( + Path scriptsDirectory, + String instanceId, + String vpcId, + String subnetIds, + Path propertiesFile, + Path configDir, + boolean deployPaused) { + + public Arguments { + if (scriptsDirectory == null) { + throw new CommandArgumentsException("scriptsDirectory must not be null"); + } + + if (instanceId == null) { + throw new CommandArgumentsException("instance-id must not be null"); + } + + if (vpcId == null) { + throw new CommandArgumentsException("vpcId must not be null"); + } + + if (subnetIds == null) { + throw new CommandArgumentsException("subnetIds must not be null"); + } + + if (propertiesFile == null && configDir == null) { + throw new CommandArgumentsException("Either --instance-properties or --config-dir must be provided"); + } } - public void deployWithClients(S3Client s3Client, DynamoDbClient dynamoClient) throws IOException, InterruptedException { - s3Client(s3Client) - .dynamoClient(dynamoClient) - .build().deploy(); + public Path resolvePropertiesFile() { + return propertiesFile != null ? propertiesFile : configDir.resolve("instance.properties"); } } } diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstanceWrk.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstanceWrk.java deleted file mode 100644 index 5f15a43d45d..00000000000 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstanceWrk.java +++ /dev/null @@ -1,187 +0,0 @@ -/* - * Copyright 2022-2026 Crown Copyright - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package sleeper.clients.deploy; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import software.amazon.awssdk.regions.PartitionMetadata; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain; -import software.amazon.awssdk.services.dynamodb.DynamoDbClient; -import software.amazon.awssdk.services.ecr.EcrClient; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.sts.StsClient; - -import sleeper.clients.table.AddTableClient; -import sleeper.clients.util.cdk.CdkCommand; -import sleeper.configuration.properties.S3InstanceProperties; -import sleeper.configuration.properties.S3TableProperties; -import sleeper.core.deploy.SleeperInstanceConfiguration; -import sleeper.core.properties.instance.InstanceProperties; -import sleeper.core.properties.model.SleeperInternalCdkApp; -import sleeper.core.properties.table.TableProperties; -import sleeper.core.util.cli.CommandArguments; -import sleeper.core.util.cli.CommandArgumentsException; -import sleeper.core.util.cli.CommandLineUsage; -import sleeper.core.util.cli.CommandOption; -import sleeper.statestore.StateStoreFactory; - -import java.io.IOException; -import java.nio.file.Path; -import java.util.List; - -import static sleeper.core.properties.instance.CommonProperty.ID; -import static sleeper.core.properties.instance.CommonProperty.SUBNETS; -import static sleeper.core.properties.instance.CommonProperty.VPC_ID; - -public class DeployNewInstanceWrk { - private static final Logger LOGGER = LoggerFactory.getLogger(DeployNewInstance.class); - - private final DeployInstance deployInstance; - private final String accountName; - private final S3Client s3Client; - private final DynamoDbClient dynamoClient; - private final SleeperInstanceConfiguration deployInstanceConfiguration; - private final SleeperInternalCdkApp cdkApp; - private final boolean deployPaused; - - public DeployNewInstanceWrk(DeployInstance deployInstance, String accountName, S3Client s3Client, - DynamoDbClient dynamoClient, SleeperInstanceConfiguration deployInstanceConfiguration, - SleeperInternalCdkApp cdkApp, boolean deployPaused) { - this.deployInstance = deployInstance; - this.accountName = accountName; - this.s3Client = s3Client; - this.dynamoClient = dynamoClient; - this.deployInstanceConfiguration = deployInstanceConfiguration; - this.cdkApp = cdkApp; - this.deployPaused = deployPaused; - } - - public static final CommandLineUsage USAGE = CommandLineUsage.builder() - .systemArguments(List.of("scriptsDirectory")) - .positionalArguments(List.of("scriptsDirectory", "instance-id", "vpcId", "subnetIds")) - .options(List.of( - CommandOption.longOption("instance-properties"), - CommandOption.longOption("config-dir"), - CommandOption.longFlag("paused"))) - .helpSummary("" + - "Deploys a new instance of Sleeper.\n" + - "\n" + - "--instance-properties \n" + - "Path to an instance properties file.\n" + - "One of --instance-properties and --config-dir must be set but not both.\n" + - "\n" + - "--config-dir \n" + - "Path to a directory containing an instance.properties file.\n" + - "One of --instance-properties and --config-dir must be set but not both.\n" + - "\n" + - "--paused\n" + - "If set, the instance will be deployed paused. Periodic background processes will not run until " + - "the instance is manually resumed.") - .build(); - - public static Arguments readArguments(CommandArguments arguments) { - return new Arguments( - Path.of(arguments.getString("scriptsDirectory")), - arguments.getString("instance-id"), - arguments.getString("vpcId"), - arguments.getString("subnetIds"), - arguments.getOptionalString("instance-properties").map(Path::of).orElse(null), - arguments.getOptionalString("config-dir").map(Path::of).orElse(null), - arguments.isFlagSet("paused")); - } - - public static void main(String[] rawArgs) throws IOException, InterruptedException { - Arguments args = CommandArguments.parseAndValidateOrExit(USAGE, rawArgs, a -> readArguments(a)); - - Path scriptsDirectory = Path.of(rawArgs[0]); - Path instancePropertiesFile = args.resolvePropertiesFile(); - boolean deployPaused = args.deployPaused(); - try (S3Client s3Client = S3Client.create(); - DynamoDbClient dynamoClient = DynamoDbClient.create(); - StsClient stsClient = StsClient.create(); - EcrClient ecrClient = EcrClient.create()) { - String accountName = stsClient.getCallerIdentity().account(); - Region region = DefaultAwsRegionProviderChain.builder().build().getRegion(); - PartitionMetadata partitionMetadata = PartitionMetadata.of(region); - - SleeperInstanceConfiguration config = SleeperInstanceConfiguration.fromLocalConfiguration(instancePropertiesFile); - - config.getInstanceProperties().set(ID, args.instanceId()); - config.getInstanceProperties().set(VPC_ID, args.vpcId()); - config.getInstanceProperties().set(SUBNETS, args.subnetIds()); - - new DeployNewInstanceWrk(DeployInstance.fromScriptsDirectory(scriptsDirectory, accountName, region, partitionMetadata, s3Client, ecrClient), - accountName, s3Client, dynamoClient, config, SleeperInternalCdkApp.STANDARD, deployPaused).deploy(); - } - } - - public void deploy() throws IOException, InterruptedException { - deployInstanceConfiguration.validate(); - - deployInstance.deploy(DeployInstanceRequest.builder() - .instanceConfig(deployInstanceConfiguration) - .cdkCommand(deployPaused ? CdkCommand.deployNewPaused() : CdkCommand.deployNew()) - .cdkApp(cdkApp) - .build()); - - InstanceProperties instanceProperties = S3InstanceProperties.loadGivenAccountAndInstanceId(s3Client, accountName, deployInstanceConfiguration.getInstanceId()); - for (TableProperties tableProperties : deployInstanceConfiguration.getTableProperties()) { - LOGGER.info("Adding table " + tableProperties.getStatus()); - new AddTableClient(tableProperties, - S3TableProperties.createStore(instanceProperties, s3Client, dynamoClient), - StateStoreFactory.createProvider(instanceProperties, s3Client, dynamoClient)) - .run(); - } - LOGGER.info("Finished deployment of new instance"); - } - - public record Arguments( - Path scriptsDirectory, - String instanceId, - String vpcId, - String subnetIds, - Path propertiesFile, - Path configDir, - boolean deployPaused) { - - public Arguments { - if (scriptsDirectory == null) { - throw new CommandArgumentsException("scriptsDirectory must not be null"); - } - - if (instanceId == null) { - throw new CommandArgumentsException("instance-id must not be null"); - } - - if (vpcId == null) { - throw new CommandArgumentsException("vpcId must not be null"); - } - - if (subnetIds == null) { - throw new CommandArgumentsException("subnetIds must not be null"); - } - - if (propertiesFile == null && configDir == null) { - throw new CommandArgumentsException("Either --instance-properties or --config-dir must be provided"); - } - } - - public Path resolvePropertiesFile() { - return propertiesFile != null ? propertiesFile : configDir.resolve("instance.properties"); - } - } -} From e11edbbdb6cf595d9cfd31077acbb76beb1b23f5 Mon Sep 17 00:00:00 2001 From: ca61688 <206189192+ca61688@users.noreply.github.com> Date: Mon, 6 Jul 2026 12:43:13 +0000 Subject: [PATCH 05/50] 6593: Add ignore table files flag --- .../clients/deploy/DeployNewInstance.java | 35 ++++++++++++++----- .../drivers/cdk/DeployNewTestInstance.java | 6 ++-- .../instance/AwsSleeperInstanceDriver.java | 6 ++-- 3 files changed, 32 insertions(+), 15 deletions(-) diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java index d44931af496..f1332b45800 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java @@ -56,17 +56,19 @@ public class DeployNewInstance { private final DynamoDbClient dynamoClient; private final SleeperInstanceConfiguration deployInstanceConfiguration; private final SleeperInternalCdkApp cdkApp; + private final boolean ignoreTableFiles; private final boolean deployPaused; public DeployNewInstance(DeployInstance deployInstance, String accountName, S3Client s3Client, DynamoDbClient dynamoClient, SleeperInstanceConfiguration deployInstanceConfiguration, - SleeperInternalCdkApp cdkApp, boolean deployPaused) { + SleeperInternalCdkApp cdkApp, boolean ignoreTableFiles, boolean deployPaused) { this.deployInstance = deployInstance; this.accountName = accountName; this.s3Client = s3Client; this.dynamoClient = dynamoClient; this.deployInstanceConfiguration = deployInstanceConfiguration; this.cdkApp = cdkApp; + this.ignoreTableFiles = ignoreTableFiles; this.deployPaused = deployPaused; } @@ -76,6 +78,7 @@ public DeployNewInstance(DeployInstance deployInstance, String accountName, S3Cl .options(List.of( CommandOption.longOption("instance-properties"), CommandOption.longOption("config-dir"), + CommandOption.longFlag("ignoreTableFiles"), CommandOption.longFlag("paused"))) .helpSummary("" + "Deploys a new instance of Sleeper.\n" + @@ -101,6 +104,7 @@ public static Arguments readArguments(CommandArguments arguments) { arguments.getString("subnetIds"), arguments.getOptionalString("instance-properties").map(Path::of).orElse(null), arguments.getOptionalString("config-dir").map(Path::of).orElse(null), + arguments.isFlagSet("ignoreTableFiles"), arguments.isFlagSet("paused")); } @@ -125,7 +129,8 @@ public static void main(String[] rawArgs) throws IOException, InterruptedExcepti config.getInstanceProperties().set(SUBNETS, args.subnetIds()); new DeployNewInstance(DeployInstance.fromScriptsDirectory(scriptsDirectory, accountName, region, partitionMetadata, s3Client, ecrClient), - accountName, s3Client, dynamoClient, config, SleeperInternalCdkApp.STANDARD, deployPaused).deploy(); + accountName, s3Client, dynamoClient, config, SleeperInternalCdkApp.STANDARD, + args.ignoreTableFiles(), deployPaused).deploy(); } } @@ -138,13 +143,16 @@ public void deploy() throws IOException, InterruptedException { .cdkApp(cdkApp) .build()); - InstanceProperties instanceProperties = S3InstanceProperties.loadGivenAccountAndInstanceId(s3Client, accountName, deployInstanceConfiguration.getInstanceId()); - for (TableProperties tableProperties : deployInstanceConfiguration.getTableProperties()) { - LOGGER.info("Adding table " + tableProperties.getStatus()); - new AddTableClient(tableProperties, - S3TableProperties.createStore(instanceProperties, s3Client, dynamoClient), - StateStoreFactory.createProvider(instanceProperties, s3Client, dynamoClient)) - .run(); + if (!ignoreTableFiles) { + InstanceProperties instanceProperties = S3InstanceProperties.loadGivenAccountAndInstanceId(s3Client, accountName, deployInstanceConfiguration.getInstanceId()); + + for (TableProperties tableProperties : deployInstanceConfiguration.getTableProperties()) { + LOGGER.info("Adding table " + tableProperties.getStatus()); + new AddTableClient(tableProperties, + S3TableProperties.createStore(instanceProperties, s3Client, dynamoClient), + StateStoreFactory.createProvider(instanceProperties, s3Client, dynamoClient)) + .run(); + } } LOGGER.info("Finished deployment of new instance"); } @@ -156,6 +164,7 @@ public record Arguments( String subnetIds, Path propertiesFile, Path configDir, + boolean ignoreTableFiles, boolean deployPaused) { public Arguments { @@ -178,6 +187,14 @@ public record Arguments( if (propertiesFile == null && configDir == null) { throw new CommandArgumentsException("Either --instance-properties or --config-dir must be provided"); } + + if (configDir == null && ignoreTableFiles) { + throw new CommandArgumentsException("ignoreTableFiles flag is only checked when --config-dir is set."); + } + + if (propertiesFile != null) { + ignoreTableFiles = true; + } } public Path resolvePropertiesFile() { diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java index 2581fb5f53c..df0ed98104e 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java @@ -24,7 +24,7 @@ import software.amazon.awssdk.services.sts.StsClient; import sleeper.clients.deploy.DeployInstance; -import sleeper.clients.deploy.DeployNewInstanceWrk; +import sleeper.clients.deploy.DeployNewInstance; import sleeper.core.deploy.SleeperInstanceConfiguration; import sleeper.core.deploy.SleeperInstanceConfigurationFromTemplates; import sleeper.core.properties.model.SleeperInternalCdkApp; @@ -68,9 +68,9 @@ public static void main(String[] args) throws IOException, InterruptedException config.getInstanceProperties().set(ID, instanceId); config.getInstanceProperties().set(VPC_ID, vpcId); config.getInstanceProperties().set(SUBNETS, subnetIds); - new DeployNewInstanceWrk(DeployInstance.fromScriptsDirectory(scriptsDirectory, accountName, region, partitionMetadata, s3Client, ecrClient), + new DeployNewInstance(DeployInstance.fromScriptsDirectory(scriptsDirectory, accountName, region, partitionMetadata, s3Client, ecrClient), accountName, s3Client, dynamoClient, config, - SleeperInternalCdkApp.DEMONSTRATION, deployPaused).deploy(); + SleeperInternalCdkApp.DEMONSTRATION, false, deployPaused).deploy(); } } diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java index 8796b146fdc..2ff004d1111 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java @@ -26,7 +26,7 @@ import sleeper.clients.deploy.DeployExistingInstance; import sleeper.clients.deploy.DeployInstance; -import sleeper.clients.deploy.DeployNewInstanceWrk; +import sleeper.clients.deploy.DeployNewInstance; import sleeper.configuration.properties.S3InstanceProperties; import sleeper.core.deploy.SleeperInstanceConfiguration; import sleeper.core.properties.instance.InstanceProperties; @@ -84,8 +84,8 @@ public boolean deployInstanceIfNotPresent(String instanceId, SleeperInstanceConf deployConfig.getInstanceProperties().set(VPC_ID, parameters.getVpcId()); deployConfig.getInstanceProperties().set(SUBNETS, parameters.getSubnetIds()); try { - new DeployNewInstanceWrk(deployInstance, parameters.getAccount(), s3, dynamoDB, deployConfig, - SleeperInternalCdkApp.STANDARD, false).deploy(); + new DeployNewInstance(deployInstance, parameters.getAccount(), s3, dynamoDB, deployConfig, + SleeperInternalCdkApp.STANDARD, false, false).deploy(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RuntimeException(e); From e348a00276cef1761c916bec2711878939c9eac1 Mon Sep 17 00:00:00 2001 From: ca61688 <206189192+ca61688@users.noreply.github.com> Date: Mon, 6 Jul 2026 13:03:17 +0000 Subject: [PATCH 06/50] 6593: Update deploy new instance help text to include ignoreTableFiles flag --- .../main/java/sleeper/clients/deploy/DeployNewInstance.java | 5 +++++ scripts/deploy/deployNew.sh | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java index f1332b45800..21b787e1759 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java @@ -91,6 +91,11 @@ public DeployNewInstance(DeployInstance deployInstance, String accountName, S3Cl "Path to a directory containing an instance.properties file.\n" + "One of --instance-properties and --config-dir must be set but not both.\n" + "\n" + + "--ignoreTableFiles\n" + + "If set, the instance will be deployed on it's own. Otherwise tables will be created based on " + + "any relevent table.properties files found in the specified --config-dir. This flag cannot be used " + + "without the --config-dir optional argument.\n" + + "\n" + "--paused\n" + "If set, the instance will be deployed paused. Periodic background processes will not run until " + "the instance is manually resumed.") diff --git a/scripts/deploy/deployNew.sh b/scripts/deploy/deployNew.sh index 4597edbc4b2..44cc40f0f93 100755 --- a/scripts/deploy/deployNew.sh +++ b/scripts/deploy/deployNew.sh @@ -19,4 +19,4 @@ unset CDPATH SCRIPTS_DIR=$(cd "$(dirname "$0")" && cd .. && pwd) VERSION=$(cat "${SCRIPTS_DIR}/templates/version.txt") -java -cp "${SCRIPTS_DIR}/jars/clients-${VERSION}-utility.jar" sleeper.clients.deploy.DeployNewInstanceWrk "${SCRIPTS_DIR}" "$@" +java -cp "${SCRIPTS_DIR}/jars/clients-${VERSION}-utility.jar" sleeper.clients.deploy.DeployNewInstance "${SCRIPTS_DIR}" "$@" From 38a7c2c5c046d74c99f8513746f66e0f6f04733c Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Mon, 6 Jul 2026 11:32:58 +0000 Subject: [PATCH 07/50] 7555 Set version number to 0.37.0 --- java/analytics-integration/athena/pom.xml | 2 +- java/analytics-integration/pom.xml | 2 +- java/analytics-integration/spark/pom.xml | 2 +- java/analytics-integration/trino/pom.xml | 2 +- java/build/pom.xml | 2 +- java/bulk-export/bulk-export-core/pom.xml | 2 +- java/bulk-export/bulk-export-planner/pom.xml | 2 +- .../bulk-export-task-creator/pom.xml | 2 +- .../bulk-export-task-execution/pom.xml | 2 +- java/bulk-export/pom.xml | 2 +- java/bulk-import/bulk-import-core/pom.xml | 2 +- java/bulk-import/bulk-import-eks/pom.xml | 2 +- java/bulk-import/bulk-import-runner/pom.xml | 2 +- java/bulk-import/bulk-import-starter/pom.xml | 2 +- java/bulk-import/pom.xml | 2 +- java/clients/pom.xml | 2 +- java/common/arrow/pom.xml | 2 +- java/common/common-invoke-tables/pom.xml | 2 +- java/common/common-job/pom.xml | 2 +- java/common/common-task/pom.xml | 2 +- java/common/docker-lambda/pom.xml | 2 +- java/common/dynamodb-tools/pom.xml | 2 +- java/common/foreign-bridge/pom.xml | 2 +- java/common/localstack-test/pom.xml | 2 +- java/common/parquet/pom.xml | 2 +- java/common/pom.xml | 2 +- java/common/sketches/pom.xml | 2 +- java/compaction/compaction-core/pom.xml | 2 +- java/compaction/compaction-datafusion/pom.xml | 2 +- .../compaction-job-creation-lambda/pom.xml | 2 +- .../compaction/compaction-job-creation/pom.xml | 2 +- .../compaction-job-execution/pom.xml | 2 +- .../compaction-task-creation/pom.xml | 2 +- java/compaction/compaction-tracker/pom.xml | 2 +- java/compaction/pom.xml | 2 +- java/configuration/pom.xml | 2 +- java/core/pom.xml | 2 +- java/deployment/build-uptime-lambda/pom.xml | 2 +- java/deployment/cdk-custom-resources/pom.xml | 2 +- java/deployment/cdk-environment/pom.xml | 2 +- java/deployment/cdk/pom.xml | 2 +- java/deployment/container-images/pom.xml | 2 +- java/deployment/pom.xml | 2 +- java/distribution/pom.xml | 2 +- java/example-iterators/pom.xml | 2 +- java/garbage-collector/pom.xml | 2 +- java/ingest/ingest-batcher-core/pom.xml | 2 +- java/ingest/ingest-batcher-job-creator/pom.xml | 2 +- java/ingest/ingest-batcher-store/pom.xml | 2 +- java/ingest/ingest-batcher-submitter/pom.xml | 2 +- java/ingest/ingest-core/pom.xml | 2 +- java/ingest/ingest-runner/pom.xml | 2 +- java/ingest/ingest-taskrunner/pom.xml | 2 +- java/ingest/ingest-tracker/pom.xml | 2 +- java/ingest/pom.xml | 2 +- java/metrics/pom.xml | 2 +- java/partitions/pom.xml | 2 +- java/partitions/splitter-lambda/pom.xml | 2 +- java/partitions/splitter/pom.xml | 2 +- java/pom.xml | 2 +- java/query/pom.xml | 2 +- java/query/query-core/pom.xml | 2 +- java/query/query-datafusion/pom.xml | 2 +- java/query/query-lambda/pom.xml | 2 +- java/query/query-runner/pom.xml | 2 +- java/rest-api/pom.xml | 2 +- java/statestore-committer-core/pom.xml | 2 +- java/statestore-committer/pom.xml | 2 +- java/statestore-lambda/pom.xml | 2 +- java/statestore/pom.xml | 2 +- java/system-test/pom.xml | 2 +- java/system-test/system-test-cdk/pom.xml | 2 +- .../system-test-configuration/pom.xml | 2 +- .../system-test-data-generation/pom.xml | 2 +- java/system-test/system-test-drivers/pom.xml | 2 +- java/system-test/system-test-dsl/pom.xml | 2 +- java/system-test/system-test-suite/pom.xml | 2 +- python/setup.py | 2 +- rust/Cargo.lock | 18 +++++++++--------- rust/Cargo.toml | 2 +- 80 files changed, 88 insertions(+), 88 deletions(-) diff --git a/java/analytics-integration/athena/pom.xml b/java/analytics-integration/athena/pom.xml index 93252c68f7a..1f4fd04dbdf 100644 --- a/java/analytics-integration/athena/pom.xml +++ b/java/analytics-integration/athena/pom.xml @@ -19,7 +19,7 @@ analytics-integration sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/analytics-integration/pom.xml b/java/analytics-integration/pom.xml index fdea85d7aeb..f55a938e866 100644 --- a/java/analytics-integration/pom.xml +++ b/java/analytics-integration/pom.xml @@ -19,7 +19,7 @@ aws sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 analytics-integration diff --git a/java/analytics-integration/spark/pom.xml b/java/analytics-integration/spark/pom.xml index dc8f3aa38ff..080d6c6514a 100644 --- a/java/analytics-integration/spark/pom.xml +++ b/java/analytics-integration/spark/pom.xml @@ -19,7 +19,7 @@ analytics-integration sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/analytics-integration/trino/pom.xml b/java/analytics-integration/trino/pom.xml index 22f5ce37d95..91d52528a04 100644 --- a/java/analytics-integration/trino/pom.xml +++ b/java/analytics-integration/trino/pom.xml @@ -19,7 +19,7 @@ analytics-integration sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/build/pom.xml b/java/build/pom.xml index a868033c770..d642f7b46b5 100644 --- a/java/build/pom.xml +++ b/java/build/pom.xml @@ -19,7 +19,7 @@ aws sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/bulk-export/bulk-export-core/pom.xml b/java/bulk-export/bulk-export-core/pom.xml index 3ea5b1e0384..76adfcabf30 100644 --- a/java/bulk-export/bulk-export-core/pom.xml +++ b/java/bulk-export/bulk-export-core/pom.xml @@ -19,7 +19,7 @@ sleeper bulk-export - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 bulk-export-core diff --git a/java/bulk-export/bulk-export-planner/pom.xml b/java/bulk-export/bulk-export-planner/pom.xml index 46af9c6211c..5a3a14bcb2e 100644 --- a/java/bulk-export/bulk-export-planner/pom.xml +++ b/java/bulk-export/bulk-export-planner/pom.xml @@ -19,7 +19,7 @@ sleeper bulk-export - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 bulk-export-planner diff --git a/java/bulk-export/bulk-export-task-creator/pom.xml b/java/bulk-export/bulk-export-task-creator/pom.xml index 6f00e8df68c..501a7939ee3 100644 --- a/java/bulk-export/bulk-export-task-creator/pom.xml +++ b/java/bulk-export/bulk-export-task-creator/pom.xml @@ -19,7 +19,7 @@ sleeper bulk-export - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 bulk-export-task-creator diff --git a/java/bulk-export/bulk-export-task-execution/pom.xml b/java/bulk-export/bulk-export-task-execution/pom.xml index 4ee386749d3..af7e43b21f6 100644 --- a/java/bulk-export/bulk-export-task-execution/pom.xml +++ b/java/bulk-export/bulk-export-task-execution/pom.xml @@ -19,7 +19,7 @@ sleeper bulk-export - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 bulk-export-task-execution diff --git a/java/bulk-export/pom.xml b/java/bulk-export/pom.xml index d8ae76336ff..33dd5281d80 100644 --- a/java/bulk-export/pom.xml +++ b/java/bulk-export/pom.xml @@ -19,7 +19,7 @@ aws sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 bulk-export diff --git a/java/bulk-import/bulk-import-core/pom.xml b/java/bulk-import/bulk-import-core/pom.xml index bab75309b23..8f7585f7679 100644 --- a/java/bulk-import/bulk-import-core/pom.xml +++ b/java/bulk-import/bulk-import-core/pom.xml @@ -19,7 +19,7 @@ bulk-import sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/bulk-import/bulk-import-eks/pom.xml b/java/bulk-import/bulk-import-eks/pom.xml index e7f3aa82ed8..43739dc87b7 100644 --- a/java/bulk-import/bulk-import-eks/pom.xml +++ b/java/bulk-import/bulk-import-eks/pom.xml @@ -19,7 +19,7 @@ sleeper bulk-import - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/bulk-import/bulk-import-runner/pom.xml b/java/bulk-import/bulk-import-runner/pom.xml index 526b48a4080..40cd530b3cd 100644 --- a/java/bulk-import/bulk-import-runner/pom.xml +++ b/java/bulk-import/bulk-import-runner/pom.xml @@ -19,7 +19,7 @@ sleeper bulk-import - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/bulk-import/bulk-import-starter/pom.xml b/java/bulk-import/bulk-import-starter/pom.xml index f8e4ad9297e..4d9ac2622bd 100644 --- a/java/bulk-import/bulk-import-starter/pom.xml +++ b/java/bulk-import/bulk-import-starter/pom.xml @@ -19,7 +19,7 @@ sleeper bulk-import - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/bulk-import/pom.xml b/java/bulk-import/pom.xml index 1e9990dab02..b058b016126 100644 --- a/java/bulk-import/pom.xml +++ b/java/bulk-import/pom.xml @@ -19,7 +19,7 @@ aws sleeper - 0.36.2-SNAPSHOT + 0.37.0 pom diff --git a/java/clients/pom.xml b/java/clients/pom.xml index 67adabb107e..248e52aca10 100644 --- a/java/clients/pom.xml +++ b/java/clients/pom.xml @@ -19,7 +19,7 @@ aws sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/common/arrow/pom.xml b/java/common/arrow/pom.xml index 0f527059301..8468e9c1a1f 100644 --- a/java/common/arrow/pom.xml +++ b/java/common/arrow/pom.xml @@ -19,7 +19,7 @@ common sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/common/common-invoke-tables/pom.xml b/java/common/common-invoke-tables/pom.xml index 72ec5fa9345..6c77dbc7565 100644 --- a/java/common/common-invoke-tables/pom.xml +++ b/java/common/common-invoke-tables/pom.xml @@ -20,7 +20,7 @@ common sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/common/common-job/pom.xml b/java/common/common-job/pom.xml index 7a1305ae7e5..7486bc3b022 100644 --- a/java/common/common-job/pom.xml +++ b/java/common/common-job/pom.xml @@ -19,7 +19,7 @@ common sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/common/common-task/pom.xml b/java/common/common-task/pom.xml index 6c51d0d541a..ece112bf98f 100644 --- a/java/common/common-task/pom.xml +++ b/java/common/common-task/pom.xml @@ -19,7 +19,7 @@ common sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/common/docker-lambda/pom.xml b/java/common/docker-lambda/pom.xml index e01cc68e916..53b3e4c6942 100644 --- a/java/common/docker-lambda/pom.xml +++ b/java/common/docker-lambda/pom.xml @@ -20,7 +20,7 @@ common sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/common/dynamodb-tools/pom.xml b/java/common/dynamodb-tools/pom.xml index 5d976ba2888..0fe5e174070 100644 --- a/java/common/dynamodb-tools/pom.xml +++ b/java/common/dynamodb-tools/pom.xml @@ -20,7 +20,7 @@ common sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/common/foreign-bridge/pom.xml b/java/common/foreign-bridge/pom.xml index d0713657b2b..719b57f095b 100644 --- a/java/common/foreign-bridge/pom.xml +++ b/java/common/foreign-bridge/pom.xml @@ -19,7 +19,7 @@ common sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/common/localstack-test/pom.xml b/java/common/localstack-test/pom.xml index 27577b4728e..550252bb3e3 100644 --- a/java/common/localstack-test/pom.xml +++ b/java/common/localstack-test/pom.xml @@ -20,7 +20,7 @@ common sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/common/parquet/pom.xml b/java/common/parquet/pom.xml index 2f36740d340..7a0c64b49f1 100644 --- a/java/common/parquet/pom.xml +++ b/java/common/parquet/pom.xml @@ -19,7 +19,7 @@ common sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/common/pom.xml b/java/common/pom.xml index 7c4ff64bdf1..27398319b8e 100644 --- a/java/common/pom.xml +++ b/java/common/pom.xml @@ -19,7 +19,7 @@ aws sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/common/sketches/pom.xml b/java/common/sketches/pom.xml index 4e47c2c87e1..326e8cac231 100644 --- a/java/common/sketches/pom.xml +++ b/java/common/sketches/pom.xml @@ -19,7 +19,7 @@ common sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/compaction/compaction-core/pom.xml b/java/compaction/compaction-core/pom.xml index 13070acd365..51d2d584af6 100644 --- a/java/compaction/compaction-core/pom.xml +++ b/java/compaction/compaction-core/pom.xml @@ -19,7 +19,7 @@ compaction sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/compaction/compaction-datafusion/pom.xml b/java/compaction/compaction-datafusion/pom.xml index ecc5e617fd6..e88d9fb75f2 100644 --- a/java/compaction/compaction-datafusion/pom.xml +++ b/java/compaction/compaction-datafusion/pom.xml @@ -19,7 +19,7 @@ compaction sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 compaction-datafusion diff --git a/java/compaction/compaction-job-creation-lambda/pom.xml b/java/compaction/compaction-job-creation-lambda/pom.xml index dbfba9ba223..05cd42d977f 100644 --- a/java/compaction/compaction-job-creation-lambda/pom.xml +++ b/java/compaction/compaction-job-creation-lambda/pom.xml @@ -19,7 +19,7 @@ compaction sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/compaction/compaction-job-creation/pom.xml b/java/compaction/compaction-job-creation/pom.xml index 805cd7b64e5..bb6748c001b 100644 --- a/java/compaction/compaction-job-creation/pom.xml +++ b/java/compaction/compaction-job-creation/pom.xml @@ -19,7 +19,7 @@ compaction sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/compaction/compaction-job-execution/pom.xml b/java/compaction/compaction-job-execution/pom.xml index 0d45090c70b..812d7924def 100644 --- a/java/compaction/compaction-job-execution/pom.xml +++ b/java/compaction/compaction-job-execution/pom.xml @@ -19,7 +19,7 @@ compaction sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/compaction/compaction-task-creation/pom.xml b/java/compaction/compaction-task-creation/pom.xml index 7abc24dcb52..31cf82a51f2 100644 --- a/java/compaction/compaction-task-creation/pom.xml +++ b/java/compaction/compaction-task-creation/pom.xml @@ -21,7 +21,7 @@ sleeper compaction - 0.36.2-SNAPSHOT + 0.37.0 compaction-task-creation diff --git a/java/compaction/compaction-tracker/pom.xml b/java/compaction/compaction-tracker/pom.xml index cdc46b97db8..ee2e66d8ce4 100644 --- a/java/compaction/compaction-tracker/pom.xml +++ b/java/compaction/compaction-tracker/pom.xml @@ -19,7 +19,7 @@ compaction sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/compaction/pom.xml b/java/compaction/pom.xml index 3352b379e79..fca0f1bf723 100644 --- a/java/compaction/pom.xml +++ b/java/compaction/pom.xml @@ -19,7 +19,7 @@ aws sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/configuration/pom.xml b/java/configuration/pom.xml index 388c968307c..3ca569b9d7b 100644 --- a/java/configuration/pom.xml +++ b/java/configuration/pom.xml @@ -19,7 +19,7 @@ aws sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/core/pom.xml b/java/core/pom.xml index 89291b7ebed..64a45d6fe24 100644 --- a/java/core/pom.xml +++ b/java/core/pom.xml @@ -19,7 +19,7 @@ aws sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/deployment/build-uptime-lambda/pom.xml b/java/deployment/build-uptime-lambda/pom.xml index 93a37650537..5d6dd59e66d 100644 --- a/java/deployment/build-uptime-lambda/pom.xml +++ b/java/deployment/build-uptime-lambda/pom.xml @@ -21,7 +21,7 @@ sleeper deployment - 0.36.2-SNAPSHOT + 0.37.0 build-uptime-lambda diff --git a/java/deployment/cdk-custom-resources/pom.xml b/java/deployment/cdk-custom-resources/pom.xml index ce31857d216..aa55da5a8ed 100644 --- a/java/deployment/cdk-custom-resources/pom.xml +++ b/java/deployment/cdk-custom-resources/pom.xml @@ -21,7 +21,7 @@ sleeper deployment - 0.36.2-SNAPSHOT + 0.37.0 cdk-custom-resources diff --git a/java/deployment/cdk-environment/pom.xml b/java/deployment/cdk-environment/pom.xml index 493447183c5..361f8a24247 100644 --- a/java/deployment/cdk-environment/pom.xml +++ b/java/deployment/cdk-environment/pom.xml @@ -19,7 +19,7 @@ deployment sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/deployment/cdk/pom.xml b/java/deployment/cdk/pom.xml index effd2191a96..c4944436e2d 100644 --- a/java/deployment/cdk/pom.xml +++ b/java/deployment/cdk/pom.xml @@ -19,7 +19,7 @@ deployment sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/deployment/container-images/pom.xml b/java/deployment/container-images/pom.xml index ee44f069cd2..e1277cba632 100644 --- a/java/deployment/container-images/pom.xml +++ b/java/deployment/container-images/pom.xml @@ -21,7 +21,7 @@ sleeper deployment - 0.36.2-SNAPSHOT + 0.37.0 container-images diff --git a/java/deployment/pom.xml b/java/deployment/pom.xml index f9570c13143..ae5708d387e 100644 --- a/java/deployment/pom.xml +++ b/java/deployment/pom.xml @@ -19,7 +19,7 @@ aws sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/distribution/pom.xml b/java/distribution/pom.xml index b1c62a50a33..53a71921c7d 100644 --- a/java/distribution/pom.xml +++ b/java/distribution/pom.xml @@ -19,7 +19,7 @@ aws sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/example-iterators/pom.xml b/java/example-iterators/pom.xml index b2941220409..5e1e2d8dcef 100644 --- a/java/example-iterators/pom.xml +++ b/java/example-iterators/pom.xml @@ -19,7 +19,7 @@ aws sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/garbage-collector/pom.xml b/java/garbage-collector/pom.xml index 95ade85f486..6187075d5aa 100644 --- a/java/garbage-collector/pom.xml +++ b/java/garbage-collector/pom.xml @@ -19,7 +19,7 @@ aws sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/ingest/ingest-batcher-core/pom.xml b/java/ingest/ingest-batcher-core/pom.xml index 07caa9eb8c2..cae1d0542d9 100644 --- a/java/ingest/ingest-batcher-core/pom.xml +++ b/java/ingest/ingest-batcher-core/pom.xml @@ -20,7 +20,7 @@ ingest sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/ingest/ingest-batcher-job-creator/pom.xml b/java/ingest/ingest-batcher-job-creator/pom.xml index 9618719b44f..cfbdc2da6ae 100644 --- a/java/ingest/ingest-batcher-job-creator/pom.xml +++ b/java/ingest/ingest-batcher-job-creator/pom.xml @@ -21,7 +21,7 @@ sleeper ingest - 0.36.2-SNAPSHOT + 0.37.0 ingest-batcher-job-creator diff --git a/java/ingest/ingest-batcher-store/pom.xml b/java/ingest/ingest-batcher-store/pom.xml index 5d87b59afa6..8ebaa12a2c0 100644 --- a/java/ingest/ingest-batcher-store/pom.xml +++ b/java/ingest/ingest-batcher-store/pom.xml @@ -21,7 +21,7 @@ sleeper ingest - 0.36.2-SNAPSHOT + 0.37.0 ingest-batcher-store diff --git a/java/ingest/ingest-batcher-submitter/pom.xml b/java/ingest/ingest-batcher-submitter/pom.xml index 4bfce8dce40..160ffa27eea 100644 --- a/java/ingest/ingest-batcher-submitter/pom.xml +++ b/java/ingest/ingest-batcher-submitter/pom.xml @@ -21,7 +21,7 @@ sleeper ingest - 0.36.2-SNAPSHOT + 0.37.0 ingest-batcher-submitter diff --git a/java/ingest/ingest-core/pom.xml b/java/ingest/ingest-core/pom.xml index 39d362a0880..00c77ad71a4 100644 --- a/java/ingest/ingest-core/pom.xml +++ b/java/ingest/ingest-core/pom.xml @@ -19,7 +19,7 @@ ingest sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/ingest/ingest-runner/pom.xml b/java/ingest/ingest-runner/pom.xml index a2cb37454b8..2b68067b473 100644 --- a/java/ingest/ingest-runner/pom.xml +++ b/java/ingest/ingest-runner/pom.xml @@ -19,7 +19,7 @@ ingest sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/ingest/ingest-taskrunner/pom.xml b/java/ingest/ingest-taskrunner/pom.xml index f2299a1def0..3ce2f748002 100644 --- a/java/ingest/ingest-taskrunner/pom.xml +++ b/java/ingest/ingest-taskrunner/pom.xml @@ -19,7 +19,7 @@ ingest sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/ingest/ingest-tracker/pom.xml b/java/ingest/ingest-tracker/pom.xml index e7d3f601038..124ad0c3cbb 100644 --- a/java/ingest/ingest-tracker/pom.xml +++ b/java/ingest/ingest-tracker/pom.xml @@ -20,7 +20,7 @@ ingest sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/ingest/pom.xml b/java/ingest/pom.xml index 468173e7358..7af440bfeba 100644 --- a/java/ingest/pom.xml +++ b/java/ingest/pom.xml @@ -19,7 +19,7 @@ aws sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/metrics/pom.xml b/java/metrics/pom.xml index d0664266c4d..48f6dbcbaa4 100644 --- a/java/metrics/pom.xml +++ b/java/metrics/pom.xml @@ -20,7 +20,7 @@ aws sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/partitions/pom.xml b/java/partitions/pom.xml index 3c0f259ce43..8f957d97539 100644 --- a/java/partitions/pom.xml +++ b/java/partitions/pom.xml @@ -19,7 +19,7 @@ aws sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/partitions/splitter-lambda/pom.xml b/java/partitions/splitter-lambda/pom.xml index 23484203368..9bb7a1fc319 100644 --- a/java/partitions/splitter-lambda/pom.xml +++ b/java/partitions/splitter-lambda/pom.xml @@ -19,7 +19,7 @@ partitions sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/partitions/splitter/pom.xml b/java/partitions/splitter/pom.xml index d8c3a4ecb7a..e2f4577cef3 100644 --- a/java/partitions/splitter/pom.xml +++ b/java/partitions/splitter/pom.xml @@ -19,7 +19,7 @@ partitions sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/pom.xml b/java/pom.xml index 6d5e0a4d836..36abe592f8c 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -21,7 +21,7 @@ sleeper aws pom - 0.36.2-SNAPSHOT + 0.37.0 analytics-integration diff --git a/java/query/pom.xml b/java/query/pom.xml index 28dd26d9ddc..7c66417ba81 100644 --- a/java/query/pom.xml +++ b/java/query/pom.xml @@ -19,7 +19,7 @@ aws sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/query/query-core/pom.xml b/java/query/query-core/pom.xml index b614073e54a..6348fae5e5f 100644 --- a/java/query/query-core/pom.xml +++ b/java/query/query-core/pom.xml @@ -19,7 +19,7 @@ query sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/query/query-datafusion/pom.xml b/java/query/query-datafusion/pom.xml index 2677432471b..c5a64a69a6d 100644 --- a/java/query/query-datafusion/pom.xml +++ b/java/query/query-datafusion/pom.xml @@ -19,7 +19,7 @@ query sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/query/query-lambda/pom.xml b/java/query/query-lambda/pom.xml index 59bcf304f99..0f40b6a4b9b 100644 --- a/java/query/query-lambda/pom.xml +++ b/java/query/query-lambda/pom.xml @@ -19,7 +19,7 @@ query sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/query/query-runner/pom.xml b/java/query/query-runner/pom.xml index 19355d09fb3..d30e1d229a4 100644 --- a/java/query/query-runner/pom.xml +++ b/java/query/query-runner/pom.xml @@ -19,7 +19,7 @@ query sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/rest-api/pom.xml b/java/rest-api/pom.xml index 3b8f31461f3..1611d2b823e 100644 --- a/java/rest-api/pom.xml +++ b/java/rest-api/pom.xml @@ -19,7 +19,7 @@ aws sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/statestore-committer-core/pom.xml b/java/statestore-committer-core/pom.xml index 520ec624065..bc372353199 100644 --- a/java/statestore-committer-core/pom.xml +++ b/java/statestore-committer-core/pom.xml @@ -19,7 +19,7 @@ aws sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/statestore-committer/pom.xml b/java/statestore-committer/pom.xml index 6a8d5844a9a..759452dd72a 100644 --- a/java/statestore-committer/pom.xml +++ b/java/statestore-committer/pom.xml @@ -19,7 +19,7 @@ aws sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/statestore-lambda/pom.xml b/java/statestore-lambda/pom.xml index ebdae2631bf..88db40d5fd5 100644 --- a/java/statestore-lambda/pom.xml +++ b/java/statestore-lambda/pom.xml @@ -19,7 +19,7 @@ aws sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/statestore/pom.xml b/java/statestore/pom.xml index 7a49d63e85d..4406e3bcc4e 100644 --- a/java/statestore/pom.xml +++ b/java/statestore/pom.xml @@ -19,7 +19,7 @@ aws sleeper - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/system-test/pom.xml b/java/system-test/pom.xml index e59b63385ae..801f8126bc2 100644 --- a/java/system-test/pom.xml +++ b/java/system-test/pom.xml @@ -21,7 +21,7 @@ sleeper aws - 0.36.2-SNAPSHOT + 0.37.0 pom diff --git a/java/system-test/system-test-cdk/pom.xml b/java/system-test/system-test-cdk/pom.xml index 5b5a60157fe..3b2bc9d5df5 100644 --- a/java/system-test/system-test-cdk/pom.xml +++ b/java/system-test/system-test-cdk/pom.xml @@ -21,7 +21,7 @@ sleeper system-test - 0.36.2-SNAPSHOT + 0.37.0 system-test-cdk diff --git a/java/system-test/system-test-configuration/pom.xml b/java/system-test/system-test-configuration/pom.xml index ef49653842f..85dc2e947ba 100644 --- a/java/system-test/system-test-configuration/pom.xml +++ b/java/system-test/system-test-configuration/pom.xml @@ -21,7 +21,7 @@ sleeper system-test - 0.36.2-SNAPSHOT + 0.37.0 system-test-configuration diff --git a/java/system-test/system-test-data-generation/pom.xml b/java/system-test/system-test-data-generation/pom.xml index 6ed9c4132a4..7e8e2edc2c0 100644 --- a/java/system-test/system-test-data-generation/pom.xml +++ b/java/system-test/system-test-data-generation/pom.xml @@ -21,7 +21,7 @@ sleeper system-test - 0.36.2-SNAPSHOT + 0.37.0 system-test-data-generation diff --git a/java/system-test/system-test-drivers/pom.xml b/java/system-test/system-test-drivers/pom.xml index cbc4e579b00..1322aff8bce 100644 --- a/java/system-test/system-test-drivers/pom.xml +++ b/java/system-test/system-test-drivers/pom.xml @@ -19,7 +19,7 @@ sleeper system-test - 0.36.2-SNAPSHOT + 0.37.0 4.0.0 diff --git a/java/system-test/system-test-dsl/pom.xml b/java/system-test/system-test-dsl/pom.xml index f94f936d1c0..b484a079ac4 100644 --- a/java/system-test/system-test-dsl/pom.xml +++ b/java/system-test/system-test-dsl/pom.xml @@ -21,7 +21,7 @@ sleeper system-test - 0.36.2-SNAPSHOT + 0.37.0 system-test-dsl diff --git a/java/system-test/system-test-suite/pom.xml b/java/system-test/system-test-suite/pom.xml index 73cf3b495b8..b77900ea34b 100644 --- a/java/system-test/system-test-suite/pom.xml +++ b/java/system-test/system-test-suite/pom.xml @@ -21,7 +21,7 @@ sleeper system-test - 0.36.2-SNAPSHOT + 0.37.0 system-test-suite diff --git a/python/setup.py b/python/setup.py index 5847ca94d9a..01a3f563c0a 100644 --- a/python/setup.py +++ b/python/setup.py @@ -15,7 +15,7 @@ setup( name="sleeper", - version="0.36.2.dev1", + version="0.37.0", description="Python client for Sleeper", install_requires=[ "pyarrow", diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 8ee31ae3817..c92df7ab117 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -19,7 +19,7 @@ checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" [[package]] name = "aggregator_udfs" -version = "0.36.2-SNAPSHOT" +version = "0.37.0" dependencies = [ "datafusion", "mockall", @@ -132,7 +132,7 @@ dependencies = [ [[package]] name = "apps" -version = "0.36.2-SNAPSHOT" +version = "0.37.0" dependencies = [ "assert_cmd", "chrono", @@ -2288,7 +2288,7 @@ checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" [[package]] name = "filter_udfs" -version = "0.36.2-SNAPSHOT" +version = "0.37.0" dependencies = [ "datafusion", ] @@ -3360,7 +3360,7 @@ dependencies = [ [[package]] name = "objectstore_ext" -version = "0.36.2-SNAPSHOT" +version = "0.37.0" dependencies = [ "async-trait", "aws-config", @@ -3651,7 +3651,7 @@ dependencies = [ [[package]] name = "query_sql" -version = "0.36.2-SNAPSHOT" +version = "0.37.0" dependencies = [ "datafusion", "tokio", @@ -3911,7 +3911,7 @@ dependencies = [ [[package]] name = "rust_sketch" -version = "0.36.2-SNAPSHOT" +version = "0.37.0" dependencies = [ "cargo_metadata", "cxx", @@ -4232,7 +4232,7 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "sleeper_core" -version = "0.36.2-SNAPSHOT" +version = "0.37.0" dependencies = [ "aggregator_udfs", "arrow", @@ -4267,7 +4267,7 @@ dependencies = [ [[package]] name = "sleeper_df" -version = "0.36.2-SNAPSHOT" +version = "0.37.0" dependencies = [ "aws-types", "cbindgen", @@ -4464,7 +4464,7 @@ dependencies = [ [[package]] name = "test_util" -version = "0.36.2-SNAPSHOT" +version = "0.37.0" dependencies = [ "bytes", "color-eyre", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 888aa76f315..ea36e8d352c 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -25,7 +25,7 @@ members = [ resolver = "2" [workspace.package] -version = "0.36.2-SNAPSHOT" +version = "0.37.0" edition = "2024" rust-version = "1.93" publish = false From fb3da307f44a1db14a8e98888706219e1e895006 Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Mon, 6 Jul 2026 11:36:37 +0000 Subject: [PATCH 08/50] 7555 Performance figures for 0.37.0 --- docs/development/system-tests.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/development/system-tests.md b/docs/development/system-tests.md index cb7ed7495c2..7a391edfa2a 100644 --- a/docs/development/system-tests.md +++ b/docs/development/system-tests.md @@ -261,3 +261,4 @@ results for each test will be at {NIGHTLY_TEST_BUCKET}/{DATE_OF_TEST}/{SUITE_NAM | 0.35.3 | 06/05/2026 | 224,064 | 3,874,728 | 186,955 | 3,500,007 | | | 0.36.0 | 14/06/2026 | 188,225 | 3,475,731 | 175,062 | | 4,581,912 | | 0.36.1 | 22/06/2026 | 211,266 | 3,504,890 | 160,144 | | 4,272,644 | +| 0.37.0 | 03/07/2026 | 207,597 | 3,477,588 | 168,637 | | 4,102,182 | From 474d46db2a15b86dbd77c6b1a7c260815441686a Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Tue, 30 Jun 2026 13:46:53 +0000 Subject: [PATCH 09/50] 7553 Changelog for 0.37.0 --- CHANGELOG.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b9ca67bd392..0f50dcc4b6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,26 @@ This page documents the releases of Sleeper. Performance figures for each releas are available [here](docs/development/system-tests.md#performance-benchmarks). A roadmap of current and future work is available [here](docs/development/roadmap.md). +## Version 0.37.0 + +### 1st July, 2026 + +This includes bulk import on EKS Auto Mode, + +Bulk import: +- Added an option to run bulk import on EKS Auto Mode, in `sleeper.bulk.import.eks.cluster.type`. + +Query: +- Increased default retries when throttled publishing results to a web socket. +- Added query processing options for retries publishing results to a web socket. + +Scripts: +- Script to add a table now takes options for configuration instead of using templates. + +Configuration: +- Made the example configuration files more representative of real usage. + + ## Version 0.36.1 ### 24th June, 2026 From c62e6e0e5ad03e77d4a979d7692bd9b19b94c5b2 Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Tue, 30 Jun 2026 13:48:02 +0000 Subject: [PATCH 10/50] 7553 Adjust release summary --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f50dcc4b6b..19d6b8efa09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ available [here](docs/development/roadmap.md). ### 1st July, 2026 -This includes bulk import on EKS Auto Mode, +This includes bulk import on EKS Auto Mode, and some improvements to usability and web socket queries. Bulk import: - Added an option to run bulk import on EKS Auto Mode, in `sleeper.bulk.import.eks.cluster.type`. From 29942a77eed361effc64527cb1fcf6daf8a04db4 Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Mon, 6 Jul 2026 09:17:15 +0000 Subject: [PATCH 11/50] 7553 Note bug fix for DataFusion calls --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 19d6b8efa09..9424707aab1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ available [here](docs/development/roadmap.md). ## Version 0.37.0 -### 1st July, 2026 +### 6th July, 2026 This includes bulk import on EKS Auto Mode, and some improvements to usability and web socket queries. @@ -24,6 +24,9 @@ Scripts: Configuration: - Made the example configuration files more representative of real usage. +Bugfixes: +- Resolved some segmentation faults that could occur during calls to DataFusion + ## Version 0.36.1 From 80a79c4b928a43485ce971031ba2b25ef2196ec0 Mon Sep 17 00:00:00 2001 From: ca61688 <206189192+ca61688@users.noreply.github.com> Date: Mon, 6 Jul 2026 15:44:49 +0000 Subject: [PATCH 12/50] 6593: Create initial unit test for deploy new instance --- .../clients/deploy/DeployInstance.java | 2 +- .../clients/deploy/DeployNewInstance.java | 64 ++++-- .../clients/deploy/InstanceDeployer.java | 22 ++ .../clients/deploy/DeployNewInstanceTest.java | 204 ++++++++++++++++++ .../drivers/cdk/DeployNewTestInstance.java | 20 +- .../instance/AwsSleeperInstanceDriver.java | 19 +- 6 files changed, 311 insertions(+), 20 deletions(-) create mode 100644 java/clients/src/main/java/sleeper/clients/deploy/InstanceDeployer.java create mode 100644 java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceTest.java diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployInstance.java index bb520adba1a..1d3151f8018 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployInstance.java @@ -45,7 +45,7 @@ import static sleeper.core.properties.instance.CommonProperty.VPC_ID; import static sleeper.core.properties.model.SleeperInternalCdkApp.ARTEFACTS; -public class DeployInstance { +public class DeployInstance implements InstanceDeployer { public static final Logger LOGGER = LoggerFactory.getLogger(DeployInstance.class); private final SyncJars syncJars; diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java index 21b787e1759..3cc3d857832 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java @@ -33,6 +33,8 @@ import sleeper.core.properties.instance.InstanceProperties; import sleeper.core.properties.model.SleeperInternalCdkApp; import sleeper.core.properties.table.TableProperties; +import sleeper.core.properties.table.TablePropertiesStore; +import sleeper.core.statestore.StateStoreProvider; import sleeper.core.util.cli.CommandArguments; import sleeper.core.util.cli.CommandArgumentsException; import sleeper.core.util.cli.CommandLineUsage; @@ -50,10 +52,9 @@ public class DeployNewInstance { private static final Logger LOGGER = LoggerFactory.getLogger(DeployNewInstance.class); - private final DeployInstance deployInstance; - private final String accountName; - private final S3Client s3Client; - private final DynamoDbClient dynamoClient; + private final InstanceDeployer deployInstance; + private final InstancePropertiesLoader propertiesLoader; + private final StoreFactory storeFactory; private final SleeperInstanceConfiguration deployInstanceConfiguration; private final SleeperInternalCdkApp cdkApp; private final boolean ignoreTableFiles; @@ -63,9 +64,22 @@ public DeployNewInstance(DeployInstance deployInstance, String accountName, S3Cl DynamoDbClient dynamoClient, SleeperInstanceConfiguration deployInstanceConfiguration, SleeperInternalCdkApp cdkApp, boolean ignoreTableFiles, boolean deployPaused) { this.deployInstance = deployInstance; - this.accountName = accountName; - this.s3Client = s3Client; - this.dynamoClient = dynamoClient; + this.deployInstanceConfiguration = deployInstanceConfiguration; + this.cdkApp = cdkApp; + this.ignoreTableFiles = ignoreTableFiles; + this.deployPaused = deployPaused; + propertiesLoader = null; + storeFactory = null; + } + + public DeployNewInstance(InstanceDeployer deployInstance, + InstancePropertiesLoader instancePropertiesLoader, + StoreFactory storeFactory, + SleeperInstanceConfiguration deployInstanceConfiguration, + SleeperInternalCdkApp cdkApp, boolean ignoreTableFiles, boolean deployPaused) { + this.deployInstance = deployInstance; + this.propertiesLoader = instancePropertiesLoader; + this.storeFactory = storeFactory; this.deployInstanceConfiguration = deployInstanceConfiguration; this.cdkApp = cdkApp; this.ignoreTableFiles = ignoreTableFiles; @@ -74,7 +88,7 @@ public DeployNewInstance(DeployInstance deployInstance, String accountName, S3Cl public static final CommandLineUsage USAGE = CommandLineUsage.builder() .systemArguments(List.of("scriptsDirectory")) - .positionalArguments(List.of("scriptsDirectory", "instance-id", "vpcId", "subnetIds")) + .positionalArguments(List.of("scriptsDirectory", "instanceId", "vpcId", "subnetIds")) .options(List.of( CommandOption.longOption("instance-properties"), CommandOption.longOption("config-dir"), @@ -104,7 +118,7 @@ public DeployNewInstance(DeployInstance deployInstance, String accountName, S3Cl public static Arguments readArguments(CommandArguments arguments) { return new Arguments( Path.of(arguments.getString("scriptsDirectory")), - arguments.getString("instance-id"), + arguments.getString("instanceId"), arguments.getString("vpcId"), arguments.getString("subnetIds"), arguments.getOptionalString("instance-properties").map(Path::of).orElse(null), @@ -134,8 +148,17 @@ public static void main(String[] rawArgs) throws IOException, InterruptedExcepti config.getInstanceProperties().set(SUBNETS, args.subnetIds()); new DeployNewInstance(DeployInstance.fromScriptsDirectory(scriptsDirectory, accountName, region, partitionMetadata, s3Client, ecrClient), - accountName, s3Client, dynamoClient, config, SleeperInternalCdkApp.STANDARD, - args.ignoreTableFiles(), deployPaused).deploy(); + instanceId -> S3InstanceProperties.loadGivenAccountAndInstanceId(s3Client, accountName, instanceId), + new StoreFactory() { + public TablePropertiesStore createTableStore(InstanceProperties p) { + return S3TableProperties.createStore(p, s3Client, dynamoClient); + } + + public StateStoreProvider createStateStore(InstanceProperties p) { + return StateStoreFactory.createProvider(p, s3Client, dynamoClient); + } + }, + config, SleeperInternalCdkApp.STANDARD, args.ignoreTableFiles(), deployPaused).deploy(); } } @@ -149,13 +172,13 @@ public void deploy() throws IOException, InterruptedException { .build()); if (!ignoreTableFiles) { - InstanceProperties instanceProperties = S3InstanceProperties.loadGivenAccountAndInstanceId(s3Client, accountName, deployInstanceConfiguration.getInstanceId()); + InstanceProperties instanceProperties = propertiesLoader.load(deployInstanceConfiguration.getInstanceId()); for (TableProperties tableProperties : deployInstanceConfiguration.getTableProperties()) { LOGGER.info("Adding table " + tableProperties.getStatus()); new AddTableClient(tableProperties, - S3TableProperties.createStore(instanceProperties, s3Client, dynamoClient), - StateStoreFactory.createProvider(instanceProperties, s3Client, dynamoClient)) + storeFactory.createTableStore(instanceProperties), + storeFactory.createStateStore(instanceProperties)) .run(); } } @@ -178,7 +201,7 @@ public record Arguments( } if (instanceId == null) { - throw new CommandArgumentsException("instance-id must not be null"); + throw new CommandArgumentsException("instanceId must not be null"); } if (vpcId == null) { @@ -206,4 +229,15 @@ public Path resolvePropertiesFile() { return propertiesFile != null ? propertiesFile : configDir.resolve("instance.properties"); } } + + @FunctionalInterface + public interface InstancePropertiesLoader { + InstanceProperties load(String instanceId); + } + + public interface StoreFactory { + TablePropertiesStore createTableStore(InstanceProperties instanceProperties); + + StateStoreProvider createStateStore(InstanceProperties instanceProperties); + } } diff --git a/java/clients/src/main/java/sleeper/clients/deploy/InstanceDeployer.java b/java/clients/src/main/java/sleeper/clients/deploy/InstanceDeployer.java new file mode 100644 index 00000000000..8bad0e003a8 --- /dev/null +++ b/java/clients/src/main/java/sleeper/clients/deploy/InstanceDeployer.java @@ -0,0 +1,22 @@ +/* + * Copyright 2022-2026 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package sleeper.clients.deploy; + +import java.io.IOException; + +public interface InstanceDeployer { + void deploy(DeployInstanceRequest request) throws IOException, InterruptedException; +} diff --git a/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceTest.java b/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceTest.java new file mode 100644 index 00000000000..8e697e6ecde --- /dev/null +++ b/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceTest.java @@ -0,0 +1,204 @@ +/* + * Copyright 2022-2026 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package sleeper.clients.deploy; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +import sleeper.clients.table.AddTableClient; +import sleeper.core.deploy.SleeperInstanceConfiguration; +import sleeper.core.properties.instance.InstanceProperties; +import sleeper.core.properties.model.SleeperInternalCdkApp; +import sleeper.core.properties.table.TableProperties; +import sleeper.core.properties.table.TablePropertiesStore; +import sleeper.core.properties.testutils.InMemoryTableProperties; +import sleeper.core.schema.Schema; +import sleeper.core.schema.SchemaSerDe; +import sleeper.core.statestore.StateStoreProvider; +import sleeper.core.statestore.testutils.InMemoryTransactionLogStateStore; +import sleeper.core.statestore.testutils.InMemoryTransactionLogsPerTable; +import sleeper.core.table.InMemoryTableIndex; +import sleeper.core.util.cli.CommandArgumentReader; +import sleeper.core.util.cli.CommandArgumentsException; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static sleeper.core.properties.testutils.InstancePropertiesTestHelper.createTestInstancePropertiesWithId; +import static sleeper.core.schema.SchemaTestHelper.createSchemaWithKey; + +public class DeployNewInstanceTest { + InstanceProperties instanceProperties = createTestInstancePropertiesWithId("my-instance"); + Schema schema = createSchemaWithKey("key"); + InMemoryTableIndex tableIndex = new InMemoryTableIndex(); + TablePropertiesStore tablePropertiesStore = InMemoryTableProperties.getStore(tableIndex); + StateStoreProvider stateStoreProvider = InMemoryTransactionLogStateStore.createProvider(instanceProperties, new InMemoryTransactionLogsPerTable()); + Map instanceIdToProperties = new HashMap<>(); + Map pathToString = new HashMap<>(); + + @BeforeEach + void setUp() { + instanceIdToProperties.put("my-instance", instanceProperties); + saveSchemaFile("./schema.json", schema); + saveFile("./table.properties", "sleeper.table.name=file-table\n"); + } + + //TODO test deploy method + + @Nested + class ArgumentsValidation { + + @Test + void shouldRejectWhenNotEnoughPositionalArguments() { + //When/Then + assertThatThrownBy(() -> deployNewInstance()) + .isInstanceOf(CommandArgumentsException.class) + .hasMessage("Expected 4 positional arguments, found 0"); + } + + @Test + void shouldRejectWhenNeitherInstancePropertiesOrConfigDirSet() { + //When/Then + assertThatThrownBy(() -> deployNewInstance("scriptsDir", "my-instance", "my-vpc", "my-subnets")) + .isInstanceOf(CommandArgumentsException.class) + .hasMessage("Either --instance-properties or --config-dir must be provided"); + } + + //TODO validate remst of argument logic + @Test + void shouldRejectWhenTableNameNotSetInPropertiesFile() throws IOException { + //Given + saveFile("other/table.properties", "sleeper.other.property=value\n"); + + //When/Then + assertThatThrownBy(() -> addTable("my-instance", "--schema", "schema.json", + "--table-properties", "other/table.properties")) + .isInstanceOf(CommandArgumentsException.class) + .hasMessage("Table name was not found. Provide --table-name, or set it in --table-properties or --config-dir."); + } + + @Test + void shouldRejectWhenTableNameNotSetInConfigDir() throws IOException { + //Given + saveFile("other/table.properties", "sleeper.other.property=value\n"); + saveSchemaFile("other/schema.json", schema); + + //When/Then + assertThatThrownBy(() -> addTable("my-instance", "--config-dir", "other/")) + .isInstanceOf(CommandArgumentsException.class) + .hasMessage("Table name was not found. Provide --table-name, or set it in --table-properties or --config-dir."); + } + + @Test + void shouldRejectWhenNoSchemaSource() { + //When/Then + assertThatThrownBy(() -> addTable("my-instance", "--table-name", "my-table")) + .isInstanceOf(CommandArgumentsException.class) + .hasMessage("Either --schema or --config-dir must be provided"); + } + + @Test + void shouldRejectWhenNoTablePropertiesInConfigDir() { + //Given + saveSchemaFile("other/schema.json", schema); + + //When/Then + assertThatThrownBy(() -> addTable("my-instance", "--config-dir", "other/")) + .isInstanceOf(UncheckedIOException.class); + } + + @Test + void shouldRejectWhenNoSchemaInConfigDir() { + //Given + saveFile("other/table.properties", "sleeper.table.name=no-schema\n"); + + //When/Then + assertThatThrownBy(() -> addTable("my-instance", "--config-dir", "other/")) + .isInstanceOf(UncheckedIOException.class); + } + + @Test + void shouldRejectWhenAllThreeFileSourcesSpecified() throws IOException { + //When/Then + assertThatThrownBy(() -> addTable("my-instance", "--table-name", "my-table", + "--schema", "schema.json", "--table-properties", "./table.properties", + "--config-dir", "./")) + .isInstanceOf(CommandArgumentsException.class) + .hasMessage("Cannot specify --schema, --table-properties, and --config-dir together"); + } + + } + + private void addTable(String... args) throws Exception { + var arguments = AddTableClient.readArguments(CommandArgumentReader.parse(AddTableClient.USAGE, args), this::readFile); + TableProperties tableProperties = AddTableClient.createTablePropertiesWithLoaders(arguments, this::loadInstanceProperties, this::readFile); + new AddTableClient(tableProperties, tablePropertiesStore, stateStoreProvider).run(); + } + + private void deployNewInstance(String... args) throws Exception { + var arguments = DeployNewInstance.readArguments(CommandArgumentReader.parse(DeployNewInstance.USAGE, args)); + SleeperInstanceConfiguration config = SleeperInstanceConfiguration.withNoTables(instanceProperties); + new DeployNewInstance( + request -> { + }, // no-op stub — no real CDK/S3/ECR + this::loadInstanceProperties, + new DeployNewInstance.StoreFactory() { + public TablePropertiesStore createTableStore(InstanceProperties p) { + return tablePropertiesStore; + } + + public StateStoreProvider createStateStore(InstanceProperties p) { + return stateStoreProvider; + } + }, + config, SleeperInternalCdkApp.STANDARD, + arguments.ignoreTableFiles(), arguments.deployPaused()).deploy(); + } + + private void saveSchemaFile(String path, Schema schema) { + pathToString.put(Path.of(path), new SchemaSerDe().toJson(schema)); + } + + private void saveFile(String path, String content) { + pathToString.put(Path.of(path), content); + } + + private String tableId(String tableName) { + return tableIndex.getTableByName(tableName) + .orElseThrow(() -> new RuntimeException("Found tables: " + tableIndex.streamAllTables().toList())) + .getTableUniqueId(); + } + + private InstanceProperties loadInstanceProperties(String instanceId) { + return Optional.ofNullable(instanceIdToProperties.get(instanceId)) + .orElseThrow(); + } + + private String readFile(Path path) throws IOException { + try { + return Optional.ofNullable(pathToString.get(path)).orElseThrow(); + } catch (NoSuchElementException e) { + throw new IOException(e); + } + } +} diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java index df0ed98104e..ae7052ebfc1 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java @@ -25,9 +25,16 @@ import sleeper.clients.deploy.DeployInstance; import sleeper.clients.deploy.DeployNewInstance; +import sleeper.clients.deploy.DeployNewInstance.StoreFactory; +import sleeper.configuration.properties.S3InstanceProperties; +import sleeper.configuration.properties.S3TableProperties; import sleeper.core.deploy.SleeperInstanceConfiguration; import sleeper.core.deploy.SleeperInstanceConfigurationFromTemplates; +import sleeper.core.properties.instance.InstanceProperties; import sleeper.core.properties.model.SleeperInternalCdkApp; +import sleeper.core.properties.table.TablePropertiesStore; +import sleeper.core.statestore.StateStoreProvider; +import sleeper.statestore.StateStoreFactory; import java.io.IOException; import java.nio.file.Path; @@ -69,8 +76,17 @@ public static void main(String[] args) throws IOException, InterruptedException config.getInstanceProperties().set(VPC_ID, vpcId); config.getInstanceProperties().set(SUBNETS, subnetIds); new DeployNewInstance(DeployInstance.fromScriptsDirectory(scriptsDirectory, accountName, region, partitionMetadata, s3Client, ecrClient), - accountName, s3Client, dynamoClient, config, - SleeperInternalCdkApp.DEMONSTRATION, false, deployPaused).deploy(); + id -> S3InstanceProperties.loadGivenAccountAndInstanceId(s3Client, accountName, id), + new StoreFactory() { + public TablePropertiesStore createTableStore(InstanceProperties p) { + return S3TableProperties.createStore(p, s3Client, dynamoClient); + } + + public StateStoreProvider createStateStore(InstanceProperties p) { + return StateStoreFactory.createProvider(p, s3Client, dynamoClient); + } + }, + config, SleeperInternalCdkApp.STANDARD, false, deployPaused).deploy(); } } diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java index 2ff004d1111..c190dbebbb7 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java @@ -27,11 +27,16 @@ import sleeper.clients.deploy.DeployExistingInstance; import sleeper.clients.deploy.DeployInstance; import sleeper.clients.deploy.DeployNewInstance; +import sleeper.clients.deploy.DeployNewInstance.StoreFactory; import sleeper.configuration.properties.S3InstanceProperties; +import sleeper.configuration.properties.S3TableProperties; import sleeper.core.deploy.SleeperInstanceConfiguration; import sleeper.core.properties.instance.InstanceProperties; import sleeper.core.properties.model.SleeperInternalCdkApp; import sleeper.core.properties.table.TableProperties; +import sleeper.core.properties.table.TablePropertiesStore; +import sleeper.core.statestore.StateStoreProvider; +import sleeper.statestore.StateStoreFactory; import sleeper.systemtest.drivers.util.SystemTestClients; import sleeper.systemtest.dsl.instance.SleeperInstanceDriver; import sleeper.systemtest.dsl.instance.SystemTestParameters; @@ -84,8 +89,18 @@ public boolean deployInstanceIfNotPresent(String instanceId, SleeperInstanceConf deployConfig.getInstanceProperties().set(VPC_ID, parameters.getVpcId()); deployConfig.getInstanceProperties().set(SUBNETS, parameters.getSubnetIds()); try { - new DeployNewInstance(deployInstance, parameters.getAccount(), s3, dynamoDB, deployConfig, - SleeperInternalCdkApp.STANDARD, false, false).deploy(); + new DeployNewInstance(deployInstance, + id -> S3InstanceProperties.loadGivenAccountAndInstanceId(s3, parameters.getAccount(), id), + new StoreFactory() { + public TablePropertiesStore createTableStore(InstanceProperties p) { + return S3TableProperties.createStore(p, s3, dynamoDB); + } + + public StateStoreProvider createStateStore(InstanceProperties p) { + return StateStoreFactory.createProvider(p, s3, dynamoDB); + } + }, + deployConfig, SleeperInternalCdkApp.STANDARD, false, false).deploy(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RuntimeException(e); From 5eea52f7fe4d0e92548abdcc387bbf94269a5e3e Mon Sep 17 00:00:00 2001 From: patchwork01 <110390516+patchwork01@users.noreply.github.com> Date: Mon, 6 Jul 2026 13:18:17 +0000 Subject: [PATCH 13/50] 7605 Set version number to 0.37.1-SNAPSHOT --- java/analytics-integration/athena/pom.xml | 2 +- java/analytics-integration/pom.xml | 2 +- java/analytics-integration/spark/pom.xml | 2 +- java/analytics-integration/trino/pom.xml | 2 +- java/build/pom.xml | 2 +- java/bulk-export/bulk-export-core/pom.xml | 2 +- java/bulk-export/bulk-export-planner/pom.xml | 2 +- .../bulk-export-task-creator/pom.xml | 2 +- .../bulk-export-task-execution/pom.xml | 2 +- java/bulk-export/pom.xml | 2 +- java/bulk-import/bulk-import-core/pom.xml | 2 +- java/bulk-import/bulk-import-eks/pom.xml | 2 +- java/bulk-import/bulk-import-runner/pom.xml | 2 +- java/bulk-import/bulk-import-starter/pom.xml | 2 +- java/bulk-import/pom.xml | 2 +- java/clients/pom.xml | 2 +- java/common/arrow/pom.xml | 2 +- java/common/common-invoke-tables/pom.xml | 2 +- java/common/common-job/pom.xml | 2 +- java/common/common-task/pom.xml | 2 +- java/common/docker-lambda/pom.xml | 2 +- java/common/dynamodb-tools/pom.xml | 2 +- java/common/foreign-bridge/pom.xml | 2 +- java/common/localstack-test/pom.xml | 2 +- java/common/parquet/pom.xml | 2 +- java/common/pom.xml | 2 +- java/common/sketches/pom.xml | 2 +- java/compaction/compaction-core/pom.xml | 2 +- java/compaction/compaction-datafusion/pom.xml | 2 +- .../compaction-job-creation-lambda/pom.xml | 2 +- .../compaction/compaction-job-creation/pom.xml | 2 +- .../compaction-job-execution/pom.xml | 2 +- .../compaction-task-creation/pom.xml | 2 +- java/compaction/compaction-tracker/pom.xml | 2 +- java/compaction/pom.xml | 2 +- java/configuration/pom.xml | 2 +- java/core/pom.xml | 2 +- java/deployment/build-uptime-lambda/pom.xml | 2 +- java/deployment/cdk-custom-resources/pom.xml | 2 +- java/deployment/cdk-environment/pom.xml | 2 +- java/deployment/cdk/pom.xml | 2 +- java/deployment/container-images/pom.xml | 2 +- java/deployment/pom.xml | 2 +- java/distribution/pom.xml | 2 +- java/example-iterators/pom.xml | 2 +- java/garbage-collector/pom.xml | 2 +- java/ingest/ingest-batcher-core/pom.xml | 2 +- java/ingest/ingest-batcher-job-creator/pom.xml | 2 +- java/ingest/ingest-batcher-store/pom.xml | 2 +- java/ingest/ingest-batcher-submitter/pom.xml | 2 +- java/ingest/ingest-core/pom.xml | 2 +- java/ingest/ingest-runner/pom.xml | 2 +- java/ingest/ingest-taskrunner/pom.xml | 2 +- java/ingest/ingest-tracker/pom.xml | 2 +- java/ingest/pom.xml | 2 +- java/metrics/pom.xml | 2 +- java/partitions/pom.xml | 2 +- java/partitions/splitter-lambda/pom.xml | 2 +- java/partitions/splitter/pom.xml | 2 +- java/pom.xml | 2 +- java/query/pom.xml | 2 +- java/query/query-core/pom.xml | 2 +- java/query/query-datafusion/pom.xml | 2 +- java/query/query-lambda/pom.xml | 2 +- java/query/query-runner/pom.xml | 2 +- java/rest-api/pom.xml | 2 +- java/statestore-committer-core/pom.xml | 2 +- java/statestore-committer/pom.xml | 2 +- java/statestore-lambda/pom.xml | 2 +- java/statestore/pom.xml | 2 +- java/system-test/pom.xml | 2 +- java/system-test/system-test-cdk/pom.xml | 2 +- .../system-test-configuration/pom.xml | 2 +- .../system-test-data-generation/pom.xml | 2 +- java/system-test/system-test-drivers/pom.xml | 2 +- java/system-test/system-test-dsl/pom.xml | 2 +- java/system-test/system-test-suite/pom.xml | 2 +- python/setup.py | 2 +- rust/Cargo.lock | 18 +++++++++--------- rust/Cargo.toml | 2 +- 80 files changed, 88 insertions(+), 88 deletions(-) diff --git a/java/analytics-integration/athena/pom.xml b/java/analytics-integration/athena/pom.xml index 1f4fd04dbdf..8106a7bd3ca 100644 --- a/java/analytics-integration/athena/pom.xml +++ b/java/analytics-integration/athena/pom.xml @@ -19,7 +19,7 @@ analytics-integration sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/analytics-integration/pom.xml b/java/analytics-integration/pom.xml index f55a938e866..85785ff418e 100644 --- a/java/analytics-integration/pom.xml +++ b/java/analytics-integration/pom.xml @@ -19,7 +19,7 @@ aws sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 analytics-integration diff --git a/java/analytics-integration/spark/pom.xml b/java/analytics-integration/spark/pom.xml index 080d6c6514a..76fa9497ed4 100644 --- a/java/analytics-integration/spark/pom.xml +++ b/java/analytics-integration/spark/pom.xml @@ -19,7 +19,7 @@ analytics-integration sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/analytics-integration/trino/pom.xml b/java/analytics-integration/trino/pom.xml index 91d52528a04..3ab27748c03 100644 --- a/java/analytics-integration/trino/pom.xml +++ b/java/analytics-integration/trino/pom.xml @@ -19,7 +19,7 @@ analytics-integration sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/build/pom.xml b/java/build/pom.xml index d642f7b46b5..6682903e6fa 100644 --- a/java/build/pom.xml +++ b/java/build/pom.xml @@ -19,7 +19,7 @@ aws sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/bulk-export/bulk-export-core/pom.xml b/java/bulk-export/bulk-export-core/pom.xml index 76adfcabf30..51ba212e98f 100644 --- a/java/bulk-export/bulk-export-core/pom.xml +++ b/java/bulk-export/bulk-export-core/pom.xml @@ -19,7 +19,7 @@ sleeper bulk-export - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 bulk-export-core diff --git a/java/bulk-export/bulk-export-planner/pom.xml b/java/bulk-export/bulk-export-planner/pom.xml index 5a3a14bcb2e..c03f5dbf265 100644 --- a/java/bulk-export/bulk-export-planner/pom.xml +++ b/java/bulk-export/bulk-export-planner/pom.xml @@ -19,7 +19,7 @@ sleeper bulk-export - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 bulk-export-planner diff --git a/java/bulk-export/bulk-export-task-creator/pom.xml b/java/bulk-export/bulk-export-task-creator/pom.xml index 501a7939ee3..f791cc39b59 100644 --- a/java/bulk-export/bulk-export-task-creator/pom.xml +++ b/java/bulk-export/bulk-export-task-creator/pom.xml @@ -19,7 +19,7 @@ sleeper bulk-export - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 bulk-export-task-creator diff --git a/java/bulk-export/bulk-export-task-execution/pom.xml b/java/bulk-export/bulk-export-task-execution/pom.xml index af7e43b21f6..88cf3837d33 100644 --- a/java/bulk-export/bulk-export-task-execution/pom.xml +++ b/java/bulk-export/bulk-export-task-execution/pom.xml @@ -19,7 +19,7 @@ sleeper bulk-export - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 bulk-export-task-execution diff --git a/java/bulk-export/pom.xml b/java/bulk-export/pom.xml index 33dd5281d80..dedb8c6f2b4 100644 --- a/java/bulk-export/pom.xml +++ b/java/bulk-export/pom.xml @@ -19,7 +19,7 @@ aws sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 bulk-export diff --git a/java/bulk-import/bulk-import-core/pom.xml b/java/bulk-import/bulk-import-core/pom.xml index 8f7585f7679..d772b56dc1c 100644 --- a/java/bulk-import/bulk-import-core/pom.xml +++ b/java/bulk-import/bulk-import-core/pom.xml @@ -19,7 +19,7 @@ bulk-import sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/bulk-import/bulk-import-eks/pom.xml b/java/bulk-import/bulk-import-eks/pom.xml index 43739dc87b7..b26463e419d 100644 --- a/java/bulk-import/bulk-import-eks/pom.xml +++ b/java/bulk-import/bulk-import-eks/pom.xml @@ -19,7 +19,7 @@ sleeper bulk-import - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/bulk-import/bulk-import-runner/pom.xml b/java/bulk-import/bulk-import-runner/pom.xml index 40cd530b3cd..fc18e08748c 100644 --- a/java/bulk-import/bulk-import-runner/pom.xml +++ b/java/bulk-import/bulk-import-runner/pom.xml @@ -19,7 +19,7 @@ sleeper bulk-import - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/bulk-import/bulk-import-starter/pom.xml b/java/bulk-import/bulk-import-starter/pom.xml index 4d9ac2622bd..3eb905eca1d 100644 --- a/java/bulk-import/bulk-import-starter/pom.xml +++ b/java/bulk-import/bulk-import-starter/pom.xml @@ -19,7 +19,7 @@ sleeper bulk-import - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/bulk-import/pom.xml b/java/bulk-import/pom.xml index b058b016126..65d6328f549 100644 --- a/java/bulk-import/pom.xml +++ b/java/bulk-import/pom.xml @@ -19,7 +19,7 @@ aws sleeper - 0.37.0 + 0.37.1-SNAPSHOT pom diff --git a/java/clients/pom.xml b/java/clients/pom.xml index 248e52aca10..a4c550e5ecf 100644 --- a/java/clients/pom.xml +++ b/java/clients/pom.xml @@ -19,7 +19,7 @@ aws sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/common/arrow/pom.xml b/java/common/arrow/pom.xml index 8468e9c1a1f..8fafdb722d5 100644 --- a/java/common/arrow/pom.xml +++ b/java/common/arrow/pom.xml @@ -19,7 +19,7 @@ common sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/common/common-invoke-tables/pom.xml b/java/common/common-invoke-tables/pom.xml index 6c77dbc7565..c48f04beddc 100644 --- a/java/common/common-invoke-tables/pom.xml +++ b/java/common/common-invoke-tables/pom.xml @@ -20,7 +20,7 @@ common sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/common/common-job/pom.xml b/java/common/common-job/pom.xml index 7486bc3b022..c54d422b44c 100644 --- a/java/common/common-job/pom.xml +++ b/java/common/common-job/pom.xml @@ -19,7 +19,7 @@ common sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/common/common-task/pom.xml b/java/common/common-task/pom.xml index ece112bf98f..15cc3edc726 100644 --- a/java/common/common-task/pom.xml +++ b/java/common/common-task/pom.xml @@ -19,7 +19,7 @@ common sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/common/docker-lambda/pom.xml b/java/common/docker-lambda/pom.xml index 53b3e4c6942..419ff7739fd 100644 --- a/java/common/docker-lambda/pom.xml +++ b/java/common/docker-lambda/pom.xml @@ -20,7 +20,7 @@ common sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/common/dynamodb-tools/pom.xml b/java/common/dynamodb-tools/pom.xml index 0fe5e174070..f79f0bfcc16 100644 --- a/java/common/dynamodb-tools/pom.xml +++ b/java/common/dynamodb-tools/pom.xml @@ -20,7 +20,7 @@ common sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/common/foreign-bridge/pom.xml b/java/common/foreign-bridge/pom.xml index 719b57f095b..fdf499272c4 100644 --- a/java/common/foreign-bridge/pom.xml +++ b/java/common/foreign-bridge/pom.xml @@ -19,7 +19,7 @@ common sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/common/localstack-test/pom.xml b/java/common/localstack-test/pom.xml index 550252bb3e3..41fa641624d 100644 --- a/java/common/localstack-test/pom.xml +++ b/java/common/localstack-test/pom.xml @@ -20,7 +20,7 @@ common sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/common/parquet/pom.xml b/java/common/parquet/pom.xml index 7a0c64b49f1..a5e2ab2a06d 100644 --- a/java/common/parquet/pom.xml +++ b/java/common/parquet/pom.xml @@ -19,7 +19,7 @@ common sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/common/pom.xml b/java/common/pom.xml index 27398319b8e..3460c606c68 100644 --- a/java/common/pom.xml +++ b/java/common/pom.xml @@ -19,7 +19,7 @@ aws sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/common/sketches/pom.xml b/java/common/sketches/pom.xml index 326e8cac231..64b54536638 100644 --- a/java/common/sketches/pom.xml +++ b/java/common/sketches/pom.xml @@ -19,7 +19,7 @@ common sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/compaction/compaction-core/pom.xml b/java/compaction/compaction-core/pom.xml index 51d2d584af6..8844482e6cb 100644 --- a/java/compaction/compaction-core/pom.xml +++ b/java/compaction/compaction-core/pom.xml @@ -19,7 +19,7 @@ compaction sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/compaction/compaction-datafusion/pom.xml b/java/compaction/compaction-datafusion/pom.xml index e88d9fb75f2..c125e94c8de 100644 --- a/java/compaction/compaction-datafusion/pom.xml +++ b/java/compaction/compaction-datafusion/pom.xml @@ -19,7 +19,7 @@ compaction sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 compaction-datafusion diff --git a/java/compaction/compaction-job-creation-lambda/pom.xml b/java/compaction/compaction-job-creation-lambda/pom.xml index 05cd42d977f..cd7e20c2b4a 100644 --- a/java/compaction/compaction-job-creation-lambda/pom.xml +++ b/java/compaction/compaction-job-creation-lambda/pom.xml @@ -19,7 +19,7 @@ compaction sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/compaction/compaction-job-creation/pom.xml b/java/compaction/compaction-job-creation/pom.xml index bb6748c001b..de160d2d5f0 100644 --- a/java/compaction/compaction-job-creation/pom.xml +++ b/java/compaction/compaction-job-creation/pom.xml @@ -19,7 +19,7 @@ compaction sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/compaction/compaction-job-execution/pom.xml b/java/compaction/compaction-job-execution/pom.xml index 812d7924def..a89c5445280 100644 --- a/java/compaction/compaction-job-execution/pom.xml +++ b/java/compaction/compaction-job-execution/pom.xml @@ -19,7 +19,7 @@ compaction sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/compaction/compaction-task-creation/pom.xml b/java/compaction/compaction-task-creation/pom.xml index 31cf82a51f2..1b5c5b4314d 100644 --- a/java/compaction/compaction-task-creation/pom.xml +++ b/java/compaction/compaction-task-creation/pom.xml @@ -21,7 +21,7 @@ sleeper compaction - 0.37.0 + 0.37.1-SNAPSHOT compaction-task-creation diff --git a/java/compaction/compaction-tracker/pom.xml b/java/compaction/compaction-tracker/pom.xml index ee2e66d8ce4..1b531a853b3 100644 --- a/java/compaction/compaction-tracker/pom.xml +++ b/java/compaction/compaction-tracker/pom.xml @@ -19,7 +19,7 @@ compaction sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/compaction/pom.xml b/java/compaction/pom.xml index fca0f1bf723..99b814fe7ae 100644 --- a/java/compaction/pom.xml +++ b/java/compaction/pom.xml @@ -19,7 +19,7 @@ aws sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/configuration/pom.xml b/java/configuration/pom.xml index 3ca569b9d7b..83adcfb7e50 100644 --- a/java/configuration/pom.xml +++ b/java/configuration/pom.xml @@ -19,7 +19,7 @@ aws sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/core/pom.xml b/java/core/pom.xml index 64a45d6fe24..caabc36bf2a 100644 --- a/java/core/pom.xml +++ b/java/core/pom.xml @@ -19,7 +19,7 @@ aws sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/deployment/build-uptime-lambda/pom.xml b/java/deployment/build-uptime-lambda/pom.xml index 5d6dd59e66d..9b123498323 100644 --- a/java/deployment/build-uptime-lambda/pom.xml +++ b/java/deployment/build-uptime-lambda/pom.xml @@ -21,7 +21,7 @@ sleeper deployment - 0.37.0 + 0.37.1-SNAPSHOT build-uptime-lambda diff --git a/java/deployment/cdk-custom-resources/pom.xml b/java/deployment/cdk-custom-resources/pom.xml index aa55da5a8ed..b65510f7e53 100644 --- a/java/deployment/cdk-custom-resources/pom.xml +++ b/java/deployment/cdk-custom-resources/pom.xml @@ -21,7 +21,7 @@ sleeper deployment - 0.37.0 + 0.37.1-SNAPSHOT cdk-custom-resources diff --git a/java/deployment/cdk-environment/pom.xml b/java/deployment/cdk-environment/pom.xml index 361f8a24247..8c4c4e133dd 100644 --- a/java/deployment/cdk-environment/pom.xml +++ b/java/deployment/cdk-environment/pom.xml @@ -19,7 +19,7 @@ deployment sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/deployment/cdk/pom.xml b/java/deployment/cdk/pom.xml index c4944436e2d..53f2d1b3f8b 100644 --- a/java/deployment/cdk/pom.xml +++ b/java/deployment/cdk/pom.xml @@ -19,7 +19,7 @@ deployment sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/deployment/container-images/pom.xml b/java/deployment/container-images/pom.xml index e1277cba632..0be22b8f9e3 100644 --- a/java/deployment/container-images/pom.xml +++ b/java/deployment/container-images/pom.xml @@ -21,7 +21,7 @@ sleeper deployment - 0.37.0 + 0.37.1-SNAPSHOT container-images diff --git a/java/deployment/pom.xml b/java/deployment/pom.xml index ae5708d387e..e107538a86e 100644 --- a/java/deployment/pom.xml +++ b/java/deployment/pom.xml @@ -19,7 +19,7 @@ aws sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/distribution/pom.xml b/java/distribution/pom.xml index 53a71921c7d..ab145c566b4 100644 --- a/java/distribution/pom.xml +++ b/java/distribution/pom.xml @@ -19,7 +19,7 @@ aws sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/example-iterators/pom.xml b/java/example-iterators/pom.xml index 5e1e2d8dcef..1ee8db6e8c4 100644 --- a/java/example-iterators/pom.xml +++ b/java/example-iterators/pom.xml @@ -19,7 +19,7 @@ aws sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/garbage-collector/pom.xml b/java/garbage-collector/pom.xml index 6187075d5aa..083206aca5f 100644 --- a/java/garbage-collector/pom.xml +++ b/java/garbage-collector/pom.xml @@ -19,7 +19,7 @@ aws sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/ingest/ingest-batcher-core/pom.xml b/java/ingest/ingest-batcher-core/pom.xml index cae1d0542d9..1135197c92a 100644 --- a/java/ingest/ingest-batcher-core/pom.xml +++ b/java/ingest/ingest-batcher-core/pom.xml @@ -20,7 +20,7 @@ ingest sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/ingest/ingest-batcher-job-creator/pom.xml b/java/ingest/ingest-batcher-job-creator/pom.xml index cfbdc2da6ae..0949b218d6f 100644 --- a/java/ingest/ingest-batcher-job-creator/pom.xml +++ b/java/ingest/ingest-batcher-job-creator/pom.xml @@ -21,7 +21,7 @@ sleeper ingest - 0.37.0 + 0.37.1-SNAPSHOT ingest-batcher-job-creator diff --git a/java/ingest/ingest-batcher-store/pom.xml b/java/ingest/ingest-batcher-store/pom.xml index 8ebaa12a2c0..47651018d42 100644 --- a/java/ingest/ingest-batcher-store/pom.xml +++ b/java/ingest/ingest-batcher-store/pom.xml @@ -21,7 +21,7 @@ sleeper ingest - 0.37.0 + 0.37.1-SNAPSHOT ingest-batcher-store diff --git a/java/ingest/ingest-batcher-submitter/pom.xml b/java/ingest/ingest-batcher-submitter/pom.xml index 160ffa27eea..7f3a9b7893c 100644 --- a/java/ingest/ingest-batcher-submitter/pom.xml +++ b/java/ingest/ingest-batcher-submitter/pom.xml @@ -21,7 +21,7 @@ sleeper ingest - 0.37.0 + 0.37.1-SNAPSHOT ingest-batcher-submitter diff --git a/java/ingest/ingest-core/pom.xml b/java/ingest/ingest-core/pom.xml index 00c77ad71a4..3afb92be43c 100644 --- a/java/ingest/ingest-core/pom.xml +++ b/java/ingest/ingest-core/pom.xml @@ -19,7 +19,7 @@ ingest sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/ingest/ingest-runner/pom.xml b/java/ingest/ingest-runner/pom.xml index 2b68067b473..fd07268da52 100644 --- a/java/ingest/ingest-runner/pom.xml +++ b/java/ingest/ingest-runner/pom.xml @@ -19,7 +19,7 @@ ingest sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/ingest/ingest-taskrunner/pom.xml b/java/ingest/ingest-taskrunner/pom.xml index 3ce2f748002..b076f959605 100644 --- a/java/ingest/ingest-taskrunner/pom.xml +++ b/java/ingest/ingest-taskrunner/pom.xml @@ -19,7 +19,7 @@ ingest sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/ingest/ingest-tracker/pom.xml b/java/ingest/ingest-tracker/pom.xml index 124ad0c3cbb..b129a4296ac 100644 --- a/java/ingest/ingest-tracker/pom.xml +++ b/java/ingest/ingest-tracker/pom.xml @@ -20,7 +20,7 @@ ingest sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/ingest/pom.xml b/java/ingest/pom.xml index 7af440bfeba..bb9a103d341 100644 --- a/java/ingest/pom.xml +++ b/java/ingest/pom.xml @@ -19,7 +19,7 @@ aws sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/metrics/pom.xml b/java/metrics/pom.xml index 48f6dbcbaa4..9f717952c92 100644 --- a/java/metrics/pom.xml +++ b/java/metrics/pom.xml @@ -20,7 +20,7 @@ aws sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/partitions/pom.xml b/java/partitions/pom.xml index 8f957d97539..15e9f1709b4 100644 --- a/java/partitions/pom.xml +++ b/java/partitions/pom.xml @@ -19,7 +19,7 @@ aws sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/partitions/splitter-lambda/pom.xml b/java/partitions/splitter-lambda/pom.xml index 9bb7a1fc319..fcb9cc32058 100644 --- a/java/partitions/splitter-lambda/pom.xml +++ b/java/partitions/splitter-lambda/pom.xml @@ -19,7 +19,7 @@ partitions sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/partitions/splitter/pom.xml b/java/partitions/splitter/pom.xml index e2f4577cef3..e5ff1e61502 100644 --- a/java/partitions/splitter/pom.xml +++ b/java/partitions/splitter/pom.xml @@ -19,7 +19,7 @@ partitions sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/pom.xml b/java/pom.xml index 36abe592f8c..7fb88e02d48 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -21,7 +21,7 @@ sleeper aws pom - 0.37.0 + 0.37.1-SNAPSHOT analytics-integration diff --git a/java/query/pom.xml b/java/query/pom.xml index 7c66417ba81..74a100af134 100644 --- a/java/query/pom.xml +++ b/java/query/pom.xml @@ -19,7 +19,7 @@ aws sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/query/query-core/pom.xml b/java/query/query-core/pom.xml index 6348fae5e5f..c0be48d295a 100644 --- a/java/query/query-core/pom.xml +++ b/java/query/query-core/pom.xml @@ -19,7 +19,7 @@ query sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/query/query-datafusion/pom.xml b/java/query/query-datafusion/pom.xml index c5a64a69a6d..057d271cc85 100644 --- a/java/query/query-datafusion/pom.xml +++ b/java/query/query-datafusion/pom.xml @@ -19,7 +19,7 @@ query sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/query/query-lambda/pom.xml b/java/query/query-lambda/pom.xml index 0f40b6a4b9b..8b64bab2b8b 100644 --- a/java/query/query-lambda/pom.xml +++ b/java/query/query-lambda/pom.xml @@ -19,7 +19,7 @@ query sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/query/query-runner/pom.xml b/java/query/query-runner/pom.xml index d30e1d229a4..f58a697c57b 100644 --- a/java/query/query-runner/pom.xml +++ b/java/query/query-runner/pom.xml @@ -19,7 +19,7 @@ query sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/rest-api/pom.xml b/java/rest-api/pom.xml index 1611d2b823e..7dbfcbcc17f 100644 --- a/java/rest-api/pom.xml +++ b/java/rest-api/pom.xml @@ -19,7 +19,7 @@ aws sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/statestore-committer-core/pom.xml b/java/statestore-committer-core/pom.xml index bc372353199..e466a751eaa 100644 --- a/java/statestore-committer-core/pom.xml +++ b/java/statestore-committer-core/pom.xml @@ -19,7 +19,7 @@ aws sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/statestore-committer/pom.xml b/java/statestore-committer/pom.xml index 759452dd72a..7d7cafd040a 100644 --- a/java/statestore-committer/pom.xml +++ b/java/statestore-committer/pom.xml @@ -19,7 +19,7 @@ aws sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/statestore-lambda/pom.xml b/java/statestore-lambda/pom.xml index 88db40d5fd5..a757a6fb3fd 100644 --- a/java/statestore-lambda/pom.xml +++ b/java/statestore-lambda/pom.xml @@ -19,7 +19,7 @@ aws sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/statestore/pom.xml b/java/statestore/pom.xml index 4406e3bcc4e..56c84ad4824 100644 --- a/java/statestore/pom.xml +++ b/java/statestore/pom.xml @@ -19,7 +19,7 @@ aws sleeper - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/system-test/pom.xml b/java/system-test/pom.xml index 801f8126bc2..4b120ba1dc0 100644 --- a/java/system-test/pom.xml +++ b/java/system-test/pom.xml @@ -21,7 +21,7 @@ sleeper aws - 0.37.0 + 0.37.1-SNAPSHOT pom diff --git a/java/system-test/system-test-cdk/pom.xml b/java/system-test/system-test-cdk/pom.xml index 3b2bc9d5df5..f63a3f1a8e9 100644 --- a/java/system-test/system-test-cdk/pom.xml +++ b/java/system-test/system-test-cdk/pom.xml @@ -21,7 +21,7 @@ sleeper system-test - 0.37.0 + 0.37.1-SNAPSHOT system-test-cdk diff --git a/java/system-test/system-test-configuration/pom.xml b/java/system-test/system-test-configuration/pom.xml index 85dc2e947ba..cd776084b2b 100644 --- a/java/system-test/system-test-configuration/pom.xml +++ b/java/system-test/system-test-configuration/pom.xml @@ -21,7 +21,7 @@ sleeper system-test - 0.37.0 + 0.37.1-SNAPSHOT system-test-configuration diff --git a/java/system-test/system-test-data-generation/pom.xml b/java/system-test/system-test-data-generation/pom.xml index 7e8e2edc2c0..129a9ca4ae5 100644 --- a/java/system-test/system-test-data-generation/pom.xml +++ b/java/system-test/system-test-data-generation/pom.xml @@ -21,7 +21,7 @@ sleeper system-test - 0.37.0 + 0.37.1-SNAPSHOT system-test-data-generation diff --git a/java/system-test/system-test-drivers/pom.xml b/java/system-test/system-test-drivers/pom.xml index 1322aff8bce..dde57a58bdd 100644 --- a/java/system-test/system-test-drivers/pom.xml +++ b/java/system-test/system-test-drivers/pom.xml @@ -19,7 +19,7 @@ sleeper system-test - 0.37.0 + 0.37.1-SNAPSHOT 4.0.0 diff --git a/java/system-test/system-test-dsl/pom.xml b/java/system-test/system-test-dsl/pom.xml index b484a079ac4..cb2e9287a40 100644 --- a/java/system-test/system-test-dsl/pom.xml +++ b/java/system-test/system-test-dsl/pom.xml @@ -21,7 +21,7 @@ sleeper system-test - 0.37.0 + 0.37.1-SNAPSHOT system-test-dsl diff --git a/java/system-test/system-test-suite/pom.xml b/java/system-test/system-test-suite/pom.xml index b77900ea34b..a777f2e1926 100644 --- a/java/system-test/system-test-suite/pom.xml +++ b/java/system-test/system-test-suite/pom.xml @@ -21,7 +21,7 @@ sleeper system-test - 0.37.0 + 0.37.1-SNAPSHOT system-test-suite diff --git a/python/setup.py b/python/setup.py index 01a3f563c0a..af0ef43b4f7 100644 --- a/python/setup.py +++ b/python/setup.py @@ -15,7 +15,7 @@ setup( name="sleeper", - version="0.37.0", + version="0.37.1.dev1", description="Python client for Sleeper", install_requires=[ "pyarrow", diff --git a/rust/Cargo.lock b/rust/Cargo.lock index c92df7ab117..9a6cdd69830 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -19,7 +19,7 @@ checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" [[package]] name = "aggregator_udfs" -version = "0.37.0" +version = "0.37.1-SNAPSHOT" dependencies = [ "datafusion", "mockall", @@ -132,7 +132,7 @@ dependencies = [ [[package]] name = "apps" -version = "0.37.0" +version = "0.37.1-SNAPSHOT" dependencies = [ "assert_cmd", "chrono", @@ -2288,7 +2288,7 @@ checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" [[package]] name = "filter_udfs" -version = "0.37.0" +version = "0.37.1-SNAPSHOT" dependencies = [ "datafusion", ] @@ -3360,7 +3360,7 @@ dependencies = [ [[package]] name = "objectstore_ext" -version = "0.37.0" +version = "0.37.1-SNAPSHOT" dependencies = [ "async-trait", "aws-config", @@ -3651,7 +3651,7 @@ dependencies = [ [[package]] name = "query_sql" -version = "0.37.0" +version = "0.37.1-SNAPSHOT" dependencies = [ "datafusion", "tokio", @@ -3911,7 +3911,7 @@ dependencies = [ [[package]] name = "rust_sketch" -version = "0.37.0" +version = "0.37.1-SNAPSHOT" dependencies = [ "cargo_metadata", "cxx", @@ -4232,7 +4232,7 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "sleeper_core" -version = "0.37.0" +version = "0.37.1-SNAPSHOT" dependencies = [ "aggregator_udfs", "arrow", @@ -4267,7 +4267,7 @@ dependencies = [ [[package]] name = "sleeper_df" -version = "0.37.0" +version = "0.37.1-SNAPSHOT" dependencies = [ "aws-types", "cbindgen", @@ -4464,7 +4464,7 @@ dependencies = [ [[package]] name = "test_util" -version = "0.37.0" +version = "0.37.1-SNAPSHOT" dependencies = [ "bytes", "color-eyre", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index ea36e8d352c..0630793b604 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -25,7 +25,7 @@ members = [ resolver = "2" [workspace.package] -version = "0.37.0" +version = "0.37.1-SNAPSHOT" edition = "2024" rust-version = "1.93" publish = false From a1411a6feb4b92b647a6ad8f6b3346e486d470fd Mon Sep 17 00:00:00 2001 From: ca61688 <206189192+ca61688@users.noreply.github.com> Date: Tue, 7 Jul 2026 08:42:20 +0100 Subject: [PATCH 14/50] 6593: Improve usage of DeployNewInstance constructor --- .../clients/deploy/DeployNewInstance.java | 22 +++--- .../clients/deploy/DeployNewInstanceTest.java | 78 ------------------- .../drivers/cdk/DeployNewTestInstance.java | 15 +--- .../instance/AwsSleeperInstanceDriver.java | 14 +--- 4 files changed, 15 insertions(+), 114 deletions(-) diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java index 3cc3d857832..6a101b9255b 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java @@ -149,15 +149,7 @@ public static void main(String[] rawArgs) throws IOException, InterruptedExcepti new DeployNewInstance(DeployInstance.fromScriptsDirectory(scriptsDirectory, accountName, region, partitionMetadata, s3Client, ecrClient), instanceId -> S3InstanceProperties.loadGivenAccountAndInstanceId(s3Client, accountName, instanceId), - new StoreFactory() { - public TablePropertiesStore createTableStore(InstanceProperties p) { - return S3TableProperties.createStore(p, s3Client, dynamoClient); - } - - public StateStoreProvider createStateStore(InstanceProperties p) { - return StateStoreFactory.createProvider(p, s3Client, dynamoClient); - } - }, + StoreFactory.withAwsClients(s3Client, dynamoClient), config, SleeperInternalCdkApp.STANDARD, args.ignoreTableFiles(), deployPaused).deploy(); } } @@ -239,5 +231,17 @@ public interface StoreFactory { TablePropertiesStore createTableStore(InstanceProperties instanceProperties); StateStoreProvider createStateStore(InstanceProperties instanceProperties); + + static StoreFactory withAwsClients(S3Client s3Client, DynamoDbClient dynamoClient) { + return new StoreFactory() { + public TablePropertiesStore createTableStore(InstanceProperties p) { + return S3TableProperties.createStore(p, s3Client, dynamoClient); + } + + public StateStoreProvider createStateStore(InstanceProperties p) { + return StateStoreFactory.createProvider(p, s3Client, dynamoClient); + } + }; + } } } diff --git a/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceTest.java b/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceTest.java index 8e697e6ecde..67a273e3254 100644 --- a/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceTest.java +++ b/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceTest.java @@ -19,11 +19,9 @@ import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; -import sleeper.clients.table.AddTableClient; import sleeper.core.deploy.SleeperInstanceConfiguration; import sleeper.core.properties.instance.InstanceProperties; import sleeper.core.properties.model.SleeperInternalCdkApp; -import sleeper.core.properties.table.TableProperties; import sleeper.core.properties.table.TablePropertiesStore; import sleeper.core.properties.testutils.InMemoryTableProperties; import sleeper.core.schema.Schema; @@ -36,7 +34,6 @@ import sleeper.core.util.cli.CommandArgumentsException; import java.io.IOException; -import java.io.UncheckedIOException; import java.nio.file.Path; import java.util.HashMap; import java.util.Map; @@ -84,75 +81,6 @@ void shouldRejectWhenNeitherInstancePropertiesOrConfigDirSet() { .hasMessage("Either --instance-properties or --config-dir must be provided"); } - //TODO validate remst of argument logic - @Test - void shouldRejectWhenTableNameNotSetInPropertiesFile() throws IOException { - //Given - saveFile("other/table.properties", "sleeper.other.property=value\n"); - - //When/Then - assertThatThrownBy(() -> addTable("my-instance", "--schema", "schema.json", - "--table-properties", "other/table.properties")) - .isInstanceOf(CommandArgumentsException.class) - .hasMessage("Table name was not found. Provide --table-name, or set it in --table-properties or --config-dir."); - } - - @Test - void shouldRejectWhenTableNameNotSetInConfigDir() throws IOException { - //Given - saveFile("other/table.properties", "sleeper.other.property=value\n"); - saveSchemaFile("other/schema.json", schema); - - //When/Then - assertThatThrownBy(() -> addTable("my-instance", "--config-dir", "other/")) - .isInstanceOf(CommandArgumentsException.class) - .hasMessage("Table name was not found. Provide --table-name, or set it in --table-properties or --config-dir."); - } - - @Test - void shouldRejectWhenNoSchemaSource() { - //When/Then - assertThatThrownBy(() -> addTable("my-instance", "--table-name", "my-table")) - .isInstanceOf(CommandArgumentsException.class) - .hasMessage("Either --schema or --config-dir must be provided"); - } - - @Test - void shouldRejectWhenNoTablePropertiesInConfigDir() { - //Given - saveSchemaFile("other/schema.json", schema); - - //When/Then - assertThatThrownBy(() -> addTable("my-instance", "--config-dir", "other/")) - .isInstanceOf(UncheckedIOException.class); - } - - @Test - void shouldRejectWhenNoSchemaInConfigDir() { - //Given - saveFile("other/table.properties", "sleeper.table.name=no-schema\n"); - - //When/Then - assertThatThrownBy(() -> addTable("my-instance", "--config-dir", "other/")) - .isInstanceOf(UncheckedIOException.class); - } - - @Test - void shouldRejectWhenAllThreeFileSourcesSpecified() throws IOException { - //When/Then - assertThatThrownBy(() -> addTable("my-instance", "--table-name", "my-table", - "--schema", "schema.json", "--table-properties", "./table.properties", - "--config-dir", "./")) - .isInstanceOf(CommandArgumentsException.class) - .hasMessage("Cannot specify --schema, --table-properties, and --config-dir together"); - } - - } - - private void addTable(String... args) throws Exception { - var arguments = AddTableClient.readArguments(CommandArgumentReader.parse(AddTableClient.USAGE, args), this::readFile); - TableProperties tableProperties = AddTableClient.createTablePropertiesWithLoaders(arguments, this::loadInstanceProperties, this::readFile); - new AddTableClient(tableProperties, tablePropertiesStore, stateStoreProvider).run(); } private void deployNewInstance(String... args) throws Exception { @@ -183,12 +111,6 @@ private void saveFile(String path, String content) { pathToString.put(Path.of(path), content); } - private String tableId(String tableName) { - return tableIndex.getTableByName(tableName) - .orElseThrow(() -> new RuntimeException("Found tables: " + tableIndex.streamAllTables().toList())) - .getTableUniqueId(); - } - private InstanceProperties loadInstanceProperties(String instanceId) { return Optional.ofNullable(instanceIdToProperties.get(instanceId)) .orElseThrow(); diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java index ae7052ebfc1..b6c6c8a7d69 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java @@ -27,14 +27,9 @@ import sleeper.clients.deploy.DeployNewInstance; import sleeper.clients.deploy.DeployNewInstance.StoreFactory; import sleeper.configuration.properties.S3InstanceProperties; -import sleeper.configuration.properties.S3TableProperties; import sleeper.core.deploy.SleeperInstanceConfiguration; import sleeper.core.deploy.SleeperInstanceConfigurationFromTemplates; -import sleeper.core.properties.instance.InstanceProperties; import sleeper.core.properties.model.SleeperInternalCdkApp; -import sleeper.core.properties.table.TablePropertiesStore; -import sleeper.core.statestore.StateStoreProvider; -import sleeper.statestore.StateStoreFactory; import java.io.IOException; import java.nio.file.Path; @@ -77,15 +72,7 @@ public static void main(String[] args) throws IOException, InterruptedException config.getInstanceProperties().set(SUBNETS, subnetIds); new DeployNewInstance(DeployInstance.fromScriptsDirectory(scriptsDirectory, accountName, region, partitionMetadata, s3Client, ecrClient), id -> S3InstanceProperties.loadGivenAccountAndInstanceId(s3Client, accountName, id), - new StoreFactory() { - public TablePropertiesStore createTableStore(InstanceProperties p) { - return S3TableProperties.createStore(p, s3Client, dynamoClient); - } - - public StateStoreProvider createStateStore(InstanceProperties p) { - return StateStoreFactory.createProvider(p, s3Client, dynamoClient); - } - }, + StoreFactory.withAwsClients(s3Client, dynamoClient), config, SleeperInternalCdkApp.STANDARD, false, deployPaused).deploy(); } } diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java index c190dbebbb7..251a90f6eb9 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java @@ -29,14 +29,10 @@ import sleeper.clients.deploy.DeployNewInstance; import sleeper.clients.deploy.DeployNewInstance.StoreFactory; import sleeper.configuration.properties.S3InstanceProperties; -import sleeper.configuration.properties.S3TableProperties; import sleeper.core.deploy.SleeperInstanceConfiguration; import sleeper.core.properties.instance.InstanceProperties; import sleeper.core.properties.model.SleeperInternalCdkApp; import sleeper.core.properties.table.TableProperties; -import sleeper.core.properties.table.TablePropertiesStore; -import sleeper.core.statestore.StateStoreProvider; -import sleeper.statestore.StateStoreFactory; import sleeper.systemtest.drivers.util.SystemTestClients; import sleeper.systemtest.dsl.instance.SleeperInstanceDriver; import sleeper.systemtest.dsl.instance.SystemTestParameters; @@ -91,15 +87,7 @@ public boolean deployInstanceIfNotPresent(String instanceId, SleeperInstanceConf try { new DeployNewInstance(deployInstance, id -> S3InstanceProperties.loadGivenAccountAndInstanceId(s3, parameters.getAccount(), id), - new StoreFactory() { - public TablePropertiesStore createTableStore(InstanceProperties p) { - return S3TableProperties.createStore(p, s3, dynamoDB); - } - - public StateStoreProvider createStateStore(InstanceProperties p) { - return StateStoreFactory.createProvider(p, s3, dynamoDB); - } - }, + StoreFactory.withAwsClients(s3, dynamoDB), deployConfig, SleeperInternalCdkApp.STANDARD, false, false).deploy(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); From 584ed75551de16d3339a48382355db18dbffa1ba Mon Sep 17 00:00:00 2001 From: ca61688 <206189192+ca61688@users.noreply.github.com> Date: Tue, 7 Jul 2026 09:47:41 +0100 Subject: [PATCH 15/50] 6593: Improve DeployNewInstance argument validation --- .../clients/deploy/DeployNewInstance.java | 20 ++-------------- .../clients/deploy/DeployNewInstanceTest.java | 23 +++++++++++++++++++ 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java index 6a101b9255b..a37f1046345 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java @@ -188,28 +188,12 @@ public record Arguments( boolean deployPaused) { public Arguments { - if (scriptsDirectory == null) { - throw new CommandArgumentsException("scriptsDirectory must not be null"); - } - - if (instanceId == null) { - throw new CommandArgumentsException("instanceId must not be null"); - } - - if (vpcId == null) { - throw new CommandArgumentsException("vpcId must not be null"); - } - - if (subnetIds == null) { - throw new CommandArgumentsException("subnetIds must not be null"); - } - if (propertiesFile == null && configDir == null) { throw new CommandArgumentsException("Either --instance-properties or --config-dir must be provided"); } - if (configDir == null && ignoreTableFiles) { - throw new CommandArgumentsException("ignoreTableFiles flag is only checked when --config-dir is set."); + if (propertiesFile != null && configDir != null) { + throw new CommandArgumentsException("Cannot use both --instance-properties and --config-dir"); } if (propertiesFile != null) { diff --git a/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceTest.java b/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceTest.java index 67a273e3254..8b84a245dad 100644 --- a/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceTest.java +++ b/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceTest.java @@ -40,6 +40,7 @@ import java.util.NoSuchElementException; import java.util.Optional; +import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static sleeper.core.properties.testutils.InstancePropertiesTestHelper.createTestInstancePropertiesWithId; import static sleeper.core.schema.SchemaTestHelper.createSchemaWithKey; @@ -81,6 +82,28 @@ void shouldRejectWhenNeitherInstancePropertiesOrConfigDirSet() { .hasMessage("Either --instance-properties or --config-dir must be provided"); } + @Test + void shouldRejectWhenBothInstancePropertiesAndConfigDirSet() { + //When/Then + assertThatThrownBy(() -> deployNewInstance("scriptsDir", "my-instance", "my-vpc", "my-subnets", "--instance-properties", "someFile", "--config-dir", "someDir")) + .isInstanceOf(CommandArgumentsException.class) + .hasMessage("Cannot use both --instance-properties and --config-dir"); + } + + @Test + void shouldSetIgnoreTableFilesTrueWhenInstancePropertiesUsed() { + var arguments = DeployNewInstance.readArguments(CommandArgumentReader.parse(DeployNewInstance.USAGE, + "scriptsDir", "my-instance", "my-vpc", "my-subnets", "--instance-properties", "someFile")); + assertThat(arguments.ignoreTableFiles()).isTrue(); + } + + @Test + void shouldResolvePropertiesFileWhenConfigDirUsed() { + var arguments = DeployNewInstance.readArguments(CommandArgumentReader.parse(DeployNewInstance.USAGE, + "scriptsDir", "my-instance", "my-vpc", "my-subnets", "--config-dir", "someDir")); + assertThat(arguments.resolvePropertiesFile()).isEqualTo(Path.of("someDir/instance.properties")); + } + } private void deployNewInstance(String... args) throws Exception { From e7a4210fb2b1a598df329fe4ee4d94ce19b86f28 Mon Sep 17 00:00:00 2001 From: ca61688 <206189192+ca61688@users.noreply.github.com> Date: Tue, 7 Jul 2026 11:24:06 +0100 Subject: [PATCH 16/50] 6593: Add remaining unit tests for DeployNewInstanceTest --- .../clients/deploy/DeployNewInstance.java | 39 ++++---- .../clients/deploy/DeployNewInstanceTest.java | 89 +++++++++++++++---- .../drivers/cdk/DeployNewTestInstance.java | 2 - .../instance/AwsSleeperInstanceDriver.java | 1 - 4 files changed, 90 insertions(+), 41 deletions(-) diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java index a37f1046345..c5d58470b9f 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java @@ -26,14 +26,16 @@ import software.amazon.awssdk.services.sts.StsClient; import sleeper.clients.table.AddTableClient; +import sleeper.clients.util.FileReader; import sleeper.clients.util.cdk.CdkCommand; -import sleeper.configuration.properties.S3InstanceProperties; import sleeper.configuration.properties.S3TableProperties; import sleeper.core.deploy.SleeperInstanceConfiguration; +import sleeper.core.properties.PropertiesUtils; import sleeper.core.properties.instance.InstanceProperties; import sleeper.core.properties.model.SleeperInternalCdkApp; import sleeper.core.properties.table.TableProperties; import sleeper.core.properties.table.TablePropertiesStore; +import sleeper.core.schema.SchemaSerDe; import sleeper.core.statestore.StateStoreProvider; import sleeper.core.util.cli.CommandArguments; import sleeper.core.util.cli.CommandArgumentsException; @@ -42,6 +44,7 @@ import sleeper.statestore.StateStoreFactory; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.util.List; @@ -53,32 +56,17 @@ public class DeployNewInstance { private static final Logger LOGGER = LoggerFactory.getLogger(DeployNewInstance.class); private final InstanceDeployer deployInstance; - private final InstancePropertiesLoader propertiesLoader; private final StoreFactory storeFactory; private final SleeperInstanceConfiguration deployInstanceConfiguration; private final SleeperInternalCdkApp cdkApp; private final boolean ignoreTableFiles; private final boolean deployPaused; - public DeployNewInstance(DeployInstance deployInstance, String accountName, S3Client s3Client, - DynamoDbClient dynamoClient, SleeperInstanceConfiguration deployInstanceConfiguration, - SleeperInternalCdkApp cdkApp, boolean ignoreTableFiles, boolean deployPaused) { - this.deployInstance = deployInstance; - this.deployInstanceConfiguration = deployInstanceConfiguration; - this.cdkApp = cdkApp; - this.ignoreTableFiles = ignoreTableFiles; - this.deployPaused = deployPaused; - propertiesLoader = null; - storeFactory = null; - } - public DeployNewInstance(InstanceDeployer deployInstance, - InstancePropertiesLoader instancePropertiesLoader, StoreFactory storeFactory, SleeperInstanceConfiguration deployInstanceConfiguration, SleeperInternalCdkApp cdkApp, boolean ignoreTableFiles, boolean deployPaused) { this.deployInstance = deployInstance; - this.propertiesLoader = instancePropertiesLoader; this.storeFactory = storeFactory; this.deployInstanceConfiguration = deployInstanceConfiguration; this.cdkApp = cdkApp; @@ -115,6 +103,19 @@ public DeployNewInstance(InstanceDeployer deployInstance, "the instance is manually resumed.") .build(); + public static SleeperInstanceConfiguration loadConfiguration(Arguments args, FileReader files) { + InstanceProperties instanceProperties = InstanceProperties.createWithoutValidation( + PropertiesUtils.loadProperties(FileReader.readFile(files, args.resolvePropertiesFile()))); + if (args.configDir() == null) { + return SleeperInstanceConfiguration.withNoTables(instanceProperties); + } + TableProperties tableProperties = new TableProperties(instanceProperties, + PropertiesUtils.loadProperties(FileReader.readFile(files, args.configDir().resolve("table.properties")))); + tableProperties.setSchema(new SchemaSerDe().fromJson( + FileReader.readFile(files, args.configDir().resolve("schema.json")))); + return new SleeperInstanceConfiguration(instanceProperties, List.of(tableProperties)); + } + public static Arguments readArguments(CommandArguments arguments) { return new Arguments( Path.of(arguments.getString("scriptsDirectory")), @@ -131,7 +132,6 @@ public static void main(String[] rawArgs) throws IOException, InterruptedExcepti Arguments args = CommandArguments.parseAndValidateOrExit(USAGE, rawArgs, a -> readArguments(a)); Path scriptsDirectory = Path.of(rawArgs[0]); - Path instancePropertiesFile = args.resolvePropertiesFile(); boolean deployPaused = args.deployPaused(); try (S3Client s3Client = S3Client.create(); DynamoDbClient dynamoClient = DynamoDbClient.create(); @@ -141,14 +141,13 @@ public static void main(String[] rawArgs) throws IOException, InterruptedExcepti Region region = DefaultAwsRegionProviderChain.builder().build().getRegion(); PartitionMetadata partitionMetadata = PartitionMetadata.of(region); - SleeperInstanceConfiguration config = SleeperInstanceConfiguration.fromLocalConfiguration(instancePropertiesFile); + SleeperInstanceConfiguration config = loadConfiguration(args, Files::readString); config.getInstanceProperties().set(ID, args.instanceId()); config.getInstanceProperties().set(VPC_ID, args.vpcId()); config.getInstanceProperties().set(SUBNETS, args.subnetIds()); new DeployNewInstance(DeployInstance.fromScriptsDirectory(scriptsDirectory, accountName, region, partitionMetadata, s3Client, ecrClient), - instanceId -> S3InstanceProperties.loadGivenAccountAndInstanceId(s3Client, accountName, instanceId), StoreFactory.withAwsClients(s3Client, dynamoClient), config, SleeperInternalCdkApp.STANDARD, args.ignoreTableFiles(), deployPaused).deploy(); } @@ -164,7 +163,7 @@ public void deploy() throws IOException, InterruptedException { .build()); if (!ignoreTableFiles) { - InstanceProperties instanceProperties = propertiesLoader.load(deployInstanceConfiguration.getInstanceId()); + InstanceProperties instanceProperties = deployInstanceConfiguration.getInstanceProperties(); for (TableProperties tableProperties : deployInstanceConfiguration.getTableProperties()) { LOGGER.info("Adding table " + tableProperties.getStatus()); diff --git a/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceTest.java b/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceTest.java index 8b84a245dad..4e6e210ae87 100644 --- a/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceTest.java +++ b/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceTest.java @@ -19,9 +19,10 @@ import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; -import sleeper.core.deploy.SleeperInstanceConfiguration; +import sleeper.clients.util.cdk.CdkCommand; import sleeper.core.properties.instance.InstanceProperties; import sleeper.core.properties.model.SleeperInternalCdkApp; +import sleeper.core.properties.table.TableProperties; import sleeper.core.properties.table.TablePropertiesStore; import sleeper.core.properties.testutils.InMemoryTableProperties; import sleeper.core.schema.Schema; @@ -42,6 +43,8 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static sleeper.core.properties.table.TableProperty.TABLE_ID; +import static sleeper.core.properties.table.TableProperty.TABLE_NAME; import static sleeper.core.properties.testutils.InstancePropertiesTestHelper.createTestInstancePropertiesWithId; import static sleeper.core.schema.SchemaTestHelper.createSchemaWithKey; @@ -50,25 +53,71 @@ public class DeployNewInstanceTest { Schema schema = createSchemaWithKey("key"); InMemoryTableIndex tableIndex = new InMemoryTableIndex(); TablePropertiesStore tablePropertiesStore = InMemoryTableProperties.getStore(tableIndex); - StateStoreProvider stateStoreProvider = InMemoryTransactionLogStateStore.createProvider(instanceProperties, new InMemoryTransactionLogsPerTable()); - Map instanceIdToProperties = new HashMap<>(); + StateStoreProvider stateStoreProvider = InMemoryTransactionLogStateStore.createProvider(instanceProperties, + new InMemoryTransactionLogsPerTable()); Map pathToString = new HashMap<>(); + DeployInstanceRequest lastDeployRequest; @BeforeEach void setUp() { - instanceIdToProperties.put("my-instance", instanceProperties); - saveSchemaFile("./schema.json", schema); - saveFile("./table.properties", "sleeper.table.name=file-table\n"); + saveFile("./instance.properties", instanceProperties); + saveFile("./configDir/instance.properties", instanceProperties); + saveFile("./configDir/table.properties", "sleeper.table.name=file-table\n"); + saveSchemaFile("./configDir/schema.json", schema); } - //TODO test deploy method + @Nested + class DeployNew { + + @Test + void shouldDeployNewInstanceWhenUsingInstanceProperties() throws Exception { + deployNewInstance("scriptsDir", "my-instance", "someVpc", "someSubnets", "--instance-properties", + "./instance.properties"); + + assertThat(tableIndex.streamAllTables()).isEmpty(); + assertThat(lastDeployRequest.getCdkCommand()).isEqualTo(CdkCommand.deployNew()); + assertThat(lastDeployRequest.getCdkApp()).isEqualTo(SleeperInternalCdkApp.STANDARD); + } + + @Test + void shouldDeployNewInstanceWhenUsingConfigDir() throws Exception { + deployNewInstance("scriptsDir", "my-instance", "someVpc", "someSubnets", "--config-dir", "./configDir"); + + TableProperties expected = new TableProperties(instanceProperties); + expected.setSchema(schema); + expected.set(TABLE_ID, tableId("file-table")); + expected.set(TABLE_NAME, "file-table"); + assertThat(tablePropertiesStore.streamAllTables()).containsExactly(expected); + assertThat(lastDeployRequest.getCdkCommand()).isEqualTo(CdkCommand.deployNew()); + assertThat(lastDeployRequest.getCdkApp()).isEqualTo(SleeperInternalCdkApp.STANDARD); + } + + @Test + void shouldDeployNewInstanceWhenUsingInstancePropertiesIgnoringTableFiles() throws Exception { + deployNewInstance("scriptsDir", "my-instance", "someVpc", "someSubnets", "--config-dir", "./configDir", + "--ignoreTableFiles"); + + assertThat(tableIndex.streamAllTables()).isEmpty(); + assertThat(lastDeployRequest.getCdkCommand()).isEqualTo(CdkCommand.deployNew()); + assertThat(lastDeployRequest.getCdkApp()).isEqualTo(SleeperInternalCdkApp.STANDARD); + } + + @Test + void shouldDeployNewInstancePaused() throws Exception { + deployNewInstance("scriptsDir", "my-instance", "someVpc", "someSubnets", "--config-dir", "./configDir", + "--paused"); + + assertThat(lastDeployRequest.getCdkCommand()).isEqualTo(CdkCommand.deployNewPaused()); + assertThat(lastDeployRequest.getCdkApp()).isEqualTo(SleeperInternalCdkApp.STANDARD); + } + } @Nested class ArgumentsValidation { @Test void shouldRejectWhenNotEnoughPositionalArguments() { - //When/Then + // When/Then assertThatThrownBy(() -> deployNewInstance()) .isInstanceOf(CommandArgumentsException.class) .hasMessage("Expected 4 positional arguments, found 0"); @@ -76,7 +125,7 @@ void shouldRejectWhenNotEnoughPositionalArguments() { @Test void shouldRejectWhenNeitherInstancePropertiesOrConfigDirSet() { - //When/Then + // When/Then assertThatThrownBy(() -> deployNewInstance("scriptsDir", "my-instance", "my-vpc", "my-subnets")) .isInstanceOf(CommandArgumentsException.class) .hasMessage("Either --instance-properties or --config-dir must be provided"); @@ -84,8 +133,9 @@ void shouldRejectWhenNeitherInstancePropertiesOrConfigDirSet() { @Test void shouldRejectWhenBothInstancePropertiesAndConfigDirSet() { - //When/Then - assertThatThrownBy(() -> deployNewInstance("scriptsDir", "my-instance", "my-vpc", "my-subnets", "--instance-properties", "someFile", "--config-dir", "someDir")) + // When/Then + assertThatThrownBy(() -> deployNewInstance("scriptsDir", "my-instance", "my-vpc", "my-subnets", + "--instance-properties", "someFile", "--config-dir", "someDir")) .isInstanceOf(CommandArgumentsException.class) .hasMessage("Cannot use both --instance-properties and --config-dir"); } @@ -108,11 +158,9 @@ void shouldResolvePropertiesFileWhenConfigDirUsed() { private void deployNewInstance(String... args) throws Exception { var arguments = DeployNewInstance.readArguments(CommandArgumentReader.parse(DeployNewInstance.USAGE, args)); - SleeperInstanceConfiguration config = SleeperInstanceConfiguration.withNoTables(instanceProperties); + var config = DeployNewInstance.loadConfiguration(arguments, this::readFile); new DeployNewInstance( - request -> { - }, // no-op stub — no real CDK/S3/ECR - this::loadInstanceProperties, + request -> lastDeployRequest = request, new DeployNewInstance.StoreFactory() { public TablePropertiesStore createTableStore(InstanceProperties p) { return tablePropertiesStore; @@ -126,6 +174,12 @@ public StateStoreProvider createStateStore(InstanceProperties p) { arguments.ignoreTableFiles(), arguments.deployPaused()).deploy(); } + private String tableId(String tableName) { + return tableIndex.getTableByName(tableName) + .orElseThrow(() -> new RuntimeException("Found tables: " + tableIndex.streamAllTables().toList())) + .getTableUniqueId(); + } + private void saveSchemaFile(String path, Schema schema) { pathToString.put(Path.of(path), new SchemaSerDe().toJson(schema)); } @@ -134,9 +188,8 @@ private void saveFile(String path, String content) { pathToString.put(Path.of(path), content); } - private InstanceProperties loadInstanceProperties(String instanceId) { - return Optional.ofNullable(instanceIdToProperties.get(instanceId)) - .orElseThrow(); + private void saveFile(String path, InstanceProperties content) { + pathToString.put(Path.of(path), content.saveAsString()); } private String readFile(Path path) throws IOException { diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java index b6c6c8a7d69..d2d80a38d4b 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java @@ -26,7 +26,6 @@ import sleeper.clients.deploy.DeployInstance; import sleeper.clients.deploy.DeployNewInstance; import sleeper.clients.deploy.DeployNewInstance.StoreFactory; -import sleeper.configuration.properties.S3InstanceProperties; import sleeper.core.deploy.SleeperInstanceConfiguration; import sleeper.core.deploy.SleeperInstanceConfigurationFromTemplates; import sleeper.core.properties.model.SleeperInternalCdkApp; @@ -71,7 +70,6 @@ public static void main(String[] args) throws IOException, InterruptedException config.getInstanceProperties().set(VPC_ID, vpcId); config.getInstanceProperties().set(SUBNETS, subnetIds); new DeployNewInstance(DeployInstance.fromScriptsDirectory(scriptsDirectory, accountName, region, partitionMetadata, s3Client, ecrClient), - id -> S3InstanceProperties.loadGivenAccountAndInstanceId(s3Client, accountName, id), StoreFactory.withAwsClients(s3Client, dynamoClient), config, SleeperInternalCdkApp.STANDARD, false, deployPaused).deploy(); } diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java index 251a90f6eb9..6aeebad9e79 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java @@ -86,7 +86,6 @@ public boolean deployInstanceIfNotPresent(String instanceId, SleeperInstanceConf deployConfig.getInstanceProperties().set(SUBNETS, parameters.getSubnetIds()); try { new DeployNewInstance(deployInstance, - id -> S3InstanceProperties.loadGivenAccountAndInstanceId(s3, parameters.getAccount(), id), StoreFactory.withAwsClients(s3, dynamoDB), deployConfig, SleeperInternalCdkApp.STANDARD, false, false).deploy(); } catch (InterruptedException e) { From e48525f7bdc1f77a8968d119d67a0bf086ff2f9f Mon Sep 17 00:00:00 2001 From: ca61688 <206189192+ca61688@users.noreply.github.com> Date: Tue, 7 Jul 2026 14:18:11 +0100 Subject: [PATCH 17/50] 6593: Add inital framework for DeployNewInstanceIT --- .../clients/deploy/DeployNewInstance.java | 5 +- .../clients/deploy/DeployNewInstanceIT.java | 99 +++++++++++++++++++ java/common/localstack-test/pom.xml | 4 + .../localstack/test/LocalStackTestBase.java | 2 + .../test/SleeperLocalStackClients.java | 2 + 5 files changed, 111 insertions(+), 1 deletion(-) create mode 100644 java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceIT.java diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java index c5d58470b9f..e92d9d89983 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java @@ -106,13 +106,16 @@ public DeployNewInstance(InstanceDeployer deployInstance, public static SleeperInstanceConfiguration loadConfiguration(Arguments args, FileReader files) { InstanceProperties instanceProperties = InstanceProperties.createWithoutValidation( PropertiesUtils.loadProperties(FileReader.readFile(files, args.resolvePropertiesFile()))); - if (args.configDir() == null) { + + if (args.ignoreTableFiles()) { return SleeperInstanceConfiguration.withNoTables(instanceProperties); } + TableProperties tableProperties = new TableProperties(instanceProperties, PropertiesUtils.loadProperties(FileReader.readFile(files, args.configDir().resolve("table.properties")))); tableProperties.setSchema(new SchemaSerDe().fromJson( FileReader.readFile(files, args.configDir().resolve("schema.json")))); + return new SleeperInstanceConfiguration(instanceProperties, List.of(tableProperties)); } diff --git a/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceIT.java b/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceIT.java new file mode 100644 index 00000000000..6f23ec387e9 --- /dev/null +++ b/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceIT.java @@ -0,0 +1,99 @@ +/* + * Copyright 2022-2026 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package sleeper.clients.deploy; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import software.amazon.awssdk.regions.PartitionMetadata; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain; + +import sleeper.clients.deploy.DeployNewInstance.StoreFactory; +import sleeper.configuration.properties.S3TableProperties; +import sleeper.configuration.table.index.DynamoDBTableIndexCreator; +import sleeper.core.deploy.SleeperInstanceConfiguration; +import sleeper.core.properties.instance.InstanceProperties; +import sleeper.core.properties.model.SleeperInternalCdkApp; +import sleeper.core.properties.table.TableProperties; +import sleeper.core.properties.table.TablePropertiesStore; +import sleeper.core.schema.Schema; +import sleeper.core.statestore.StateStore; +import sleeper.localstack.test.LocalStackTestBase; +import sleeper.statestore.StateStoreFactory; +import sleeper.statestore.transactionlog.TransactionLogStateStoreCreator; + +import java.io.IOException; +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.CONFIG_BUCKET; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.DATA_BUCKET; +import static sleeper.core.properties.testutils.InstancePropertiesTestHelper.createTestInstanceProperties; +import static sleeper.core.schema.SchemaTestHelper.createSchemaWithKey; + +public class DeployNewInstanceIT extends LocalStackTestBase { + private final InstanceProperties instanceProperties = createTestInstanceProperties(); + private final Schema schema = createSchemaWithKey("key1"); + private final TablePropertiesStore propertiesStore = S3TableProperties.createStore(instanceProperties, s3Client, dynamoClient); + + @TempDir + private Path tempDir; + + @BeforeEach + void setUp() { + createBucket(instanceProperties.get(CONFIG_BUCKET)); + createBucket(instanceProperties.get(DATA_BUCKET)); + new TransactionLogStateStoreCreator(instanceProperties, dynamoClient).create(); + DynamoDBTableIndexCreator.create(dynamoClient, instanceProperties); + } + + @Test + void shouldDeployInstanceByInstanceProperties() throws IOException, InterruptedException { + // Given + SleeperInstanceConfiguration config = SleeperInstanceConfiguration.withNoTables(instanceProperties); + + // When + deployInstance(config); + + // Then + assertThat(propertiesStore.streamAllTables()).isEmpty(); + } + + @Test + void shouldDeployInstanceByConfigDirectoryWithTables() { + + } + + @Test + void shouldDeployInstanceByConfigDirectoryIgnoringTables() { + + } + + private StateStore stateStore(TableProperties tableProperties) { + return new StateStoreFactory(instanceProperties, s3Client, dynamoClient).getStateStore(tableProperties); + } + + private void deployInstance(SleeperInstanceConfiguration config) throws IOException, InterruptedException { + String accountName = stsClient.getCallerIdentity().account(); + Region region = DefaultAwsRegionProviderChain.builder().build().getRegion(); + PartitionMetadata partitionMetadata = PartitionMetadata.of(region); + + new DeployNewInstance(DeployInstance.fromScriptsDirectory(Path.of("TODO"), accountName, region, partitionMetadata, s3Client, ecrClient), + StoreFactory.withAwsClients(s3Client, dynamoClient), + config, SleeperInternalCdkApp.STANDARD, false, false).deploy(); + } +} diff --git a/java/common/localstack-test/pom.xml b/java/common/localstack-test/pom.xml index 41fa641624d..cace0ffeb55 100644 --- a/java/common/localstack-test/pom.xml +++ b/java/common/localstack-test/pom.xml @@ -68,6 +68,10 @@ software.amazon.awssdk cloudwatch + + software.amazon.awssdk + ecr + software.amazon.awssdk.crt aws-crt diff --git a/java/common/localstack-test/src/main/java/sleeper/localstack/test/LocalStackTestBase.java b/java/common/localstack-test/src/main/java/sleeper/localstack/test/LocalStackTestBase.java index d86737e8892..1a2d2a93841 100644 --- a/java/common/localstack-test/src/main/java/sleeper/localstack/test/LocalStackTestBase.java +++ b/java/common/localstack-test/src/main/java/sleeper/localstack/test/LocalStackTestBase.java @@ -23,6 +23,7 @@ import software.amazon.awssdk.core.sync.ResponseTransformer; import software.amazon.awssdk.services.cloudwatch.CloudWatchClient; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.ecr.EcrClient; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.PutObjectResponse; @@ -62,6 +63,7 @@ public abstract class LocalStackTestBase { protected final Configuration hadoopConf = SleeperLocalStackClients.HADOOP_CONF; protected final CloudWatchClient cloudWatchClient = SleeperLocalStackClients.CLOUDWATCH_CLIENT; protected final AwsCredentialsProvider credentialsProvider = SleeperLocalStackClients.CREDENTIALS_PROVIDER; + protected final EcrClient ecrClient = SleeperLocalStackClients.ECR_CLIENT; public static void createBucket(String bucketName) { S3_CLIENT.createBucket(builder -> builder.bucket(bucketName)); diff --git a/java/common/localstack-test/src/main/java/sleeper/localstack/test/SleeperLocalStackClients.java b/java/common/localstack-test/src/main/java/sleeper/localstack/test/SleeperLocalStackClients.java index a8ad60fef9d..6112ee2c22b 100644 --- a/java/common/localstack-test/src/main/java/sleeper/localstack/test/SleeperLocalStackClients.java +++ b/java/common/localstack-test/src/main/java/sleeper/localstack/test/SleeperLocalStackClients.java @@ -20,6 +20,7 @@ import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.services.cloudwatch.CloudWatchClient; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; +import software.amazon.awssdk.services.ecr.EcrClient; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.sqs.SqsClient; @@ -48,5 +49,6 @@ private SleeperLocalStackClients() { public static final CloudWatchClient CLOUDWATCH_CLIENT = buildAwsV2Client(CONTAINER, CloudWatchClient.builder()); public static final Configuration HADOOP_CONF = getHadoopConfiguration(CONTAINER); public static final AwsCredentialsProvider CREDENTIALS_PROVIDER = buildAwsCredentialsProvider(); + public static final EcrClient ECR_CLIENT = buildAwsV2Client(CONTAINER, EcrClient.builder()); } From 7593e6dcc8d122ff94f1bbfe978c0935698a5592 Mon Sep 17 00:00:00 2001 From: ca61688 <206189192+ca61688@users.noreply.github.com> Date: Tue, 7 Jul 2026 15:35:00 +0100 Subject: [PATCH 18/50] 6593: Begin to unpick file reader changes --- .../clients/deploy/DeployNewInstance.java | 10 ++++- .../clients/deploy/DeployNewInstanceTest.java | 37 +++++++++---------- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java index e92d9d89983..9e5fae7c1bd 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java @@ -106,11 +106,11 @@ public DeployNewInstance(InstanceDeployer deployInstance, public static SleeperInstanceConfiguration loadConfiguration(Arguments args, FileReader files) { InstanceProperties instanceProperties = InstanceProperties.createWithoutValidation( PropertiesUtils.loadProperties(FileReader.readFile(files, args.resolvePropertiesFile()))); - if (args.ignoreTableFiles()) { return SleeperInstanceConfiguration.withNoTables(instanceProperties); } + //TODO, this is wrong and doesn't load the table properties correctly TableProperties tableProperties = new TableProperties(instanceProperties, PropertiesUtils.loadProperties(FileReader.readFile(files, args.configDir().resolve("table.properties")))); tableProperties.setSchema(new SchemaSerDe().fromJson( @@ -119,6 +119,14 @@ public static SleeperInstanceConfiguration loadConfiguration(Arguments args, Fil return new SleeperInstanceConfiguration(instanceProperties, List.of(tableProperties)); } + public static SleeperInstanceConfiguration loadConfiguration2(Arguments args, FileReader files) { + if (args.ignoreTableFiles()) { + return SleeperInstanceConfiguration.fromLocalConfiguration(args.resolvePropertiesFile()); + } + + return SleeperInstanceConfiguration.fromLocalConfigurationDirectory(args.configDir()); + } + public static Arguments readArguments(CommandArguments arguments) { return new Arguments( Path.of(arguments.getString("scriptsDirectory")), diff --git a/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceTest.java b/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceTest.java index 4e6e210ae87..a51a91e515d 100644 --- a/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceTest.java +++ b/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceTest.java @@ -18,6 +18,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import sleeper.clients.util.cdk.CdkCommand; import sleeper.core.properties.instance.InstanceProperties; @@ -35,12 +36,15 @@ import sleeper.core.util.cli.CommandArgumentsException; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.util.HashMap; import java.util.Map; import java.util.NoSuchElementException; import java.util.Optional; +import static java.nio.file.Files.createDirectory; +import static java.nio.file.Files.createTempDirectory; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static sleeper.core.properties.table.TableProperty.TABLE_ID; @@ -58,12 +62,19 @@ public class DeployNewInstanceTest { Map pathToString = new HashMap<>(); DeployInstanceRequest lastDeployRequest; + @TempDir + private Path tempDir; + @BeforeEach - void setUp() { - saveFile("./instance.properties", instanceProperties); - saveFile("./configDir/instance.properties", instanceProperties); - saveFile("./configDir/table.properties", "sleeper.table.name=file-table\n"); - saveSchemaFile("./configDir/schema.json", schema); + void setUp() throws IOException { + createTempDirectory(tempDir, null); + Files.writeString(tempDir.resolve("instance.properties"), instanceProperties.saveAsString()); + Path tables = tempDir.resolve("tables"); + Path table1 = tables.resolve("table1"); + createDirectory(tables); + createDirectory(table1); + Files.writeString(table1.resolve("table.properties"), "sleeper.table.name=file-table\n"); + Files.writeString(table1.resolve("schema.json"), new SchemaSerDe().toJson(schema)); } @Nested @@ -72,7 +83,7 @@ class DeployNew { @Test void shouldDeployNewInstanceWhenUsingInstanceProperties() throws Exception { deployNewInstance("scriptsDir", "my-instance", "someVpc", "someSubnets", "--instance-properties", - "./instance.properties"); + tempDir.resolve("./instance.properties").toString()); assertThat(tableIndex.streamAllTables()).isEmpty(); assertThat(lastDeployRequest.getCdkCommand()).isEqualTo(CdkCommand.deployNew()); @@ -158,7 +169,7 @@ void shouldResolvePropertiesFileWhenConfigDirUsed() { private void deployNewInstance(String... args) throws Exception { var arguments = DeployNewInstance.readArguments(CommandArgumentReader.parse(DeployNewInstance.USAGE, args)); - var config = DeployNewInstance.loadConfiguration(arguments, this::readFile); + var config = DeployNewInstance.loadConfiguration(arguments, Files::readString); new DeployNewInstance( request -> lastDeployRequest = request, new DeployNewInstance.StoreFactory() { @@ -180,18 +191,6 @@ private String tableId(String tableName) { .getTableUniqueId(); } - private void saveSchemaFile(String path, Schema schema) { - pathToString.put(Path.of(path), new SchemaSerDe().toJson(schema)); - } - - private void saveFile(String path, String content) { - pathToString.put(Path.of(path), content); - } - - private void saveFile(String path, InstanceProperties content) { - pathToString.put(Path.of(path), content.saveAsString()); - } - private String readFile(Path path) throws IOException { try { return Optional.ofNullable(pathToString.get(path)).orElseThrow(); From 3925abc1dafd970fd957eadee2bfc807d1615461 Mon Sep 17 00:00:00 2001 From: ca61688 <206189192+ca61688@users.noreply.github.com> Date: Wed, 8 Jul 2026 09:26:07 +0100 Subject: [PATCH 19/50] 6593: Make DeployNewInstanceTest use tempDir --- .../clients/deploy/DeployNewInstance.java | 24 +---- .../clients/deploy/DeployNewInstanceIT.java | 99 ------------------- .../clients/deploy/DeployNewInstanceTest.java | 34 +++---- .../localstack/test/LocalStackTestBase.java | 2 - .../test/SleeperLocalStackClients.java | 2 - 5 files changed, 18 insertions(+), 143 deletions(-) delete mode 100644 java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceIT.java diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java index 9e5fae7c1bd..4ac8fb02923 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java @@ -26,16 +26,13 @@ import software.amazon.awssdk.services.sts.StsClient; import sleeper.clients.table.AddTableClient; -import sleeper.clients.util.FileReader; import sleeper.clients.util.cdk.CdkCommand; import sleeper.configuration.properties.S3TableProperties; import sleeper.core.deploy.SleeperInstanceConfiguration; -import sleeper.core.properties.PropertiesUtils; import sleeper.core.properties.instance.InstanceProperties; import sleeper.core.properties.model.SleeperInternalCdkApp; import sleeper.core.properties.table.TableProperties; import sleeper.core.properties.table.TablePropertiesStore; -import sleeper.core.schema.SchemaSerDe; import sleeper.core.statestore.StateStoreProvider; import sleeper.core.util.cli.CommandArguments; import sleeper.core.util.cli.CommandArgumentsException; @@ -44,7 +41,6 @@ import sleeper.statestore.StateStoreFactory; import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; import java.util.List; @@ -103,23 +99,7 @@ public DeployNewInstance(InstanceDeployer deployInstance, "the instance is manually resumed.") .build(); - public static SleeperInstanceConfiguration loadConfiguration(Arguments args, FileReader files) { - InstanceProperties instanceProperties = InstanceProperties.createWithoutValidation( - PropertiesUtils.loadProperties(FileReader.readFile(files, args.resolvePropertiesFile()))); - if (args.ignoreTableFiles()) { - return SleeperInstanceConfiguration.withNoTables(instanceProperties); - } - - //TODO, this is wrong and doesn't load the table properties correctly - TableProperties tableProperties = new TableProperties(instanceProperties, - PropertiesUtils.loadProperties(FileReader.readFile(files, args.configDir().resolve("table.properties")))); - tableProperties.setSchema(new SchemaSerDe().fromJson( - FileReader.readFile(files, args.configDir().resolve("schema.json")))); - - return new SleeperInstanceConfiguration(instanceProperties, List.of(tableProperties)); - } - - public static SleeperInstanceConfiguration loadConfiguration2(Arguments args, FileReader files) { + public static SleeperInstanceConfiguration loadConfiguration(Arguments args) { if (args.ignoreTableFiles()) { return SleeperInstanceConfiguration.fromLocalConfiguration(args.resolvePropertiesFile()); } @@ -152,7 +132,7 @@ public static void main(String[] rawArgs) throws IOException, InterruptedExcepti Region region = DefaultAwsRegionProviderChain.builder().build().getRegion(); PartitionMetadata partitionMetadata = PartitionMetadata.of(region); - SleeperInstanceConfiguration config = loadConfiguration(args, Files::readString); + SleeperInstanceConfiguration config = loadConfiguration(args); config.getInstanceProperties().set(ID, args.instanceId()); config.getInstanceProperties().set(VPC_ID, args.vpcId()); diff --git a/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceIT.java b/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceIT.java deleted file mode 100644 index 6f23ec387e9..00000000000 --- a/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceIT.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright 2022-2026 Crown Copyright - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package sleeper.clients.deploy; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; -import software.amazon.awssdk.regions.PartitionMetadata; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain; - -import sleeper.clients.deploy.DeployNewInstance.StoreFactory; -import sleeper.configuration.properties.S3TableProperties; -import sleeper.configuration.table.index.DynamoDBTableIndexCreator; -import sleeper.core.deploy.SleeperInstanceConfiguration; -import sleeper.core.properties.instance.InstanceProperties; -import sleeper.core.properties.model.SleeperInternalCdkApp; -import sleeper.core.properties.table.TableProperties; -import sleeper.core.properties.table.TablePropertiesStore; -import sleeper.core.schema.Schema; -import sleeper.core.statestore.StateStore; -import sleeper.localstack.test.LocalStackTestBase; -import sleeper.statestore.StateStoreFactory; -import sleeper.statestore.transactionlog.TransactionLogStateStoreCreator; - -import java.io.IOException; -import java.nio.file.Path; - -import static org.assertj.core.api.Assertions.assertThat; -import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.CONFIG_BUCKET; -import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.DATA_BUCKET; -import static sleeper.core.properties.testutils.InstancePropertiesTestHelper.createTestInstanceProperties; -import static sleeper.core.schema.SchemaTestHelper.createSchemaWithKey; - -public class DeployNewInstanceIT extends LocalStackTestBase { - private final InstanceProperties instanceProperties = createTestInstanceProperties(); - private final Schema schema = createSchemaWithKey("key1"); - private final TablePropertiesStore propertiesStore = S3TableProperties.createStore(instanceProperties, s3Client, dynamoClient); - - @TempDir - private Path tempDir; - - @BeforeEach - void setUp() { - createBucket(instanceProperties.get(CONFIG_BUCKET)); - createBucket(instanceProperties.get(DATA_BUCKET)); - new TransactionLogStateStoreCreator(instanceProperties, dynamoClient).create(); - DynamoDBTableIndexCreator.create(dynamoClient, instanceProperties); - } - - @Test - void shouldDeployInstanceByInstanceProperties() throws IOException, InterruptedException { - // Given - SleeperInstanceConfiguration config = SleeperInstanceConfiguration.withNoTables(instanceProperties); - - // When - deployInstance(config); - - // Then - assertThat(propertiesStore.streamAllTables()).isEmpty(); - } - - @Test - void shouldDeployInstanceByConfigDirectoryWithTables() { - - } - - @Test - void shouldDeployInstanceByConfigDirectoryIgnoringTables() { - - } - - private StateStore stateStore(TableProperties tableProperties) { - return new StateStoreFactory(instanceProperties, s3Client, dynamoClient).getStateStore(tableProperties); - } - - private void deployInstance(SleeperInstanceConfiguration config) throws IOException, InterruptedException { - String accountName = stsClient.getCallerIdentity().account(); - Region region = DefaultAwsRegionProviderChain.builder().build().getRegion(); - PartitionMetadata partitionMetadata = PartitionMetadata.of(region); - - new DeployNewInstance(DeployInstance.fromScriptsDirectory(Path.of("TODO"), accountName, region, partitionMetadata, s3Client, ecrClient), - StoreFactory.withAwsClients(s3Client, dynamoClient), - config, SleeperInternalCdkApp.STANDARD, false, false).deploy(); - } -} diff --git a/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceTest.java b/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceTest.java index a51a91e515d..53cedb801d0 100644 --- a/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceTest.java +++ b/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceTest.java @@ -40,8 +40,6 @@ import java.nio.file.Path; import java.util.HashMap; import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Optional; import static java.nio.file.Files.createDirectory; import static java.nio.file.Files.createTempDirectory; @@ -61,6 +59,7 @@ public class DeployNewInstanceTest { new InMemoryTransactionLogsPerTable()); Map pathToString = new HashMap<>(); DeployInstanceRequest lastDeployRequest; + String configDir; @TempDir private Path tempDir; @@ -75,6 +74,7 @@ void setUp() throws IOException { createDirectory(table1); Files.writeString(table1.resolve("table.properties"), "sleeper.table.name=file-table\n"); Files.writeString(table1.resolve("schema.json"), new SchemaSerDe().toJson(schema)); + configDir = tempDir.toString(); } @Nested @@ -85,41 +85,47 @@ void shouldDeployNewInstanceWhenUsingInstanceProperties() throws Exception { deployNewInstance("scriptsDir", "my-instance", "someVpc", "someSubnets", "--instance-properties", tempDir.resolve("./instance.properties").toString()); - assertThat(tableIndex.streamAllTables()).isEmpty(); assertThat(lastDeployRequest.getCdkCommand()).isEqualTo(CdkCommand.deployNew()); assertThat(lastDeployRequest.getCdkApp()).isEqualTo(SleeperInternalCdkApp.STANDARD); + assertThat(tableIndex.streamAllTables()).isEmpty(); } @Test void shouldDeployNewInstanceWhenUsingConfigDir() throws Exception { - deployNewInstance("scriptsDir", "my-instance", "someVpc", "someSubnets", "--config-dir", "./configDir"); + deployNewInstance("scriptsDir", "my-instance", "someVpc", "someSubnets", "--config-dir", + configDir); + assertThat(lastDeployRequest.getCdkCommand()).isEqualTo(CdkCommand.deployNew()); + assertThat(lastDeployRequest.getCdkApp()).isEqualTo(SleeperInternalCdkApp.STANDARD); TableProperties expected = new TableProperties(instanceProperties); expected.setSchema(schema); expected.set(TABLE_ID, tableId("file-table")); expected.set(TABLE_NAME, "file-table"); assertThat(tablePropertiesStore.streamAllTables()).containsExactly(expected); - assertThat(lastDeployRequest.getCdkCommand()).isEqualTo(CdkCommand.deployNew()); - assertThat(lastDeployRequest.getCdkApp()).isEqualTo(SleeperInternalCdkApp.STANDARD); } @Test void shouldDeployNewInstanceWhenUsingInstancePropertiesIgnoringTableFiles() throws Exception { - deployNewInstance("scriptsDir", "my-instance", "someVpc", "someSubnets", "--config-dir", "./configDir", + deployNewInstance("scriptsDir", "my-instance", "someVpc", "someSubnets", "--config-dir", configDir, "--ignoreTableFiles"); - assertThat(tableIndex.streamAllTables()).isEmpty(); assertThat(lastDeployRequest.getCdkCommand()).isEqualTo(CdkCommand.deployNew()); assertThat(lastDeployRequest.getCdkApp()).isEqualTo(SleeperInternalCdkApp.STANDARD); + assertThat(tableIndex.streamAllTables()).isEmpty(); } @Test void shouldDeployNewInstancePaused() throws Exception { - deployNewInstance("scriptsDir", "my-instance", "someVpc", "someSubnets", "--config-dir", "./configDir", + deployNewInstance("scriptsDir", "my-instance", "someVpc", "someSubnets", "--config-dir", configDir, "--paused"); assertThat(lastDeployRequest.getCdkCommand()).isEqualTo(CdkCommand.deployNewPaused()); assertThat(lastDeployRequest.getCdkApp()).isEqualTo(SleeperInternalCdkApp.STANDARD); + TableProperties expected = new TableProperties(instanceProperties); + expected.setSchema(schema); + expected.set(TABLE_ID, tableId("file-table")); + expected.set(TABLE_NAME, "file-table"); + assertThat(tablePropertiesStore.streamAllTables()).containsExactly(expected); } } @@ -169,7 +175,7 @@ void shouldResolvePropertiesFileWhenConfigDirUsed() { private void deployNewInstance(String... args) throws Exception { var arguments = DeployNewInstance.readArguments(CommandArgumentReader.parse(DeployNewInstance.USAGE, args)); - var config = DeployNewInstance.loadConfiguration(arguments, Files::readString); + var config = DeployNewInstance.loadConfiguration(arguments); new DeployNewInstance( request -> lastDeployRequest = request, new DeployNewInstance.StoreFactory() { @@ -190,12 +196,4 @@ private String tableId(String tableName) { .orElseThrow(() -> new RuntimeException("Found tables: " + tableIndex.streamAllTables().toList())) .getTableUniqueId(); } - - private String readFile(Path path) throws IOException { - try { - return Optional.ofNullable(pathToString.get(path)).orElseThrow(); - } catch (NoSuchElementException e) { - throw new IOException(e); - } - } } diff --git a/java/common/localstack-test/src/main/java/sleeper/localstack/test/LocalStackTestBase.java b/java/common/localstack-test/src/main/java/sleeper/localstack/test/LocalStackTestBase.java index 1a2d2a93841..d86737e8892 100644 --- a/java/common/localstack-test/src/main/java/sleeper/localstack/test/LocalStackTestBase.java +++ b/java/common/localstack-test/src/main/java/sleeper/localstack/test/LocalStackTestBase.java @@ -23,7 +23,6 @@ import software.amazon.awssdk.core.sync.ResponseTransformer; import software.amazon.awssdk.services.cloudwatch.CloudWatchClient; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; -import software.amazon.awssdk.services.ecr.EcrClient; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.PutObjectResponse; @@ -63,7 +62,6 @@ public abstract class LocalStackTestBase { protected final Configuration hadoopConf = SleeperLocalStackClients.HADOOP_CONF; protected final CloudWatchClient cloudWatchClient = SleeperLocalStackClients.CLOUDWATCH_CLIENT; protected final AwsCredentialsProvider credentialsProvider = SleeperLocalStackClients.CREDENTIALS_PROVIDER; - protected final EcrClient ecrClient = SleeperLocalStackClients.ECR_CLIENT; public static void createBucket(String bucketName) { S3_CLIENT.createBucket(builder -> builder.bucket(bucketName)); diff --git a/java/common/localstack-test/src/main/java/sleeper/localstack/test/SleeperLocalStackClients.java b/java/common/localstack-test/src/main/java/sleeper/localstack/test/SleeperLocalStackClients.java index 6112ee2c22b..a8ad60fef9d 100644 --- a/java/common/localstack-test/src/main/java/sleeper/localstack/test/SleeperLocalStackClients.java +++ b/java/common/localstack-test/src/main/java/sleeper/localstack/test/SleeperLocalStackClients.java @@ -20,7 +20,6 @@ import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.services.cloudwatch.CloudWatchClient; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; -import software.amazon.awssdk.services.ecr.EcrClient; import software.amazon.awssdk.services.s3.S3AsyncClient; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.sqs.SqsClient; @@ -49,6 +48,5 @@ private SleeperLocalStackClients() { public static final CloudWatchClient CLOUDWATCH_CLIENT = buildAwsV2Client(CONTAINER, CloudWatchClient.builder()); public static final Configuration HADOOP_CONF = getHadoopConfiguration(CONTAINER); public static final AwsCredentialsProvider CREDENTIALS_PROVIDER = buildAwsCredentialsProvider(); - public static final EcrClient ECR_CLIENT = buildAwsV2Client(CONTAINER, EcrClient.builder()); } From 972747f40d3e708aef55ded494ba827e438a9297 Mon Sep 17 00:00:00 2001 From: ca61688 <206189192+ca61688@users.noreply.github.com> Date: Wed, 8 Jul 2026 10:53:19 +0100 Subject: [PATCH 20/50] 6593: Remove unused dependency --- java/common/localstack-test/pom.xml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/java/common/localstack-test/pom.xml b/java/common/localstack-test/pom.xml index cace0ffeb55..41fa641624d 100644 --- a/java/common/localstack-test/pom.xml +++ b/java/common/localstack-test/pom.xml @@ -68,10 +68,6 @@ software.amazon.awssdk cloudwatch - - software.amazon.awssdk - ecr - software.amazon.awssdk.crt aws-crt From ac6c85b5c7c2944fa5dabb42b303a8a955dec947 Mon Sep 17 00:00:00 2001 From: ca61688 <206189192+ca61688@users.noreply.github.com> Date: Fri, 10 Jul 2026 10:19:19 +0100 Subject: [PATCH 21/50] 6593: Have DeployNewInstance save update instance properties file --- .../clients/deploy/DeployInstance.java | 7 +- .../clients/deploy/DeployInstanceRequest.java | 25 +++++++ .../clients/deploy/DeployNewInstance.java | 49 +++++++++--- .../clients/deploy/DeployNewInstanceTest.java | 74 +++++++++++++++++-- 4 files changed, 136 insertions(+), 19 deletions(-) diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployInstance.java index 1d3151f8018..4c994d2fccc 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployInstance.java @@ -87,11 +87,14 @@ public void deploy(DeployInstanceRequest request) throws IOException, Interrupte syncJars.sync(SyncJarsRequest.from(instanceProperties)); dockerImageUploader.upload( UploadDockerImagesToEcrRequest.forDeployment(instanceProperties, request.getCdkApp(), DockerImageConfiguration.getDefault())); - Path configurationDirectory = writeLocalProperties.write(instanceConfig); LOGGER.info("-------------------------------------------------------"); LOGGER.info("Deploying Stacks"); LOGGER.info("-------------------------------------------------------"); - invokeCdk.invoke(request.getCdkApp(), request.getCdkCommand().withConfigurationDirectory(configurationDirectory)); + if (request.getPropertiesFile() != null) { + invokeCdk.invoke(request.getCdkApp(), request.getCdkCommand().withPropertiesFile(request.getPropertiesFile())); + } else { + invokeCdk.invoke(request.getCdkApp(), request.getCdkCommand().withConfigurationDirectory(request.getConfigDir())); + } } public interface WriteLocalProperties { diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployInstanceRequest.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployInstanceRequest.java index 2ab88c3cb9c..e576595cd5a 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployInstanceRequest.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployInstanceRequest.java @@ -19,6 +19,7 @@ import sleeper.core.deploy.SleeperInstanceConfiguration; import sleeper.core.properties.model.SleeperInternalCdkApp; +import java.nio.file.Path; import java.util.Objects; public class DeployInstanceRequest { @@ -26,11 +27,15 @@ public class DeployInstanceRequest { private final SleeperInstanceConfiguration instanceConfig; private final CdkCommand cdkCommand; private final SleeperInternalCdkApp cdkApp; + private Path propertiesFile; + private Path configDir; private DeployInstanceRequest(Builder builder) { instanceConfig = Objects.requireNonNull(builder.instanceConfig, "instanceConfig must not be null"); cdkCommand = Objects.requireNonNull(builder.cdkCommand, "cdkCommand must not be null"); cdkApp = Objects.requireNonNull(builder.cdkApp, "cdkApp must not be null"); + propertiesFile = builder.propertiesFile; + configDir = builder.configDir; } public static Builder builder() { @@ -49,10 +54,20 @@ public CdkCommand getCdkCommand() { return cdkCommand; } + public Path getPropertiesFile() { + return propertiesFile; + } + + public Path getConfigDir() { + return configDir; + } + public static class Builder { private SleeperInstanceConfiguration instanceConfig; private CdkCommand cdkCommand; private SleeperInternalCdkApp cdkApp; + private Path propertiesFile; + private Path configDir; public Builder instanceConfig(SleeperInstanceConfiguration instanceConfig) { this.instanceConfig = instanceConfig; @@ -69,6 +84,16 @@ public Builder cdkApp(SleeperInternalCdkApp cdkApp) { return this; } + public Builder propertiesFile(Path propertiesFile) { + this.propertiesFile = propertiesFile; + return this; + } + + public Builder configDir(Path configDir) { + this.configDir = configDir; + return this; + } + public DeployInstanceRequest build() { return new DeployInstanceRequest(this); } diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java index 4ac8fb02923..31ae7d252cf 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java @@ -40,7 +40,10 @@ import sleeper.core.util.cli.CommandOption; import sleeper.statestore.StateStoreFactory; +import java.io.BufferedWriter; import java.io.IOException; +import java.io.PrintWriter; +import java.nio.file.Files; import java.nio.file.Path; import java.util.List; @@ -55,6 +58,8 @@ public class DeployNewInstance { private final StoreFactory storeFactory; private final SleeperInstanceConfiguration deployInstanceConfiguration; private final SleeperInternalCdkApp cdkApp; + private final Path propertiesFile; + private final Path configDir; private final boolean ignoreTableFiles; private final boolean deployPaused; @@ -66,6 +71,22 @@ public DeployNewInstance(InstanceDeployer deployInstance, this.storeFactory = storeFactory; this.deployInstanceConfiguration = deployInstanceConfiguration; this.cdkApp = cdkApp; + this.propertiesFile = null; + this.configDir = null; + this.ignoreTableFiles = ignoreTableFiles; + this.deployPaused = deployPaused; + } + + public DeployNewInstance(InstanceDeployer deployInstance, + StoreFactory storeFactory, + SleeperInstanceConfiguration deployInstanceConfiguration, + SleeperInternalCdkApp cdkApp, Path propertiesFile, Path configDir, boolean ignoreTableFiles, boolean deployPaused) { + this.deployInstance = deployInstance; + this.storeFactory = storeFactory; + this.deployInstanceConfiguration = deployInstanceConfiguration; + this.cdkApp = cdkApp; + this.propertiesFile = propertiesFile; + this.configDir = configDir; this.ignoreTableFiles = ignoreTableFiles; this.deployPaused = deployPaused; } @@ -99,12 +120,24 @@ public DeployNewInstance(InstanceDeployer deployInstance, "the instance is manually resumed.") .build(); - public static SleeperInstanceConfiguration loadConfiguration(Arguments args) { + public static SleeperInstanceConfiguration loadAndUpdateConfiguration(Arguments args) throws IOException { + SleeperInstanceConfiguration config; if (args.ignoreTableFiles()) { - return SleeperInstanceConfiguration.fromLocalConfiguration(args.resolvePropertiesFile()); + config = SleeperInstanceConfiguration.fromLocalConfiguration(args.resolvePropertiesFile()); + } else { + config = SleeperInstanceConfiguration.fromLocalConfigurationDirectory(args.configDir()); } - return SleeperInstanceConfiguration.fromLocalConfigurationDirectory(args.configDir()); + config.getInstanceProperties().set(ID, args.instanceId()); + config.getInstanceProperties().set(VPC_ID, args.vpcId()); + config.getInstanceProperties().set(SUBNETS, args.subnetIds()); + + try (BufferedWriter writer = Files.newBufferedWriter(args.resolvePropertiesFile())) { + InstanceProperties.createPrettyPrinter(new PrintWriter(writer)) + .print(config.getInstanceProperties()); + } + + return config; } public static Arguments readArguments(CommandArguments arguments) { @@ -132,15 +165,11 @@ public static void main(String[] rawArgs) throws IOException, InterruptedExcepti Region region = DefaultAwsRegionProviderChain.builder().build().getRegion(); PartitionMetadata partitionMetadata = PartitionMetadata.of(region); - SleeperInstanceConfiguration config = loadConfiguration(args); - - config.getInstanceProperties().set(ID, args.instanceId()); - config.getInstanceProperties().set(VPC_ID, args.vpcId()); - config.getInstanceProperties().set(SUBNETS, args.subnetIds()); + SleeperInstanceConfiguration config = loadAndUpdateConfiguration(args); new DeployNewInstance(DeployInstance.fromScriptsDirectory(scriptsDirectory, accountName, region, partitionMetadata, s3Client, ecrClient), StoreFactory.withAwsClients(s3Client, dynamoClient), - config, SleeperInternalCdkApp.STANDARD, args.ignoreTableFiles(), deployPaused).deploy(); + config, SleeperInternalCdkApp.STANDARD, args.propertiesFile(), args.configDir(), args.ignoreTableFiles(), deployPaused).deploy(); } } @@ -151,6 +180,8 @@ public void deploy() throws IOException, InterruptedException { .instanceConfig(deployInstanceConfiguration) .cdkCommand(deployPaused ? CdkCommand.deployNewPaused() : CdkCommand.deployNew()) .cdkApp(cdkApp) + .propertiesFile(propertiesFile) + .configDir(configDir) .build()); if (!ignoreTableFiles) { diff --git a/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceTest.java b/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceTest.java index 53cedb801d0..1838f565f12 100644 --- a/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceTest.java +++ b/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceTest.java @@ -21,6 +21,7 @@ import org.junit.jupiter.api.io.TempDir; import sleeper.clients.util.cdk.CdkCommand; +import sleeper.core.deploy.SleeperInstanceConfiguration; import sleeper.core.properties.instance.InstanceProperties; import sleeper.core.properties.model.SleeperInternalCdkApp; import sleeper.core.properties.table.TableProperties; @@ -45,6 +46,9 @@ import static java.nio.file.Files.createTempDirectory; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static sleeper.core.properties.instance.CommonProperty.ID; +import static sleeper.core.properties.instance.CommonProperty.SUBNETS; +import static sleeper.core.properties.instance.CommonProperty.VPC_ID; import static sleeper.core.properties.table.TableProperty.TABLE_ID; import static sleeper.core.properties.table.TableProperty.TABLE_NAME; import static sleeper.core.properties.testutils.InstancePropertiesTestHelper.createTestInstancePropertiesWithId; @@ -59,6 +63,7 @@ public class DeployNewInstanceTest { new InMemoryTransactionLogsPerTable()); Map pathToString = new HashMap<>(); DeployInstanceRequest lastDeployRequest; + Path instancePropertiesFile; String configDir; @TempDir @@ -67,7 +72,8 @@ public class DeployNewInstanceTest { @BeforeEach void setUp() throws IOException { createTempDirectory(tempDir, null); - Files.writeString(tempDir.resolve("instance.properties"), instanceProperties.saveAsString()); + instancePropertiesFile = tempDir.resolve("instance.properties"); + Files.writeString(instancePropertiesFile, instanceProperties.saveAsString()); Path tables = tempDir.resolve("tables"); Path table1 = tables.resolve("table1"); createDirectory(tables); @@ -82,21 +88,47 @@ class DeployNew { @Test void shouldDeployNewInstanceWhenUsingInstanceProperties() throws Exception { - deployNewInstance("scriptsDir", "my-instance", "someVpc", "someSubnets", "--instance-properties", - tempDir.resolve("./instance.properties").toString()); + //When + deployNewInstance("scriptsDir", "someInstance", "someVpc", "someSubnets", "--instance-properties", + instancePropertiesFile.toString()); + //Then + //Verify Instance Properties file updates + instanceProperties.set(ID, "someInstance"); + instanceProperties.set(VPC_ID, "someVpc"); + instanceProperties.set(SUBNETS, "someSubnets"); + assertThat(instanceProperties) + .isEqualTo(SleeperInstanceConfiguration.fromLocalConfiguration( + instancePropertiesFile).getInstanceProperties()); + + //Verify CDK Command assertThat(lastDeployRequest.getCdkCommand()).isEqualTo(CdkCommand.deployNew()); assertThat(lastDeployRequest.getCdkApp()).isEqualTo(SleeperInternalCdkApp.STANDARD); + + //Verify no table properties stored assertThat(tableIndex.streamAllTables()).isEmpty(); } @Test void shouldDeployNewInstanceWhenUsingConfigDir() throws Exception { - deployNewInstance("scriptsDir", "my-instance", "someVpc", "someSubnets", "--config-dir", + //When + deployNewInstance("scriptsDir", "someInstance", "someVpc", "someSubnets", "--config-dir", configDir); + //Then + //Verify Instance Properties file updates + instanceProperties.set(ID, "someInstance"); + instanceProperties.set(VPC_ID, "someVpc"); + instanceProperties.set(SUBNETS, "someSubnets"); + assertThat(instanceProperties) + .isEqualTo(SleeperInstanceConfiguration.fromLocalConfiguration( + instancePropertiesFile).getInstanceProperties()); + + //Verify CDK Command assertThat(lastDeployRequest.getCdkCommand()).isEqualTo(CdkCommand.deployNew()); assertThat(lastDeployRequest.getCdkApp()).isEqualTo(SleeperInternalCdkApp.STANDARD); + + //Verify Table properties store saved TableProperties expected = new TableProperties(instanceProperties); expected.setSchema(schema); expected.set(TABLE_ID, tableId("file-table")); @@ -106,21 +138,47 @@ void shouldDeployNewInstanceWhenUsingConfigDir() throws Exception { @Test void shouldDeployNewInstanceWhenUsingInstancePropertiesIgnoringTableFiles() throws Exception { - deployNewInstance("scriptsDir", "my-instance", "someVpc", "someSubnets", "--config-dir", configDir, + //When + deployNewInstance("scriptsDir", "someInstance", "someVpc", "someSubnets", "--config-dir", configDir, "--ignoreTableFiles"); + //Then + //Verify Instance Properties file updates + instanceProperties.set(ID, "someInstance"); + instanceProperties.set(VPC_ID, "someVpc"); + instanceProperties.set(SUBNETS, "someSubnets"); + assertThat(instanceProperties) + .isEqualTo(SleeperInstanceConfiguration.fromLocalConfiguration( + instancePropertiesFile).getInstanceProperties()); + + //Verify CDK Command assertThat(lastDeployRequest.getCdkCommand()).isEqualTo(CdkCommand.deployNew()); assertThat(lastDeployRequest.getCdkApp()).isEqualTo(SleeperInternalCdkApp.STANDARD); + + //Verify no table properties stored assertThat(tableIndex.streamAllTables()).isEmpty(); } @Test void shouldDeployNewInstancePaused() throws Exception { - deployNewInstance("scriptsDir", "my-instance", "someVpc", "someSubnets", "--config-dir", configDir, + //When + deployNewInstance("scriptsDir", "someInstance", "someVpc", "someSubnets", "--config-dir", configDir, "--paused"); + //Then + //Verify Instance Properties file updates + instanceProperties.set(ID, "someInstance"); + instanceProperties.set(VPC_ID, "someVpc"); + instanceProperties.set(SUBNETS, "someSubnets"); + assertThat(instanceProperties) + .isEqualTo(SleeperInstanceConfiguration.fromLocalConfiguration( + instancePropertiesFile).getInstanceProperties()); + + //Verify CDK Command assertThat(lastDeployRequest.getCdkCommand()).isEqualTo(CdkCommand.deployNewPaused()); assertThat(lastDeployRequest.getCdkApp()).isEqualTo(SleeperInternalCdkApp.STANDARD); + + //Verify Table properties store saved TableProperties expected = new TableProperties(instanceProperties); expected.setSchema(schema); expected.set(TABLE_ID, tableId("file-table")); @@ -175,7 +233,7 @@ void shouldResolvePropertiesFileWhenConfigDirUsed() { private void deployNewInstance(String... args) throws Exception { var arguments = DeployNewInstance.readArguments(CommandArgumentReader.parse(DeployNewInstance.USAGE, args)); - var config = DeployNewInstance.loadConfiguration(arguments); + var config = DeployNewInstance.loadAndUpdateConfiguration(arguments); new DeployNewInstance( request -> lastDeployRequest = request, new DeployNewInstance.StoreFactory() { @@ -187,7 +245,7 @@ public StateStoreProvider createStateStore(InstanceProperties p) { return stateStoreProvider; } }, - config, SleeperInternalCdkApp.STANDARD, + config, SleeperInternalCdkApp.STANDARD, instancePropertiesFile, tempDir, arguments.ignoreTableFiles(), arguments.deployPaused()).deploy(); } From 7d3f05b1b59adebfee3dd3a7a930f30dba27f90f Mon Sep 17 00:00:00 2001 From: ca61688 <206189192+ca61688@users.noreply.github.com> Date: Mon, 13 Jul 2026 10:51:04 +0100 Subject: [PATCH 22/50] 6593: Remove old DeployNewInstance constructor --- .../sleeper/clients/deploy/DeployNewInstance.java | 14 -------------- .../drivers/cdk/DeployNewTestInstance.java | 6 +++--- .../drivers/instance/AwsSleeperInstanceDriver.java | 14 +++++++++++++- 3 files changed, 16 insertions(+), 18 deletions(-) diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java index 31ae7d252cf..eebc9345cca 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java @@ -63,20 +63,6 @@ public class DeployNewInstance { private final boolean ignoreTableFiles; private final boolean deployPaused; - public DeployNewInstance(InstanceDeployer deployInstance, - StoreFactory storeFactory, - SleeperInstanceConfiguration deployInstanceConfiguration, - SleeperInternalCdkApp cdkApp, boolean ignoreTableFiles, boolean deployPaused) { - this.deployInstance = deployInstance; - this.storeFactory = storeFactory; - this.deployInstanceConfiguration = deployInstanceConfiguration; - this.cdkApp = cdkApp; - this.propertiesFile = null; - this.configDir = null; - this.ignoreTableFiles = ignoreTableFiles; - this.deployPaused = deployPaused; - } - public DeployNewInstance(InstanceDeployer deployInstance, StoreFactory storeFactory, SleeperInstanceConfiguration deployInstanceConfiguration, diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java index d2d80a38d4b..fe4e3c8f88a 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java @@ -49,7 +49,7 @@ public static void main(String[] args) throws IOException, InterruptedException " "); } Path scriptsDirectory = Path.of(args[0]); - Path propertiesFile = Path.of(args[1]); + Path configurationPath = Path.of(args[1]); String instanceId = args[2]; String vpcId = args[3]; String subnetIds = args[4]; @@ -65,13 +65,13 @@ public static void main(String[] args) throws IOException, InterruptedException PartitionMetadata partitionMetadata = PartitionMetadata.of(region); SleeperInstanceConfiguration config = SleeperInstanceConfiguration.forNewInstanceDefaultingTables( - propertiesFile, templates(scriptsDirectory, splitPointsFileForTemplate)); + configurationPath, templates(scriptsDirectory, splitPointsFileForTemplate)); config.getInstanceProperties().set(ID, instanceId); config.getInstanceProperties().set(VPC_ID, vpcId); config.getInstanceProperties().set(SUBNETS, subnetIds); new DeployNewInstance(DeployInstance.fromScriptsDirectory(scriptsDirectory, accountName, region, partitionMetadata, s3Client, ecrClient), StoreFactory.withAwsClients(s3Client, dynamoClient), - config, SleeperInternalCdkApp.STANDARD, false, deployPaused).deploy(); + config, SleeperInternalCdkApp.STANDARD, null, configurationPath, false, deployPaused).deploy(); } } diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java index 6aeebad9e79..ab7534595e3 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java @@ -31,6 +31,7 @@ import sleeper.configuration.properties.S3InstanceProperties; import sleeper.core.deploy.SleeperInstanceConfiguration; import sleeper.core.properties.instance.InstanceProperties; +import sleeper.core.properties.local.SaveLocalProperties; import sleeper.core.properties.model.SleeperInternalCdkApp; import sleeper.core.properties.table.TableProperties; import sleeper.systemtest.drivers.util.SystemTestClients; @@ -39,6 +40,7 @@ import java.io.IOException; import java.io.UncheckedIOException; +import java.nio.file.Path; import java.util.List; import java.util.Set; @@ -84,10 +86,20 @@ public boolean deployInstanceIfNotPresent(String instanceId, SleeperInstanceConf deployConfig.getInstanceProperties().set(ID, instanceId); deployConfig.getInstanceProperties().set(VPC_ID, parameters.getVpcId()); deployConfig.getInstanceProperties().set(SUBNETS, parameters.getSubnetIds()); + + Path configDir = parameters.getScriptsDirectory().resolve("example"); + try { + SaveLocalProperties.saveToDirectory(configDir, + deployConfig.getInstanceProperties(), + deployConfig.getTableProperties().stream()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + try { new DeployNewInstance(deployInstance, StoreFactory.withAwsClients(s3, dynamoDB), - deployConfig, SleeperInternalCdkApp.STANDARD, false, false).deploy(); + deployConfig, SleeperInternalCdkApp.STANDARD, null, configDir, false, false).deploy(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RuntimeException(e); From fa1fa51cd6da723bf51583a689d25b705b7b8d36 Mon Sep 17 00:00:00 2001 From: ca61688 <206189192+ca61688@users.noreply.github.com> Date: Mon, 13 Jul 2026 11:51:37 +0100 Subject: [PATCH 23/50] 6593: Remove unused writeLocalProperties interface --- .../clients/deploy/DeployInstance.java | 28 +------------------ .../instance/SystemTestDeploymentFactory.java | 1 - 2 files changed, 1 insertion(+), 28 deletions(-) diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployInstance.java index 4c994d2fccc..3147e36f677 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployInstance.java @@ -28,15 +28,12 @@ import sleeper.clients.deploy.container.UploadDockerImagesToEcrRequest; import sleeper.clients.deploy.jar.SyncJars; import sleeper.clients.deploy.jar.SyncJarsRequest; -import sleeper.clients.util.ClientUtils; import sleeper.clients.util.cdk.CdkCommand; import sleeper.clients.util.cdk.InvokeCdk; import sleeper.core.deploy.SleeperInstanceConfiguration; import sleeper.core.properties.instance.InstanceProperties; -import sleeper.core.properties.local.SaveLocalProperties; import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; import static sleeper.core.properties.instance.CommonProperty.ARTEFACTS_DEPLOYMENT_ID; @@ -50,13 +47,11 @@ public class DeployInstance implements InstanceDeployer { private final SyncJars syncJars; private final UploadDockerImagesToEcr dockerImageUploader; - private final WriteLocalProperties writeLocalProperties; private final InvokeCdk invokeCdk; - public DeployInstance(SyncJars syncJars, UploadDockerImagesToEcr dockerImageUploader, WriteLocalProperties writeLocalProperties, InvokeCdk invokeCdk) { + public DeployInstance(SyncJars syncJars, UploadDockerImagesToEcr dockerImageUploader, InvokeCdk invokeCdk) { this.syncJars = syncJars; this.dockerImageUploader = dockerImageUploader; - this.writeLocalProperties = writeLocalProperties; this.invokeCdk = invokeCdk; } @@ -67,7 +62,6 @@ public static DeployInstance fromScriptsDirectory( new UploadDockerImagesToEcr( UploadDockerImages.fromScriptsDirectory(scriptsDirectory, ecrClient), account, region, partitionMetadata), - DeployInstance.WriteLocalProperties.underScriptsDirectory(scriptsDirectory), InvokeCdk.fromScriptsDirectory(scriptsDirectory)); } @@ -96,24 +90,4 @@ public void deploy(DeployInstanceRequest request) throws IOException, Interrupte invokeCdk.invoke(request.getCdkApp(), request.getCdkCommand().withConfigurationDirectory(request.getConfigDir())); } } - - public interface WriteLocalProperties { - Path write(SleeperInstanceConfiguration instanceConfig) throws IOException; - - static WriteLocalProperties underScriptsDirectory(Path scriptsDirectory) { - return toDirectory(scriptsDirectory.resolve("generated")); - } - - static WriteLocalProperties toDirectory(Path directory) { - return instanceConfig -> { - LOGGER.info("Writing instance configuration to local directory: {}", directory); - Files.createDirectories(directory); - ClientUtils.clearDirectory(directory); - SaveLocalProperties.saveToDirectory(directory, - instanceConfig.getInstanceProperties(), - instanceConfig.getTableProperties().stream()); - return directory; - }; - } - } } diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/SystemTestDeploymentFactory.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/SystemTestDeploymentFactory.java index f81a3d691c1..6542bf3b474 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/SystemTestDeploymentFactory.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/SystemTestDeploymentFactory.java @@ -37,7 +37,6 @@ public static DeployInstance createDeployInstance(SystemTestParameters parameter return new DeployInstance( createSyncJars(parameters, clients), createDockerUploader(parameters, clients), - DeployInstance.WriteLocalProperties.underScriptsDirectory(parameters.getScriptsDirectory()), createInvokeCdk(parameters, clients)); } From 5848f60834af6b40ee6e25703b2af2a9e569ef9e Mon Sep 17 00:00:00 2001 From: ca61688 <206189192+ca61688@users.noreply.github.com> Date: Mon, 13 Jul 2026 14:18:00 +0100 Subject: [PATCH 24/50] 6593: Fix DeployExistingInstance --- .../deploy/DeployExistingInstance.java | 25 +++++++++++++++++++ .../instance/AwsSleeperInstanceDriver.java | 4 +++ 2 files changed, 29 insertions(+) diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java index fd8578e072d..e8801b524ad 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java @@ -26,11 +26,13 @@ import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.sts.StsClient; +import sleeper.clients.util.ClientUtils; import sleeper.clients.util.cdk.CdkCommand; import sleeper.configuration.properties.S3InstanceProperties; import sleeper.configuration.properties.S3TableProperties; import sleeper.core.deploy.SleeperInstanceConfiguration; import sleeper.core.properties.instance.InstanceProperties; +import sleeper.core.properties.local.SaveLocalProperties; import sleeper.core.properties.model.SleeperInternalCdkApp; import sleeper.core.properties.table.TableProperties; import sleeper.core.util.cli.CommandArguments; @@ -38,6 +40,8 @@ import sleeper.core.util.cli.CommandOption; import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; import java.nio.file.Path; import java.util.List; import java.util.stream.Collectors; @@ -53,6 +57,7 @@ public class DeployExistingInstance { private final List tablePropertiesList; private final boolean deployPaused; private final SleeperInternalCdkApp forceCdkApp; + private final Path configDir; private DeployExistingInstance(Builder builder) { deployInstance = builder.deployInstance; @@ -60,6 +65,7 @@ private DeployExistingInstance(Builder builder) { tablePropertiesList = builder.tablePropertiesList; deployPaused = builder.deployPaused; forceCdkApp = builder.forceCdkApp; + configDir = builder.configDir; } public static Builder builder() { @@ -107,6 +113,7 @@ public static void main(String[] rawArgs) throws IOException, InterruptedExcepti .deployPaused(args.deployPaused()) .forceCdkApp(args.forceCdkApp()) .loadPropertiesFromS3(accountName, s3Client, dynamoClient) + .configDir(args.scriptsDirectory().resolve("generated")) .build().update(); } } @@ -115,6 +122,18 @@ public record Arguments(Path scriptsDirectory, String instanceId, boolean deploy } public void update() throws IOException, InterruptedException { + SleeperInstanceConfiguration deployConfig = SleeperInstanceConfiguration.builder().instanceProperties(properties).tableProperties(tablePropertiesList).build(); + + try { + Files.createDirectories(configDir); + ClientUtils.clearDirectory(configDir); + SaveLocalProperties.saveToDirectory(configDir, + deployConfig.getInstanceProperties(), + deployConfig.getTableProperties().stream()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + deployInstance.deploy(DeployInstanceRequest.builder() .instanceConfig(SleeperInstanceConfiguration.builder().instanceProperties(properties).tableProperties(tablePropertiesList).build()) .cdkCommand(deployPaused ? CdkCommand.deployExistingPaused() : CdkCommand.deployExisting()) @@ -143,6 +162,7 @@ public static final class Builder { private List tablePropertiesList; private boolean deployPaused; private SleeperInternalCdkApp forceCdkApp; + private Path configDir; private Builder() { } @@ -181,6 +201,11 @@ public Builder forceCdkApp(SleeperInternalCdkApp forceCdkApp) { return this; } + public Builder configDir(Path configDir) { + this.configDir = configDir; + return this; + } + public Builder loadPropertiesFromS3(String accountName, S3Client s3Client, DynamoDbClient dynamoCient) { properties = S3InstanceProperties.loadGivenAccountAndInstanceId(s3Client, accountName, instanceId); tablePropertiesList = S3TableProperties.createStore(properties, s3Client, dynamoCient) diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java index ab7534595e3..987dab55ade 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java @@ -28,6 +28,7 @@ import sleeper.clients.deploy.DeployInstance; import sleeper.clients.deploy.DeployNewInstance; import sleeper.clients.deploy.DeployNewInstance.StoreFactory; +import sleeper.clients.util.ClientUtils; import sleeper.configuration.properties.S3InstanceProperties; import sleeper.core.deploy.SleeperInstanceConfiguration; import sleeper.core.properties.instance.InstanceProperties; @@ -40,6 +41,7 @@ import java.io.IOException; import java.io.UncheckedIOException; +import java.nio.file.Files; import java.nio.file.Path; import java.util.List; import java.util.Set; @@ -89,6 +91,8 @@ public boolean deployInstanceIfNotPresent(String instanceId, SleeperInstanceConf Path configDir = parameters.getScriptsDirectory().resolve("example"); try { + Files.createDirectories(configDir); + ClientUtils.clearDirectory(configDir); SaveLocalProperties.saveToDirectory(configDir, deployConfig.getInstanceProperties(), deployConfig.getTableProperties().stream()); From 353f694e8b14882a29a44dc0d32e940315360cad Mon Sep 17 00:00:00 2001 From: ca61688 <206189192+ca61688@users.noreply.github.com> Date: Mon, 13 Jul 2026 14:28:50 +0100 Subject: [PATCH 25/50] 6593: Fix DeployExistingInstance --- .../main/java/sleeper/clients/deploy/DeployExistingInstance.java | 1 + 1 file changed, 1 insertion(+) diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java index e8801b524ad..3d4afe5ce46 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java @@ -138,6 +138,7 @@ public void update() throws IOException, InterruptedException { .instanceConfig(SleeperInstanceConfiguration.builder().instanceProperties(properties).tableProperties(tablePropertiesList).build()) .cdkCommand(deployPaused ? CdkCommand.deployExistingPaused() : CdkCommand.deployExisting()) .cdkApp(getCdkApp()) + .configDir(configDir) .build()); LOGGER.info("Finished deployment of existing instance"); From 9e85d3c0167a89cc6755119532c9e6cf2722e410 Mon Sep 17 00:00:00 2001 From: ca61688 <206189192+ca61688@users.noreply.github.com> Date: Mon, 13 Jul 2026 16:26:10 +0100 Subject: [PATCH 26/50] 6593: Add logging --- .../sleeper/core/properties/table/TablePropertiesStore.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/java/core/src/main/java/sleeper/core/properties/table/TablePropertiesStore.java b/java/core/src/main/java/sleeper/core/properties/table/TablePropertiesStore.java index 44d5351a73c..68cd5d3e3ef 100644 --- a/java/core/src/main/java/sleeper/core/properties/table/TablePropertiesStore.java +++ b/java/core/src/main/java/sleeper/core/properties/table/TablePropertiesStore.java @@ -16,6 +16,9 @@ package sleeper.core.properties.table; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import sleeper.core.table.TableAlreadyExistsException; import sleeper.core.table.TableIdGenerator; import sleeper.core.table.TableIndex; @@ -34,6 +37,7 @@ * A store to load and save table properties via the table index of the Sleeper instance. */ public class TablePropertiesStore { + private static final Logger LOGGER = LoggerFactory.getLogger(TablePropertiesStore.class); private static final TableIdGenerator ID_GENERATOR = new TableIdGenerator(); @@ -141,6 +145,7 @@ public Stream streamOnlineTableIds() { */ public void createTable(TableProperties tableProperties) { String tableName = tableProperties.get(TableProperty.TABLE_NAME); + LOGGER.info("Table name: " + tableName); tableIndex.getTableByName(tableName).ifPresent(tableId -> { throw new TableAlreadyExistsException(tableId); }); From 314e76efe8942f74f7fe02480d31a72eea0e7576 Mon Sep 17 00:00:00 2001 From: ca61688 <206189192+ca61688@users.noreply.github.com> Date: Tue, 14 Jul 2026 09:24:00 +0100 Subject: [PATCH 27/50] 6593: Add logging for testing --- .../src/main/java/sleeper/clients/deploy/DeployNewInstance.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java index eebc9345cca..a50a324f36f 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java @@ -33,6 +33,7 @@ import sleeper.core.properties.model.SleeperInternalCdkApp; import sleeper.core.properties.table.TableProperties; import sleeper.core.properties.table.TablePropertiesStore; +import sleeper.core.properties.table.TableProperty; import sleeper.core.statestore.StateStoreProvider; import sleeper.core.util.cli.CommandArguments; import sleeper.core.util.cli.CommandArgumentsException; @@ -175,6 +176,7 @@ public void deploy() throws IOException, InterruptedException { for (TableProperties tableProperties : deployInstanceConfiguration.getTableProperties()) { LOGGER.info("Adding table " + tableProperties.getStatus()); + LOGGER.info("Table name: " + tableProperties.get(TableProperty.TABLE_NAME)); new AddTableClient(tableProperties, storeFactory.createTableStore(instanceProperties), storeFactory.createStateStore(instanceProperties)) From b4ef23bf7debc043fe7b2a4cb1e3914b3dd8022c Mon Sep 17 00:00:00 2001 From: ca61688 <206189192+ca61688@users.noreply.github.com> Date: Tue, 14 Jul 2026 10:08:53 +0100 Subject: [PATCH 28/50] 6593: Change AddTableClient call in DeployNewInstance to match ATC-main method --- .../clients/deploy/DeployNewInstance.java | 22 ++++++++++--------- .../table/TablePropertiesStore.java | 5 ----- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java index a50a324f36f..6382a610935 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java @@ -33,7 +33,6 @@ import sleeper.core.properties.model.SleeperInternalCdkApp; import sleeper.core.properties.table.TableProperties; import sleeper.core.properties.table.TablePropertiesStore; -import sleeper.core.properties.table.TableProperty; import sleeper.core.statestore.StateStoreProvider; import sleeper.core.util.cli.CommandArguments; import sleeper.core.util.cli.CommandArgumentsException; @@ -48,6 +47,7 @@ import java.nio.file.Path; import java.util.List; +import static sleeper.configuration.utils.AwsV2ClientHelper.buildAwsV2Client; import static sleeper.core.properties.instance.CommonProperty.ID; import static sleeper.core.properties.instance.CommonProperty.SUBNETS; import static sleeper.core.properties.instance.CommonProperty.VPC_ID; @@ -172,15 +172,17 @@ public void deploy() throws IOException, InterruptedException { .build()); if (!ignoreTableFiles) { - InstanceProperties instanceProperties = deployInstanceConfiguration.getInstanceProperties(); - - for (TableProperties tableProperties : deployInstanceConfiguration.getTableProperties()) { - LOGGER.info("Adding table " + tableProperties.getStatus()); - LOGGER.info("Table name: " + tableProperties.get(TableProperty.TABLE_NAME)); - new AddTableClient(tableProperties, - storeFactory.createTableStore(instanceProperties), - storeFactory.createStateStore(instanceProperties)) - .run(); + try (S3Client s3Client = buildAwsV2Client(S3Client.builder()); + DynamoDbClient dynamoClient = buildAwsV2Client(DynamoDbClient.builder())) { + InstanceProperties instanceProperties = deployInstanceConfiguration.getInstanceProperties(); + TablePropertiesStore tablePropertiesStore = S3TableProperties.createStore(instanceProperties, s3Client, dynamoClient); + StateStoreProvider stateStoreProvider = StateStoreFactory.createProvider(instanceProperties, s3Client, dynamoClient); + + for (TableProperties tableProperties : deployInstanceConfiguration.getTableProperties()) { + LOGGER.info("Adding table " + tableProperties.getStatus()); + new AddTableClient(tableProperties, tablePropertiesStore, stateStoreProvider).run(); + + } } } LOGGER.info("Finished deployment of new instance"); diff --git a/java/core/src/main/java/sleeper/core/properties/table/TablePropertiesStore.java b/java/core/src/main/java/sleeper/core/properties/table/TablePropertiesStore.java index 68cd5d3e3ef..44d5351a73c 100644 --- a/java/core/src/main/java/sleeper/core/properties/table/TablePropertiesStore.java +++ b/java/core/src/main/java/sleeper/core/properties/table/TablePropertiesStore.java @@ -16,9 +16,6 @@ package sleeper.core.properties.table; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import sleeper.core.table.TableAlreadyExistsException; import sleeper.core.table.TableIdGenerator; import sleeper.core.table.TableIndex; @@ -37,7 +34,6 @@ * A store to load and save table properties via the table index of the Sleeper instance. */ public class TablePropertiesStore { - private static final Logger LOGGER = LoggerFactory.getLogger(TablePropertiesStore.class); private static final TableIdGenerator ID_GENERATOR = new TableIdGenerator(); @@ -145,7 +141,6 @@ public Stream streamOnlineTableIds() { */ public void createTable(TableProperties tableProperties) { String tableName = tableProperties.get(TableProperty.TABLE_NAME); - LOGGER.info("Table name: " + tableName); tableIndex.getTableByName(tableName).ifPresent(tableId -> { throw new TableAlreadyExistsException(tableId); }); From d786778806412d0e2b8f504442b091957538559b Mon Sep 17 00:00:00 2001 From: ca61688 <206189192+ca61688@users.noreply.github.com> Date: Tue, 14 Jul 2026 13:29:06 +0100 Subject: [PATCH 29/50] 6593: Reload properties to get CDK defined ones --- .../clients/deploy/DeployNewInstance.java | 32 +++++++++++-------- .../clients/deploy/DeployNewInstanceTest.java | 3 ++ 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java index 6382a610935..12d541cb50d 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java @@ -27,6 +27,7 @@ import sleeper.clients.table.AddTableClient; import sleeper.clients.util.cdk.CdkCommand; +import sleeper.configuration.properties.S3InstanceProperties; import sleeper.configuration.properties.S3TableProperties; import sleeper.core.deploy.SleeperInstanceConfiguration; import sleeper.core.properties.instance.InstanceProperties; @@ -47,7 +48,6 @@ import java.nio.file.Path; import java.util.List; -import static sleeper.configuration.utils.AwsV2ClientHelper.buildAwsV2Client; import static sleeper.core.properties.instance.CommonProperty.ID; import static sleeper.core.properties.instance.CommonProperty.SUBNETS; import static sleeper.core.properties.instance.CommonProperty.VPC_ID; @@ -155,7 +155,7 @@ public static void main(String[] rawArgs) throws IOException, InterruptedExcepti SleeperInstanceConfiguration config = loadAndUpdateConfiguration(args); new DeployNewInstance(DeployInstance.fromScriptsDirectory(scriptsDirectory, accountName, region, partitionMetadata, s3Client, ecrClient), - StoreFactory.withAwsClients(s3Client, dynamoClient), + StoreFactory.withAwsClients(s3Client, dynamoClient, accountName), config, SleeperInternalCdkApp.STANDARD, args.propertiesFile(), args.configDir(), args.ignoreTableFiles(), deployPaused).deploy(); } } @@ -172,17 +172,15 @@ public void deploy() throws IOException, InterruptedException { .build()); if (!ignoreTableFiles) { - try (S3Client s3Client = buildAwsV2Client(S3Client.builder()); - DynamoDbClient dynamoClient = buildAwsV2Client(DynamoDbClient.builder())) { - InstanceProperties instanceProperties = deployInstanceConfiguration.getInstanceProperties(); - TablePropertiesStore tablePropertiesStore = S3TableProperties.createStore(instanceProperties, s3Client, dynamoClient); - StateStoreProvider stateStoreProvider = StateStoreFactory.createProvider(instanceProperties, s3Client, dynamoClient); - - for (TableProperties tableProperties : deployInstanceConfiguration.getTableProperties()) { - LOGGER.info("Adding table " + tableProperties.getStatus()); - new AddTableClient(tableProperties, tablePropertiesStore, stateStoreProvider).run(); - - } + InstanceProperties instanceProperties = deployInstanceConfiguration.getInstanceProperties(); + storeFactory.reloadInstanceProperties(instanceProperties); + + for (TableProperties tableProperties : deployInstanceConfiguration.getTableProperties()) { + LOGGER.info("Adding table " + tableProperties.getStatus()); + new AddTableClient(tableProperties, + storeFactory.createTableStore(instanceProperties), + storeFactory.createStateStore(instanceProperties)) + .run(); } } LOGGER.info("Finished deployment of new instance"); @@ -227,7 +225,9 @@ public interface StoreFactory { StateStoreProvider createStateStore(InstanceProperties instanceProperties); - static StoreFactory withAwsClients(S3Client s3Client, DynamoDbClient dynamoClient) { + void reloadInstanceProperties(InstanceProperties instanceProperties); + + static StoreFactory withAwsClients(S3Client s3Client, DynamoDbClient dynamoClient, String accountName) { return new StoreFactory() { public TablePropertiesStore createTableStore(InstanceProperties p) { return S3TableProperties.createStore(p, s3Client, dynamoClient); @@ -236,6 +236,10 @@ public TablePropertiesStore createTableStore(InstanceProperties p) { public StateStoreProvider createStateStore(InstanceProperties p) { return StateStoreFactory.createProvider(p, s3Client, dynamoClient); } + + public void reloadInstanceProperties(InstanceProperties p) { + S3InstanceProperties.reloadGivenAccountAndInstanceId(s3Client, p, accountName, p.get(ID)); + } }; } } diff --git a/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceTest.java b/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceTest.java index 1838f565f12..d5b0f8c4504 100644 --- a/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceTest.java +++ b/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceTest.java @@ -244,6 +244,9 @@ public TablePropertiesStore createTableStore(InstanceProperties p) { public StateStoreProvider createStateStore(InstanceProperties p) { return stateStoreProvider; } + + public void reloadInstanceProperties(InstanceProperties p) { + } }, config, SleeperInternalCdkApp.STANDARD, instancePropertiesFile, tempDir, arguments.ignoreTableFiles(), arguments.deployPaused()).deploy(); From 0ff19a3b91fec18f270b279a5d7ec988ae002804 Mon Sep 17 00:00:00 2001 From: ca61688 <206189192+ca61688@users.noreply.github.com> Date: Tue, 14 Jul 2026 13:37:42 +0100 Subject: [PATCH 30/50] 6593: Fix build error --- .../systemtest/drivers/cdk/DeployNewTestInstance.java | 2 +- .../drivers/instance/AwsSleeperInstanceDriver.java | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java index fe4e3c8f88a..b5658756f86 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java @@ -70,7 +70,7 @@ public static void main(String[] args) throws IOException, InterruptedException config.getInstanceProperties().set(VPC_ID, vpcId); config.getInstanceProperties().set(SUBNETS, subnetIds); new DeployNewInstance(DeployInstance.fromScriptsDirectory(scriptsDirectory, accountName, region, partitionMetadata, s3Client, ecrClient), - StoreFactory.withAwsClients(s3Client, dynamoClient), + StoreFactory.withAwsClients(s3Client, dynamoClient, accountName), config, SleeperInternalCdkApp.STANDARD, null, configurationPath, false, deployPaused).deploy(); } } diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java index 987dab55ade..3bf60e1559b 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java @@ -23,6 +23,7 @@ import software.amazon.awssdk.services.cloudformation.model.Stack; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.sts.StsClient; import sleeper.clients.deploy.DeployExistingInstance; import sleeper.clients.deploy.DeployInstance; @@ -57,6 +58,7 @@ public class AwsSleeperInstanceDriver implements SleeperInstanceDriver { private final SystemTestParameters parameters; private final S3Client s3; + private final StsClient sts; private final DynamoDbClient dynamoDB; private final CloudFormationClient cloudFormationClient; private final AwsResetInstanceOnFirstConnect resetInstance; @@ -64,7 +66,8 @@ public class AwsSleeperInstanceDriver implements SleeperInstanceDriver { public AwsSleeperInstanceDriver(SystemTestParameters parameters, SystemTestClients clients) { this.parameters = parameters; - this.s3 = clients.getS3(); + this.s3 = clients.getS3() + this.sts = clients.getSts(); this.dynamoDB = clients.getDynamo(); this.cloudFormationClient = clients.getCloudFormation(); this.resetInstance = new AwsResetInstanceOnFirstConnect(clients); @@ -102,7 +105,7 @@ public boolean deployInstanceIfNotPresent(String instanceId, SleeperInstanceConf try { new DeployNewInstance(deployInstance, - StoreFactory.withAwsClients(s3, dynamoDB), + StoreFactory.withAwsClients(s3, dynamoDB, sts.getCallerIdentity().account()), deployConfig, SleeperInternalCdkApp.STANDARD, null, configDir, false, false).deploy(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); From fbef581a3507d6ecb930a6a028504f7c9d892ad4 Mon Sep 17 00:00:00 2001 From: ca61688 <206189192+ca61688@users.noreply.github.com> Date: Tue, 14 Jul 2026 13:40:58 +0100 Subject: [PATCH 31/50] 6593: Fix build error --- .../systemtest/drivers/instance/AwsSleeperInstanceDriver.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java index 3bf60e1559b..646397e0316 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java @@ -66,7 +66,7 @@ public class AwsSleeperInstanceDriver implements SleeperInstanceDriver { public AwsSleeperInstanceDriver(SystemTestParameters parameters, SystemTestClients clients) { this.parameters = parameters; - this.s3 = clients.getS3() + this.s3 = clients.getS3(); this.sts = clients.getSts(); this.dynamoDB = clients.getDynamo(); this.cloudFormationClient = clients.getCloudFormation(); From 3c26600fb35f5f97aeafeeb1cde0c980c00ac554 Mon Sep 17 00:00:00 2001 From: ca61688 <206189192+ca61688@users.noreply.github.com> Date: Mon, 20 Jul 2026 13:40:16 +0100 Subject: [PATCH 32/50] 6593: Add check in DeployInstanceRequest constructor for propertiesFile or confgDir being set --- .../clients/deploy/DeployExistingInstance.java | 15 +++++---------- .../clients/deploy/DeployInstanceRequest.java | 7 +++++-- ...InstanceTest.java => DeployNewInstanceIT.java} | 2 +- 3 files changed, 11 insertions(+), 13 deletions(-) rename java/clients/src/test/java/sleeper/clients/deploy/{DeployNewInstanceTest.java => DeployNewInstanceIT.java} (99%) diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java index 3d4afe5ce46..8a78ba9c7e0 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java @@ -40,7 +40,6 @@ import sleeper.core.util.cli.CommandOption; import java.io.IOException; -import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; @@ -124,15 +123,11 @@ public record Arguments(Path scriptsDirectory, String instanceId, boolean deploy public void update() throws IOException, InterruptedException { SleeperInstanceConfiguration deployConfig = SleeperInstanceConfiguration.builder().instanceProperties(properties).tableProperties(tablePropertiesList).build(); - try { - Files.createDirectories(configDir); - ClientUtils.clearDirectory(configDir); - SaveLocalProperties.saveToDirectory(configDir, - deployConfig.getInstanceProperties(), - deployConfig.getTableProperties().stream()); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + Files.createDirectories(configDir); + ClientUtils.clearDirectory(configDir); + SaveLocalProperties.saveToDirectory(configDir, + deployConfig.getInstanceProperties(), + deployConfig.getTableProperties().stream()); deployInstance.deploy(DeployInstanceRequest.builder() .instanceConfig(SleeperInstanceConfiguration.builder().instanceProperties(properties).tableProperties(tablePropertiesList).build()) diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployInstanceRequest.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployInstanceRequest.java index e576595cd5a..fccb6d21aaa 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployInstanceRequest.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployInstanceRequest.java @@ -27,8 +27,8 @@ public class DeployInstanceRequest { private final SleeperInstanceConfiguration instanceConfig; private final CdkCommand cdkCommand; private final SleeperInternalCdkApp cdkApp; - private Path propertiesFile; - private Path configDir; + private final Path propertiesFile; + private final Path configDir; private DeployInstanceRequest(Builder builder) { instanceConfig = Objects.requireNonNull(builder.instanceConfig, "instanceConfig must not be null"); @@ -36,6 +36,9 @@ private DeployInstanceRequest(Builder builder) { cdkApp = Objects.requireNonNull(builder.cdkApp, "cdkApp must not be null"); propertiesFile = builder.propertiesFile; configDir = builder.configDir; + if (propertiesFile == null && configDir == null) { + throw new NullPointerException("One of propertiesFile and configDir must not be null"); + } } public static Builder builder() { diff --git a/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceTest.java b/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceIT.java similarity index 99% rename from java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceTest.java rename to java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceIT.java index d5b0f8c4504..6d1a6f35675 100644 --- a/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceTest.java +++ b/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceIT.java @@ -54,7 +54,7 @@ import static sleeper.core.properties.testutils.InstancePropertiesTestHelper.createTestInstancePropertiesWithId; import static sleeper.core.schema.SchemaTestHelper.createSchemaWithKey; -public class DeployNewInstanceTest { +public class DeployNewInstanceIT { InstanceProperties instanceProperties = createTestInstancePropertiesWithId("my-instance"); Schema schema = createSchemaWithKey("key"); InMemoryTableIndex tableIndex = new InMemoryTableIndex(); From 8189067eede2918dfe90d96d12fb21b89a64bb22 Mon Sep 17 00:00:00 2001 From: ca61688 <206189192+ca61688@users.noreply.github.com> Date: Mon, 20 Jul 2026 14:06:00 +0100 Subject: [PATCH 33/50] 6593: Move clear directories method into FilesUtil and allow SaveLocalProperties to create directories --- .../AdminClientPropertiesStore.java | 4 +- .../deploy/DeployExistingInstance.java | 8 +--- .../clients/teardown/TearDownInstance.java | 4 +- .../sleeper/clients/util/ClientUtils.java | 14 ------ .../properties/local/SaveLocalProperties.java | 16 +++++++ .../java/sleeper/core/util/FilesUtil.java | 47 +++++++++++++++++++ .../sleeper/core/util/FilesUtilTest.java} | 11 ++--- .../instance/AwsSleeperInstanceDriver.java | 8 +--- 8 files changed, 74 insertions(+), 38 deletions(-) create mode 100644 java/core/src/main/java/sleeper/core/util/FilesUtil.java rename java/{clients/src/test/java/sleeper/clients/util/ClientUtilsTest.java => core/src/test/java/sleeper/core/util/FilesUtilTest.java} (91%) diff --git a/java/clients/src/main/java/sleeper/clients/admin/properties/AdminClientPropertiesStore.java b/java/clients/src/main/java/sleeper/clients/admin/properties/AdminClientPropertiesStore.java index 020fdea9a20..4644af7aed3 100644 --- a/java/clients/src/main/java/sleeper/clients/admin/properties/AdminClientPropertiesStore.java +++ b/java/clients/src/main/java/sleeper/clients/admin/properties/AdminClientPropertiesStore.java @@ -23,7 +23,6 @@ import sleeper.clients.deploy.container.DockerImageConfiguration; import sleeper.clients.deploy.container.UploadDockerImagesToEcr; import sleeper.clients.deploy.container.UploadDockerImagesToEcrRequest; -import sleeper.clients.util.ClientUtils; import sleeper.clients.util.cdk.CdkCommand; import sleeper.clients.util.cdk.InvokeCdk; import sleeper.clients.util.console.ConsoleOutput; @@ -38,6 +37,7 @@ import sleeper.core.properties.table.TablePropertiesStore; import sleeper.core.statestore.StateStore; import sleeper.core.table.TableIndex; +import sleeper.core.util.FilesUtil; import sleeper.statestore.StateStoreFactory; import java.io.IOException; @@ -305,7 +305,7 @@ public void saveInstanceProperties(InstanceProperties instanceProperties) { @Override public void saveLocalProperties(InstanceProperties instanceProperties, Stream tablePropertiesStream) throws IOException { Files.createDirectories(localDirectory); - ClientUtils.clearDirectory(localDirectory); + FilesUtil.clearDirectory(localDirectory); SaveLocalProperties.saveToDirectory(localDirectory, instanceProperties, tablePropertiesStream); } diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java index 8a78ba9c7e0..02089801877 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java @@ -26,7 +26,6 @@ import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.sts.StsClient; -import sleeper.clients.util.ClientUtils; import sleeper.clients.util.cdk.CdkCommand; import sleeper.configuration.properties.S3InstanceProperties; import sleeper.configuration.properties.S3TableProperties; @@ -40,7 +39,6 @@ import sleeper.core.util.cli.CommandOption; import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; import java.util.List; import java.util.stream.Collectors; @@ -123,11 +121,7 @@ public record Arguments(Path scriptsDirectory, String instanceId, boolean deploy public void update() throws IOException, InterruptedException { SleeperInstanceConfiguration deployConfig = SleeperInstanceConfiguration.builder().instanceProperties(properties).tableProperties(tablePropertiesList).build(); - Files.createDirectories(configDir); - ClientUtils.clearDirectory(configDir); - SaveLocalProperties.saveToDirectory(configDir, - deployConfig.getInstanceProperties(), - deployConfig.getTableProperties().stream()); + SaveLocalProperties.createDirectoryAndSaveProperties(configDir, deployConfig); deployInstance.deploy(DeployInstanceRequest.builder() .instanceConfig(SleeperInstanceConfiguration.builder().instanceProperties(properties).tableProperties(tablePropertiesList).build()) diff --git a/java/clients/src/main/java/sleeper/clients/teardown/TearDownInstance.java b/java/clients/src/main/java/sleeper/clients/teardown/TearDownInstance.java index a682ae2c770..7b879e8229d 100644 --- a/java/clients/src/main/java/sleeper/clients/teardown/TearDownInstance.java +++ b/java/clients/src/main/java/sleeper/clients/teardown/TearDownInstance.java @@ -19,9 +19,9 @@ import org.slf4j.LoggerFactory; import software.amazon.awssdk.services.cloudformation.CloudFormationClient; -import sleeper.clients.util.ClientUtils; import sleeper.core.properties.instance.InstanceProperties; import sleeper.core.properties.local.LoadLocalProperties; +import sleeper.core.util.FilesUtil; import java.io.IOException; import java.nio.file.Files; @@ -137,7 +137,7 @@ private static void removeGeneratedDir(Path scriptsDir) throws IOException { Path generatedDir = scriptsDir.resolve("generated"); if (Files.isDirectory(generatedDir)) { LOGGER.info("Removing generated files"); - ClientUtils.clearDirectory(generatedDir); + FilesUtil.clearDirectory(generatedDir); } else { LOGGER.info("Generated directory not found"); } diff --git a/java/clients/src/main/java/sleeper/clients/util/ClientUtils.java b/java/clients/src/main/java/sleeper/clients/util/ClientUtils.java index 698184f7aae..027f8bdda5b 100644 --- a/java/clients/src/main/java/sleeper/clients/util/ClientUtils.java +++ b/java/clients/src/main/java/sleeper/clients/util/ClientUtils.java @@ -18,12 +18,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Comparator; import java.util.Optional; -import java.util.stream.Stream; import static sleeper.core.util.NumberFormatUtils.countWithCommas; @@ -59,13 +54,4 @@ public static String abbreviatedRowCount(long rows) { return countWithCommas(Math.round((double) rows / T_COUNT)) + "T (" + countWithCommas(rows) + ")"; } } - - public static void clearDirectory(Path tempDir) throws IOException { - try (Stream paths = Files.walk(tempDir)) { - Stream nestedPaths = paths.skip(1).sorted(Comparator.reverseOrder()); - for (Path path : (Iterable) nestedPaths::iterator) { - Files.delete(path); - } - } - } } diff --git a/java/core/src/main/java/sleeper/core/properties/local/SaveLocalProperties.java b/java/core/src/main/java/sleeper/core/properties/local/SaveLocalProperties.java index 90bf3acb51a..9ccad4e74ea 100644 --- a/java/core/src/main/java/sleeper/core/properties/local/SaveLocalProperties.java +++ b/java/core/src/main/java/sleeper/core/properties/local/SaveLocalProperties.java @@ -16,8 +16,10 @@ package sleeper.core.properties.local; +import sleeper.core.deploy.SleeperInstanceConfiguration; import sleeper.core.properties.instance.InstanceProperties; import sleeper.core.properties.table.TableProperties; +import sleeper.core.util.FilesUtil; import java.io.BufferedWriter; import java.io.IOException; @@ -37,6 +39,20 @@ public class SaveLocalProperties { private SaveLocalProperties() { } + /** + * Creates a given directory and saves instance and table properties to the given directory. + * + * @param directory the directory + * @param config the Sleeper instance configuration that contains instance properties and table properties + * @throws IOException if we could not write to the file system + */ + public static void createDirectoryAndSaveProperties( + Path directory, SleeperInstanceConfiguration config) throws IOException { + Files.createDirectories(directory); + FilesUtil.clearDirectory(directory); + saveToDirectory(directory, config.getInstanceProperties(), config.getTableProperties().stream()); + } + /** * Saves instance and table properties to the given directory. * diff --git a/java/core/src/main/java/sleeper/core/util/FilesUtil.java b/java/core/src/main/java/sleeper/core/util/FilesUtil.java new file mode 100644 index 00000000000..bbf8fa3cba5 --- /dev/null +++ b/java/core/src/main/java/sleeper/core/util/FilesUtil.java @@ -0,0 +1,47 @@ +/* + * Copyright 2022-2026 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package sleeper.core.util; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.stream.Stream; + +/** + * Utils to interact with files and directories. + */ +public class FilesUtil { + + private FilesUtil() { + } + + /** + * Clears the provided directory of files. + * + * @param directory the directory to clear + * @throws IOException if an I/O error occurs + */ + public static void clearDirectory(Path directory) throws IOException { + try (Stream paths = Files.walk(directory)) { + Stream nestedPaths = paths.skip(1).sorted(Comparator.reverseOrder()); + for (Path path : (Iterable) nestedPaths::iterator) { + Files.delete(path); + } + } + } + +} diff --git a/java/clients/src/test/java/sleeper/clients/util/ClientUtilsTest.java b/java/core/src/test/java/sleeper/core/util/FilesUtilTest.java similarity index 91% rename from java/clients/src/test/java/sleeper/clients/util/ClientUtilsTest.java rename to java/core/src/test/java/sleeper/core/util/FilesUtilTest.java index fcba8d71e52..b4b7517e5cb 100644 --- a/java/clients/src/test/java/sleeper/clients/util/ClientUtilsTest.java +++ b/java/core/src/test/java/sleeper/core/util/FilesUtilTest.java @@ -13,8 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -package sleeper.clients.util; +package sleeper.core.util; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Nested; @@ -27,7 +26,7 @@ import static org.assertj.core.api.Assertions.assertThat; -class ClientUtilsTest { +public class FilesUtilTest { @DisplayName("Clear directories") @Nested @@ -42,7 +41,7 @@ void shouldRemoveFileIfExists() throws IOException { Files.createFile(newFile); // When - ClientUtils.clearDirectory(tempDir); + FilesUtil.clearDirectory(tempDir); // Then assertThat(newFile).doesNotExist(); @@ -51,7 +50,7 @@ void shouldRemoveFileIfExists() throws IOException { @Test void shouldNotRemoveRootDirectory() throws IOException { // Given/When - ClientUtils.clearDirectory(tempDir); + FilesUtil.clearDirectory(tempDir); // Then assertThat(tempDir).exists(); @@ -68,7 +67,7 @@ void shouldRemoveMultipleDirectoriesAndFiles() throws IOException { Files.createFile(tempDir.resolve("dir2/nested2/file2")); // When - ClientUtils.clearDirectory(tempDir); + FilesUtil.clearDirectory(tempDir); // Then assertThat(tempDir).isEmptyDirectory(); diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java index 646397e0316..3a2f41b6d09 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java @@ -29,7 +29,6 @@ import sleeper.clients.deploy.DeployInstance; import sleeper.clients.deploy.DeployNewInstance; import sleeper.clients.deploy.DeployNewInstance.StoreFactory; -import sleeper.clients.util.ClientUtils; import sleeper.configuration.properties.S3InstanceProperties; import sleeper.core.deploy.SleeperInstanceConfiguration; import sleeper.core.properties.instance.InstanceProperties; @@ -42,7 +41,6 @@ import java.io.IOException; import java.io.UncheckedIOException; -import java.nio.file.Files; import java.nio.file.Path; import java.util.List; import java.util.Set; @@ -94,11 +92,7 @@ public boolean deployInstanceIfNotPresent(String instanceId, SleeperInstanceConf Path configDir = parameters.getScriptsDirectory().resolve("example"); try { - Files.createDirectories(configDir); - ClientUtils.clearDirectory(configDir); - SaveLocalProperties.saveToDirectory(configDir, - deployConfig.getInstanceProperties(), - deployConfig.getTableProperties().stream()); + SaveLocalProperties.createDirectoryAndSaveProperties(configDir, deployConfig); } catch (IOException e) { throw new UncheckedIOException(e); } From dcb9f88061acda1844115ab7e68a9008a7ee7024 Mon Sep 17 00:00:00 2001 From: ca61688 <206189192+ca61688@users.noreply.github.com> Date: Mon, 20 Jul 2026 14:17:20 +0100 Subject: [PATCH 34/50] 6593: Build SleeperInstanceConfiguration once in DeployExistingInstance --- .../clients/deploy/DeployExistingInstance.java | 4 +--- .../core/properties/local/SaveLocalProperties.java | 13 +++++++------ .../drivers/instance/AwsSleeperInstanceDriver.java | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java index 02089801877..d1a6dfb50ea 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java @@ -119,9 +119,7 @@ public record Arguments(Path scriptsDirectory, String instanceId, boolean deploy } public void update() throws IOException, InterruptedException { - SleeperInstanceConfiguration deployConfig = SleeperInstanceConfiguration.builder().instanceProperties(properties).tableProperties(tablePropertiesList).build(); - - SaveLocalProperties.createDirectoryAndSaveProperties(configDir, deployConfig); + SaveLocalProperties.createDirectoryAndSaveProperties(configDir, properties, tablePropertiesList.stream()); deployInstance.deploy(DeployInstanceRequest.builder() .instanceConfig(SleeperInstanceConfiguration.builder().instanceProperties(properties).tableProperties(tablePropertiesList).build()) diff --git a/java/core/src/main/java/sleeper/core/properties/local/SaveLocalProperties.java b/java/core/src/main/java/sleeper/core/properties/local/SaveLocalProperties.java index 9ccad4e74ea..4d2c0952a93 100644 --- a/java/core/src/main/java/sleeper/core/properties/local/SaveLocalProperties.java +++ b/java/core/src/main/java/sleeper/core/properties/local/SaveLocalProperties.java @@ -16,7 +16,6 @@ package sleeper.core.properties.local; -import sleeper.core.deploy.SleeperInstanceConfiguration; import sleeper.core.properties.instance.InstanceProperties; import sleeper.core.properties.table.TableProperties; import sleeper.core.util.FilesUtil; @@ -42,15 +41,17 @@ private SaveLocalProperties() { /** * Creates a given directory and saves instance and table properties to the given directory. * - * @param directory the directory - * @param config the Sleeper instance configuration that contains instance properties and table properties - * @throws IOException if we could not write to the file system + * @param directory the directory + * @param instanceProperties the instance properties + * @param tablePropertiesStream the table properties + * @throws IOException if we could not write to the file system */ public static void createDirectoryAndSaveProperties( - Path directory, SleeperInstanceConfiguration config) throws IOException { + Path directory, InstanceProperties instanceProperties, + Stream tablePropertiesStream) throws IOException { Files.createDirectories(directory); FilesUtil.clearDirectory(directory); - saveToDirectory(directory, config.getInstanceProperties(), config.getTableProperties().stream()); + saveToDirectory(directory, instanceProperties, tablePropertiesStream); } /** diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java index 3a2f41b6d09..851a399ce9d 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java @@ -92,7 +92,7 @@ public boolean deployInstanceIfNotPresent(String instanceId, SleeperInstanceConf Path configDir = parameters.getScriptsDirectory().resolve("example"); try { - SaveLocalProperties.createDirectoryAndSaveProperties(configDir, deployConfig); + SaveLocalProperties.createDirectoryAndSaveProperties(configDir, deployConfig.getInstanceProperties(), deployConfig.getTableProperties().stream()); } catch (IOException e) { throw new UncheckedIOException(e); } From 158a54e44e169507ceee8e304e6246418945a7a1 Mon Sep 17 00:00:00 2001 From: ca61688 <206189192+ca61688@users.noreply.github.com> Date: Mon, 20 Jul 2026 15:48:58 +0100 Subject: [PATCH 35/50] 6593: Assert on full DeployNewInstanceRequest --- .../clients/deploy/DeployNewInstanceIT.java | 111 +++++++++++++----- 1 file changed, 80 insertions(+), 31 deletions(-) diff --git a/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceIT.java b/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceIT.java index 6d1a6f35675..2e36eedd448 100644 --- a/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceIT.java +++ b/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceIT.java @@ -15,6 +15,7 @@ */ package sleeper.clients.deploy; +import org.assertj.core.api.recursive.comparison.RecursiveComparisonConfiguration; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -39,7 +40,9 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import static java.nio.file.Files.createDirectory; @@ -62,7 +65,7 @@ public class DeployNewInstanceIT { StateStoreProvider stateStoreProvider = InMemoryTransactionLogStateStore.createProvider(instanceProperties, new InMemoryTransactionLogsPerTable()); Map pathToString = new HashMap<>(); - DeployInstanceRequest lastDeployRequest; + List deployRequests = new ArrayList<>(); Path instancePropertiesFile; String configDir; @@ -86,10 +89,13 @@ void setUp() throws IOException { @Nested class DeployNew { + protected static final RecursiveComparisonConfiguration IGNORE_ID = RecursiveComparisonConfiguration.builder() + .withIgnoredFields("instanceConfig.tableProperties").build(); + @Test void shouldDeployNewInstanceWhenUsingInstanceProperties() throws Exception { //When - deployNewInstance("scriptsDir", "someInstance", "someVpc", "someSubnets", "--instance-properties", + deployNewInstanceByPropertiesFile("scriptsDir", "someInstance", "someVpc", "someSubnets", "--instance-properties", instancePropertiesFile.toString()); //Then @@ -97,13 +103,20 @@ void shouldDeployNewInstanceWhenUsingInstanceProperties() throws Exception { instanceProperties.set(ID, "someInstance"); instanceProperties.set(VPC_ID, "someVpc"); instanceProperties.set(SUBNETS, "someSubnets"); - assertThat(instanceProperties) - .isEqualTo(SleeperInstanceConfiguration.fromLocalConfiguration( - instancePropertiesFile).getInstanceProperties()); + SleeperInstanceConfiguration config = SleeperInstanceConfiguration.fromLocalConfiguration(instancePropertiesFile); + assertThat(config.getInstanceProperties()).isEqualTo(instanceProperties); //Verify CDK Command - assertThat(lastDeployRequest.getCdkCommand()).isEqualTo(CdkCommand.deployNew()); - assertThat(lastDeployRequest.getCdkApp()).isEqualTo(SleeperInternalCdkApp.STANDARD); + assertThat(deployRequests.size()).isEqualTo(1); + DeployInstanceRequest lastDeployRequest = deployRequests.get(0); + assertThat(lastDeployRequest).usingRecursiveComparison() + .isEqualTo(DeployInstanceRequest.builder() + .instanceConfig(config) + .cdkCommand(CdkCommand.deployNew()) + .cdkApp(SleeperInternalCdkApp.STANDARD) + .propertiesFile(instancePropertiesFile) + .configDir(null) + .build()); //Verify no table properties stored assertThat(tableIndex.streamAllTables()).isEmpty(); @@ -112,7 +125,7 @@ void shouldDeployNewInstanceWhenUsingInstanceProperties() throws Exception { @Test void shouldDeployNewInstanceWhenUsingConfigDir() throws Exception { //When - deployNewInstance("scriptsDir", "someInstance", "someVpc", "someSubnets", "--config-dir", + deployNewInstanceByConfigDir("scriptsDir", "someInstance", "someVpc", "someSubnets", "--config-dir", configDir); //Then @@ -120,13 +133,15 @@ void shouldDeployNewInstanceWhenUsingConfigDir() throws Exception { instanceProperties.set(ID, "someInstance"); instanceProperties.set(VPC_ID, "someVpc"); instanceProperties.set(SUBNETS, "someSubnets"); - assertThat(instanceProperties) - .isEqualTo(SleeperInstanceConfiguration.fromLocalConfiguration( - instancePropertiesFile).getInstanceProperties()); + SleeperInstanceConfiguration config = SleeperInstanceConfiguration.fromLocalConfigurationDirectory(tempDir); + config.getTableProperties().get(0).set(TABLE_ID, tableId("file-table")); + assertThat(config.getInstanceProperties()).isEqualTo(instanceProperties); //Verify CDK Command - assertThat(lastDeployRequest.getCdkCommand()).isEqualTo(CdkCommand.deployNew()); - assertThat(lastDeployRequest.getCdkApp()).isEqualTo(SleeperInternalCdkApp.STANDARD); + assertThat(deployRequests.size()).isEqualTo(1); + DeployInstanceRequest lastDeployRequest = deployRequests.get(0); + assertThat(lastDeployRequest).usingRecursiveComparison() + .isEqualTo(buildExpectedCDKCommandWithConfigDir(config, false)); //Verify Table properties store saved TableProperties expected = new TableProperties(instanceProperties); @@ -139,7 +154,7 @@ void shouldDeployNewInstanceWhenUsingConfigDir() throws Exception { @Test void shouldDeployNewInstanceWhenUsingInstancePropertiesIgnoringTableFiles() throws Exception { //When - deployNewInstance("scriptsDir", "someInstance", "someVpc", "someSubnets", "--config-dir", configDir, + deployNewInstanceByConfigDir("scriptsDir", "someInstance", "someVpc", "someSubnets", "--config-dir", configDir, "--ignoreTableFiles"); //Then @@ -147,13 +162,14 @@ void shouldDeployNewInstanceWhenUsingInstancePropertiesIgnoringTableFiles() thro instanceProperties.set(ID, "someInstance"); instanceProperties.set(VPC_ID, "someVpc"); instanceProperties.set(SUBNETS, "someSubnets"); - assertThat(instanceProperties) - .isEqualTo(SleeperInstanceConfiguration.fromLocalConfiguration( - instancePropertiesFile).getInstanceProperties()); + SleeperInstanceConfiguration config = SleeperInstanceConfiguration.fromLocalConfiguration(instancePropertiesFile); + assertThat(config.getInstanceProperties()).isEqualTo(instanceProperties); //Verify CDK Command - assertThat(lastDeployRequest.getCdkCommand()).isEqualTo(CdkCommand.deployNew()); - assertThat(lastDeployRequest.getCdkApp()).isEqualTo(SleeperInternalCdkApp.STANDARD); + assertThat(deployRequests.size()).isEqualTo(1); + DeployInstanceRequest lastDeployRequest = deployRequests.get(0); + assertThat(lastDeployRequest).usingRecursiveComparison() + .isEqualTo(buildExpectedCDKCommandWithConfigDir(config, false)); //Verify no table properties stored assertThat(tableIndex.streamAllTables()).isEmpty(); @@ -162,7 +178,7 @@ void shouldDeployNewInstanceWhenUsingInstancePropertiesIgnoringTableFiles() thro @Test void shouldDeployNewInstancePaused() throws Exception { //When - deployNewInstance("scriptsDir", "someInstance", "someVpc", "someSubnets", "--config-dir", configDir, + deployNewInstanceByConfigDir("scriptsDir", "someInstance", "someVpc", "someSubnets", "--config-dir", configDir, "--paused"); //Then @@ -170,13 +186,15 @@ void shouldDeployNewInstancePaused() throws Exception { instanceProperties.set(ID, "someInstance"); instanceProperties.set(VPC_ID, "someVpc"); instanceProperties.set(SUBNETS, "someSubnets"); - assertThat(instanceProperties) - .isEqualTo(SleeperInstanceConfiguration.fromLocalConfiguration( - instancePropertiesFile).getInstanceProperties()); + SleeperInstanceConfiguration config = SleeperInstanceConfiguration.fromLocalConfigurationDirectory(instancePropertiesFile); + config.getTableProperties().get(0).set(TABLE_ID, tableId("file-table")); + assertThat(config.getInstanceProperties()).isEqualTo(instanceProperties); //Verify CDK Command - assertThat(lastDeployRequest.getCdkCommand()).isEqualTo(CdkCommand.deployNewPaused()); - assertThat(lastDeployRequest.getCdkApp()).isEqualTo(SleeperInternalCdkApp.STANDARD); + assertThat(deployRequests.size()).isEqualTo(1); + DeployInstanceRequest lastDeployRequest = deployRequests.get(0); + assertThat(lastDeployRequest).usingRecursiveComparison() + .isEqualTo(buildExpectedCDKCommandWithConfigDir(config, true)); //Verify Table properties store saved TableProperties expected = new TableProperties(instanceProperties); @@ -185,6 +203,16 @@ void shouldDeployNewInstancePaused() throws Exception { expected.set(TABLE_NAME, "file-table"); assertThat(tablePropertiesStore.streamAllTables()).containsExactly(expected); } + + private DeployInstanceRequest buildExpectedCDKCommandWithConfigDir(SleeperInstanceConfiguration config, boolean deployPaused) { + return DeployInstanceRequest.builder() + .instanceConfig(config) + .cdkCommand(deployPaused ? CdkCommand.deployNewPaused() : CdkCommand.deployNew()) + .cdkApp(SleeperInternalCdkApp.STANDARD) + .propertiesFile(null) + .configDir(tempDir) + .build(); + } } @Nested @@ -193,7 +221,7 @@ class ArgumentsValidation { @Test void shouldRejectWhenNotEnoughPositionalArguments() { // When/Then - assertThatThrownBy(() -> deployNewInstance()) + assertThatThrownBy(() -> deployNewInstanceByPropertiesFile()) .isInstanceOf(CommandArgumentsException.class) .hasMessage("Expected 4 positional arguments, found 0"); } @@ -201,7 +229,7 @@ void shouldRejectWhenNotEnoughPositionalArguments() { @Test void shouldRejectWhenNeitherInstancePropertiesOrConfigDirSet() { // When/Then - assertThatThrownBy(() -> deployNewInstance("scriptsDir", "my-instance", "my-vpc", "my-subnets")) + assertThatThrownBy(() -> deployNewInstanceByPropertiesFile("scriptsDir", "my-instance", "my-vpc", "my-subnets")) .isInstanceOf(CommandArgumentsException.class) .hasMessage("Either --instance-properties or --config-dir must be provided"); } @@ -209,7 +237,7 @@ void shouldRejectWhenNeitherInstancePropertiesOrConfigDirSet() { @Test void shouldRejectWhenBothInstancePropertiesAndConfigDirSet() { // When/Then - assertThatThrownBy(() -> deployNewInstance("scriptsDir", "my-instance", "my-vpc", "my-subnets", + assertThatThrownBy(() -> deployNewInstanceByPropertiesFile("scriptsDir", "my-instance", "my-vpc", "my-subnets", "--instance-properties", "someFile", "--config-dir", "someDir")) .isInstanceOf(CommandArgumentsException.class) .hasMessage("Cannot use both --instance-properties and --config-dir"); @@ -231,11 +259,32 @@ void shouldResolvePropertiesFileWhenConfigDirUsed() { } - private void deployNewInstance(String... args) throws Exception { + private void deployNewInstanceByPropertiesFile(String... args) throws Exception { + var arguments = DeployNewInstance.readArguments(CommandArgumentReader.parse(DeployNewInstance.USAGE, args)); + var config = DeployNewInstance.loadAndUpdateConfiguration(arguments); + new DeployNewInstance( + request -> deployRequests.add(request), + new DeployNewInstance.StoreFactory() { + public TablePropertiesStore createTableStore(InstanceProperties p) { + return tablePropertiesStore; + } + + public StateStoreProvider createStateStore(InstanceProperties p) { + return stateStoreProvider; + } + + public void reloadInstanceProperties(InstanceProperties p) { + } + }, + config, SleeperInternalCdkApp.STANDARD, instancePropertiesFile, null, + arguments.ignoreTableFiles(), arguments.deployPaused()).deploy(); + } + + private void deployNewInstanceByConfigDir(String... args) throws Exception { var arguments = DeployNewInstance.readArguments(CommandArgumentReader.parse(DeployNewInstance.USAGE, args)); var config = DeployNewInstance.loadAndUpdateConfiguration(arguments); new DeployNewInstance( - request -> lastDeployRequest = request, + request -> deployRequests.add(request), new DeployNewInstance.StoreFactory() { public TablePropertiesStore createTableStore(InstanceProperties p) { return tablePropertiesStore; @@ -248,7 +297,7 @@ public StateStoreProvider createStateStore(InstanceProperties p) { public void reloadInstanceProperties(InstanceProperties p) { } }, - config, SleeperInternalCdkApp.STANDARD, instancePropertiesFile, tempDir, + config, SleeperInternalCdkApp.STANDARD, null, tempDir, arguments.ignoreTableFiles(), arguments.deployPaused()).deploy(); } From 203c3751c41dbfb694ba8b4dfe0356bda668b647 Mon Sep 17 00:00:00 2001 From: ca61688 <206189192+ca61688@users.noreply.github.com> Date: Mon, 20 Jul 2026 16:44:54 +0100 Subject: [PATCH 36/50] 6593: Add builder for DeployNewInstance class --- .../clients/deploy/DeployNewInstance.java | 99 ++++++++++++++++--- .../clients/deploy/DeployNewInstanceIT.java | 69 +++++++------ .../drivers/cdk/DeployNewTestInstance.java | 12 ++- .../instance/AwsSleeperInstanceDriver.java | 10 +- 4 files changed, 132 insertions(+), 58 deletions(-) diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java index 12d541cb50d..f429c054690 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java @@ -47,6 +47,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.List; +import java.util.Objects; import static sleeper.core.properties.instance.CommonProperty.ID; import static sleeper.core.properties.instance.CommonProperty.SUBNETS; @@ -64,18 +65,19 @@ public class DeployNewInstance { private final boolean ignoreTableFiles; private final boolean deployPaused; - public DeployNewInstance(InstanceDeployer deployInstance, - StoreFactory storeFactory, - SleeperInstanceConfiguration deployInstanceConfiguration, - SleeperInternalCdkApp cdkApp, Path propertiesFile, Path configDir, boolean ignoreTableFiles, boolean deployPaused) { - this.deployInstance = deployInstance; - this.storeFactory = storeFactory; - this.deployInstanceConfiguration = deployInstanceConfiguration; - this.cdkApp = cdkApp; - this.propertiesFile = propertiesFile; - this.configDir = configDir; - this.ignoreTableFiles = ignoreTableFiles; - this.deployPaused = deployPaused; + public static Builder builder() { + return new Builder(); + } + + public DeployNewInstance(Builder builder) { + this.deployInstance = Objects.requireNonNull(builder.deployInstance, "deployInstance must not be null"); + this.storeFactory = Objects.requireNonNull(builder.storeFactory, "storeFactory must not be null"); + this.deployInstanceConfiguration = Objects.requireNonNull(builder.deployInstanceConfiguration, "deployInstanceConfiguration must not be null"); + this.cdkApp = Objects.requireNonNull(builder.cdkApp, "cdkApp must not be null"); + this.propertiesFile = builder.propertiesFile; + this.configDir = builder.configDir; + this.ignoreTableFiles = builder.ignoreTableFiles; + this.deployPaused = builder.deployPaused; } public static final CommandLineUsage USAGE = CommandLineUsage.builder() @@ -143,7 +145,6 @@ public static void main(String[] rawArgs) throws IOException, InterruptedExcepti Arguments args = CommandArguments.parseAndValidateOrExit(USAGE, rawArgs, a -> readArguments(a)); Path scriptsDirectory = Path.of(rawArgs[0]); - boolean deployPaused = args.deployPaused(); try (S3Client s3Client = S3Client.create(); DynamoDbClient dynamoClient = DynamoDbClient.create(); StsClient stsClient = StsClient.create(); @@ -154,9 +155,16 @@ public static void main(String[] rawArgs) throws IOException, InterruptedExcepti SleeperInstanceConfiguration config = loadAndUpdateConfiguration(args); - new DeployNewInstance(DeployInstance.fromScriptsDirectory(scriptsDirectory, accountName, region, partitionMetadata, s3Client, ecrClient), - StoreFactory.withAwsClients(s3Client, dynamoClient, accountName), - config, SleeperInternalCdkApp.STANDARD, args.propertiesFile(), args.configDir(), args.ignoreTableFiles(), deployPaused).deploy(); + DeployNewInstance.builder() + .deployInstance(DeployInstance.fromScriptsDirectory(scriptsDirectory, accountName, region, partitionMetadata, s3Client, ecrClient)) + .storeFactory(StoreFactory.withAwsClients(s3Client, dynamoClient, accountName)) + .deployInstanceConfiguration(config) + .cdkApp(SleeperInternalCdkApp.STANDARD) + .propertiesFile(args.propertiesFile()) + .configDir(args.configDir()) + .ignoreTableFiles(args.ignoreTableFiles()) + .deployPaused(args.deployPaused()) + .build().deploy(); } } @@ -215,6 +223,65 @@ public Path resolvePropertiesFile() { } } + public static final class Builder { + private InstanceDeployer deployInstance; + private StoreFactory storeFactory; + private SleeperInstanceConfiguration deployInstanceConfiguration; + private SleeperInternalCdkApp cdkApp; + private Path propertiesFile; + private Path configDir; + private boolean ignoreTableFiles = false; + private boolean deployPaused = false; + + private Builder() { + + } + + public Builder deployInstance(InstanceDeployer deployInstance) { + this.deployInstance = deployInstance; + return this; + } + + public Builder storeFactory(StoreFactory storeFactory) { + this.storeFactory = storeFactory; + return this; + } + + public Builder deployInstanceConfiguration(SleeperInstanceConfiguration deployInstanceConfiguration) { + this.deployInstanceConfiguration = deployInstanceConfiguration; + return this; + } + + public Builder cdkApp(SleeperInternalCdkApp cdkApp) { + this.cdkApp = cdkApp; + return this; + } + + public Builder propertiesFile(Path propertiesFile) { + this.propertiesFile = propertiesFile; + return this; + } + + public Builder configDir(Path configDir) { + this.configDir = configDir; + return this; + } + + public Builder ignoreTableFiles(boolean ignoreTableFiles) { + this.ignoreTableFiles = ignoreTableFiles; + return this; + } + + public Builder deployPaused(boolean deployPaused) { + this.deployPaused = deployPaused; + return this; + } + + public DeployNewInstance build() { + return new DeployNewInstance(this); + } + } + @FunctionalInterface public interface InstancePropertiesLoader { InstanceProperties load(String instanceId); diff --git a/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceIT.java b/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceIT.java index 2e36eedd448..3e4c28b4ac7 100644 --- a/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceIT.java +++ b/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceIT.java @@ -15,7 +15,6 @@ */ package sleeper.clients.deploy; -import org.assertj.core.api.recursive.comparison.RecursiveComparisonConfiguration; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -41,9 +40,11 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Stream; import static java.nio.file.Files.createDirectory; import static java.nio.file.Files.createTempDirectory; @@ -89,13 +90,10 @@ void setUp() throws IOException { @Nested class DeployNew { - protected static final RecursiveComparisonConfiguration IGNORE_ID = RecursiveComparisonConfiguration.builder() - .withIgnoredFields("instanceConfig.tableProperties").build(); - @Test void shouldDeployNewInstanceWhenUsingInstanceProperties() throws Exception { //When - deployNewInstanceByPropertiesFile("scriptsDir", "someInstance", "someVpc", "someSubnets", "--instance-properties", + deployNewInstanceByPropertiesFile("someInstance", "someVpc", "someSubnets", "--instance-properties", instancePropertiesFile.toString()); //Then @@ -125,7 +123,7 @@ void shouldDeployNewInstanceWhenUsingInstanceProperties() throws Exception { @Test void shouldDeployNewInstanceWhenUsingConfigDir() throws Exception { //When - deployNewInstanceByConfigDir("scriptsDir", "someInstance", "someVpc", "someSubnets", "--config-dir", + deployNewInstanceByConfigDir("someInstance", "someVpc", "someSubnets", "--config-dir", configDir); //Then @@ -154,7 +152,7 @@ void shouldDeployNewInstanceWhenUsingConfigDir() throws Exception { @Test void shouldDeployNewInstanceWhenUsingInstancePropertiesIgnoringTableFiles() throws Exception { //When - deployNewInstanceByConfigDir("scriptsDir", "someInstance", "someVpc", "someSubnets", "--config-dir", configDir, + deployNewInstanceByConfigDir("someInstance", "someVpc", "someSubnets", "--config-dir", configDir, "--ignoreTableFiles"); //Then @@ -178,7 +176,7 @@ void shouldDeployNewInstanceWhenUsingInstancePropertiesIgnoringTableFiles() thro @Test void shouldDeployNewInstancePaused() throws Exception { //When - deployNewInstanceByConfigDir("scriptsDir", "someInstance", "someVpc", "someSubnets", "--config-dir", configDir, + deployNewInstanceByConfigDir("someInstance", "someVpc", "someSubnets", "--config-dir", configDir, "--paused"); //Then @@ -223,13 +221,13 @@ void shouldRejectWhenNotEnoughPositionalArguments() { // When/Then assertThatThrownBy(() -> deployNewInstanceByPropertiesFile()) .isInstanceOf(CommandArgumentsException.class) - .hasMessage("Expected 4 positional arguments, found 0"); + .hasMessage("Expected 4 positional arguments, found 1"); } @Test void shouldRejectWhenNeitherInstancePropertiesOrConfigDirSet() { // When/Then - assertThatThrownBy(() -> deployNewInstanceByPropertiesFile("scriptsDir", "my-instance", "my-vpc", "my-subnets")) + assertThatThrownBy(() -> deployNewInstanceByPropertiesFile("my-instance", "my-vpc", "my-subnets")) .isInstanceOf(CommandArgumentsException.class) .hasMessage("Either --instance-properties or --config-dir must be provided"); } @@ -237,7 +235,7 @@ void shouldRejectWhenNeitherInstancePropertiesOrConfigDirSet() { @Test void shouldRejectWhenBothInstancePropertiesAndConfigDirSet() { // When/Then - assertThatThrownBy(() -> deployNewInstanceByPropertiesFile("scriptsDir", "my-instance", "my-vpc", "my-subnets", + assertThatThrownBy(() -> deployNewInstanceByPropertiesFile("my-instance", "my-vpc", "my-subnets", "--instance-properties", "someFile", "--config-dir", "someDir")) .isInstanceOf(CommandArgumentsException.class) .hasMessage("Cannot use both --instance-properties and --config-dir"); @@ -260,32 +258,21 @@ void shouldResolvePropertiesFileWhenConfigDirUsed() { } private void deployNewInstanceByPropertiesFile(String... args) throws Exception { - var arguments = DeployNewInstance.readArguments(CommandArgumentReader.parse(DeployNewInstance.USAGE, args)); - var config = DeployNewInstance.loadAndUpdateConfiguration(arguments); - new DeployNewInstance( - request -> deployRequests.add(request), - new DeployNewInstance.StoreFactory() { - public TablePropertiesStore createTableStore(InstanceProperties p) { - return tablePropertiesStore; - } - - public StateStoreProvider createStateStore(InstanceProperties p) { - return stateStoreProvider; - } - - public void reloadInstanceProperties(InstanceProperties p) { - } - }, - config, SleeperInternalCdkApp.STANDARD, instancePropertiesFile, null, - arguments.ignoreTableFiles(), arguments.deployPaused()).deploy(); + deployNewInstance(true, args); } private void deployNewInstanceByConfigDir(String... args) throws Exception { - var arguments = DeployNewInstance.readArguments(CommandArgumentReader.parse(DeployNewInstance.USAGE, args)); + deployNewInstance(false, args); + } + + private void deployNewInstance(boolean isByPropFile, String... args) throws Exception { + var arguments = DeployNewInstance.readArguments(CommandArgumentReader.parse(DeployNewInstance.USAGE, + Stream.concat(Stream.of("scriptsDir"), Arrays.stream(args)).toArray(String[]::new))); var config = DeployNewInstance.loadAndUpdateConfiguration(arguments); - new DeployNewInstance( - request -> deployRequests.add(request), - new DeployNewInstance.StoreFactory() { + + DeployNewInstance.Builder builder = DeployNewInstance.builder() + .deployInstance(request -> deployRequests.add(request)) + .storeFactory(new DeployNewInstance.StoreFactory() { public TablePropertiesStore createTableStore(InstanceProperties p) { return tablePropertiesStore; } @@ -296,9 +283,19 @@ public StateStoreProvider createStateStore(InstanceProperties p) { public void reloadInstanceProperties(InstanceProperties p) { } - }, - config, SleeperInternalCdkApp.STANDARD, null, tempDir, - arguments.ignoreTableFiles(), arguments.deployPaused()).deploy(); + }) + .deployInstanceConfiguration(config) + .cdkApp(SleeperInternalCdkApp.STANDARD) + .ignoreTableFiles(arguments.ignoreTableFiles()) + .deployPaused(arguments.deployPaused()); + + if (isByPropFile) { + builder.propertiesFile(instancePropertiesFile); + } else { + builder.configDir(tempDir); + } + + builder.build().deploy(); } private String tableId(String tableName) { diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java index b5658756f86..44acfb298ec 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java @@ -69,9 +69,15 @@ public static void main(String[] args) throws IOException, InterruptedException config.getInstanceProperties().set(ID, instanceId); config.getInstanceProperties().set(VPC_ID, vpcId); config.getInstanceProperties().set(SUBNETS, subnetIds); - new DeployNewInstance(DeployInstance.fromScriptsDirectory(scriptsDirectory, accountName, region, partitionMetadata, s3Client, ecrClient), - StoreFactory.withAwsClients(s3Client, dynamoClient, accountName), - config, SleeperInternalCdkApp.STANDARD, null, configurationPath, false, deployPaused).deploy(); + + DeployNewInstance.builder() + .deployInstance(DeployInstance.fromScriptsDirectory(scriptsDirectory, accountName, region, partitionMetadata, s3Client, ecrClient)) + .storeFactory(StoreFactory.withAwsClients(s3Client, dynamoClient, accountName)) + .deployInstanceConfiguration(config) + .cdkApp(SleeperInternalCdkApp.STANDARD) + .configDir(configurationPath) + .deployPaused(deployPaused) + .build().deploy(); } } diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java index 851a399ce9d..7f553ac0efb 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java @@ -98,9 +98,13 @@ public boolean deployInstanceIfNotPresent(String instanceId, SleeperInstanceConf } try { - new DeployNewInstance(deployInstance, - StoreFactory.withAwsClients(s3, dynamoDB, sts.getCallerIdentity().account()), - deployConfig, SleeperInternalCdkApp.STANDARD, null, configDir, false, false).deploy(); + DeployNewInstance.builder() + .deployInstance(deployInstance) + .storeFactory(StoreFactory.withAwsClients(s3, dynamoDB, sts.getCallerIdentity().account())) + .deployInstanceConfiguration(deployConfig) + .cdkApp(SleeperInternalCdkApp.STANDARD) + .configDir(configDir) + .build().deploy(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RuntimeException(e); From 2c03d432a046471172aebbb13ebcd91b58ce7b12 Mon Sep 17 00:00:00 2001 From: ca61688 <206189192+ca61688@users.noreply.github.com> Date: Mon, 20 Jul 2026 17:02:17 +0100 Subject: [PATCH 37/50] 6593: Remove unused variable --- .../main/java/sleeper/clients/deploy/DeployNewInstance.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java index f429c054690..5fbc6329846 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java @@ -144,7 +144,6 @@ public static Arguments readArguments(CommandArguments arguments) { public static void main(String[] rawArgs) throws IOException, InterruptedException { Arguments args = CommandArguments.parseAndValidateOrExit(USAGE, rawArgs, a -> readArguments(a)); - Path scriptsDirectory = Path.of(rawArgs[0]); try (S3Client s3Client = S3Client.create(); DynamoDbClient dynamoClient = DynamoDbClient.create(); StsClient stsClient = StsClient.create(); @@ -156,7 +155,7 @@ public static void main(String[] rawArgs) throws IOException, InterruptedExcepti SleeperInstanceConfiguration config = loadAndUpdateConfiguration(args); DeployNewInstance.builder() - .deployInstance(DeployInstance.fromScriptsDirectory(scriptsDirectory, accountName, region, partitionMetadata, s3Client, ecrClient)) + .deployInstance(DeployInstance.fromScriptsDirectory(args.scriptsDirectory(), accountName, region, partitionMetadata, s3Client, ecrClient)) .storeFactory(StoreFactory.withAwsClients(s3Client, dynamoClient, accountName)) .deployInstanceConfiguration(config) .cdkApp(SleeperInternalCdkApp.STANDARD) From 26c7fa374c14fbf9f2ae136c927189cbc45e5df4 Mon Sep 17 00:00:00 2001 From: ca61688 <206189192+ca61688@users.noreply.github.com> Date: Mon, 20 Jul 2026 17:07:58 +0100 Subject: [PATCH 38/50] 6593: Seperate InstancePropertiesLoader from FileReader --- .../sleeper/clients/table/AddTableClient.java | 3 ++- .../java/sleeper/clients/util/FileReader.java | 6 ----- .../util/InstancePropertiesLoader.java | 22 +++++++++++++++++++ 3 files changed, 24 insertions(+), 7 deletions(-) create mode 100644 java/clients/src/main/java/sleeper/clients/util/InstancePropertiesLoader.java diff --git a/java/clients/src/main/java/sleeper/clients/table/AddTableClient.java b/java/clients/src/main/java/sleeper/clients/table/AddTableClient.java index af2ba8e7ece..4f778c53d9c 100644 --- a/java/clients/src/main/java/sleeper/clients/table/AddTableClient.java +++ b/java/clients/src/main/java/sleeper/clients/table/AddTableClient.java @@ -22,6 +22,7 @@ import software.amazon.awssdk.services.sts.StsClient; import sleeper.clients.util.FileReader; +import sleeper.clients.util.InstancePropertiesLoader; import sleeper.configuration.properties.S3InstanceProperties; import sleeper.configuration.properties.S3TableProperties; import sleeper.core.properties.PropertiesUtils; @@ -132,7 +133,7 @@ public static void main(String[] rawArgs) throws IOException { } } - public static TableProperties createTablePropertiesWithLoaders(Arguments args, FileReader.InstancePropertiesLoader instance, FileReader files) { + public static TableProperties createTablePropertiesWithLoaders(Arguments args, InstancePropertiesLoader instance, FileReader files) { TableProperties tableProperties = createTableProperties(instance.load(args.instanceId()), args); tableProperties.setSchema(new SchemaSerDe().fromJson(FileReader.readFile(files, args.resolveSchemaFile()))); return tableProperties; diff --git a/java/clients/src/main/java/sleeper/clients/util/FileReader.java b/java/clients/src/main/java/sleeper/clients/util/FileReader.java index 666cf726ad2..92cf7ceea90 100644 --- a/java/clients/src/main/java/sleeper/clients/util/FileReader.java +++ b/java/clients/src/main/java/sleeper/clients/util/FileReader.java @@ -15,8 +15,6 @@ */ package sleeper.clients.util; -import sleeper.core.properties.instance.InstanceProperties; - import java.io.IOException; import java.io.UncheckedIOException; import java.nio.file.Path; @@ -31,8 +29,4 @@ static String readFile(FileReader reader, Path path) { throw new UncheckedIOException(e); } } - - interface InstancePropertiesLoader { - InstanceProperties load(String instanceId); - } } diff --git a/java/clients/src/main/java/sleeper/clients/util/InstancePropertiesLoader.java b/java/clients/src/main/java/sleeper/clients/util/InstancePropertiesLoader.java new file mode 100644 index 00000000000..2f61f5afc20 --- /dev/null +++ b/java/clients/src/main/java/sleeper/clients/util/InstancePropertiesLoader.java @@ -0,0 +1,22 @@ +/* + * Copyright 2022-2026 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package sleeper.clients.util; + +import sleeper.core.properties.instance.InstanceProperties; + +public interface InstancePropertiesLoader { + InstanceProperties load(String instanceId); +} From 5e46c4030b0edc97d21383e5b79288185cb9ba03 Mon Sep 17 00:00:00 2001 From: ca61688 <206189192+ca61688@users.noreply.github.com> Date: Mon, 20 Jul 2026 17:29:29 +0100 Subject: [PATCH 39/50] 6593: Make FileReader readFile default method --- .../src/main/java/sleeper/clients/table/AddTableClient.java | 6 +++--- .../src/main/java/sleeper/clients/util/FileReader.java | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/java/clients/src/main/java/sleeper/clients/table/AddTableClient.java b/java/clients/src/main/java/sleeper/clients/table/AddTableClient.java index 4f778c53d9c..e8b9db54f77 100644 --- a/java/clients/src/main/java/sleeper/clients/table/AddTableClient.java +++ b/java/clients/src/main/java/sleeper/clients/table/AddTableClient.java @@ -99,9 +99,9 @@ public static Arguments readArguments(CommandArguments arguments, FileReader fil Properties rawTableProperties; rawTableProperties = tablePropertiesFile.isPresent() - ? PropertiesUtils.loadProperties(FileReader.readFile(files, tablePropertiesFile.get())) + ? PropertiesUtils.loadProperties(files.readFile(tablePropertiesFile.get())) : configDir.isPresent() - ? PropertiesUtils.loadProperties(FileReader.readFile(files, configDir.get().resolve("table.properties"))) + ? PropertiesUtils.loadProperties(files.readFile(configDir.get().resolve("table.properties"))) : null; return new Arguments( @@ -135,7 +135,7 @@ public static void main(String[] rawArgs) throws IOException { public static TableProperties createTablePropertiesWithLoaders(Arguments args, InstancePropertiesLoader instance, FileReader files) { TableProperties tableProperties = createTableProperties(instance.load(args.instanceId()), args); - tableProperties.setSchema(new SchemaSerDe().fromJson(FileReader.readFile(files, args.resolveSchemaFile()))); + tableProperties.setSchema(new SchemaSerDe().fromJson(files.readFile(args.resolveSchemaFile()))); return tableProperties; } diff --git a/java/clients/src/main/java/sleeper/clients/util/FileReader.java b/java/clients/src/main/java/sleeper/clients/util/FileReader.java index 92cf7ceea90..96088fefa7a 100644 --- a/java/clients/src/main/java/sleeper/clients/util/FileReader.java +++ b/java/clients/src/main/java/sleeper/clients/util/FileReader.java @@ -22,9 +22,9 @@ public interface FileReader { String readStringChecked(Path path) throws IOException; - static String readFile(FileReader reader, Path path) { + default String readFile(Path path) { try { - return reader.readStringChecked(path); + return readStringChecked(path); } catch (IOException e) { throw new UncheckedIOException(e); } From 640411d5b5e6716ec7ed355d8865c08380f87b2e Mon Sep 17 00:00:00 2001 From: ca61688 <206189192+ca61688@users.noreply.github.com> Date: Wed, 22 Jul 2026 15:12:16 +0100 Subject: [PATCH 40/50] 6593: Remove id,vpcId and subnet from DeployNewInstanceIT instance properties --- .../sleeper/clients/deploy/DeployNewInstanceIT.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceIT.java b/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceIT.java index 3e4c28b4ac7..00bd0cdc332 100644 --- a/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceIT.java +++ b/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceIT.java @@ -55,11 +55,11 @@ import static sleeper.core.properties.instance.CommonProperty.VPC_ID; import static sleeper.core.properties.table.TableProperty.TABLE_ID; import static sleeper.core.properties.table.TableProperty.TABLE_NAME; -import static sleeper.core.properties.testutils.InstancePropertiesTestHelper.createTestInstancePropertiesWithId; +import static sleeper.core.properties.testutils.InstancePropertiesTestHelper.createTestInstanceProperties; import static sleeper.core.schema.SchemaTestHelper.createSchemaWithKey; public class DeployNewInstanceIT { - InstanceProperties instanceProperties = createTestInstancePropertiesWithId("my-instance"); + InstanceProperties instanceProperties = generateInstancePropertiesForFile(); Schema schema = createSchemaWithKey("key"); InMemoryTableIndex tableIndex = new InMemoryTableIndex(); TablePropertiesStore tablePropertiesStore = InMemoryTableProperties.getStore(tableIndex); @@ -303,4 +303,12 @@ private String tableId(String tableName) { .orElseThrow(() -> new RuntimeException("Found tables: " + tableIndex.streamAllTables().toList())) .getTableUniqueId(); } + + private static InstanceProperties generateInstancePropertiesForFile() { + InstanceProperties instanceProperties = createTestInstanceProperties(); + instanceProperties.unset(ID); + instanceProperties.unset(VPC_ID); + instanceProperties.unset(SUBNETS); + return instanceProperties; + } } From 51e50ccf3f27b7bc7b247f313785c8538f691fc9 Mon Sep 17 00:00:00 2001 From: ca61688 <206189192+ca61688@users.noreply.github.com> Date: Wed, 22 Jul 2026 16:17:54 +0100 Subject: [PATCH 41/50] 6593: Remove propertiesFile and configDir from DeployInstanceRequest --- .../deploy/DeployExistingInstance.java | 5 +- .../clients/deploy/DeployInstance.java | 6 +-- .../clients/deploy/DeployInstanceRequest.java | 28 ----------- .../clients/deploy/DeployNewInstance.java | 13 ++++-- .../clients/deploy/DeployNewInstanceIT.java | 46 ++++++++++--------- 5 files changed, 36 insertions(+), 62 deletions(-) diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java index d1a6dfb50ea..4fd0feb2cef 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java @@ -120,12 +120,11 @@ public record Arguments(Path scriptsDirectory, String instanceId, boolean deploy public void update() throws IOException, InterruptedException { SaveLocalProperties.createDirectoryAndSaveProperties(configDir, properties, tablePropertiesList.stream()); - + CdkCommand cdkCommand = deployPaused ? CdkCommand.deployExistingPaused() : CdkCommand.deployExisting(); deployInstance.deploy(DeployInstanceRequest.builder() .instanceConfig(SleeperInstanceConfiguration.builder().instanceProperties(properties).tableProperties(tablePropertiesList).build()) - .cdkCommand(deployPaused ? CdkCommand.deployExistingPaused() : CdkCommand.deployExisting()) + .cdkCommand(cdkCommand.withConfigurationDirectory(configDir)) .cdkApp(getCdkApp()) - .configDir(configDir) .build()); LOGGER.info("Finished deployment of existing instance"); diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployInstance.java index 3147e36f677..6af355605a8 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployInstance.java @@ -84,10 +84,6 @@ public void deploy(DeployInstanceRequest request) throws IOException, Interrupte LOGGER.info("-------------------------------------------------------"); LOGGER.info("Deploying Stacks"); LOGGER.info("-------------------------------------------------------"); - if (request.getPropertiesFile() != null) { - invokeCdk.invoke(request.getCdkApp(), request.getCdkCommand().withPropertiesFile(request.getPropertiesFile())); - } else { - invokeCdk.invoke(request.getCdkApp(), request.getCdkCommand().withConfigurationDirectory(request.getConfigDir())); - } + invokeCdk.invoke(request.getCdkApp(), request.getCdkCommand()); } } diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployInstanceRequest.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployInstanceRequest.java index fccb6d21aaa..2ab88c3cb9c 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployInstanceRequest.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployInstanceRequest.java @@ -19,7 +19,6 @@ import sleeper.core.deploy.SleeperInstanceConfiguration; import sleeper.core.properties.model.SleeperInternalCdkApp; -import java.nio.file.Path; import java.util.Objects; public class DeployInstanceRequest { @@ -27,18 +26,11 @@ public class DeployInstanceRequest { private final SleeperInstanceConfiguration instanceConfig; private final CdkCommand cdkCommand; private final SleeperInternalCdkApp cdkApp; - private final Path propertiesFile; - private final Path configDir; private DeployInstanceRequest(Builder builder) { instanceConfig = Objects.requireNonNull(builder.instanceConfig, "instanceConfig must not be null"); cdkCommand = Objects.requireNonNull(builder.cdkCommand, "cdkCommand must not be null"); cdkApp = Objects.requireNonNull(builder.cdkApp, "cdkApp must not be null"); - propertiesFile = builder.propertiesFile; - configDir = builder.configDir; - if (propertiesFile == null && configDir == null) { - throw new NullPointerException("One of propertiesFile and configDir must not be null"); - } } public static Builder builder() { @@ -57,20 +49,10 @@ public CdkCommand getCdkCommand() { return cdkCommand; } - public Path getPropertiesFile() { - return propertiesFile; - } - - public Path getConfigDir() { - return configDir; - } - public static class Builder { private SleeperInstanceConfiguration instanceConfig; private CdkCommand cdkCommand; private SleeperInternalCdkApp cdkApp; - private Path propertiesFile; - private Path configDir; public Builder instanceConfig(SleeperInstanceConfiguration instanceConfig) { this.instanceConfig = instanceConfig; @@ -87,16 +69,6 @@ public Builder cdkApp(SleeperInternalCdkApp cdkApp) { return this; } - public Builder propertiesFile(Path propertiesFile) { - this.propertiesFile = propertiesFile; - return this; - } - - public Builder configDir(Path configDir) { - this.configDir = configDir; - return this; - } - public DeployInstanceRequest build() { return new DeployInstanceRequest(this); } diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java index 5fbc6329846..3fa05a46127 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java @@ -159,7 +159,7 @@ public static void main(String[] rawArgs) throws IOException, InterruptedExcepti .storeFactory(StoreFactory.withAwsClients(s3Client, dynamoClient, accountName)) .deployInstanceConfiguration(config) .cdkApp(SleeperInternalCdkApp.STANDARD) - .propertiesFile(args.propertiesFile()) + .propertiesFile(args.resolvePropertiesFile()) .configDir(args.configDir()) .ignoreTableFiles(args.ignoreTableFiles()) .deployPaused(args.deployPaused()) @@ -170,12 +170,17 @@ public static void main(String[] rawArgs) throws IOException, InterruptedExcepti public void deploy() throws IOException, InterruptedException { deployInstanceConfiguration.validate(); + CdkCommand cdkCommand = deployPaused ? CdkCommand.deployNewPaused() : CdkCommand.deployNew(); + if (ignoreTableFiles) { + cdkCommand = cdkCommand.withPropertiesFile(propertiesFile); + } else { + cdkCommand = cdkCommand.withConfigurationDirectory(configDir); + } + deployInstance.deploy(DeployInstanceRequest.builder() .instanceConfig(deployInstanceConfiguration) - .cdkCommand(deployPaused ? CdkCommand.deployNewPaused() : CdkCommand.deployNew()) + .cdkCommand(cdkCommand) .cdkApp(cdkApp) - .propertiesFile(propertiesFile) - .configDir(configDir) .build()); if (!ignoreTableFiles) { diff --git a/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceIT.java b/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceIT.java index 00bd0cdc332..16c7a7fd5c1 100644 --- a/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceIT.java +++ b/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceIT.java @@ -93,7 +93,7 @@ class DeployNew { @Test void shouldDeployNewInstanceWhenUsingInstanceProperties() throws Exception { //When - deployNewInstanceByPropertiesFile("someInstance", "someVpc", "someSubnets", "--instance-properties", + deployNewInstanceWithoutTables("someInstance", "someVpc", "someSubnets", "--instance-properties", instancePropertiesFile.toString()); //Then @@ -108,13 +108,7 @@ void shouldDeployNewInstanceWhenUsingInstanceProperties() throws Exception { assertThat(deployRequests.size()).isEqualTo(1); DeployInstanceRequest lastDeployRequest = deployRequests.get(0); assertThat(lastDeployRequest).usingRecursiveComparison() - .isEqualTo(DeployInstanceRequest.builder() - .instanceConfig(config) - .cdkCommand(CdkCommand.deployNew()) - .cdkApp(SleeperInternalCdkApp.STANDARD) - .propertiesFile(instancePropertiesFile) - .configDir(null) - .build()); + .isEqualTo(buildExpectedCDKCommandWithPropertyFile(config, false)); //Verify no table properties stored assertThat(tableIndex.streamAllTables()).isEmpty(); @@ -123,7 +117,7 @@ void shouldDeployNewInstanceWhenUsingInstanceProperties() throws Exception { @Test void shouldDeployNewInstanceWhenUsingConfigDir() throws Exception { //When - deployNewInstanceByConfigDir("someInstance", "someVpc", "someSubnets", "--config-dir", + deployNewInstanceWithTables("someInstance", "someVpc", "someSubnets", "--config-dir", configDir); //Then @@ -152,7 +146,7 @@ void shouldDeployNewInstanceWhenUsingConfigDir() throws Exception { @Test void shouldDeployNewInstanceWhenUsingInstancePropertiesIgnoringTableFiles() throws Exception { //When - deployNewInstanceByConfigDir("someInstance", "someVpc", "someSubnets", "--config-dir", configDir, + deployNewInstanceWithoutTables("someInstance", "someVpc", "someSubnets", "--config-dir", configDir, "--ignoreTableFiles"); //Then @@ -167,7 +161,7 @@ void shouldDeployNewInstanceWhenUsingInstancePropertiesIgnoringTableFiles() thro assertThat(deployRequests.size()).isEqualTo(1); DeployInstanceRequest lastDeployRequest = deployRequests.get(0); assertThat(lastDeployRequest).usingRecursiveComparison() - .isEqualTo(buildExpectedCDKCommandWithConfigDir(config, false)); + .isEqualTo(buildExpectedCDKCommandWithPropertyFile(config, false)); //Verify no table properties stored assertThat(tableIndex.streamAllTables()).isEmpty(); @@ -176,7 +170,7 @@ void shouldDeployNewInstanceWhenUsingInstancePropertiesIgnoringTableFiles() thro @Test void shouldDeployNewInstancePaused() throws Exception { //When - deployNewInstanceByConfigDir("someInstance", "someVpc", "someSubnets", "--config-dir", configDir, + deployNewInstanceWithTables("someInstance", "someVpc", "someSubnets", "--config-dir", configDir, "--paused"); //Then @@ -202,13 +196,21 @@ void shouldDeployNewInstancePaused() throws Exception { assertThat(tablePropertiesStore.streamAllTables()).containsExactly(expected); } + private DeployInstanceRequest buildExpectedCDKCommandWithPropertyFile(SleeperInstanceConfiguration config, boolean deployPaused) { + CdkCommand cdkCommand = deployPaused ? CdkCommand.deployNewPaused() : CdkCommand.deployNew(); + return DeployInstanceRequest.builder() + .instanceConfig(config) + .cdkCommand(cdkCommand.withPropertiesFile(instancePropertiesFile)) + .cdkApp(SleeperInternalCdkApp.STANDARD) + .build(); + } + private DeployInstanceRequest buildExpectedCDKCommandWithConfigDir(SleeperInstanceConfiguration config, boolean deployPaused) { + CdkCommand cdkCommand = deployPaused ? CdkCommand.deployNewPaused() : CdkCommand.deployNew(); return DeployInstanceRequest.builder() .instanceConfig(config) - .cdkCommand(deployPaused ? CdkCommand.deployNewPaused() : CdkCommand.deployNew()) + .cdkCommand(cdkCommand.withConfigurationDirectory(tempDir)) .cdkApp(SleeperInternalCdkApp.STANDARD) - .propertiesFile(null) - .configDir(tempDir) .build(); } } @@ -219,7 +221,7 @@ class ArgumentsValidation { @Test void shouldRejectWhenNotEnoughPositionalArguments() { // When/Then - assertThatThrownBy(() -> deployNewInstanceByPropertiesFile()) + assertThatThrownBy(() -> deployNewInstanceWithoutTables()) .isInstanceOf(CommandArgumentsException.class) .hasMessage("Expected 4 positional arguments, found 1"); } @@ -227,7 +229,7 @@ void shouldRejectWhenNotEnoughPositionalArguments() { @Test void shouldRejectWhenNeitherInstancePropertiesOrConfigDirSet() { // When/Then - assertThatThrownBy(() -> deployNewInstanceByPropertiesFile("my-instance", "my-vpc", "my-subnets")) + assertThatThrownBy(() -> deployNewInstanceWithoutTables("my-instance", "my-vpc", "my-subnets")) .isInstanceOf(CommandArgumentsException.class) .hasMessage("Either --instance-properties or --config-dir must be provided"); } @@ -235,7 +237,7 @@ void shouldRejectWhenNeitherInstancePropertiesOrConfigDirSet() { @Test void shouldRejectWhenBothInstancePropertiesAndConfigDirSet() { // When/Then - assertThatThrownBy(() -> deployNewInstanceByPropertiesFile("my-instance", "my-vpc", "my-subnets", + assertThatThrownBy(() -> deployNewInstanceWithoutTables("my-instance", "my-vpc", "my-subnets", "--instance-properties", "someFile", "--config-dir", "someDir")) .isInstanceOf(CommandArgumentsException.class) .hasMessage("Cannot use both --instance-properties and --config-dir"); @@ -257,15 +259,15 @@ void shouldResolvePropertiesFileWhenConfigDirUsed() { } - private void deployNewInstanceByPropertiesFile(String... args) throws Exception { + private void deployNewInstanceWithoutTables(String... args) throws Exception { deployNewInstance(true, args); } - private void deployNewInstanceByConfigDir(String... args) throws Exception { + private void deployNewInstanceWithTables(String... args) throws Exception { deployNewInstance(false, args); } - private void deployNewInstance(boolean isByPropFile, String... args) throws Exception { + private void deployNewInstance(boolean isWithTables, String... args) throws Exception { var arguments = DeployNewInstance.readArguments(CommandArgumentReader.parse(DeployNewInstance.USAGE, Stream.concat(Stream.of("scriptsDir"), Arrays.stream(args)).toArray(String[]::new))); var config = DeployNewInstance.loadAndUpdateConfiguration(arguments); @@ -289,7 +291,7 @@ public void reloadInstanceProperties(InstanceProperties p) { .ignoreTableFiles(arguments.ignoreTableFiles()) .deployPaused(arguments.deployPaused()); - if (isByPropFile) { + if (isWithTables) { builder.propertiesFile(instancePropertiesFile); } else { builder.configDir(tempDir); From 5110eb6ae8428882968e0695152302cb66e7d9bf Mon Sep 17 00:00:00 2001 From: ca61688 <206189192+ca61688@users.noreply.github.com> Date: Thu, 23 Jul 2026 10:53:08 +0100 Subject: [PATCH 42/50] 6593: Pass network parameters to CDK cml instead of resaving congif file --- .../clients/deploy/DeployNewInstance.java | 17 +++--- .../sleeper/clients/util/cdk/CdkCommand.java | 16 ++++++ .../clients/deploy/DeployNewInstanceIT.java | 52 +++++++++---------- 3 files changed, 47 insertions(+), 38 deletions(-) diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java index 3fa05a46127..27f5c76d890 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java @@ -41,10 +41,7 @@ import sleeper.core.util.cli.CommandOption; import sleeper.statestore.StateStoreFactory; -import java.io.BufferedWriter; import java.io.IOException; -import java.io.PrintWriter; -import java.nio.file.Files; import java.nio.file.Path; import java.util.List; import java.util.Objects; @@ -109,7 +106,7 @@ public DeployNewInstance(Builder builder) { "the instance is manually resumed.") .build(); - public static SleeperInstanceConfiguration loadAndUpdateConfiguration(Arguments args) throws IOException { + public static SleeperInstanceConfiguration loadConfiguration(Arguments args) throws IOException { SleeperInstanceConfiguration config; if (args.ignoreTableFiles()) { config = SleeperInstanceConfiguration.fromLocalConfiguration(args.resolvePropertiesFile()); @@ -121,11 +118,6 @@ public static SleeperInstanceConfiguration loadAndUpdateConfiguration(Arguments config.getInstanceProperties().set(VPC_ID, args.vpcId()); config.getInstanceProperties().set(SUBNETS, args.subnetIds()); - try (BufferedWriter writer = Files.newBufferedWriter(args.resolvePropertiesFile())) { - InstanceProperties.createPrettyPrinter(new PrintWriter(writer)) - .print(config.getInstanceProperties()); - } - return config; } @@ -152,7 +144,7 @@ public static void main(String[] rawArgs) throws IOException, InterruptedExcepti Region region = DefaultAwsRegionProviderChain.builder().build().getRegion(); PartitionMetadata partitionMetadata = PartitionMetadata.of(region); - SleeperInstanceConfiguration config = loadAndUpdateConfiguration(args); + SleeperInstanceConfiguration config = loadConfiguration(args); DeployNewInstance.builder() .deployInstance(DeployInstance.fromScriptsDirectory(args.scriptsDirectory(), accountName, region, partitionMetadata, s3Client, ecrClient)) @@ -171,6 +163,10 @@ public void deploy() throws IOException, InterruptedException { deployInstanceConfiguration.validate(); CdkCommand cdkCommand = deployPaused ? CdkCommand.deployNewPaused() : CdkCommand.deployNew(); + + InstanceProperties instanceProperties = deployInstanceConfiguration.getInstanceProperties(); + cdkCommand = cdkCommand.withNetworkConfiguration(instanceProperties.get(ID), instanceProperties.get(VPC_ID), instanceProperties.get(SUBNETS)); + if (ignoreTableFiles) { cdkCommand = cdkCommand.withPropertiesFile(propertiesFile); } else { @@ -184,7 +180,6 @@ public void deploy() throws IOException, InterruptedException { .build()); if (!ignoreTableFiles) { - InstanceProperties instanceProperties = deployInstanceConfiguration.getInstanceProperties(); storeFactory.reloadInstanceProperties(instanceProperties); for (TableProperties tableProperties : deployInstanceConfiguration.getTableProperties()) { diff --git a/java/clients/src/main/java/sleeper/clients/util/cdk/CdkCommand.java b/java/clients/src/main/java/sleeper/clients/util/cdk/CdkCommand.java index 38ef155dc39..c99c897e68d 100644 --- a/java/clients/src/main/java/sleeper/clients/util/cdk/CdkCommand.java +++ b/java/clients/src/main/java/sleeper/clients/util/cdk/CdkCommand.java @@ -67,6 +67,10 @@ public CdkCommand withConfigurationDirectory(Path configurationDirectory) { return builder().command(command).configurationDirectory(configurationDirectory).arguments(arguments).build(); } + public CdkCommand withNetworkConfiguration(String instanceId, String vpcId, String subnets) { + return builder().command(command).arguments(arguments).instanceId(instanceId).vpcId(vpcId).subnets(subnets).build(); + } + public static final class Builder { private List command; private List arguments = new ArrayList<>(); @@ -93,6 +97,18 @@ public Builder arguments(List arguments) { return this; } + public Builder instanceId(String id) { + return context("id", id); + } + + public Builder vpcId(String vpcId) { + return context("vpc", vpcId); + } + + public Builder subnets(String subnets) { + return context("subnets", subnets); + } + public Builder propertiesFile(Path propertiesFile) { return context("propertiesFile", propertiesFile.toString()); } diff --git a/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceIT.java b/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceIT.java index 16c7a7fd5c1..b9dabeacaa4 100644 --- a/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceIT.java +++ b/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceIT.java @@ -69,6 +69,9 @@ public class DeployNewInstanceIT { List deployRequests = new ArrayList<>(); Path instancePropertiesFile; String configDir; + String instanceId = "someInstance"; + String vpcId = "someVpc"; + String subnets = "someSubnet1,someSubnet2"; @TempDir private Path tempDir; @@ -93,16 +96,12 @@ class DeployNew { @Test void shouldDeployNewInstanceWhenUsingInstanceProperties() throws Exception { //When - deployNewInstanceWithoutTables("someInstance", "someVpc", "someSubnets", "--instance-properties", + deployNewInstanceWithoutTables(instanceId, vpcId, subnets, "--instance-properties", instancePropertiesFile.toString()); //Then - //Verify Instance Properties file updates - instanceProperties.set(ID, "someInstance"); - instanceProperties.set(VPC_ID, "someVpc"); - instanceProperties.set(SUBNETS, "someSubnets"); SleeperInstanceConfiguration config = SleeperInstanceConfiguration.fromLocalConfiguration(instancePropertiesFile); - assertThat(config.getInstanceProperties()).isEqualTo(instanceProperties); + updatePropertyFiles(config); //Verify CDK Command assertThat(deployRequests.size()).isEqualTo(1); @@ -117,17 +116,13 @@ void shouldDeployNewInstanceWhenUsingInstanceProperties() throws Exception { @Test void shouldDeployNewInstanceWhenUsingConfigDir() throws Exception { //When - deployNewInstanceWithTables("someInstance", "someVpc", "someSubnets", "--config-dir", + deployNewInstanceWithTables(instanceId, vpcId, subnets, "--config-dir", configDir); //Then - //Verify Instance Properties file updates - instanceProperties.set(ID, "someInstance"); - instanceProperties.set(VPC_ID, "someVpc"); - instanceProperties.set(SUBNETS, "someSubnets"); SleeperInstanceConfiguration config = SleeperInstanceConfiguration.fromLocalConfigurationDirectory(tempDir); + updatePropertyFiles(config); config.getTableProperties().get(0).set(TABLE_ID, tableId("file-table")); - assertThat(config.getInstanceProperties()).isEqualTo(instanceProperties); //Verify CDK Command assertThat(deployRequests.size()).isEqualTo(1); @@ -146,16 +141,12 @@ void shouldDeployNewInstanceWhenUsingConfigDir() throws Exception { @Test void shouldDeployNewInstanceWhenUsingInstancePropertiesIgnoringTableFiles() throws Exception { //When - deployNewInstanceWithoutTables("someInstance", "someVpc", "someSubnets", "--config-dir", configDir, + deployNewInstanceWithoutTables(instanceId, vpcId, subnets, "--config-dir", configDir, "--ignoreTableFiles"); //Then - //Verify Instance Properties file updates - instanceProperties.set(ID, "someInstance"); - instanceProperties.set(VPC_ID, "someVpc"); - instanceProperties.set(SUBNETS, "someSubnets"); SleeperInstanceConfiguration config = SleeperInstanceConfiguration.fromLocalConfiguration(instancePropertiesFile); - assertThat(config.getInstanceProperties()).isEqualTo(instanceProperties); + updatePropertyFiles(config); //Verify CDK Command assertThat(deployRequests.size()).isEqualTo(1); @@ -170,17 +161,13 @@ void shouldDeployNewInstanceWhenUsingInstancePropertiesIgnoringTableFiles() thro @Test void shouldDeployNewInstancePaused() throws Exception { //When - deployNewInstanceWithTables("someInstance", "someVpc", "someSubnets", "--config-dir", configDir, + deployNewInstanceWithTables(instanceId, vpcId, subnets, "--config-dir", configDir, "--paused"); //Then - //Verify Instance Properties file updates - instanceProperties.set(ID, "someInstance"); - instanceProperties.set(VPC_ID, "someVpc"); - instanceProperties.set(SUBNETS, "someSubnets"); SleeperInstanceConfiguration config = SleeperInstanceConfiguration.fromLocalConfigurationDirectory(instancePropertiesFile); + updatePropertyFiles(config); config.getTableProperties().get(0).set(TABLE_ID, tableId("file-table")); - assertThat(config.getInstanceProperties()).isEqualTo(instanceProperties); //Verify CDK Command assertThat(deployRequests.size()).isEqualTo(1); @@ -196,11 +183,21 @@ void shouldDeployNewInstancePaused() throws Exception { assertThat(tablePropertiesStore.streamAllTables()).containsExactly(expected); } + private void updatePropertyFiles(SleeperInstanceConfiguration config) { + instanceProperties.set(ID, instanceId); + instanceProperties.set(VPC_ID, vpcId); + instanceProperties.set(SUBNETS, subnets); + config.getInstanceProperties().set(ID, instanceId); + config.getInstanceProperties().set(VPC_ID, vpcId); + config.getInstanceProperties().set(SUBNETS, subnets); + } + private DeployInstanceRequest buildExpectedCDKCommandWithPropertyFile(SleeperInstanceConfiguration config, boolean deployPaused) { CdkCommand cdkCommand = deployPaused ? CdkCommand.deployNewPaused() : CdkCommand.deployNew(); return DeployInstanceRequest.builder() .instanceConfig(config) - .cdkCommand(cdkCommand.withPropertiesFile(instancePropertiesFile)) + .cdkCommand(cdkCommand.withPropertiesFile(instancePropertiesFile) + .withNetworkConfiguration(instanceId, vpcId, subnets)) .cdkApp(SleeperInternalCdkApp.STANDARD) .build(); } @@ -209,7 +206,8 @@ private DeployInstanceRequest buildExpectedCDKCommandWithConfigDir(SleeperInstan CdkCommand cdkCommand = deployPaused ? CdkCommand.deployNewPaused() : CdkCommand.deployNew(); return DeployInstanceRequest.builder() .instanceConfig(config) - .cdkCommand(cdkCommand.withConfigurationDirectory(tempDir)) + .cdkCommand(cdkCommand.withConfigurationDirectory(tempDir) + .withNetworkConfiguration(instanceId, vpcId, subnets)) .cdkApp(SleeperInternalCdkApp.STANDARD) .build(); } @@ -270,7 +268,7 @@ private void deployNewInstanceWithTables(String... args) throws Exception { private void deployNewInstance(boolean isWithTables, String... args) throws Exception { var arguments = DeployNewInstance.readArguments(CommandArgumentReader.parse(DeployNewInstance.USAGE, Stream.concat(Stream.of("scriptsDir"), Arrays.stream(args)).toArray(String[]::new))); - var config = DeployNewInstance.loadAndUpdateConfiguration(arguments); + var config = DeployNewInstance.loadConfiguration(arguments); DeployNewInstance.Builder builder = DeployNewInstance.builder() .deployInstance(request -> deployRequests.add(request)) From a08283fa17292f754107dba170ff0968d16ede90 Mon Sep 17 00:00:00 2001 From: ca61688 <206189192+ca61688@users.noreply.github.com> Date: Thu, 23 Jul 2026 14:46:17 +0100 Subject: [PATCH 43/50] 6593: Put FileReader and InstancePropertiesLoader back in AddTableCLient --- .../sleeper/clients/table/AddTableClient.java | 25 ++++++++++++--- .../java/sleeper/clients/util/FileReader.java | 32 ------------------- .../util/InstancePropertiesLoader.java | 22 ------------- 3 files changed, 20 insertions(+), 59 deletions(-) delete mode 100644 java/clients/src/main/java/sleeper/clients/util/FileReader.java delete mode 100644 java/clients/src/main/java/sleeper/clients/util/InstancePropertiesLoader.java diff --git a/java/clients/src/main/java/sleeper/clients/table/AddTableClient.java b/java/clients/src/main/java/sleeper/clients/table/AddTableClient.java index e8b9db54f77..5df30181030 100644 --- a/java/clients/src/main/java/sleeper/clients/table/AddTableClient.java +++ b/java/clients/src/main/java/sleeper/clients/table/AddTableClient.java @@ -21,8 +21,6 @@ import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.sts.StsClient; -import sleeper.clients.util.FileReader; -import sleeper.clients.util.InstancePropertiesLoader; import sleeper.configuration.properties.S3InstanceProperties; import sleeper.configuration.properties.S3TableProperties; import sleeper.core.properties.PropertiesUtils; @@ -39,6 +37,7 @@ import sleeper.statestore.StateStoreFactory; import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; @@ -99,9 +98,9 @@ public static Arguments readArguments(CommandArguments arguments, FileReader fil Properties rawTableProperties; rawTableProperties = tablePropertiesFile.isPresent() - ? PropertiesUtils.loadProperties(files.readFile(tablePropertiesFile.get())) + ? PropertiesUtils.loadProperties(readFile(files, tablePropertiesFile.get())) : configDir.isPresent() - ? PropertiesUtils.loadProperties(files.readFile(configDir.get().resolve("table.properties"))) + ? PropertiesUtils.loadProperties(readFile(files, configDir.get().resolve("table.properties"))) : null; return new Arguments( @@ -135,7 +134,7 @@ public static void main(String[] rawArgs) throws IOException { public static TableProperties createTablePropertiesWithLoaders(Arguments args, InstancePropertiesLoader instance, FileReader files) { TableProperties tableProperties = createTableProperties(instance.load(args.instanceId()), args); - tableProperties.setSchema(new SchemaSerDe().fromJson(files.readFile(args.resolveSchemaFile()))); + tableProperties.setSchema(new SchemaSerDe().fromJson(readFile(files, args.resolveSchemaFile()))); return tableProperties; } @@ -181,4 +180,20 @@ public Path resolveSchemaFile() { return schemaFile != null ? schemaFile : configDir.resolve("schema.json"); } } + + private static String readFile(FileReader reader, Path path) { + try { + return reader.readStringChecked(path); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public interface InstancePropertiesLoader { + InstanceProperties load(String instanceId); + } + + public interface FileReader { + String readStringChecked(Path path) throws IOException; + } } diff --git a/java/clients/src/main/java/sleeper/clients/util/FileReader.java b/java/clients/src/main/java/sleeper/clients/util/FileReader.java deleted file mode 100644 index 96088fefa7a..00000000000 --- a/java/clients/src/main/java/sleeper/clients/util/FileReader.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2022-2026 Crown Copyright - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package sleeper.clients.util; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.file.Path; - -public interface FileReader { - String readStringChecked(Path path) throws IOException; - - default String readFile(Path path) { - try { - return readStringChecked(path); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } -} diff --git a/java/clients/src/main/java/sleeper/clients/util/InstancePropertiesLoader.java b/java/clients/src/main/java/sleeper/clients/util/InstancePropertiesLoader.java deleted file mode 100644 index 2f61f5afc20..00000000000 --- a/java/clients/src/main/java/sleeper/clients/util/InstancePropertiesLoader.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright 2022-2026 Crown Copyright - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package sleeper.clients.util; - -import sleeper.core.properties.instance.InstanceProperties; - -public interface InstancePropertiesLoader { - InstanceProperties load(String instanceId); -} From c7386e02b8b583a26082d0c1b0145327a0b38a49 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Tue, 4 Aug 2026 06:56:24 +0000 Subject: [PATCH 44/50] 6593: Update assertions --- .../clients/deploy/DeployInstanceRequest.java | 20 +++++++++++++++++++ .../clients/deploy/DeployNewInstanceIT.java | 20 ++++--------------- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployInstanceRequest.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployInstanceRequest.java index 2ab88c3cb9c..66035a2ea76 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployInstanceRequest.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployInstanceRequest.java @@ -49,6 +49,26 @@ public CdkCommand getCdkCommand() { return cdkCommand; } + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + DeployInstanceRequest instanceRequest = (DeployInstanceRequest) o; + + return Objects.equals(instanceConfig, instanceRequest.instanceConfig) && + Objects.equals(cdkCommand, instanceRequest.cdkCommand) && + Objects.equals(cdkApp, instanceRequest.cdkApp); + } + + @Override + public int hashCode() { + return Objects.hash(instanceConfig, cdkCommand, cdkApp); + } + public static class Builder { private SleeperInstanceConfiguration instanceConfig; private CdkCommand cdkCommand; diff --git a/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceIT.java b/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceIT.java index b9dabeacaa4..10d8b6f40c7 100644 --- a/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceIT.java +++ b/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceIT.java @@ -104,10 +104,7 @@ void shouldDeployNewInstanceWhenUsingInstanceProperties() throws Exception { updatePropertyFiles(config); //Verify CDK Command - assertThat(deployRequests.size()).isEqualTo(1); - DeployInstanceRequest lastDeployRequest = deployRequests.get(0); - assertThat(lastDeployRequest).usingRecursiveComparison() - .isEqualTo(buildExpectedCDKCommandWithPropertyFile(config, false)); + assertThat(deployRequests).containsExactly(buildExpectedCDKCommandWithPropertyFile(config, false)); //Verify no table properties stored assertThat(tableIndex.streamAllTables()).isEmpty(); @@ -125,10 +122,7 @@ void shouldDeployNewInstanceWhenUsingConfigDir() throws Exception { config.getTableProperties().get(0).set(TABLE_ID, tableId("file-table")); //Verify CDK Command - assertThat(deployRequests.size()).isEqualTo(1); - DeployInstanceRequest lastDeployRequest = deployRequests.get(0); - assertThat(lastDeployRequest).usingRecursiveComparison() - .isEqualTo(buildExpectedCDKCommandWithConfigDir(config, false)); + assertThat(deployRequests).containsExactly(buildExpectedCDKCommandWithConfigDir(config, false)); //Verify Table properties store saved TableProperties expected = new TableProperties(instanceProperties); @@ -149,10 +143,7 @@ void shouldDeployNewInstanceWhenUsingInstancePropertiesIgnoringTableFiles() thro updatePropertyFiles(config); //Verify CDK Command - assertThat(deployRequests.size()).isEqualTo(1); - DeployInstanceRequest lastDeployRequest = deployRequests.get(0); - assertThat(lastDeployRequest).usingRecursiveComparison() - .isEqualTo(buildExpectedCDKCommandWithPropertyFile(config, false)); + assertThat(deployRequests).containsExactly(buildExpectedCDKCommandWithPropertyFile(config, false)); //Verify no table properties stored assertThat(tableIndex.streamAllTables()).isEmpty(); @@ -170,10 +161,7 @@ void shouldDeployNewInstancePaused() throws Exception { config.getTableProperties().get(0).set(TABLE_ID, tableId("file-table")); //Verify CDK Command - assertThat(deployRequests.size()).isEqualTo(1); - DeployInstanceRequest lastDeployRequest = deployRequests.get(0); - assertThat(lastDeployRequest).usingRecursiveComparison() - .isEqualTo(buildExpectedCDKCommandWithConfigDir(config, true)); + assertThat(deployRequests).containsExactly(buildExpectedCDKCommandWithConfigDir(config, true)); //Verify Table properties store saved TableProperties expected = new TableProperties(instanceProperties); From 688ae7b5d3051cccc7152f44570db61ca29f801f Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Tue, 4 Aug 2026 06:58:34 +0000 Subject: [PATCH 45/50] 6593: Adjust instanceProperties creation --- .../test/java/sleeper/clients/deploy/DeployNewInstanceIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceIT.java b/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceIT.java index 10d8b6f40c7..b27c675c4e2 100644 --- a/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceIT.java +++ b/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceIT.java @@ -59,7 +59,7 @@ import static sleeper.core.schema.SchemaTestHelper.createSchemaWithKey; public class DeployNewInstanceIT { - InstanceProperties instanceProperties = generateInstancePropertiesForFile(); + InstanceProperties instanceProperties = new InstanceProperties(); Schema schema = createSchemaWithKey("key"); InMemoryTableIndex tableIndex = new InMemoryTableIndex(); TablePropertiesStore tablePropertiesStore = InMemoryTableProperties.getStore(tableIndex); From 32606b4adfecf3bb0c3fe4d8328a3dd2f771881e Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Tue, 4 Aug 2026 06:59:54 +0000 Subject: [PATCH 46/50] 6539: Re-ordered methods --- .../java/sleeper/clients/deploy/DeployNewInstance.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java index 27f5c76d890..3387993fe6c 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java @@ -62,10 +62,6 @@ public class DeployNewInstance { private final boolean ignoreTableFiles; private final boolean deployPaused; - public static Builder builder() { - return new Builder(); - } - public DeployNewInstance(Builder builder) { this.deployInstance = Objects.requireNonNull(builder.deployInstance, "deployInstance must not be null"); this.storeFactory = Objects.requireNonNull(builder.storeFactory, "storeFactory must not be null"); @@ -77,6 +73,10 @@ public DeployNewInstance(Builder builder) { this.deployPaused = builder.deployPaused; } + public static Builder builder() { + return new Builder(); + } + public static final CommandLineUsage USAGE = CommandLineUsage.builder() .systemArguments(List.of("scriptsDirectory")) .positionalArguments(List.of("scriptsDirectory", "instanceId", "vpcId", "subnetIds")) From 1edcb4e5f5cfc9d268f9f9be9e4457bc262fa75d Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Tue, 4 Aug 2026 07:10:42 +0000 Subject: [PATCH 47/50] 6539: Adjusted constructor accessibility --- .../src/main/java/sleeper/clients/deploy/DeployNewInstance.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java index 3387993fe6c..096469c4303 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java @@ -62,7 +62,7 @@ public class DeployNewInstance { private final boolean ignoreTableFiles; private final boolean deployPaused; - public DeployNewInstance(Builder builder) { + private DeployNewInstance(Builder builder) { this.deployInstance = Objects.requireNonNull(builder.deployInstance, "deployInstance must not be null"); this.storeFactory = Objects.requireNonNull(builder.storeFactory, "storeFactory must not be null"); this.deployInstanceConfiguration = Objects.requireNonNull(builder.deployInstanceConfiguration, "deployInstanceConfiguration must not be null"); From 31fe3a4fcb6031d6cf299f8a020ddc49232ada58 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Tue, 4 Aug 2026 07:13:22 +0000 Subject: [PATCH 48/50] 6539: Re-order methods --- .../clients/deploy/DeployNewInstance.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java index 096469c4303..ed137fc2f2a 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java @@ -106,6 +106,18 @@ public static Builder builder() { "the instance is manually resumed.") .build(); + public static Arguments readArguments(CommandArguments arguments) { + return new Arguments( + Path.of(arguments.getString("scriptsDirectory")), + arguments.getString("instanceId"), + arguments.getString("vpcId"), + arguments.getString("subnetIds"), + arguments.getOptionalString("instance-properties").map(Path::of).orElse(null), + arguments.getOptionalString("config-dir").map(Path::of).orElse(null), + arguments.isFlagSet("ignoreTableFiles"), + arguments.isFlagSet("paused")); + } + public static SleeperInstanceConfiguration loadConfiguration(Arguments args) throws IOException { SleeperInstanceConfiguration config; if (args.ignoreTableFiles()) { @@ -121,18 +133,6 @@ public static SleeperInstanceConfiguration loadConfiguration(Arguments args) thr return config; } - public static Arguments readArguments(CommandArguments arguments) { - return new Arguments( - Path.of(arguments.getString("scriptsDirectory")), - arguments.getString("instanceId"), - arguments.getString("vpcId"), - arguments.getString("subnetIds"), - arguments.getOptionalString("instance-properties").map(Path::of).orElse(null), - arguments.getOptionalString("config-dir").map(Path::of).orElse(null), - arguments.isFlagSet("ignoreTableFiles"), - arguments.isFlagSet("paused")); - } - public static void main(String[] rawArgs) throws IOException, InterruptedException { Arguments args = CommandArguments.parseAndValidateOrExit(USAGE, rawArgs, a -> readArguments(a)); From a6f4311890c5372959c4baf95fa66b08e0a98aca Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Tue, 4 Aug 2026 07:18:25 +0000 Subject: [PATCH 49/50] 6593: Remove un-needed validate --- .../src/main/java/sleeper/clients/deploy/DeployNewInstance.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java index ed137fc2f2a..650eee6fec5 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java @@ -160,8 +160,6 @@ public static void main(String[] rawArgs) throws IOException, InterruptedExcepti } public void deploy() throws IOException, InterruptedException { - deployInstanceConfiguration.validate(); - CdkCommand cdkCommand = deployPaused ? CdkCommand.deployNewPaused() : CdkCommand.deployNew(); InstanceProperties instanceProperties = deployInstanceConfiguration.getInstanceProperties(); From 3c9488e23c5fc8ac755181b78cef52ab3ac0a43c Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Tue, 4 Aug 2026 08:16:14 +0000 Subject: [PATCH 50/50] 6593: Re-adjust the cdkapp setting --- .../sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java index 44acfb298ec..b8c5fe3187c 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java @@ -74,7 +74,7 @@ public static void main(String[] args) throws IOException, InterruptedException .deployInstance(DeployInstance.fromScriptsDirectory(scriptsDirectory, accountName, region, partitionMetadata, s3Client, ecrClient)) .storeFactory(StoreFactory.withAwsClients(s3Client, dynamoClient, accountName)) .deployInstanceConfiguration(config) - .cdkApp(SleeperInternalCdkApp.STANDARD) + .cdkApp(SleeperInternalCdkApp.DEMONSTRATION) .configDir(configurationPath) .deployPaused(deployPaused) .build().deploy();