diff --git a/java/clients/src/main/java/sleeper/clients/admin/properties/AdminClientPropertiesStore.java b/java/clients/src/main/java/sleeper/clients/admin/properties/AdminClientPropertiesStore.java index 020fdea9a20..4644af7aed3 100644 --- a/java/clients/src/main/java/sleeper/clients/admin/properties/AdminClientPropertiesStore.java +++ b/java/clients/src/main/java/sleeper/clients/admin/properties/AdminClientPropertiesStore.java @@ -23,7 +23,6 @@ import sleeper.clients.deploy.container.DockerImageConfiguration; import sleeper.clients.deploy.container.UploadDockerImagesToEcr; import sleeper.clients.deploy.container.UploadDockerImagesToEcrRequest; -import sleeper.clients.util.ClientUtils; import sleeper.clients.util.cdk.CdkCommand; import sleeper.clients.util.cdk.InvokeCdk; import sleeper.clients.util.console.ConsoleOutput; @@ -38,6 +37,7 @@ import sleeper.core.properties.table.TablePropertiesStore; import sleeper.core.statestore.StateStore; import sleeper.core.table.TableIndex; +import sleeper.core.util.FilesUtil; import sleeper.statestore.StateStoreFactory; import java.io.IOException; @@ -305,7 +305,7 @@ public void saveInstanceProperties(InstanceProperties instanceProperties) { @Override public void saveLocalProperties(InstanceProperties instanceProperties, Stream tablePropertiesStream) throws IOException { Files.createDirectories(localDirectory); - ClientUtils.clearDirectory(localDirectory); + FilesUtil.clearDirectory(localDirectory); SaveLocalProperties.saveToDirectory(localDirectory, instanceProperties, tablePropertiesStream); } diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java index fd8578e072d..4fd0feb2cef 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java @@ -31,6 +31,7 @@ import sleeper.configuration.properties.S3TableProperties; import sleeper.core.deploy.SleeperInstanceConfiguration; import sleeper.core.properties.instance.InstanceProperties; +import sleeper.core.properties.local.SaveLocalProperties; import sleeper.core.properties.model.SleeperInternalCdkApp; import sleeper.core.properties.table.TableProperties; import sleeper.core.util.cli.CommandArguments; @@ -53,6 +54,7 @@ public class DeployExistingInstance { private final List tablePropertiesList; private final boolean deployPaused; private final SleeperInternalCdkApp forceCdkApp; + private final Path configDir; private DeployExistingInstance(Builder builder) { deployInstance = builder.deployInstance; @@ -60,6 +62,7 @@ private DeployExistingInstance(Builder builder) { tablePropertiesList = builder.tablePropertiesList; deployPaused = builder.deployPaused; forceCdkApp = builder.forceCdkApp; + configDir = builder.configDir; } public static Builder builder() { @@ -107,6 +110,7 @@ public static void main(String[] rawArgs) throws IOException, InterruptedExcepti .deployPaused(args.deployPaused()) .forceCdkApp(args.forceCdkApp()) .loadPropertiesFromS3(accountName, s3Client, dynamoClient) + .configDir(args.scriptsDirectory().resolve("generated")) .build().update(); } } @@ -115,9 +119,11 @@ public record Arguments(Path scriptsDirectory, String instanceId, boolean deploy } public void update() throws IOException, InterruptedException { + SaveLocalProperties.createDirectoryAndSaveProperties(configDir, properties, tablePropertiesList.stream()); + CdkCommand cdkCommand = deployPaused ? CdkCommand.deployExistingPaused() : CdkCommand.deployExisting(); deployInstance.deploy(DeployInstanceRequest.builder() .instanceConfig(SleeperInstanceConfiguration.builder().instanceProperties(properties).tableProperties(tablePropertiesList).build()) - .cdkCommand(deployPaused ? CdkCommand.deployExistingPaused() : CdkCommand.deployExisting()) + .cdkCommand(cdkCommand.withConfigurationDirectory(configDir)) .cdkApp(getCdkApp()) .build()); @@ -143,6 +149,7 @@ public static final class Builder { private List tablePropertiesList; private boolean deployPaused; private SleeperInternalCdkApp forceCdkApp; + private Path configDir; private Builder() { } @@ -181,6 +188,11 @@ public Builder forceCdkApp(SleeperInternalCdkApp forceCdkApp) { return this; } + public Builder configDir(Path configDir) { + this.configDir = configDir; + return this; + } + public Builder loadPropertiesFromS3(String accountName, S3Client s3Client, DynamoDbClient dynamoCient) { properties = S3InstanceProperties.loadGivenAccountAndInstanceId(s3Client, accountName, instanceId); tablePropertiesList = S3TableProperties.createStore(properties, s3Client, dynamoCient) diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployInstance.java index bb520adba1a..6af355605a8 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployInstance.java @@ -28,15 +28,12 @@ import sleeper.clients.deploy.container.UploadDockerImagesToEcrRequest; import sleeper.clients.deploy.jar.SyncJars; import sleeper.clients.deploy.jar.SyncJarsRequest; -import sleeper.clients.util.ClientUtils; import sleeper.clients.util.cdk.CdkCommand; import sleeper.clients.util.cdk.InvokeCdk; import sleeper.core.deploy.SleeperInstanceConfiguration; import sleeper.core.properties.instance.InstanceProperties; -import sleeper.core.properties.local.SaveLocalProperties; import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; import static sleeper.core.properties.instance.CommonProperty.ARTEFACTS_DEPLOYMENT_ID; @@ -45,18 +42,16 @@ import static sleeper.core.properties.instance.CommonProperty.VPC_ID; import static sleeper.core.properties.model.SleeperInternalCdkApp.ARTEFACTS; -public class DeployInstance { +public class DeployInstance implements InstanceDeployer { public static final Logger LOGGER = LoggerFactory.getLogger(DeployInstance.class); private final SyncJars syncJars; private final UploadDockerImagesToEcr dockerImageUploader; - private final WriteLocalProperties writeLocalProperties; private final InvokeCdk invokeCdk; - public DeployInstance(SyncJars syncJars, UploadDockerImagesToEcr dockerImageUploader, WriteLocalProperties writeLocalProperties, InvokeCdk invokeCdk) { + public DeployInstance(SyncJars syncJars, UploadDockerImagesToEcr dockerImageUploader, InvokeCdk invokeCdk) { this.syncJars = syncJars; this.dockerImageUploader = dockerImageUploader; - this.writeLocalProperties = writeLocalProperties; this.invokeCdk = invokeCdk; } @@ -67,7 +62,6 @@ public static DeployInstance fromScriptsDirectory( new UploadDockerImagesToEcr( UploadDockerImages.fromScriptsDirectory(scriptsDirectory, ecrClient), account, region, partitionMetadata), - DeployInstance.WriteLocalProperties.underScriptsDirectory(scriptsDirectory), InvokeCdk.fromScriptsDirectory(scriptsDirectory)); } @@ -87,30 +81,9 @@ public void deploy(DeployInstanceRequest request) throws IOException, Interrupte syncJars.sync(SyncJarsRequest.from(instanceProperties)); dockerImageUploader.upload( UploadDockerImagesToEcrRequest.forDeployment(instanceProperties, request.getCdkApp(), DockerImageConfiguration.getDefault())); - Path configurationDirectory = writeLocalProperties.write(instanceConfig); LOGGER.info("-------------------------------------------------------"); LOGGER.info("Deploying Stacks"); LOGGER.info("-------------------------------------------------------"); - invokeCdk.invoke(request.getCdkApp(), request.getCdkCommand().withConfigurationDirectory(configurationDirectory)); - } - - public interface WriteLocalProperties { - Path write(SleeperInstanceConfiguration instanceConfig) throws IOException; - - static WriteLocalProperties underScriptsDirectory(Path scriptsDirectory) { - return toDirectory(scriptsDirectory.resolve("generated")); - } - - static WriteLocalProperties toDirectory(Path directory) { - return instanceConfig -> { - LOGGER.info("Writing instance configuration to local directory: {}", directory); - Files.createDirectories(directory); - ClientUtils.clearDirectory(directory); - SaveLocalProperties.saveToDirectory(directory, - instanceConfig.getInstanceProperties(), - instanceConfig.getTableProperties().stream()); - return directory; - }; - } + invokeCdk.invoke(request.getCdkApp(), request.getCdkCommand()); } } diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployInstanceRequest.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployInstanceRequest.java index 2ab88c3cb9c..66035a2ea76 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployInstanceRequest.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployInstanceRequest.java @@ -49,6 +49,26 @@ public CdkCommand getCdkCommand() { return cdkCommand; } + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + DeployInstanceRequest instanceRequest = (DeployInstanceRequest) o; + + return Objects.equals(instanceConfig, instanceRequest.instanceConfig) && + Objects.equals(cdkCommand, instanceRequest.cdkCommand) && + Objects.equals(cdkApp, instanceRequest.cdkApp); + } + + @Override + public int hashCode() { + return Objects.hash(instanceConfig, cdkCommand, cdkApp); + } + public static class Builder { private SleeperInstanceConfiguration instanceConfig; private CdkCommand cdkCommand; diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java index ad4325258e3..650eee6fec5 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java @@ -33,12 +33,19 @@ import sleeper.core.properties.instance.InstanceProperties; import sleeper.core.properties.model.SleeperInternalCdkApp; import sleeper.core.properties.table.TableProperties; +import sleeper.core.properties.table.TablePropertiesStore; +import sleeper.core.statestore.StateStoreProvider; +import sleeper.core.util.cli.CommandArguments; +import sleeper.core.util.cli.CommandArgumentsException; +import sleeper.core.util.cli.CommandLineUsage; +import sleeper.core.util.cli.CommandOption; import sleeper.statestore.StateStoreFactory; import java.io.IOException; import java.nio.file.Path; +import java.util.List; +import java.util.Objects; -import static sleeper.clients.util.ClientUtils.optionalArgument; import static sleeper.core.properties.instance.CommonProperty.ID; import static sleeper.core.properties.instance.CommonProperty.SUBNETS; import static sleeper.core.properties.instance.CommonProperty.VPC_ID; @@ -46,39 +53,89 @@ public class DeployNewInstance { private static final Logger LOGGER = LoggerFactory.getLogger(DeployNewInstance.class); - private final DeployInstance deployInstance; - private final String accountName; - private final S3Client s3Client; - private final DynamoDbClient dynamoClient; + private final InstanceDeployer deployInstance; + private final StoreFactory storeFactory; private final SleeperInstanceConfiguration deployInstanceConfiguration; private final SleeperInternalCdkApp cdkApp; + private final Path propertiesFile; + private final Path configDir; + private final boolean ignoreTableFiles; private final boolean deployPaused; private DeployNewInstance(Builder builder) { - deployInstance = builder.deployInstance; - accountName = builder.accountName; - s3Client = builder.s3Client; - dynamoClient = builder.dynamoClient; - deployInstanceConfiguration = builder.deployInstanceConfiguration; - cdkApp = builder.cdkApp; - deployPaused = builder.deployPaused; + this.deployInstance = Objects.requireNonNull(builder.deployInstance, "deployInstance must not be null"); + this.storeFactory = Objects.requireNonNull(builder.storeFactory, "storeFactory must not be null"); + this.deployInstanceConfiguration = Objects.requireNonNull(builder.deployInstanceConfiguration, "deployInstanceConfiguration must not be null"); + this.cdkApp = Objects.requireNonNull(builder.cdkApp, "cdkApp must not be null"); + this.propertiesFile = builder.propertiesFile; + this.configDir = builder.configDir; + this.ignoreTableFiles = builder.ignoreTableFiles; + this.deployPaused = builder.deployPaused; } public static Builder builder() { return new Builder(); } - public static void main(String[] args) throws IOException, InterruptedException { - if (args.length < 4 || args.length > 6) { - throw new IllegalArgumentException("Usage: " + - " "); - } - Path scriptsDirectory = Path.of(args[0]); - String instanceId = args[1]; - String vpcId = args[2]; - String subnetIds = args[3]; - Path instancePropertiesFile = optionalArgument(args, 4).map(Path::of).orElse(null); - boolean deployPaused = "true".equalsIgnoreCase(optionalArgument(args, 5).orElse("false")); + public static final CommandLineUsage USAGE = CommandLineUsage.builder() + .systemArguments(List.of("scriptsDirectory")) + .positionalArguments(List.of("scriptsDirectory", "instanceId", "vpcId", "subnetIds")) + .options(List.of( + CommandOption.longOption("instance-properties"), + CommandOption.longOption("config-dir"), + CommandOption.longFlag("ignoreTableFiles"), + CommandOption.longFlag("paused"))) + .helpSummary("" + + "Deploys a new instance of Sleeper.\n" + + "\n" + + "--instance-properties \n" + + "Path to an instance properties file.\n" + + "One of --instance-properties and --config-dir must be set but not both.\n" + + "\n" + + "--config-dir \n" + + "Path to a directory containing an instance.properties file.\n" + + "One of --instance-properties and --config-dir must be set but not both.\n" + + "\n" + + "--ignoreTableFiles\n" + + "If set, the instance will be deployed on it's own. Otherwise tables will be created based on " + + "any relevent table.properties files found in the specified --config-dir. This flag cannot be used " + + "without the --config-dir optional argument.\n" + + "\n" + + "--paused\n" + + "If set, the instance will be deployed paused. Periodic background processes will not run until " + + "the instance is manually resumed.") + .build(); + + public static Arguments readArguments(CommandArguments arguments) { + return new Arguments( + Path.of(arguments.getString("scriptsDirectory")), + arguments.getString("instanceId"), + arguments.getString("vpcId"), + arguments.getString("subnetIds"), + arguments.getOptionalString("instance-properties").map(Path::of).orElse(null), + arguments.getOptionalString("config-dir").map(Path::of).orElse(null), + arguments.isFlagSet("ignoreTableFiles"), + arguments.isFlagSet("paused")); + } + + public static SleeperInstanceConfiguration loadConfiguration(Arguments args) throws IOException { + SleeperInstanceConfiguration config; + if (args.ignoreTableFiles()) { + config = SleeperInstanceConfiguration.fromLocalConfiguration(args.resolvePropertiesFile()); + } else { + config = SleeperInstanceConfiguration.fromLocalConfigurationDirectory(args.configDir()); + } + + config.getInstanceProperties().set(ID, args.instanceId()); + config.getInstanceProperties().set(VPC_ID, args.vpcId()); + config.getInstanceProperties().set(SUBNETS, args.subnetIds()); + + return config; + } + + public static void main(String[] rawArgs) throws IOException, InterruptedException { + Arguments args = CommandArguments.parseAndValidateOrExit(USAGE, rawArgs, a -> readArguments(a)); + try (S3Client s3Client = S3Client.create(); DynamoDbClient dynamoClient = DynamoDbClient.create(); StsClient stsClient = StsClient.create(); @@ -87,84 +144,128 @@ public static void main(String[] args) throws IOException, InterruptedException Region region = DefaultAwsRegionProviderChain.builder().build().getRegion(); PartitionMetadata partitionMetadata = PartitionMetadata.of(region); - SleeperInstanceConfiguration config = SleeperInstanceConfiguration.forNewInstanceDefaultingInstance( - instancePropertiesFile, scriptsDirectory.resolve("templates")); + SleeperInstanceConfiguration config = loadConfiguration(args); - config.getInstanceProperties().set(ID, instanceId); - config.getInstanceProperties().set(VPC_ID, vpcId); - config.getInstanceProperties().set(SUBNETS, subnetIds); - - builder() - .deployInstance(DeployInstance.fromScriptsDirectory(scriptsDirectory, accountName, region, partitionMetadata, s3Client, ecrClient)) - .accountName(accountName) - .s3Client(s3Client) - .dynamoClient(dynamoClient) + DeployNewInstance.builder() + .deployInstance(DeployInstance.fromScriptsDirectory(args.scriptsDirectory(), accountName, region, partitionMetadata, s3Client, ecrClient)) + .storeFactory(StoreFactory.withAwsClients(s3Client, dynamoClient, accountName)) .deployInstanceConfiguration(config) - .deployPaused(deployPaused) .cdkApp(SleeperInternalCdkApp.STANDARD) + .propertiesFile(args.resolvePropertiesFile()) + .configDir(args.configDir()) + .ignoreTableFiles(args.ignoreTableFiles()) + .deployPaused(args.deployPaused()) .build().deploy(); } } public void deploy() throws IOException, InterruptedException { - deployInstanceConfiguration.validate(); + CdkCommand cdkCommand = deployPaused ? CdkCommand.deployNewPaused() : CdkCommand.deployNew(); + + InstanceProperties instanceProperties = deployInstanceConfiguration.getInstanceProperties(); + cdkCommand = cdkCommand.withNetworkConfiguration(instanceProperties.get(ID), instanceProperties.get(VPC_ID), instanceProperties.get(SUBNETS)); + + if (ignoreTableFiles) { + cdkCommand = cdkCommand.withPropertiesFile(propertiesFile); + } else { + cdkCommand = cdkCommand.withConfigurationDirectory(configDir); + } deployInstance.deploy(DeployInstanceRequest.builder() .instanceConfig(deployInstanceConfiguration) - .cdkCommand(deployPaused ? CdkCommand.deployNewPaused() : CdkCommand.deployNew()) + .cdkCommand(cdkCommand) .cdkApp(cdkApp) .build()); - InstanceProperties instanceProperties = S3InstanceProperties.loadGivenAccountAndInstanceId(s3Client, accountName, deployInstanceConfiguration.getInstanceId()); - for (TableProperties tableProperties : deployInstanceConfiguration.getTableProperties()) { - LOGGER.info("Adding table " + tableProperties.getStatus()); - new AddTableClient(tableProperties, - S3TableProperties.createStore(instanceProperties, s3Client, dynamoClient), - StateStoreFactory.createProvider(instanceProperties, s3Client, dynamoClient)) - .run(); + if (!ignoreTableFiles) { + storeFactory.reloadInstanceProperties(instanceProperties); + + for (TableProperties tableProperties : deployInstanceConfiguration.getTableProperties()) { + LOGGER.info("Adding table " + tableProperties.getStatus()); + new AddTableClient(tableProperties, + storeFactory.createTableStore(instanceProperties), + storeFactory.createStateStore(instanceProperties)) + .run(); + } } LOGGER.info("Finished deployment of new instance"); } + public record Arguments( + Path scriptsDirectory, + String instanceId, + String vpcId, + String subnetIds, + Path propertiesFile, + Path configDir, + boolean ignoreTableFiles, + boolean deployPaused) { + + public Arguments { + if (propertiesFile == null && configDir == null) { + throw new CommandArgumentsException("Either --instance-properties or --config-dir must be provided"); + } + + if (propertiesFile != null && configDir != null) { + throw new CommandArgumentsException("Cannot use both --instance-properties and --config-dir"); + } + + if (propertiesFile != null) { + ignoreTableFiles = true; + } + } + + public Path resolvePropertiesFile() { + return propertiesFile != null ? propertiesFile : configDir.resolve("instance.properties"); + } + } + public static final class Builder { - private DeployInstance deployInstance; - private String accountName; - private S3Client s3Client; - private DynamoDbClient dynamoClient; + private InstanceDeployer deployInstance; + private StoreFactory storeFactory; private SleeperInstanceConfiguration deployInstanceConfiguration; private SleeperInternalCdkApp cdkApp; - private boolean deployPaused; + private Path propertiesFile; + private Path configDir; + private boolean ignoreTableFiles = false; + private boolean deployPaused = false; private Builder() { + } - public Builder deployInstance(DeployInstance deployInstance) { + public Builder deployInstance(InstanceDeployer deployInstance) { this.deployInstance = deployInstance; return this; } - public Builder accountName(String accountName) { - this.accountName = accountName; + public Builder storeFactory(StoreFactory storeFactory) { + this.storeFactory = storeFactory; return this; } - public Builder s3Client(S3Client s3Client) { - this.s3Client = s3Client; + public Builder deployInstanceConfiguration(SleeperInstanceConfiguration deployInstanceConfiguration) { + this.deployInstanceConfiguration = deployInstanceConfiguration; return this; } - public Builder dynamoClient(DynamoDbClient dynamoClient) { - this.dynamoClient = dynamoClient; + public Builder cdkApp(SleeperInternalCdkApp cdkApp) { + this.cdkApp = cdkApp; return this; } - public Builder deployInstanceConfiguration(SleeperInstanceConfiguration deployInstanceConfiguration) { - this.deployInstanceConfiguration = deployInstanceConfiguration; + public Builder propertiesFile(Path propertiesFile) { + this.propertiesFile = propertiesFile; return this; } - public Builder cdkApp(SleeperInternalCdkApp cdkApp) { - this.cdkApp = cdkApp; + public Builder configDir(Path configDir) { + this.configDir = configDir; + return this; + } + + public Builder ignoreTableFiles(boolean ignoreTableFiles) { + this.ignoreTableFiles = ignoreTableFiles; return this; } @@ -176,11 +277,34 @@ public Builder deployPaused(boolean deployPaused) { public DeployNewInstance build() { return new DeployNewInstance(this); } + } - public void deployWithClients(S3Client s3Client, DynamoDbClient dynamoClient) throws IOException, InterruptedException { - s3Client(s3Client) - .dynamoClient(dynamoClient) - .build().deploy(); + @FunctionalInterface + public interface InstancePropertiesLoader { + InstanceProperties load(String instanceId); + } + + public interface StoreFactory { + TablePropertiesStore createTableStore(InstanceProperties instanceProperties); + + StateStoreProvider createStateStore(InstanceProperties instanceProperties); + + void reloadInstanceProperties(InstanceProperties instanceProperties); + + static StoreFactory withAwsClients(S3Client s3Client, DynamoDbClient dynamoClient, String accountName) { + return new StoreFactory() { + public TablePropertiesStore createTableStore(InstanceProperties p) { + return S3TableProperties.createStore(p, s3Client, dynamoClient); + } + + public StateStoreProvider createStateStore(InstanceProperties p) { + return StateStoreFactory.createProvider(p, s3Client, dynamoClient); + } + + public void reloadInstanceProperties(InstanceProperties p) { + S3InstanceProperties.reloadGivenAccountAndInstanceId(s3Client, p, accountName, p.get(ID)); + } + }; } } } diff --git a/java/clients/src/main/java/sleeper/clients/deploy/InstanceDeployer.java b/java/clients/src/main/java/sleeper/clients/deploy/InstanceDeployer.java new file mode 100644 index 00000000000..8bad0e003a8 --- /dev/null +++ b/java/clients/src/main/java/sleeper/clients/deploy/InstanceDeployer.java @@ -0,0 +1,22 @@ +/* + * Copyright 2022-2026 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package sleeper.clients.deploy; + +import java.io.IOException; + +public interface InstanceDeployer { + void deploy(DeployInstanceRequest request) throws IOException, InterruptedException; +} diff --git a/java/clients/src/main/java/sleeper/clients/teardown/TearDownInstance.java b/java/clients/src/main/java/sleeper/clients/teardown/TearDownInstance.java index a682ae2c770..7b879e8229d 100644 --- a/java/clients/src/main/java/sleeper/clients/teardown/TearDownInstance.java +++ b/java/clients/src/main/java/sleeper/clients/teardown/TearDownInstance.java @@ -19,9 +19,9 @@ import org.slf4j.LoggerFactory; import software.amazon.awssdk.services.cloudformation.CloudFormationClient; -import sleeper.clients.util.ClientUtils; import sleeper.core.properties.instance.InstanceProperties; import sleeper.core.properties.local.LoadLocalProperties; +import sleeper.core.util.FilesUtil; import java.io.IOException; import java.nio.file.Files; @@ -137,7 +137,7 @@ private static void removeGeneratedDir(Path scriptsDir) throws IOException { Path generatedDir = scriptsDir.resolve("generated"); if (Files.isDirectory(generatedDir)) { LOGGER.info("Removing generated files"); - ClientUtils.clearDirectory(generatedDir); + FilesUtil.clearDirectory(generatedDir); } else { LOGGER.info("Generated directory not found"); } diff --git a/java/clients/src/main/java/sleeper/clients/util/ClientUtils.java b/java/clients/src/main/java/sleeper/clients/util/ClientUtils.java index 698184f7aae..027f8bdda5b 100644 --- a/java/clients/src/main/java/sleeper/clients/util/ClientUtils.java +++ b/java/clients/src/main/java/sleeper/clients/util/ClientUtils.java @@ -18,12 +18,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Comparator; import java.util.Optional; -import java.util.stream.Stream; import static sleeper.core.util.NumberFormatUtils.countWithCommas; @@ -59,13 +54,4 @@ public static String abbreviatedRowCount(long rows) { return countWithCommas(Math.round((double) rows / T_COUNT)) + "T (" + countWithCommas(rows) + ")"; } } - - public static void clearDirectory(Path tempDir) throws IOException { - try (Stream paths = Files.walk(tempDir)) { - Stream nestedPaths = paths.skip(1).sorted(Comparator.reverseOrder()); - for (Path path : (Iterable) nestedPaths::iterator) { - Files.delete(path); - } - } - } } diff --git a/java/clients/src/main/java/sleeper/clients/util/cdk/CdkCommand.java b/java/clients/src/main/java/sleeper/clients/util/cdk/CdkCommand.java index 38ef155dc39..c99c897e68d 100644 --- a/java/clients/src/main/java/sleeper/clients/util/cdk/CdkCommand.java +++ b/java/clients/src/main/java/sleeper/clients/util/cdk/CdkCommand.java @@ -67,6 +67,10 @@ public CdkCommand withConfigurationDirectory(Path configurationDirectory) { return builder().command(command).configurationDirectory(configurationDirectory).arguments(arguments).build(); } + public CdkCommand withNetworkConfiguration(String instanceId, String vpcId, String subnets) { + return builder().command(command).arguments(arguments).instanceId(instanceId).vpcId(vpcId).subnets(subnets).build(); + } + public static final class Builder { private List command; private List arguments = new ArrayList<>(); @@ -93,6 +97,18 @@ public Builder arguments(List arguments) { return this; } + public Builder instanceId(String id) { + return context("id", id); + } + + public Builder vpcId(String vpcId) { + return context("vpc", vpcId); + } + + public Builder subnets(String subnets) { + return context("subnets", subnets); + } + public Builder propertiesFile(Path propertiesFile) { return context("propertiesFile", propertiesFile.toString()); } diff --git a/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceIT.java b/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceIT.java new file mode 100644 index 00000000000..b27c675c4e2 --- /dev/null +++ b/java/clients/src/test/java/sleeper/clients/deploy/DeployNewInstanceIT.java @@ -0,0 +1,302 @@ +/* + * Copyright 2022-2026 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package sleeper.clients.deploy; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import sleeper.clients.util.cdk.CdkCommand; +import sleeper.core.deploy.SleeperInstanceConfiguration; +import sleeper.core.properties.instance.InstanceProperties; +import sleeper.core.properties.model.SleeperInternalCdkApp; +import sleeper.core.properties.table.TableProperties; +import sleeper.core.properties.table.TablePropertiesStore; +import sleeper.core.properties.testutils.InMemoryTableProperties; +import sleeper.core.schema.Schema; +import sleeper.core.schema.SchemaSerDe; +import sleeper.core.statestore.StateStoreProvider; +import sleeper.core.statestore.testutils.InMemoryTransactionLogStateStore; +import sleeper.core.statestore.testutils.InMemoryTransactionLogsPerTable; +import sleeper.core.table.InMemoryTableIndex; +import sleeper.core.util.cli.CommandArgumentReader; +import sleeper.core.util.cli.CommandArgumentsException; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +import static java.nio.file.Files.createDirectory; +import static java.nio.file.Files.createTempDirectory; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static sleeper.core.properties.instance.CommonProperty.ID; +import static sleeper.core.properties.instance.CommonProperty.SUBNETS; +import static sleeper.core.properties.instance.CommonProperty.VPC_ID; +import static sleeper.core.properties.table.TableProperty.TABLE_ID; +import static sleeper.core.properties.table.TableProperty.TABLE_NAME; +import static sleeper.core.properties.testutils.InstancePropertiesTestHelper.createTestInstanceProperties; +import static sleeper.core.schema.SchemaTestHelper.createSchemaWithKey; + +public class DeployNewInstanceIT { + InstanceProperties instanceProperties = new InstanceProperties(); + Schema schema = createSchemaWithKey("key"); + InMemoryTableIndex tableIndex = new InMemoryTableIndex(); + TablePropertiesStore tablePropertiesStore = InMemoryTableProperties.getStore(tableIndex); + StateStoreProvider stateStoreProvider = InMemoryTransactionLogStateStore.createProvider(instanceProperties, + new InMemoryTransactionLogsPerTable()); + Map pathToString = new HashMap<>(); + List deployRequests = new ArrayList<>(); + Path instancePropertiesFile; + String configDir; + String instanceId = "someInstance"; + String vpcId = "someVpc"; + String subnets = "someSubnet1,someSubnet2"; + + @TempDir + private Path tempDir; + + @BeforeEach + void setUp() throws IOException { + createTempDirectory(tempDir, null); + instancePropertiesFile = tempDir.resolve("instance.properties"); + Files.writeString(instancePropertiesFile, instanceProperties.saveAsString()); + Path tables = tempDir.resolve("tables"); + Path table1 = tables.resolve("table1"); + createDirectory(tables); + createDirectory(table1); + Files.writeString(table1.resolve("table.properties"), "sleeper.table.name=file-table\n"); + Files.writeString(table1.resolve("schema.json"), new SchemaSerDe().toJson(schema)); + configDir = tempDir.toString(); + } + + @Nested + class DeployNew { + + @Test + void shouldDeployNewInstanceWhenUsingInstanceProperties() throws Exception { + //When + deployNewInstanceWithoutTables(instanceId, vpcId, subnets, "--instance-properties", + instancePropertiesFile.toString()); + + //Then + SleeperInstanceConfiguration config = SleeperInstanceConfiguration.fromLocalConfiguration(instancePropertiesFile); + updatePropertyFiles(config); + + //Verify CDK Command + assertThat(deployRequests).containsExactly(buildExpectedCDKCommandWithPropertyFile(config, false)); + + //Verify no table properties stored + assertThat(tableIndex.streamAllTables()).isEmpty(); + } + + @Test + void shouldDeployNewInstanceWhenUsingConfigDir() throws Exception { + //When + deployNewInstanceWithTables(instanceId, vpcId, subnets, "--config-dir", + configDir); + + //Then + SleeperInstanceConfiguration config = SleeperInstanceConfiguration.fromLocalConfigurationDirectory(tempDir); + updatePropertyFiles(config); + config.getTableProperties().get(0).set(TABLE_ID, tableId("file-table")); + + //Verify CDK Command + assertThat(deployRequests).containsExactly(buildExpectedCDKCommandWithConfigDir(config, false)); + + //Verify Table properties store saved + TableProperties expected = new TableProperties(instanceProperties); + expected.setSchema(schema); + expected.set(TABLE_ID, tableId("file-table")); + expected.set(TABLE_NAME, "file-table"); + assertThat(tablePropertiesStore.streamAllTables()).containsExactly(expected); + } + + @Test + void shouldDeployNewInstanceWhenUsingInstancePropertiesIgnoringTableFiles() throws Exception { + //When + deployNewInstanceWithoutTables(instanceId, vpcId, subnets, "--config-dir", configDir, + "--ignoreTableFiles"); + + //Then + SleeperInstanceConfiguration config = SleeperInstanceConfiguration.fromLocalConfiguration(instancePropertiesFile); + updatePropertyFiles(config); + + //Verify CDK Command + assertThat(deployRequests).containsExactly(buildExpectedCDKCommandWithPropertyFile(config, false)); + + //Verify no table properties stored + assertThat(tableIndex.streamAllTables()).isEmpty(); + } + + @Test + void shouldDeployNewInstancePaused() throws Exception { + //When + deployNewInstanceWithTables(instanceId, vpcId, subnets, "--config-dir", configDir, + "--paused"); + + //Then + SleeperInstanceConfiguration config = SleeperInstanceConfiguration.fromLocalConfigurationDirectory(instancePropertiesFile); + updatePropertyFiles(config); + config.getTableProperties().get(0).set(TABLE_ID, tableId("file-table")); + + //Verify CDK Command + assertThat(deployRequests).containsExactly(buildExpectedCDKCommandWithConfigDir(config, true)); + + //Verify Table properties store saved + TableProperties expected = new TableProperties(instanceProperties); + expected.setSchema(schema); + expected.set(TABLE_ID, tableId("file-table")); + expected.set(TABLE_NAME, "file-table"); + assertThat(tablePropertiesStore.streamAllTables()).containsExactly(expected); + } + + private void updatePropertyFiles(SleeperInstanceConfiguration config) { + instanceProperties.set(ID, instanceId); + instanceProperties.set(VPC_ID, vpcId); + instanceProperties.set(SUBNETS, subnets); + config.getInstanceProperties().set(ID, instanceId); + config.getInstanceProperties().set(VPC_ID, vpcId); + config.getInstanceProperties().set(SUBNETS, subnets); + } + + private DeployInstanceRequest buildExpectedCDKCommandWithPropertyFile(SleeperInstanceConfiguration config, boolean deployPaused) { + CdkCommand cdkCommand = deployPaused ? CdkCommand.deployNewPaused() : CdkCommand.deployNew(); + return DeployInstanceRequest.builder() + .instanceConfig(config) + .cdkCommand(cdkCommand.withPropertiesFile(instancePropertiesFile) + .withNetworkConfiguration(instanceId, vpcId, subnets)) + .cdkApp(SleeperInternalCdkApp.STANDARD) + .build(); + } + + private DeployInstanceRequest buildExpectedCDKCommandWithConfigDir(SleeperInstanceConfiguration config, boolean deployPaused) { + CdkCommand cdkCommand = deployPaused ? CdkCommand.deployNewPaused() : CdkCommand.deployNew(); + return DeployInstanceRequest.builder() + .instanceConfig(config) + .cdkCommand(cdkCommand.withConfigurationDirectory(tempDir) + .withNetworkConfiguration(instanceId, vpcId, subnets)) + .cdkApp(SleeperInternalCdkApp.STANDARD) + .build(); + } + } + + @Nested + class ArgumentsValidation { + + @Test + void shouldRejectWhenNotEnoughPositionalArguments() { + // When/Then + assertThatThrownBy(() -> deployNewInstanceWithoutTables()) + .isInstanceOf(CommandArgumentsException.class) + .hasMessage("Expected 4 positional arguments, found 1"); + } + + @Test + void shouldRejectWhenNeitherInstancePropertiesOrConfigDirSet() { + // When/Then + assertThatThrownBy(() -> deployNewInstanceWithoutTables("my-instance", "my-vpc", "my-subnets")) + .isInstanceOf(CommandArgumentsException.class) + .hasMessage("Either --instance-properties or --config-dir must be provided"); + } + + @Test + void shouldRejectWhenBothInstancePropertiesAndConfigDirSet() { + // When/Then + assertThatThrownBy(() -> deployNewInstanceWithoutTables("my-instance", "my-vpc", "my-subnets", + "--instance-properties", "someFile", "--config-dir", "someDir")) + .isInstanceOf(CommandArgumentsException.class) + .hasMessage("Cannot use both --instance-properties and --config-dir"); + } + + @Test + void shouldSetIgnoreTableFilesTrueWhenInstancePropertiesUsed() { + var arguments = DeployNewInstance.readArguments(CommandArgumentReader.parse(DeployNewInstance.USAGE, + "scriptsDir", "my-instance", "my-vpc", "my-subnets", "--instance-properties", "someFile")); + assertThat(arguments.ignoreTableFiles()).isTrue(); + } + + @Test + void shouldResolvePropertiesFileWhenConfigDirUsed() { + var arguments = DeployNewInstance.readArguments(CommandArgumentReader.parse(DeployNewInstance.USAGE, + "scriptsDir", "my-instance", "my-vpc", "my-subnets", "--config-dir", "someDir")); + assertThat(arguments.resolvePropertiesFile()).isEqualTo(Path.of("someDir/instance.properties")); + } + + } + + private void deployNewInstanceWithoutTables(String... args) throws Exception { + deployNewInstance(true, args); + } + + private void deployNewInstanceWithTables(String... args) throws Exception { + deployNewInstance(false, args); + } + + private void deployNewInstance(boolean isWithTables, String... args) throws Exception { + var arguments = DeployNewInstance.readArguments(CommandArgumentReader.parse(DeployNewInstance.USAGE, + Stream.concat(Stream.of("scriptsDir"), Arrays.stream(args)).toArray(String[]::new))); + var config = DeployNewInstance.loadConfiguration(arguments); + + DeployNewInstance.Builder builder = DeployNewInstance.builder() + .deployInstance(request -> deployRequests.add(request)) + .storeFactory(new DeployNewInstance.StoreFactory() { + public TablePropertiesStore createTableStore(InstanceProperties p) { + return tablePropertiesStore; + } + + public StateStoreProvider createStateStore(InstanceProperties p) { + return stateStoreProvider; + } + + public void reloadInstanceProperties(InstanceProperties p) { + } + }) + .deployInstanceConfiguration(config) + .cdkApp(SleeperInternalCdkApp.STANDARD) + .ignoreTableFiles(arguments.ignoreTableFiles()) + .deployPaused(arguments.deployPaused()); + + if (isWithTables) { + builder.propertiesFile(instancePropertiesFile); + } else { + builder.configDir(tempDir); + } + + builder.build().deploy(); + } + + private String tableId(String tableName) { + return tableIndex.getTableByName(tableName) + .orElseThrow(() -> new RuntimeException("Found tables: " + tableIndex.streamAllTables().toList())) + .getTableUniqueId(); + } + + private static InstanceProperties generateInstancePropertiesForFile() { + InstanceProperties instanceProperties = createTestInstanceProperties(); + instanceProperties.unset(ID); + instanceProperties.unset(VPC_ID); + instanceProperties.unset(SUBNETS); + return instanceProperties; + } +} diff --git a/java/core/src/main/java/sleeper/core/properties/local/SaveLocalProperties.java b/java/core/src/main/java/sleeper/core/properties/local/SaveLocalProperties.java index 90bf3acb51a..4d2c0952a93 100644 --- a/java/core/src/main/java/sleeper/core/properties/local/SaveLocalProperties.java +++ b/java/core/src/main/java/sleeper/core/properties/local/SaveLocalProperties.java @@ -18,6 +18,7 @@ import sleeper.core.properties.instance.InstanceProperties; import sleeper.core.properties.table.TableProperties; +import sleeper.core.util.FilesUtil; import java.io.BufferedWriter; import java.io.IOException; @@ -37,6 +38,22 @@ public class SaveLocalProperties { private SaveLocalProperties() { } + /** + * Creates a given directory and saves instance and table properties to the given directory. + * + * @param directory the directory + * @param instanceProperties the instance properties + * @param tablePropertiesStream the table properties + * @throws IOException if we could not write to the file system + */ + public static void createDirectoryAndSaveProperties( + Path directory, InstanceProperties instanceProperties, + Stream tablePropertiesStream) throws IOException { + Files.createDirectories(directory); + FilesUtil.clearDirectory(directory); + saveToDirectory(directory, instanceProperties, tablePropertiesStream); + } + /** * Saves instance and table properties to the given directory. * diff --git a/java/core/src/main/java/sleeper/core/util/FilesUtil.java b/java/core/src/main/java/sleeper/core/util/FilesUtil.java new file mode 100644 index 00000000000..bbf8fa3cba5 --- /dev/null +++ b/java/core/src/main/java/sleeper/core/util/FilesUtil.java @@ -0,0 +1,47 @@ +/* + * Copyright 2022-2026 Crown Copyright + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package sleeper.core.util; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.stream.Stream; + +/** + * Utils to interact with files and directories. + */ +public class FilesUtil { + + private FilesUtil() { + } + + /** + * Clears the provided directory of files. + * + * @param directory the directory to clear + * @throws IOException if an I/O error occurs + */ + public static void clearDirectory(Path directory) throws IOException { + try (Stream paths = Files.walk(directory)) { + Stream nestedPaths = paths.skip(1).sorted(Comparator.reverseOrder()); + for (Path path : (Iterable) nestedPaths::iterator) { + Files.delete(path); + } + } + } + +} diff --git a/java/clients/src/test/java/sleeper/clients/util/ClientUtilsTest.java b/java/core/src/test/java/sleeper/core/util/FilesUtilTest.java similarity index 91% rename from java/clients/src/test/java/sleeper/clients/util/ClientUtilsTest.java rename to java/core/src/test/java/sleeper/core/util/FilesUtilTest.java index fcba8d71e52..b4b7517e5cb 100644 --- a/java/clients/src/test/java/sleeper/clients/util/ClientUtilsTest.java +++ b/java/core/src/test/java/sleeper/core/util/FilesUtilTest.java @@ -13,8 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - -package sleeper.clients.util; +package sleeper.core.util; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Nested; @@ -27,7 +26,7 @@ import static org.assertj.core.api.Assertions.assertThat; -class ClientUtilsTest { +public class FilesUtilTest { @DisplayName("Clear directories") @Nested @@ -42,7 +41,7 @@ void shouldRemoveFileIfExists() throws IOException { Files.createFile(newFile); // When - ClientUtils.clearDirectory(tempDir); + FilesUtil.clearDirectory(tempDir); // Then assertThat(newFile).doesNotExist(); @@ -51,7 +50,7 @@ void shouldRemoveFileIfExists() throws IOException { @Test void shouldNotRemoveRootDirectory() throws IOException { // Given/When - ClientUtils.clearDirectory(tempDir); + FilesUtil.clearDirectory(tempDir); // Then assertThat(tempDir).exists(); @@ -68,7 +67,7 @@ void shouldRemoveMultipleDirectoriesAndFiles() throws IOException { Files.createFile(tempDir.resolve("dir2/nested2/file2")); // When - ClientUtils.clearDirectory(tempDir); + FilesUtil.clearDirectory(tempDir); // Then assertThat(tempDir).isEmptyDirectory(); diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java index ff6ba0519c8..b8c5fe3187c 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java @@ -25,6 +25,7 @@ import sleeper.clients.deploy.DeployInstance; import sleeper.clients.deploy.DeployNewInstance; +import sleeper.clients.deploy.DeployNewInstance.StoreFactory; import sleeper.core.deploy.SleeperInstanceConfiguration; import sleeper.core.deploy.SleeperInstanceConfigurationFromTemplates; import sleeper.core.properties.model.SleeperInternalCdkApp; @@ -48,7 +49,7 @@ public static void main(String[] args) throws IOException, InterruptedException " "); } Path scriptsDirectory = Path.of(args[0]); - Path propertiesFile = Path.of(args[1]); + Path configurationPath = Path.of(args[1]); String instanceId = args[2]; String vpcId = args[3]; String subnetIds = args[4]; @@ -64,17 +65,17 @@ public static void main(String[] args) throws IOException, InterruptedException PartitionMetadata partitionMetadata = PartitionMetadata.of(region); SleeperInstanceConfiguration config = SleeperInstanceConfiguration.forNewInstanceDefaultingTables( - propertiesFile, templates(scriptsDirectory, splitPointsFileForTemplate)); + configurationPath, templates(scriptsDirectory, splitPointsFileForTemplate)); config.getInstanceProperties().set(ID, instanceId); config.getInstanceProperties().set(VPC_ID, vpcId); config.getInstanceProperties().set(SUBNETS, subnetIds); + DeployNewInstance.builder() .deployInstance(DeployInstance.fromScriptsDirectory(scriptsDirectory, accountName, region, partitionMetadata, s3Client, ecrClient)) - .accountName(accountName) - .s3Client(s3Client) - .dynamoClient(dynamoClient) + .storeFactory(StoreFactory.withAwsClients(s3Client, dynamoClient, accountName)) .deployInstanceConfiguration(config) .cdkApp(SleeperInternalCdkApp.DEMONSTRATION) + .configDir(configurationPath) .deployPaused(deployPaused) .build().deploy(); } diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java index fb85316ca03..7f553ac0efb 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java @@ -23,13 +23,16 @@ import software.amazon.awssdk.services.cloudformation.model.Stack; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.sts.StsClient; import sleeper.clients.deploy.DeployExistingInstance; import sleeper.clients.deploy.DeployInstance; import sleeper.clients.deploy.DeployNewInstance; +import sleeper.clients.deploy.DeployNewInstance.StoreFactory; import sleeper.configuration.properties.S3InstanceProperties; import sleeper.core.deploy.SleeperInstanceConfiguration; import sleeper.core.properties.instance.InstanceProperties; +import sleeper.core.properties.local.SaveLocalProperties; import sleeper.core.properties.model.SleeperInternalCdkApp; import sleeper.core.properties.table.TableProperties; import sleeper.systemtest.drivers.util.SystemTestClients; @@ -38,6 +41,7 @@ import java.io.IOException; import java.io.UncheckedIOException; +import java.nio.file.Path; import java.util.List; import java.util.Set; @@ -52,6 +56,7 @@ public class AwsSleeperInstanceDriver implements SleeperInstanceDriver { private final SystemTestParameters parameters; private final S3Client s3; + private final StsClient sts; private final DynamoDbClient dynamoDB; private final CloudFormationClient cloudFormationClient; private final AwsResetInstanceOnFirstConnect resetInstance; @@ -60,6 +65,7 @@ public class AwsSleeperInstanceDriver implements SleeperInstanceDriver { public AwsSleeperInstanceDriver(SystemTestParameters parameters, SystemTestClients clients) { this.parameters = parameters; this.s3 = clients.getS3(); + this.sts = clients.getSts(); this.dynamoDB = clients.getDynamo(); this.cloudFormationClient = clients.getCloudFormation(); this.resetInstance = new AwsResetInstanceOnFirstConnect(clients); @@ -83,14 +89,21 @@ public boolean deployInstanceIfNotPresent(String instanceId, SleeperInstanceConf deployConfig.getInstanceProperties().set(ID, instanceId); deployConfig.getInstanceProperties().set(VPC_ID, parameters.getVpcId()); deployConfig.getInstanceProperties().set(SUBNETS, parameters.getSubnetIds()); + + Path configDir = parameters.getScriptsDirectory().resolve("example"); + try { + SaveLocalProperties.createDirectoryAndSaveProperties(configDir, deployConfig.getInstanceProperties(), deployConfig.getTableProperties().stream()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + try { DeployNewInstance.builder() .deployInstance(deployInstance) - .accountName(parameters.getAccount()) - .s3Client(s3) - .dynamoClient(dynamoDB) + .storeFactory(StoreFactory.withAwsClients(s3, dynamoDB, sts.getCallerIdentity().account())) .deployInstanceConfiguration(deployConfig) .cdkApp(SleeperInternalCdkApp.STANDARD) + .configDir(configDir) .build().deploy(); } catch (InterruptedException e) { Thread.currentThread().interrupt(); diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/SystemTestDeploymentFactory.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/SystemTestDeploymentFactory.java index f81a3d691c1..6542bf3b474 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/SystemTestDeploymentFactory.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/SystemTestDeploymentFactory.java @@ -37,7 +37,6 @@ public static DeployInstance createDeployInstance(SystemTestParameters parameter return new DeployInstance( createSyncJars(parameters, clients), createDockerUploader(parameters, clients), - DeployInstance.WriteLocalProperties.underScriptsDirectory(parameters.getScriptsDirectory()), createInvokeCdk(parameters, clients)); } diff --git a/scripts/deploy/deployNew.sh b/scripts/deploy/deployNew.sh index 990b14c3e8a..44cc40f0f93 100755 --- a/scripts/deploy/deployNew.sh +++ b/scripts/deploy/deployNew.sh @@ -16,11 +16,6 @@ set -e unset CDPATH -if [ "$#" -lt 3 ] || [ "$#" -gt 5 ]; then - echo "Usage: $0 " - exit 1 -fi - SCRIPTS_DIR=$(cd "$(dirname "$0")" && cd .. && pwd) VERSION=$(cat "${SCRIPTS_DIR}/templates/version.txt")