Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
63 commits
Select commit Hold shift + click to select a range
18fbcfe
6593: Add CommandArguments to DeployNewInstance
ca61688 Jul 6, 2026
cc33173
6593: Make deployNew script use new DeployNewInstance file
ca61688 Jul 6, 2026
c74b7d4
6593: Move uses of DeployNewIsntance to new Class
ca61688 Jul 6, 2026
e6b6070
6593: Replace old deployNewInstance with new version
ca61688 Jul 6, 2026
e11edbb
6593: Add ignore table files flag
ca61688 Jul 6, 2026
e348a00
6593: Update deploy new instance help text to include ignoreTableFile…
ca61688 Jul 6, 2026
38a7c2c
7555 Set version number to 0.37.0
patchwork01 Jul 6, 2026
fb3da30
7555 Performance figures for 0.37.0
patchwork01 Jul 6, 2026
474d46d
7553 Changelog for 0.37.0
patchwork01 Jun 30, 2026
c62e6e0
7553 Adjust release summary
patchwork01 Jun 30, 2026
29942a7
7553 Note bug fix for DataFusion calls
patchwork01 Jul 6, 2026
80a79c4
6593: Create initial unit test for deploy new instance
ca61688 Jul 6, 2026
5eea52f
7605 Set version number to 0.37.1-SNAPSHOT
patchwork01 Jul 6, 2026
a1411a6
6593: Improve usage of DeployNewInstance constructor
ca61688 Jul 7, 2026
584ed75
6593: Improve DeployNewInstance argument validation
ca61688 Jul 7, 2026
e7a4210
6593: Add remaining unit tests for DeployNewInstanceTest
ca61688 Jul 7, 2026
0262eb7
Merge branch 'develop' into 6593-improve-usability-of-deploy-new-script
ca61688 Jul 7, 2026
e48525f
6593: Add inital framework for DeployNewInstanceIT
ca61688 Jul 7, 2026
7593e6d
6593: Begin to unpick file reader changes
ca61688 Jul 7, 2026
3925abc
6593: Make DeployNewInstanceTest use tempDir
ca61688 Jul 8, 2026
972747f
6593: Remove unused dependency
ca61688 Jul 8, 2026
1ca79b6
Merge branch 'develop' into 6593-improve-usability-of-deploy-new-script
ca61688 Jul 8, 2026
876c869
Merge branch 'develop' into 6593-improve-usability-of-deploy-new-script
ca61688 Jul 9, 2026
9b3b08c
Merge branch 'develop' into 6593-improve-usability-of-deploy-new-script
ca61688 Jul 10, 2026
ac6c85b
6593: Have DeployNewInstance save update instance properties file
ca61688 Jul 10, 2026
3b3d60b
Merge branch '6593-improve-usability-of-deploy-new-script' of github.…
ca61688 Jul 10, 2026
1300e94
Merge branch 'develop' into 6593-improve-usability-of-deploy-new-script
ca61688 Jul 10, 2026
7d3f05b
6593: Remove old DeployNewInstance constructor
ca61688 Jul 13, 2026
c10561b
Merge branch 'develop' into 6593-improve-usability-of-deploy-new-script
ca61688 Jul 13, 2026
fa1fa51
6593: Remove unused writeLocalProperties interface
ca61688 Jul 13, 2026
5848f60
6593: Fix DeployExistingInstance
ca61688 Jul 13, 2026
353f694
6593: Fix DeployExistingInstance
ca61688 Jul 13, 2026
9e85d3c
6593: Add logging
ca61688 Jul 13, 2026
314e76e
6593: Add logging for testing
ca61688 Jul 14, 2026
b4ef23b
6593: Change AddTableClient call in DeployNewInstance to match ATC-ma…
ca61688 Jul 14, 2026
d786778
6593: Reload properties to get CDK defined ones
ca61688 Jul 14, 2026
0ff19a3
6593: Fix build error
ca61688 Jul 14, 2026
fbef581
6593: Fix build error
ca61688 Jul 14, 2026
ae9ff7e
Merge branch 'develop' into 6593-improve-usability-of-deploy-new-script
ca61688 Jul 14, 2026
ec9ba16
Merge branch 'develop' into 6593-improve-usability-of-deploy-new-script
ca61688 Jul 15, 2026
3c26600
6593: Add check in DeployInstanceRequest constructor for propertiesFi…
ca61688 Jul 20, 2026
8189067
6593: Move clear directories method into FilesUtil and allow SaveLoca…
ca61688 Jul 20, 2026
b8b722f
Merge branch 'develop' into 6593-improve-usability-of-deploy-new-script
ca61688 Jul 20, 2026
dcb9f88
6593: Build SleeperInstanceConfiguration once in DeployExistingInstance
ca61688 Jul 20, 2026
158a54e
6593: Assert on full DeployNewInstanceRequest
ca61688 Jul 20, 2026
203c375
6593: Add builder for DeployNewInstance class
ca61688 Jul 20, 2026
2c03d43
6593: Remove unused variable
ca61688 Jul 20, 2026
26c7fa3
6593: Seperate InstancePropertiesLoader from FileReader
ca61688 Jul 20, 2026
5e46c40
6593: Make FileReader readFile default method
ca61688 Jul 20, 2026
bbea9da
Merge branch 'develop' into 6593-improve-usability-of-deploy-new-script
ca61688 Jul 22, 2026
640411d
6593: Remove id,vpcId and subnet from DeployNewInstanceIT instance pr…
ca61688 Jul 22, 2026
51e50cc
6593: Remove propertiesFile and configDir from DeployInstanceRequest
ca61688 Jul 22, 2026
5110eb6
6593: Pass network parameters to CDK cml instead of resaving congif file
ca61688 Jul 23, 2026
a08283f
6593: Put FileReader and InstancePropertiesLoader back in AddTableCLient
ca61688 Jul 23, 2026
d82ce18
Merge branch 'develop' into 6593-improve-usability-of-deploy-new-script
ca61688 Jul 23, 2026
c7386e0
6593: Update assertions
rtjd6554 Aug 4, 2026
688ae7b
6593: Adjust instanceProperties creation
rtjd6554 Aug 4, 2026
32606b4
6539: Re-ordered methods
rtjd6554 Aug 4, 2026
1edcb4e
6539: Adjusted constructor accessibility
rtjd6554 Aug 4, 2026
31fe3a4
6539: Re-order methods
rtjd6554 Aug 4, 2026
a6f4311
6593: Remove un-needed validate
rtjd6554 Aug 4, 2026
cc0d9a9
Merge branch 'develop' into 6593-improve-usability-of-deploy-new-script
rtjd6554 Aug 4, 2026
3c9488e
6593: Re-adjust the cdkapp setting
rtjd6554 Aug 4, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
import sleeper.clients.deploy.container.DockerImageConfiguration;
import sleeper.clients.deploy.container.UploadDockerImagesToEcr;
import sleeper.clients.deploy.container.UploadDockerImagesToEcrRequest;
import sleeper.clients.util.ClientUtils;
import sleeper.clients.util.cdk.CdkCommand;
import sleeper.clients.util.cdk.InvokeCdk;
import sleeper.clients.util.console.ConsoleOutput;
Expand All @@ -38,6 +37,7 @@
import sleeper.core.properties.table.TablePropertiesStore;
import sleeper.core.statestore.StateStore;
import sleeper.core.table.TableIndex;
import sleeper.core.util.FilesUtil;
import sleeper.statestore.StateStoreFactory;

import java.io.IOException;
Expand Down Expand Up @@ -305,7 +305,7 @@ public void saveInstanceProperties(InstanceProperties instanceProperties) {
@Override
public void saveLocalProperties(InstanceProperties instanceProperties, Stream<TableProperties> tablePropertiesStream) throws IOException {
Files.createDirectories(localDirectory);
ClientUtils.clearDirectory(localDirectory);
FilesUtil.clearDirectory(localDirectory);
SaveLocalProperties.saveToDirectory(localDirectory, instanceProperties, tablePropertiesStream);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import sleeper.configuration.properties.S3TableProperties;
import sleeper.core.deploy.SleeperInstanceConfiguration;
import sleeper.core.properties.instance.InstanceProperties;
import sleeper.core.properties.local.SaveLocalProperties;
import sleeper.core.properties.model.SleeperInternalCdkApp;
import sleeper.core.properties.table.TableProperties;
import sleeper.core.util.cli.CommandArguments;
Expand All @@ -53,13 +54,15 @@ public class DeployExistingInstance {
private final List<TableProperties> tablePropertiesList;
private final boolean deployPaused;
private final SleeperInternalCdkApp forceCdkApp;
private final Path configDir;

private DeployExistingInstance(Builder builder) {
deployInstance = builder.deployInstance;
properties = builder.properties;
tablePropertiesList = builder.tablePropertiesList;
deployPaused = builder.deployPaused;
forceCdkApp = builder.forceCdkApp;
configDir = builder.configDir;
}

public static Builder builder() {
Expand Down Expand Up @@ -107,6 +110,7 @@ public static void main(String[] rawArgs) throws IOException, InterruptedExcepti
.deployPaused(args.deployPaused())
.forceCdkApp(args.forceCdkApp())
.loadPropertiesFromS3(accountName, s3Client, dynamoClient)
.configDir(args.scriptsDirectory().resolve("generated"))
.build().update();
}
}
Expand All @@ -115,9 +119,11 @@ public record Arguments(Path scriptsDirectory, String instanceId, boolean deploy
}

public void update() throws IOException, InterruptedException {
SaveLocalProperties.createDirectoryAndSaveProperties(configDir, properties, tablePropertiesList.stream());
CdkCommand cdkCommand = deployPaused ? CdkCommand.deployExistingPaused() : CdkCommand.deployExisting();
deployInstance.deploy(DeployInstanceRequest.builder()
.instanceConfig(SleeperInstanceConfiguration.builder().instanceProperties(properties).tableProperties(tablePropertiesList).build())
.cdkCommand(deployPaused ? CdkCommand.deployExistingPaused() : CdkCommand.deployExisting())
.cdkCommand(cdkCommand.withConfigurationDirectory(configDir))
.cdkApp(getCdkApp())
.build());

Expand All @@ -143,6 +149,7 @@ public static final class Builder {
private List<TableProperties> tablePropertiesList;
private boolean deployPaused;
private SleeperInternalCdkApp forceCdkApp;
private Path configDir;

private Builder() {
}
Expand Down Expand Up @@ -181,6 +188,11 @@ public Builder forceCdkApp(SleeperInternalCdkApp forceCdkApp) {
return this;
}

public Builder configDir(Path configDir) {
this.configDir = configDir;
return this;
}

public Builder loadPropertiesFromS3(String accountName, S3Client s3Client, DynamoDbClient dynamoCient) {
properties = S3InstanceProperties.loadGivenAccountAndInstanceId(s3Client, accountName, instanceId);
tablePropertiesList = S3TableProperties.createStore(properties, s3Client, dynamoCient)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,15 +28,12 @@
import sleeper.clients.deploy.container.UploadDockerImagesToEcrRequest;
import sleeper.clients.deploy.jar.SyncJars;
import sleeper.clients.deploy.jar.SyncJarsRequest;
import sleeper.clients.util.ClientUtils;
import sleeper.clients.util.cdk.CdkCommand;
import sleeper.clients.util.cdk.InvokeCdk;
import sleeper.core.deploy.SleeperInstanceConfiguration;
import sleeper.core.properties.instance.InstanceProperties;
import sleeper.core.properties.local.SaveLocalProperties;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;

import static sleeper.core.properties.instance.CommonProperty.ARTEFACTS_DEPLOYMENT_ID;
Expand All @@ -45,18 +42,16 @@
import static sleeper.core.properties.instance.CommonProperty.VPC_ID;
import static sleeper.core.properties.model.SleeperInternalCdkApp.ARTEFACTS;

public class DeployInstance {
public class DeployInstance implements InstanceDeployer {
public static final Logger LOGGER = LoggerFactory.getLogger(DeployInstance.class);

private final SyncJars syncJars;
private final UploadDockerImagesToEcr dockerImageUploader;
private final WriteLocalProperties writeLocalProperties;
private final InvokeCdk invokeCdk;

public DeployInstance(SyncJars syncJars, UploadDockerImagesToEcr dockerImageUploader, WriteLocalProperties writeLocalProperties, InvokeCdk invokeCdk) {
public DeployInstance(SyncJars syncJars, UploadDockerImagesToEcr dockerImageUploader, InvokeCdk invokeCdk) {
this.syncJars = syncJars;
this.dockerImageUploader = dockerImageUploader;
this.writeLocalProperties = writeLocalProperties;
this.invokeCdk = invokeCdk;
}

Expand All @@ -67,7 +62,6 @@ public static DeployInstance fromScriptsDirectory(
new UploadDockerImagesToEcr(
UploadDockerImages.fromScriptsDirectory(scriptsDirectory, ecrClient),
account, region, partitionMetadata),
DeployInstance.WriteLocalProperties.underScriptsDirectory(scriptsDirectory),
InvokeCdk.fromScriptsDirectory(scriptsDirectory));
}

Expand All @@ -87,30 +81,9 @@ public void deploy(DeployInstanceRequest request) throws IOException, Interrupte
syncJars.sync(SyncJarsRequest.from(instanceProperties));
dockerImageUploader.upload(
UploadDockerImagesToEcrRequest.forDeployment(instanceProperties, request.getCdkApp(), DockerImageConfiguration.getDefault()));
Path configurationDirectory = writeLocalProperties.write(instanceConfig);
LOGGER.info("-------------------------------------------------------");
LOGGER.info("Deploying Stacks");
LOGGER.info("-------------------------------------------------------");
invokeCdk.invoke(request.getCdkApp(), request.getCdkCommand().withConfigurationDirectory(configurationDirectory));
}

public interface WriteLocalProperties {
Path write(SleeperInstanceConfiguration instanceConfig) throws IOException;

static WriteLocalProperties underScriptsDirectory(Path scriptsDirectory) {
return toDirectory(scriptsDirectory.resolve("generated"));
}

static WriteLocalProperties toDirectory(Path directory) {
return instanceConfig -> {
LOGGER.info("Writing instance configuration to local directory: {}", directory);
Files.createDirectories(directory);
ClientUtils.clearDirectory(directory);
SaveLocalProperties.saveToDirectory(directory,
instanceConfig.getInstanceProperties(),
instanceConfig.getTableProperties().stream());
return directory;
};
}
invokeCdk.invoke(request.getCdkApp(), request.getCdkCommand());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,26 @@ public CdkCommand getCdkCommand() {
return cdkCommand;
}

@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
DeployInstanceRequest instanceRequest = (DeployInstanceRequest) o;

return Objects.equals(instanceConfig, instanceRequest.instanceConfig) &&
Objects.equals(cdkCommand, instanceRequest.cdkCommand) &&
Objects.equals(cdkApp, instanceRequest.cdkApp);
}

@Override
public int hashCode() {
return Objects.hash(instanceConfig, cdkCommand, cdkApp);
}

public static class Builder {
private SleeperInstanceConfiguration instanceConfig;
private CdkCommand cdkCommand;
Expand Down
Loading
Loading