From 96cf7d36264f66d36031a0ac5cd243a18a7cef7f Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Fri, 14 Nov 2025 08:27:21 +0000 Subject: [PATCH 01/65] 5793: Create CDKDefinedVariants of properties --- .../instance/CdkDefinedInstanceProperty.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/java/core/src/main/java/sleeper/core/properties/instance/CdkDefinedInstanceProperty.java b/java/core/src/main/java/sleeper/core/properties/instance/CdkDefinedInstanceProperty.java index 909ceb57254..b0c8f858461 100644 --- a/java/core/src/main/java/sleeper/core/properties/instance/CdkDefinedInstanceProperty.java +++ b/java/core/src/main/java/sleeper/core/properties/instance/CdkDefinedInstanceProperty.java @@ -765,6 +765,16 @@ static List getAllInGroup(PropertyGroup group) { .propertyGroup(InstancePropertyGroup.BULK_IMPORT) .build(); + //To move to better location + CdkDefinedInstanceProperty ACCOUNT = Index.propertyBuilder("sleeper.account") + .description("The AWS account number. This is the AWS account that the instance will be deployed to.") + .propertyGroup(InstancePropertyGroup.COMMON) + .build(); + CdkDefinedInstanceProperty REGION = Index.propertyBuilder("sleeper.region") + .description("The AWS region to deploy to.") + .propertyGroup(InstancePropertyGroup.COMMON) + .build(); + @Override default boolean isSetByCdk() { return true; From 8e6c763ecb1cbac1c5d8fc29984e890931f27455 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Fri, 14 Nov 2025 11:18:28 +0000 Subject: [PATCH 02/65] 5793: Application of Accout and Region --- .../main/java/sleeper/cdk/SleeperCdkApp.java | 8 +++----- .../java/sleeper/cdk/SleeperInstance.java | 5 +++++ .../instance/CdkDefinedInstanceProperty.java | 20 +++++++++---------- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/java/cdk/src/main/java/sleeper/cdk/SleeperCdkApp.java b/java/cdk/src/main/java/sleeper/cdk/SleeperCdkApp.java index 20a9f43e832..fca1df718b5 100644 --- a/java/cdk/src/main/java/sleeper/cdk/SleeperCdkApp.java +++ b/java/cdk/src/main/java/sleeper/cdk/SleeperCdkApp.java @@ -25,9 +25,7 @@ import sleeper.core.properties.instance.InstanceProperties; -import static sleeper.core.properties.instance.CommonProperty.ACCOUNT; import static sleeper.core.properties.instance.CommonProperty.ID; -import static sleeper.core.properties.instance.CommonProperty.REGION; /** * Deploys an instance of Sleeper, including any configured optional stacks. @@ -49,15 +47,15 @@ public static void main(String[] args) { String id = instanceProperties.get(ID); Environment environment = Environment.builder() - .account(instanceProperties.get(ACCOUNT)) - .region(instanceProperties.get(REGION)) + .account(System.getenv("CDK_DEFAULT_ACCOUNT")) + .region(System.getenv("CDK_DEFAULT_REGION")) .build(); SleeperInstance.createAsRootStack(app, id, StackProps.builder() .stackName(id) .env(environment) .build(), - SleeperInstanceProps.fromContext(app, s3Client, dynamoClient)); + props); instanceProperties.getTags() .forEach((key, value) -> Tags.of(app).add(key, value)); diff --git a/java/cdk/src/main/java/sleeper/cdk/SleeperInstance.java b/java/cdk/src/main/java/sleeper/cdk/SleeperInstance.java index c981bbde178..450235220f6 100644 --- a/java/cdk/src/main/java/sleeper/cdk/SleeperInstance.java +++ b/java/cdk/src/main/java/sleeper/cdk/SleeperInstance.java @@ -27,6 +27,9 @@ import sleeper.cdk.stack.core.SleeperInstanceRoles; import sleeper.core.properties.instance.InstanceProperties; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; + /** * Deploys an instance of Sleeper, including any configured optional stacks. *

@@ -75,6 +78,8 @@ public static SleeperInstance createAsNestedStack(Construct scope, String id, Ne */ public static void createAsRootStack(Construct scope, String id, StackProps stackProps, SleeperInstanceProps sleeperProps) { Stack stack = new Stack(scope, id, stackProps); + sleeperProps.getInstanceProperties().set(ACCOUNT, stack.getAccount()); + sleeperProps.getInstanceProperties().set(REGION, stack.getRegion()); addNestedStacks(stack, sleeperProps); } diff --git a/java/core/src/main/java/sleeper/core/properties/instance/CdkDefinedInstanceProperty.java b/java/core/src/main/java/sleeper/core/properties/instance/CdkDefinedInstanceProperty.java index b0c8f858461..313d0af2c5c 100644 --- a/java/core/src/main/java/sleeper/core/properties/instance/CdkDefinedInstanceProperty.java +++ b/java/core/src/main/java/sleeper/core/properties/instance/CdkDefinedInstanceProperty.java @@ -63,6 +63,16 @@ static List getAllInGroup(PropertyGroup group) { .propertyGroup(InstancePropertyGroup.COMMON) .build(); + //AWS Config + CdkDefinedInstanceProperty ACCOUNT = Index.propertyBuilder("sleeper.account") + .description("The AWS account number. This is the AWS account that the instance will be deployed to.") + .propertyGroup(InstancePropertyGroup.COMMON) + .build(); + CdkDefinedInstanceProperty REGION = Index.propertyBuilder("sleeper.region") + .description("The AWS region to deploy to.") + .propertyGroup(InstancePropertyGroup.COMMON) + .build(); + // Data CdkDefinedInstanceProperty DATA_BUCKET = Index.propertyBuilder("sleeper.data.bucket") .description("The S3 bucket name used to store table data.") @@ -765,16 +775,6 @@ static List getAllInGroup(PropertyGroup group) { .propertyGroup(InstancePropertyGroup.BULK_IMPORT) .build(); - //To move to better location - CdkDefinedInstanceProperty ACCOUNT = Index.propertyBuilder("sleeper.account") - .description("The AWS account number. This is the AWS account that the instance will be deployed to.") - .propertyGroup(InstancePropertyGroup.COMMON) - .build(); - CdkDefinedInstanceProperty REGION = Index.propertyBuilder("sleeper.region") - .description("The AWS region to deploy to.") - .propertyGroup(InstancePropertyGroup.COMMON) - .build(); - @Override default boolean isSetByCdk() { return true; From 3be913d31d8251897ec3e50c2c83b4fba533a1fa Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Fri, 14 Nov 2025 11:23:20 +0000 Subject: [PATCH 03/65] 5793: Remove Account from CommonProperty and repoint imports --- .../bulkimport/runner/dataframe/WriteParquetFilesIT.java | 2 +- .../java/sleeper/cdk/custom/PropertiesWriterLambdaIT.java | 2 +- java/cdk/src/main/java/sleeper/cdk/stack/AthenaStack.java | 2 +- .../cdk/stack/bulkimport/CommonEmrBulkImportStack.java | 2 +- .../cdk/stack/compaction/CompactionOnEc2Resources.java | 2 +- .../java/sleeper/cdk/stack/SleeperInstanceStacksPropsIT.java | 2 +- .../deploy/container/UploadDockerImagesToEcrRequest.java | 2 +- .../deploy/documentation/GeneratePropertiesTemplates.java | 2 +- .../clients/deploy/localstack/DeployDockerInstance.java | 2 +- .../admin/properties/AdminClientPropertiesStoreIT.java | 2 +- .../deploy/container/UploadDockerImagesToEcrTestBase.java | 2 +- .../deploy/properties/PopulateInstancePropertiesAwsIT.java | 2 +- .../documentation/GeneratePropertiesTemplatesTest.java | 2 +- .../src/main/java/sleeper/core/deploy/DockerDeployment.java | 2 +- .../java/sleeper/core/deploy/PopulateInstanceProperties.java | 2 +- .../sleeper/core/properties/instance/CommonProperty.java | 5 ----- .../test/java/sleeper/core/deploy/DockerDeploymentTest.java | 2 +- .../sleeper/core/deploy/PopulateInstancePropertiesTest.java | 2 +- .../java/sleeper/core/properties/SleeperPropertiesTest.java | 2 +- .../core/properties/SleeperPropertiesValidationTest.java | 2 +- .../core/properties/instance/InstancePropertiesTest.java | 2 +- .../properties/testutils/InstancePropertiesTestHelper.java | 2 +- .../src/main/java/sleeper/systemtest/cdk/SystemTestApp.java | 2 +- 23 files changed, 22 insertions(+), 27 deletions(-) diff --git a/java/bulk-import/bulk-import-runner/src/test/java/sleeper/bulkimport/runner/dataframe/WriteParquetFilesIT.java b/java/bulk-import/bulk-import-runner/src/test/java/sleeper/bulkimport/runner/dataframe/WriteParquetFilesIT.java index 71a630871ae..359022b7d5b 100644 --- a/java/bulk-import/bulk-import-runner/src/test/java/sleeper/bulkimport/runner/dataframe/WriteParquetFilesIT.java +++ b/java/bulk-import/bulk-import-runner/src/test/java/sleeper/bulkimport/runner/dataframe/WriteParquetFilesIT.java @@ -42,10 +42,10 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.tuple; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.CONFIG_BUCKET; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.DATA_BUCKET; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.VERSION; -import static sleeper.core.properties.instance.CommonProperty.ACCOUNT; import static sleeper.core.properties.instance.CommonProperty.FILE_SYSTEM; import static sleeper.core.properties.instance.CommonProperty.ID; import static sleeper.core.properties.instance.CommonProperty.JARS_BUCKET; diff --git a/java/cdk-custom-resources/src/test/java/sleeper/cdk/custom/PropertiesWriterLambdaIT.java b/java/cdk-custom-resources/src/test/java/sleeper/cdk/custom/PropertiesWriterLambdaIT.java index 0bce0ade333..ff75931e2fb 100644 --- a/java/cdk-custom-resources/src/test/java/sleeper/cdk/custom/PropertiesWriterLambdaIT.java +++ b/java/cdk-custom-resources/src/test/java/sleeper/cdk/custom/PropertiesWriterLambdaIT.java @@ -27,9 +27,9 @@ import java.util.UUID; import static org.assertj.core.api.Assertions.assertThat; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.CONFIG_BUCKET; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.VERSION; -import static sleeper.core.properties.instance.CommonProperty.ACCOUNT; import static sleeper.core.properties.instance.CommonProperty.ID; import static sleeper.core.properties.instance.CommonProperty.JARS_BUCKET; import static sleeper.core.properties.instance.CommonProperty.REGION; diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/AthenaStack.java b/java/cdk/src/main/java/sleeper/cdk/stack/AthenaStack.java index ca4539e65b1..64dfebe3887 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/AthenaStack.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/AthenaStack.java @@ -50,7 +50,7 @@ import static sleeper.core.properties.instance.AthenaProperty.ATHENA_COMPOSITE_HANDLER_TIMEOUT_IN_SECONDS; import static sleeper.core.properties.instance.AthenaProperty.ATHENA_SPILL_MASTER_KEY_ARN; import static sleeper.core.properties.instance.AthenaProperty.SPILL_BUCKET_AGE_OFF_IN_DAYS; -import static sleeper.core.properties.instance.CommonProperty.ACCOUNT; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; import static sleeper.core.properties.instance.CommonProperty.REGION; @SuppressFBWarnings("NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE") diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/bulkimport/CommonEmrBulkImportStack.java b/java/cdk/src/main/java/sleeper/cdk/stack/bulkimport/CommonEmrBulkImportStack.java index 4dbfc382c3c..2660158c5e3 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/bulkimport/CommonEmrBulkImportStack.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/bulkimport/CommonEmrBulkImportStack.java @@ -52,9 +52,9 @@ import java.util.Map; import java.util.stream.Collectors; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.BULK_IMPORT_EMR_CLUSTER_ROLE_NAME; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.BULK_IMPORT_EMR_EC2_ROLE_NAME; -import static sleeper.core.properties.instance.CommonProperty.ACCOUNT; import static sleeper.core.properties.instance.CommonProperty.JARS_BUCKET; import static sleeper.core.properties.instance.CommonProperty.REGION; import static sleeper.core.properties.instance.CommonProperty.SUBNETS; diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/compaction/CompactionOnEc2Resources.java b/java/cdk/src/main/java/sleeper/cdk/stack/compaction/CompactionOnEc2Resources.java index a16b009075e..6eaaf1ae21c 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/compaction/CompactionOnEc2Resources.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/compaction/CompactionOnEc2Resources.java @@ -74,9 +74,9 @@ import java.util.function.Predicate; import java.util.stream.Collectors; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.COMPACTION_AUTO_SCALING_GROUP; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.COMPACTION_TASK_EC2_DEFINITION_FAMILY; -import static sleeper.core.properties.instance.CommonProperty.ACCOUNT; import static sleeper.core.properties.instance.CommonProperty.ECS_SECURITY_GROUPS; import static sleeper.core.properties.instance.CompactionProperty.COMPACTION_EC2_POOL_MAXIMUM; import static sleeper.core.properties.instance.CompactionProperty.COMPACTION_EC2_POOL_MINIMUM; diff --git a/java/cdk/src/test/java/sleeper/cdk/stack/SleeperInstanceStacksPropsIT.java b/java/cdk/src/test/java/sleeper/cdk/stack/SleeperInstanceStacksPropsIT.java index f8270f91214..06fe0ec8334 100644 --- a/java/cdk/src/test/java/sleeper/cdk/stack/SleeperInstanceStacksPropsIT.java +++ b/java/cdk/src/test/java/sleeper/cdk/stack/SleeperInstanceStacksPropsIT.java @@ -36,9 +36,9 @@ import static org.assertj.core.api.Assertions.assertThatCode; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static sleeper.core.SleeperVersion.getVersion; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.BULK_IMPORT_BUCKET; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.VERSION; -import static sleeper.core.properties.instance.CommonProperty.ACCOUNT; import static sleeper.core.properties.instance.CommonProperty.ID; import static sleeper.core.properties.instance.CommonProperty.JARS_BUCKET; import static sleeper.core.properties.instance.CommonProperty.REGION; diff --git a/java/clients/src/main/java/sleeper/clients/deploy/container/UploadDockerImagesToEcrRequest.java b/java/clients/src/main/java/sleeper/clients/deploy/container/UploadDockerImagesToEcrRequest.java index 0c152d83faa..9411736c9a3 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/container/UploadDockerImagesToEcrRequest.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/container/UploadDockerImagesToEcrRequest.java @@ -25,8 +25,8 @@ import java.util.Optional; import static java.util.Objects.requireNonNull; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.VERSION; -import static sleeper.core.properties.instance.CommonProperty.ACCOUNT; import static sleeper.core.properties.instance.CommonProperty.ECR_REPOSITORY_PREFIX; import static sleeper.core.properties.instance.CommonProperty.REGION; diff --git a/java/clients/src/main/java/sleeper/clients/deploy/documentation/GeneratePropertiesTemplates.java b/java/clients/src/main/java/sleeper/clients/deploy/documentation/GeneratePropertiesTemplates.java index fc138b53bd5..4c120c9a871 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/documentation/GeneratePropertiesTemplates.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/documentation/GeneratePropertiesTemplates.java @@ -39,7 +39,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static sleeper.core.properties.instance.CommonProperty.ACCOUNT; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; import static sleeper.core.properties.instance.CommonProperty.ID; import static sleeper.core.properties.instance.CommonProperty.REGION; import static sleeper.core.properties.instance.CommonProperty.SUBNETS; diff --git a/java/clients/src/main/java/sleeper/clients/deploy/localstack/DeployDockerInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/localstack/DeployDockerInstance.java index 28fa9dc42cb..c13a800617d 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/localstack/DeployDockerInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/localstack/DeployDockerInstance.java @@ -43,9 +43,9 @@ import java.util.function.Consumer; import static sleeper.configuration.utils.AwsV2ClientHelper.buildAwsV2Client; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.CONFIG_BUCKET; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.QUERY_RESULTS_BUCKET; -import static sleeper.core.properties.instance.CommonProperty.ACCOUNT; import static sleeper.core.properties.instance.CommonProperty.ENDPOINT_URL; import static sleeper.core.properties.instance.CommonProperty.ID; import static sleeper.core.properties.instance.CommonProperty.OPTIONAL_STACKS; diff --git a/java/clients/src/test/java/sleeper/clients/admin/properties/AdminClientPropertiesStoreIT.java b/java/clients/src/test/java/sleeper/clients/admin/properties/AdminClientPropertiesStoreIT.java index 9c41f413efd..22f4f6cdfac 100644 --- a/java/clients/src/test/java/sleeper/clients/admin/properties/AdminClientPropertiesStoreIT.java +++ b/java/clients/src/test/java/sleeper/clients/admin/properties/AdminClientPropertiesStoreIT.java @@ -50,8 +50,8 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static sleeper.clients.deploy.container.StackDockerImage.dockerBuildImage; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.VERSION; -import static sleeper.core.properties.instance.CommonProperty.ACCOUNT; import static sleeper.core.properties.instance.CommonProperty.FARGATE_VERSION; import static sleeper.core.properties.instance.CommonProperty.FORCE_RELOAD_PROPERTIES; import static sleeper.core.properties.instance.CommonProperty.ID; diff --git a/java/clients/src/test/java/sleeper/clients/deploy/container/UploadDockerImagesToEcrTestBase.java b/java/clients/src/test/java/sleeper/clients/deploy/container/UploadDockerImagesToEcrTestBase.java index d71daab78a8..8a9875b3290 100644 --- a/java/clients/src/test/java/sleeper/clients/deploy/container/UploadDockerImagesToEcrTestBase.java +++ b/java/clients/src/test/java/sleeper/clients/deploy/container/UploadDockerImagesToEcrTestBase.java @@ -26,8 +26,8 @@ import static sleeper.clients.util.command.Command.command; import static sleeper.clients.util.command.CommandPipeline.pipeline; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.VERSION; -import static sleeper.core.properties.instance.CommonProperty.ACCOUNT; import static sleeper.core.properties.instance.CommonProperty.ID; import static sleeper.core.properties.instance.CommonProperty.REGION; import static sleeper.core.properties.testutils.InstancePropertiesTestHelper.createTestInstanceProperties; diff --git a/java/clients/src/test/java/sleeper/clients/deploy/properties/PopulateInstancePropertiesAwsIT.java b/java/clients/src/test/java/sleeper/clients/deploy/properties/PopulateInstancePropertiesAwsIT.java index 797b18d1ce2..cad4015b646 100644 --- a/java/clients/src/test/java/sleeper/clients/deploy/properties/PopulateInstancePropertiesAwsIT.java +++ b/java/clients/src/test/java/sleeper/clients/deploy/properties/PopulateInstancePropertiesAwsIT.java @@ -25,7 +25,7 @@ import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; -import static sleeper.core.properties.instance.CommonProperty.ACCOUNT; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; import static sleeper.core.properties.instance.CommonProperty.ID; import static sleeper.core.properties.instance.CommonProperty.REGION; import static sleeper.core.properties.instance.CommonProperty.SUBNETS; diff --git a/java/clients/src/test/java/sleeper/clients/documentation/GeneratePropertiesTemplatesTest.java b/java/clients/src/test/java/sleeper/clients/documentation/GeneratePropertiesTemplatesTest.java index cd72991f558..27d422f7722 100644 --- a/java/clients/src/test/java/sleeper/clients/documentation/GeneratePropertiesTemplatesTest.java +++ b/java/clients/src/test/java/sleeper/clients/documentation/GeneratePropertiesTemplatesTest.java @@ -42,7 +42,7 @@ import static java.util.regex.Pattern.DOTALL; import static org.assertj.core.api.Assertions.assertThat; import static sleeper.core.properties.PropertiesUtils.loadProperties; -import static sleeper.core.properties.instance.CommonProperty.ACCOUNT; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; import static sleeper.core.properties.instance.CommonProperty.ID; import static sleeper.core.properties.instance.CommonProperty.REGION; import static sleeper.core.properties.instance.CommonProperty.SUBNETS; diff --git a/java/core/src/main/java/sleeper/core/deploy/DockerDeployment.java b/java/core/src/main/java/sleeper/core/deploy/DockerDeployment.java index 81ae36377f8..298af2fbb7d 100644 --- a/java/core/src/main/java/sleeper/core/deploy/DockerDeployment.java +++ b/java/core/src/main/java/sleeper/core/deploy/DockerDeployment.java @@ -22,8 +22,8 @@ import java.util.Collections; import java.util.List; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.VERSION; -import static sleeper.core.properties.instance.CommonProperty.ACCOUNT; import static sleeper.core.properties.instance.CommonProperty.ECR_REPOSITORY_PREFIX; import static sleeper.core.properties.instance.CommonProperty.REGION; diff --git a/java/core/src/main/java/sleeper/core/deploy/PopulateInstanceProperties.java b/java/core/src/main/java/sleeper/core/deploy/PopulateInstanceProperties.java index 28ddac34a43..368da30ef74 100644 --- a/java/core/src/main/java/sleeper/core/deploy/PopulateInstanceProperties.java +++ b/java/core/src/main/java/sleeper/core/deploy/PopulateInstanceProperties.java @@ -28,9 +28,9 @@ import java.util.function.Consumer; import java.util.function.Supplier; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.CONFIG_BUCKET; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.QUERY_RESULTS_BUCKET; -import static sleeper.core.properties.instance.CommonProperty.ACCOUNT; import static sleeper.core.properties.instance.CommonProperty.ID; import static sleeper.core.properties.instance.CommonProperty.LAMBDA_DEPLOY_TYPE; import static sleeper.core.properties.instance.CommonProperty.REGION; diff --git a/java/core/src/main/java/sleeper/core/properties/instance/CommonProperty.java b/java/core/src/main/java/sleeper/core/properties/instance/CommonProperty.java index b9d79d273e3..e9913536ed0 100644 --- a/java/core/src/main/java/sleeper/core/properties/instance/CommonProperty.java +++ b/java/core/src/main/java/sleeper/core/properties/instance/CommonProperty.java @@ -115,11 +115,6 @@ public interface CommonProperty { .propertyGroup(InstancePropertyGroup.COMMON) .runCdkDeployWhenChanged(true) .build(); - UserDefinedInstanceProperty ACCOUNT = Index.propertyBuilder("sleeper.account") - .description("The AWS account number. This is the AWS account that the instance will be deployed to.") - .validationPredicate(Objects::nonNull) - .propertyGroup(InstancePropertyGroup.COMMON) - .editable(false).build(); UserDefinedInstanceProperty REGION = Index.propertyBuilder("sleeper.region") .description("The AWS region to deploy to.") .validationPredicate(Objects::nonNull) diff --git a/java/core/src/test/java/sleeper/core/deploy/DockerDeploymentTest.java b/java/core/src/test/java/sleeper/core/deploy/DockerDeploymentTest.java index 933af245fc9..a7b4703f21a 100644 --- a/java/core/src/test/java/sleeper/core/deploy/DockerDeploymentTest.java +++ b/java/core/src/test/java/sleeper/core/deploy/DockerDeploymentTest.java @@ -20,8 +20,8 @@ import sleeper.core.properties.instance.InstanceProperties; import static org.assertj.core.api.Assertions.assertThat; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.VERSION; -import static sleeper.core.properties.instance.CommonProperty.ACCOUNT; import static sleeper.core.properties.instance.CommonProperty.ECR_REPOSITORY_PREFIX; import static sleeper.core.properties.instance.CommonProperty.ID; import static sleeper.core.properties.instance.CommonProperty.REGION; diff --git a/java/core/src/test/java/sleeper/core/deploy/PopulateInstancePropertiesTest.java b/java/core/src/test/java/sleeper/core/deploy/PopulateInstancePropertiesTest.java index 0e08b0a5ae0..9bab5a3ffa1 100644 --- a/java/core/src/test/java/sleeper/core/deploy/PopulateInstancePropertiesTest.java +++ b/java/core/src/test/java/sleeper/core/deploy/PopulateInstancePropertiesTest.java @@ -25,6 +25,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static sleeper.core.deploy.PopulatePropertiesTestHelper.createTestPopulateInstanceProperties; import static sleeper.core.deploy.PopulatePropertiesTestHelper.testPopulateInstancePropertiesBuilder; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.BULK_EXPORT_TASK_CREATION_CLOUDWATCH_RULE; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.COMPACTION_JOB_CREATION_CLOUDWATCH_RULE; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.COMPACTION_TASK_CREATION_CLOUDWATCH_RULE; @@ -39,7 +40,6 @@ import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.TRANSACTION_LOG_SNAPSHOT_CREATION_RULE; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.TRANSACTION_LOG_SNAPSHOT_DELETION_RULE; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.TRANSACTION_LOG_TRANSACTION_DELETION_RULE; -import static sleeper.core.properties.instance.CommonProperty.ACCOUNT; import static sleeper.core.properties.instance.CommonProperty.ID; import static sleeper.core.properties.instance.CommonProperty.LAMBDA_DEPLOY_TYPE; import static sleeper.core.properties.instance.CommonProperty.REGION; diff --git a/java/core/src/test/java/sleeper/core/properties/SleeperPropertiesTest.java b/java/core/src/test/java/sleeper/core/properties/SleeperPropertiesTest.java index fc692afd285..805180a3293 100644 --- a/java/core/src/test/java/sleeper/core/properties/SleeperPropertiesTest.java +++ b/java/core/src/test/java/sleeper/core/properties/SleeperPropertiesTest.java @@ -29,7 +29,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static sleeper.core.properties.PropertiesUtils.loadProperties; -import static sleeper.core.properties.instance.CommonProperty.ACCOUNT; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; import static sleeper.core.properties.instance.CommonProperty.OPTIONAL_STACKS; import static sleeper.core.properties.instance.CommonProperty.SUBNETS; import static sleeper.core.properties.instance.CommonProperty.USER_JARS; diff --git a/java/core/src/test/java/sleeper/core/properties/SleeperPropertiesValidationTest.java b/java/core/src/test/java/sleeper/core/properties/SleeperPropertiesValidationTest.java index ced58581375..b678a58c5a3 100644 --- a/java/core/src/test/java/sleeper/core/properties/SleeperPropertiesValidationTest.java +++ b/java/core/src/test/java/sleeper/core/properties/SleeperPropertiesValidationTest.java @@ -29,8 +29,8 @@ import static org.assertj.core.api.Assertions.assertThatCode; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static sleeper.core.properties.PropertiesUtils.loadProperties; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.VERSION; -import static sleeper.core.properties.instance.CommonProperty.ACCOUNT; import static sleeper.core.properties.instance.CommonProperty.ID; import static sleeper.core.properties.instance.CommonProperty.JARS_BUCKET; import static sleeper.core.properties.instance.CommonProperty.LOG_RETENTION_IN_DAYS; diff --git a/java/core/src/test/java/sleeper/core/properties/instance/InstancePropertiesTest.java b/java/core/src/test/java/sleeper/core/properties/instance/InstancePropertiesTest.java index f030ef250cd..0be826a3070 100644 --- a/java/core/src/test/java/sleeper/core/properties/instance/InstancePropertiesTest.java +++ b/java/core/src/test/java/sleeper/core/properties/instance/InstancePropertiesTest.java @@ -33,6 +33,7 @@ import static sleeper.core.properties.PropertiesUtils.loadProperties; import static sleeper.core.properties.instance.ArrayListIngestProperty.MAX_IN_MEMORY_BATCH_SIZE; import static sleeper.core.properties.instance.ArrayListIngestProperty.MAX_ROWS_TO_WRITE_LOCALLY; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.COMPACTION_AUTO_SCALING_GROUP; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.COMPACTION_CLUSTER; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.COMPACTION_JOB_DLQ_URL; @@ -47,7 +48,6 @@ import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.QUERY_QUEUE_URL; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.QUERY_RESULTS_QUEUE_URL; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.VERSION; -import static sleeper.core.properties.instance.CommonProperty.ACCOUNT; import static sleeper.core.properties.instance.CommonProperty.DEFAULT_LAMBDA_CONCURRENCY_RESERVED; import static sleeper.core.properties.instance.CommonProperty.EMAIL_ADDRESS_FOR_ERROR_NOTIFICATION; import static sleeper.core.properties.instance.CommonProperty.FARGATE_VERSION; diff --git a/java/core/src/test/java/sleeper/core/properties/testutils/InstancePropertiesTestHelper.java b/java/core/src/test/java/sleeper/core/properties/testutils/InstancePropertiesTestHelper.java index 2ce8e9bfbc8..2b90e10698a 100644 --- a/java/core/src/test/java/sleeper/core/properties/testutils/InstancePropertiesTestHelper.java +++ b/java/core/src/test/java/sleeper/core/properties/testutils/InstancePropertiesTestHelper.java @@ -31,6 +31,7 @@ import static sleeper.core.properties.instance.ArrowIngestProperty.ARROW_INGEST_MAX_LOCAL_STORE_BYTES; import static sleeper.core.properties.instance.ArrowIngestProperty.ARROW_INGEST_MAX_SINGLE_WRITE_TO_FILE_ROWS; import static sleeper.core.properties.instance.ArrowIngestProperty.ARROW_INGEST_WORKING_BUFFER_BYTES; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.CONFIG_BUCKET; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.DATA_BUCKET; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.QUERY_RESULTS_BUCKET; @@ -44,7 +45,6 @@ import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.TRANSACTION_LOG_LATEST_SNAPSHOTS_TABLENAME; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.TRANSACTION_LOG_PARTITIONS_TABLENAME; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.VERSION; -import static sleeper.core.properties.instance.CommonProperty.ACCOUNT; import static sleeper.core.properties.instance.CommonProperty.ID; import static sleeper.core.properties.instance.CommonProperty.JARS_BUCKET; import static sleeper.core.properties.instance.CommonProperty.MAXIMUM_CONNECTIONS_TO_S3; diff --git a/java/system-test/system-test-cdk/src/main/java/sleeper/systemtest/cdk/SystemTestApp.java b/java/system-test/system-test-cdk/src/main/java/sleeper/systemtest/cdk/SystemTestApp.java index 1f831410ff8..413e93d82a3 100644 --- a/java/system-test/system-test-cdk/src/main/java/sleeper/systemtest/cdk/SystemTestApp.java +++ b/java/system-test/system-test-cdk/src/main/java/sleeper/systemtest/cdk/SystemTestApp.java @@ -33,7 +33,7 @@ import sleeper.core.properties.instance.InstanceProperties; import sleeper.systemtest.configuration.SystemTestProperties; -import static sleeper.core.properties.instance.CommonProperty.ACCOUNT; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; import static sleeper.core.properties.instance.CommonProperty.ID; import static sleeper.core.properties.instance.CommonProperty.REGION; import static sleeper.systemtest.configuration.SystemTestProperty.SYSTEM_TEST_CLUSTER_ENABLED; From de6e28cb0661907b2a5f017bdd59588f556797cf Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Fri, 14 Nov 2025 11:46:10 +0000 Subject: [PATCH 04/65] 5793: Update missing account reference --- .../java/sleeper/core/deploy/PopulateInstanceProperties.java | 2 +- .../sleeper/core/deploy/PopulateInstancePropertiesTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/java/core/src/main/java/sleeper/core/deploy/PopulateInstanceProperties.java b/java/core/src/main/java/sleeper/core/deploy/PopulateInstanceProperties.java index 62b655d0b2a..befb2eac799 100644 --- a/java/core/src/main/java/sleeper/core/deploy/PopulateInstanceProperties.java +++ b/java/core/src/main/java/sleeper/core/deploy/PopulateInstanceProperties.java @@ -27,7 +27,7 @@ import java.util.function.Consumer; import java.util.function.Supplier; -import static sleeper.core.properties.instance.CommonProperty.ACCOUNT; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; import static sleeper.core.properties.instance.CommonProperty.ID; import static sleeper.core.properties.instance.CommonProperty.REGION; import static sleeper.core.properties.instance.CommonProperty.SUBNETS; diff --git a/java/core/src/test/java/sleeper/core/deploy/PopulateInstancePropertiesTest.java b/java/core/src/test/java/sleeper/core/deploy/PopulateInstancePropertiesTest.java index 05016eb72ab..32a67344ad8 100644 --- a/java/core/src/test/java/sleeper/core/deploy/PopulateInstancePropertiesTest.java +++ b/java/core/src/test/java/sleeper/core/deploy/PopulateInstancePropertiesTest.java @@ -24,7 +24,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static sleeper.core.deploy.PopulatePropertiesTestHelper.createTestPopulateInstanceProperties; import static sleeper.core.deploy.PopulatePropertiesTestHelper.testPopulateInstancePropertiesBuilder; -import static sleeper.core.properties.instance.CommonProperty.ACCOUNT; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; import static sleeper.core.properties.instance.CommonProperty.ID; import static sleeper.core.properties.instance.CommonProperty.REGION; import static sleeper.core.properties.instance.CommonProperty.SUBNETS; From 3c4a13501a8d54799305c97b92119153b7b7419c Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Fri, 14 Nov 2025 12:03:12 +0000 Subject: [PATCH 05/65] 5793: Adjust common property Region --- .../bulkimport/runner/dataframe/WriteParquetFilesIT.java | 2 +- .../java/sleeper/cdk/custom/PropertiesWriterLambdaIT.java | 2 +- java/cdk/src/main/java/sleeper/cdk/stack/AthenaStack.java | 2 +- java/cdk/src/main/java/sleeper/cdk/stack/DashboardStack.java | 2 +- .../cdk/stack/bulkexport/BulkExportTaskResources.java | 2 +- .../cdk/stack/bulkimport/CommonEmrBulkImportStack.java | 2 +- .../sleeper/cdk/stack/bulkimport/EksBulkImportStack.java | 2 +- .../cdk/stack/compaction/CompactionTaskResources.java | 2 +- .../src/main/java/sleeper/cdk/stack/core/VpcCheckStack.java | 2 +- .../java/sleeper/cdk/stack/SleeperInstanceStacksPropsIT.java | 2 +- .../java/sleeper/clients/api/role/AssumeSleeperRole.java | 2 +- .../deploy/container/UploadDockerImagesToEcrRequest.java | 2 +- .../deploy/documentation/GeneratePropertiesTemplates.java | 2 +- .../java/sleeper/clients/deploy/jar/JarsBucketCreator.java | 2 +- .../java/sleeper/clients/deploy/jar/SyncJarsRequest.java | 2 +- .../clients/deploy/localstack/DeployDockerInstance.java | 2 +- .../java/sleeper/clients/query/QueryWebSocketConnection.java | 2 +- .../admin/properties/AdminClientPropertiesStoreIT.java | 2 +- .../deploy/container/UploadDockerImagesToEcrTestBase.java | 2 +- .../deploy/properties/PopulateInstancePropertiesAwsIT.java | 2 +- .../deploy/properties/SleeperPropertyMarkdownTableTest.java | 2 +- .../documentation/GeneratePropertiesTemplatesTest.java | 2 +- .../src/main/java/sleeper/core/deploy/DockerDeployment.java | 2 +- .../java/sleeper/core/deploy/PopulateInstanceProperties.java | 2 +- .../sleeper/core/properties/instance/CommonProperty.java | 5 ----- .../test/java/sleeper/core/deploy/DockerDeploymentTest.java | 2 +- .../sleeper/core/deploy/PopulateInstancePropertiesTest.java | 2 +- .../core/properties/SleeperPropertiesValidationTest.java | 2 +- .../core/properties/instance/InstancePropertiesTest.java | 2 +- .../properties/testutils/InstancePropertiesTestHelper.java | 2 +- 30 files changed, 29 insertions(+), 34 deletions(-) diff --git a/java/bulk-import/bulk-import-runner/src/test/java/sleeper/bulkimport/runner/dataframe/WriteParquetFilesIT.java b/java/bulk-import/bulk-import-runner/src/test/java/sleeper/bulkimport/runner/dataframe/WriteParquetFilesIT.java index 359022b7d5b..ac5d9a9f442 100644 --- a/java/bulk-import/bulk-import-runner/src/test/java/sleeper/bulkimport/runner/dataframe/WriteParquetFilesIT.java +++ b/java/bulk-import/bulk-import-runner/src/test/java/sleeper/bulkimport/runner/dataframe/WriteParquetFilesIT.java @@ -45,11 +45,11 @@ import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.CONFIG_BUCKET; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.DATA_BUCKET; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.VERSION; import static sleeper.core.properties.instance.CommonProperty.FILE_SYSTEM; import static sleeper.core.properties.instance.CommonProperty.ID; import static sleeper.core.properties.instance.CommonProperty.JARS_BUCKET; -import static sleeper.core.properties.instance.CommonProperty.REGION; import static sleeper.core.properties.instance.CommonProperty.SUBNETS; import static sleeper.core.properties.instance.CommonProperty.VPC_ID; import static sleeper.core.properties.table.TableProperty.TABLE_NAME; diff --git a/java/cdk-custom-resources/src/test/java/sleeper/cdk/custom/PropertiesWriterLambdaIT.java b/java/cdk-custom-resources/src/test/java/sleeper/cdk/custom/PropertiesWriterLambdaIT.java index ff75931e2fb..5e5d6ed6269 100644 --- a/java/cdk-custom-resources/src/test/java/sleeper/cdk/custom/PropertiesWriterLambdaIT.java +++ b/java/cdk-custom-resources/src/test/java/sleeper/cdk/custom/PropertiesWriterLambdaIT.java @@ -29,10 +29,10 @@ import static org.assertj.core.api.Assertions.assertThat; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.CONFIG_BUCKET; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.VERSION; import static sleeper.core.properties.instance.CommonProperty.ID; import static sleeper.core.properties.instance.CommonProperty.JARS_BUCKET; -import static sleeper.core.properties.instance.CommonProperty.REGION; import static sleeper.core.properties.instance.CommonProperty.SUBNETS; import static sleeper.core.properties.instance.CommonProperty.VPC_ID; diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/AthenaStack.java b/java/cdk/src/main/java/sleeper/cdk/stack/AthenaStack.java index 64dfebe3887..70503639a22 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/AthenaStack.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/AthenaStack.java @@ -51,7 +51,7 @@ import static sleeper.core.properties.instance.AthenaProperty.ATHENA_SPILL_MASTER_KEY_ARN; import static sleeper.core.properties.instance.AthenaProperty.SPILL_BUCKET_AGE_OFF_IN_DAYS; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; -import static sleeper.core.properties.instance.CommonProperty.REGION; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; @SuppressFBWarnings("NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE") public class AthenaStack extends NestedStack { diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/DashboardStack.java b/java/cdk/src/main/java/sleeper/cdk/stack/DashboardStack.java index 1a0eafeb17d..dce21743290 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/DashboardStack.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/DashboardStack.java @@ -43,8 +43,8 @@ import java.util.Map; import java.util.stream.IntStream; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; import static sleeper.core.properties.instance.CommonProperty.ID; -import static sleeper.core.properties.instance.CommonProperty.REGION; import static sleeper.core.properties.instance.MetricsProperty.DASHBOARD_TIME_WINDOW_MINUTES; import static sleeper.core.properties.instance.MetricsProperty.METRICS_NAMESPACE; diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/bulkexport/BulkExportTaskResources.java b/java/cdk/src/main/java/sleeper/cdk/stack/bulkexport/BulkExportTaskResources.java index bc2b83e803d..c56efac3a1f 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/bulkexport/BulkExportTaskResources.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/bulkexport/BulkExportTaskResources.java @@ -60,8 +60,8 @@ import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.BULK_EXPORT_CLUSTER; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.BULK_EXPORT_TASK_CREATION_CLOUDWATCH_RULE; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.BULK_EXPORT_TASK_CREATION_LAMBDA_FUNCTION; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.VERSION; -import static sleeper.core.properties.instance.CommonProperty.REGION; import static sleeper.core.properties.instance.CommonProperty.TASK_RUNNER_LAMBDA_MEMORY_IN_MB; import static sleeper.core.properties.instance.CommonProperty.TASK_RUNNER_LAMBDA_TIMEOUT_IN_SECONDS; import static sleeper.core.properties.instance.CommonProperty.VPC_ID; diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/bulkimport/CommonEmrBulkImportStack.java b/java/cdk/src/main/java/sleeper/cdk/stack/bulkimport/CommonEmrBulkImportStack.java index 2660158c5e3..fbdfe6b68a5 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/bulkimport/CommonEmrBulkImportStack.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/bulkimport/CommonEmrBulkImportStack.java @@ -55,8 +55,8 @@ import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.BULK_IMPORT_EMR_CLUSTER_ROLE_NAME; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.BULK_IMPORT_EMR_EC2_ROLE_NAME; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; import static sleeper.core.properties.instance.CommonProperty.JARS_BUCKET; -import static sleeper.core.properties.instance.CommonProperty.REGION; import static sleeper.core.properties.instance.CommonProperty.SUBNETS; import static sleeper.core.properties.instance.CommonProperty.VPC_ID; import static sleeper.core.properties.instance.EMRProperty.BULK_IMPORT_EMR_EBS_ENCRYPTION_KEY_ARN; diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/bulkimport/EksBulkImportStack.java b/java/cdk/src/main/java/sleeper/cdk/stack/bulkimport/EksBulkImportStack.java index 463ad64d735..2c1c01c2bbf 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/bulkimport/EksBulkImportStack.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/bulkimport/EksBulkImportStack.java @@ -83,8 +83,8 @@ import static sleeper.core.properties.instance.BulkImportProperty.BULK_IMPORT_STARTER_LAMBDA_MEMORY; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.BULK_IMPORT_EKS_JOB_QUEUE_ARN; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.BULK_IMPORT_EKS_JOB_QUEUE_URL; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; import static sleeper.core.properties.instance.CommonProperty.JARS_BUCKET; -import static sleeper.core.properties.instance.CommonProperty.REGION; import static sleeper.core.properties.instance.CommonProperty.SUBNETS; import static sleeper.core.properties.instance.CommonProperty.VPC_ID; import static sleeper.core.properties.instance.EKSProperty.EKS_CLUSTER_ADMIN_ROLES; diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/compaction/CompactionTaskResources.java b/java/cdk/src/main/java/sleeper/cdk/stack/compaction/CompactionTaskResources.java index 2ec63a40286..6a7c8d335b5 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/compaction/CompactionTaskResources.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/compaction/CompactionTaskResources.java @@ -61,8 +61,8 @@ import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.COMPACTION_CLUSTER; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.COMPACTION_TASK_CREATION_CLOUDWATCH_RULE; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.COMPACTION_TASK_CREATION_LAMBDA_FUNCTION; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.VERSION; -import static sleeper.core.properties.instance.CommonProperty.REGION; import static sleeper.core.properties.instance.CommonProperty.TASK_RUNNER_LAMBDA_MEMORY_IN_MB; import static sleeper.core.properties.instance.CommonProperty.TASK_RUNNER_LAMBDA_TIMEOUT_IN_SECONDS; import static sleeper.core.properties.instance.CommonProperty.VPC_ID; diff --git a/java/cdk/src/main/java/sleeper/cdk/stack/core/VpcCheckStack.java b/java/cdk/src/main/java/sleeper/cdk/stack/core/VpcCheckStack.java index 4a275454857..168c01dea32 100644 --- a/java/cdk/src/main/java/sleeper/cdk/stack/core/VpcCheckStack.java +++ b/java/cdk/src/main/java/sleeper/cdk/stack/core/VpcCheckStack.java @@ -38,7 +38,7 @@ import java.util.List; import java.util.Map; -import static sleeper.core.properties.instance.CommonProperty.REGION; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; import static sleeper.core.properties.instance.CommonProperty.VPC_ID; public class VpcCheckStack extends NestedStack { diff --git a/java/cdk/src/test/java/sleeper/cdk/stack/SleeperInstanceStacksPropsIT.java b/java/cdk/src/test/java/sleeper/cdk/stack/SleeperInstanceStacksPropsIT.java index 06fe0ec8334..8a0f170a76e 100644 --- a/java/cdk/src/test/java/sleeper/cdk/stack/SleeperInstanceStacksPropsIT.java +++ b/java/cdk/src/test/java/sleeper/cdk/stack/SleeperInstanceStacksPropsIT.java @@ -38,10 +38,10 @@ import static sleeper.core.SleeperVersion.getVersion; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.BULK_IMPORT_BUCKET; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.VERSION; import static sleeper.core.properties.instance.CommonProperty.ID; import static sleeper.core.properties.instance.CommonProperty.JARS_BUCKET; -import static sleeper.core.properties.instance.CommonProperty.REGION; import static sleeper.core.properties.instance.CommonProperty.SUBNETS; import static sleeper.core.properties.instance.CommonProperty.VPC_ID; diff --git a/java/clients/src/main/java/sleeper/clients/api/role/AssumeSleeperRole.java b/java/clients/src/main/java/sleeper/clients/api/role/AssumeSleeperRole.java index 8ed07cfc4d8..0fc3e336a13 100644 --- a/java/clients/src/main/java/sleeper/clients/api/role/AssumeSleeperRole.java +++ b/java/clients/src/main/java/sleeper/clients/api/role/AssumeSleeperRole.java @@ -29,8 +29,8 @@ import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ADMIN_ROLE_ARN; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.INGEST_BY_QUEUE_ROLE_ARN; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.INGEST_DIRECT_ROLE_ARN; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; import static sleeper.core.properties.instance.CommonProperty.ENDPOINT_URL; -import static sleeper.core.properties.instance.CommonProperty.REGION; public class AssumeSleeperRole { public static final Logger LOGGER = LoggerFactory.getLogger(AssumeSleeperRole.class); diff --git a/java/clients/src/main/java/sleeper/clients/deploy/container/UploadDockerImagesToEcrRequest.java b/java/clients/src/main/java/sleeper/clients/deploy/container/UploadDockerImagesToEcrRequest.java index 9411736c9a3..47caadd6eaa 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/container/UploadDockerImagesToEcrRequest.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/container/UploadDockerImagesToEcrRequest.java @@ -26,9 +26,9 @@ import static java.util.Objects.requireNonNull; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.VERSION; import static sleeper.core.properties.instance.CommonProperty.ECR_REPOSITORY_PREFIX; -import static sleeper.core.properties.instance.CommonProperty.REGION; public class UploadDockerImagesToEcrRequest { private final String ecrPrefix; diff --git a/java/clients/src/main/java/sleeper/clients/deploy/documentation/GeneratePropertiesTemplates.java b/java/clients/src/main/java/sleeper/clients/deploy/documentation/GeneratePropertiesTemplates.java index 4c120c9a871..48ad5c1dddb 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/documentation/GeneratePropertiesTemplates.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/documentation/GeneratePropertiesTemplates.java @@ -40,8 +40,8 @@ import java.util.stream.Stream; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; import static sleeper.core.properties.instance.CommonProperty.ID; -import static sleeper.core.properties.instance.CommonProperty.REGION; import static sleeper.core.properties.instance.CommonProperty.SUBNETS; import static sleeper.core.properties.instance.CommonProperty.VPC_ID; import static sleeper.core.properties.instance.CompactionProperty.DEFAULT_SIZERATIO_COMPACTION_STRATEGY_MAX_CONCURRENT_JOBS_PER_PARTITION; diff --git a/java/clients/src/main/java/sleeper/clients/deploy/jar/JarsBucketCreator.java b/java/clients/src/main/java/sleeper/clients/deploy/jar/JarsBucketCreator.java index f6a030aa267..5bf40a65688 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/jar/JarsBucketCreator.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/jar/JarsBucketCreator.java @@ -24,8 +24,8 @@ import sleeper.core.properties.instance.InstanceProperties; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; import static sleeper.core.properties.instance.CommonProperty.JARS_BUCKET; -import static sleeper.core.properties.instance.CommonProperty.REGION; /** * A tool to create a jars bucket that matches how it is created by the CDK. Usually only used for testing. diff --git a/java/clients/src/main/java/sleeper/clients/deploy/jar/SyncJarsRequest.java b/java/clients/src/main/java/sleeper/clients/deploy/jar/SyncJarsRequest.java index 0cc0d1bd0a1..8a56df32fd1 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/jar/SyncJarsRequest.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/jar/SyncJarsRequest.java @@ -21,8 +21,8 @@ import java.util.Objects; import java.util.function.Predicate; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; import static sleeper.core.properties.instance.CommonProperty.JARS_BUCKET; -import static sleeper.core.properties.instance.CommonProperty.REGION; public class SyncJarsRequest { diff --git a/java/clients/src/main/java/sleeper/clients/deploy/localstack/DeployDockerInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/localstack/DeployDockerInstance.java index c13a800617d..c175e3465b7 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/localstack/DeployDockerInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/localstack/DeployDockerInstance.java @@ -46,10 +46,10 @@ import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.CONFIG_BUCKET; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.QUERY_RESULTS_BUCKET; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; import static sleeper.core.properties.instance.CommonProperty.ENDPOINT_URL; import static sleeper.core.properties.instance.CommonProperty.ID; import static sleeper.core.properties.instance.CommonProperty.OPTIONAL_STACKS; -import static sleeper.core.properties.instance.CommonProperty.REGION; import static sleeper.core.properties.instance.CommonProperty.SUBNETS; import static sleeper.core.properties.instance.CommonProperty.VPC_ID; import static sleeper.core.properties.instance.CompactionProperty.COMPACTION_TASK_DELAY_BEFORE_RETRY_IN_SECONDS; diff --git a/java/clients/src/main/java/sleeper/clients/query/QueryWebSocketConnection.java b/java/clients/src/main/java/sleeper/clients/query/QueryWebSocketConnection.java index d20c95162cd..a8ec82d0708 100644 --- a/java/clients/src/main/java/sleeper/clients/query/QueryWebSocketConnection.java +++ b/java/clients/src/main/java/sleeper/clients/query/QueryWebSocketConnection.java @@ -32,7 +32,7 @@ import java.net.URI; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.QUERY_WEBSOCKET_API_URL; -import static sleeper.core.properties.instance.CommonProperty.REGION; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; class QueryWebSocketConnection extends WebSocketClient implements QueryWebSocketClient.Connection { public static final Logger LOGGER = LoggerFactory.getLogger(QueryWebSocketConnection.class); diff --git a/java/clients/src/test/java/sleeper/clients/admin/properties/AdminClientPropertiesStoreIT.java b/java/clients/src/test/java/sleeper/clients/admin/properties/AdminClientPropertiesStoreIT.java index 22f4f6cdfac..e9647c94775 100644 --- a/java/clients/src/test/java/sleeper/clients/admin/properties/AdminClientPropertiesStoreIT.java +++ b/java/clients/src/test/java/sleeper/clients/admin/properties/AdminClientPropertiesStoreIT.java @@ -51,12 +51,12 @@ import static org.mockito.Mockito.verifyNoInteractions; import static sleeper.clients.deploy.container.StackDockerImage.dockerBuildImage; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.VERSION; import static sleeper.core.properties.instance.CommonProperty.FARGATE_VERSION; import static sleeper.core.properties.instance.CommonProperty.FORCE_RELOAD_PROPERTIES; import static sleeper.core.properties.instance.CommonProperty.ID; import static sleeper.core.properties.instance.CommonProperty.OPTIONAL_STACKS; -import static sleeper.core.properties.instance.CommonProperty.REGION; import static sleeper.core.properties.instance.CommonProperty.TASK_RUNNER_LAMBDA_MEMORY_IN_MB; import static sleeper.core.properties.local.LoadLocalProperties.loadInstancePropertiesFromDirectory; import static sleeper.core.properties.local.LoadLocalProperties.loadTablesFromDirectory; diff --git a/java/clients/src/test/java/sleeper/clients/deploy/container/UploadDockerImagesToEcrTestBase.java b/java/clients/src/test/java/sleeper/clients/deploy/container/UploadDockerImagesToEcrTestBase.java index 8a9875b3290..6e2e1121634 100644 --- a/java/clients/src/test/java/sleeper/clients/deploy/container/UploadDockerImagesToEcrTestBase.java +++ b/java/clients/src/test/java/sleeper/clients/deploy/container/UploadDockerImagesToEcrTestBase.java @@ -27,9 +27,9 @@ import static sleeper.clients.util.command.Command.command; import static sleeper.clients.util.command.CommandPipeline.pipeline; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.VERSION; import static sleeper.core.properties.instance.CommonProperty.ID; -import static sleeper.core.properties.instance.CommonProperty.REGION; import static sleeper.core.properties.testutils.InstancePropertiesTestHelper.createTestInstanceProperties; public abstract class UploadDockerImagesToEcrTestBase extends DockerImagesTestBase { diff --git a/java/clients/src/test/java/sleeper/clients/deploy/properties/PopulateInstancePropertiesAwsIT.java b/java/clients/src/test/java/sleeper/clients/deploy/properties/PopulateInstancePropertiesAwsIT.java index cad4015b646..2ac06995f35 100644 --- a/java/clients/src/test/java/sleeper/clients/deploy/properties/PopulateInstancePropertiesAwsIT.java +++ b/java/clients/src/test/java/sleeper/clients/deploy/properties/PopulateInstancePropertiesAwsIT.java @@ -26,8 +26,8 @@ import static org.assertj.core.api.Assertions.assertThat; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; import static sleeper.core.properties.instance.CommonProperty.ID; -import static sleeper.core.properties.instance.CommonProperty.REGION; import static sleeper.core.properties.instance.CommonProperty.SUBNETS; import static sleeper.core.properties.instance.CommonProperty.VPC_ID; diff --git a/java/clients/src/test/java/sleeper/clients/deploy/properties/SleeperPropertyMarkdownTableTest.java b/java/clients/src/test/java/sleeper/clients/deploy/properties/SleeperPropertyMarkdownTableTest.java index 30f632097d3..5b86f22486f 100644 --- a/java/clients/src/test/java/sleeper/clients/deploy/properties/SleeperPropertyMarkdownTableTest.java +++ b/java/clients/src/test/java/sleeper/clients/deploy/properties/SleeperPropertyMarkdownTableTest.java @@ -36,7 +36,7 @@ public class SleeperPropertyMarkdownTableTest { void shouldWriteTable() throws IOException { // Given TableWriter writer = SleeperPropertyMarkdownTable.createTableWriterForUserDefinedProperties(List.of( - CommonProperty.REGION, + CdkDefinedInstanceProperty.REGION, CommonProperty.FILE_SYSTEM)); // When diff --git a/java/clients/src/test/java/sleeper/clients/documentation/GeneratePropertiesTemplatesTest.java b/java/clients/src/test/java/sleeper/clients/documentation/GeneratePropertiesTemplatesTest.java index 27d422f7722..73aee0e6a01 100644 --- a/java/clients/src/test/java/sleeper/clients/documentation/GeneratePropertiesTemplatesTest.java +++ b/java/clients/src/test/java/sleeper/clients/documentation/GeneratePropertiesTemplatesTest.java @@ -43,8 +43,8 @@ import static org.assertj.core.api.Assertions.assertThat; import static sleeper.core.properties.PropertiesUtils.loadProperties; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; import static sleeper.core.properties.instance.CommonProperty.ID; -import static sleeper.core.properties.instance.CommonProperty.REGION; import static sleeper.core.properties.instance.CommonProperty.SUBNETS; import static sleeper.core.properties.instance.CommonProperty.VPC_ID; import static sleeper.core.properties.table.TableProperty.ITERATOR_CLASS_NAME; diff --git a/java/core/src/main/java/sleeper/core/deploy/DockerDeployment.java b/java/core/src/main/java/sleeper/core/deploy/DockerDeployment.java index 298af2fbb7d..b7e6361a298 100644 --- a/java/core/src/main/java/sleeper/core/deploy/DockerDeployment.java +++ b/java/core/src/main/java/sleeper/core/deploy/DockerDeployment.java @@ -23,9 +23,9 @@ import java.util.List; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.VERSION; import static sleeper.core.properties.instance.CommonProperty.ECR_REPOSITORY_PREFIX; -import static sleeper.core.properties.instance.CommonProperty.REGION; /** * A deployment of a Sleeper component that is deployed with Docker. Can be used to derive the Docker image names and diff --git a/java/core/src/main/java/sleeper/core/deploy/PopulateInstanceProperties.java b/java/core/src/main/java/sleeper/core/deploy/PopulateInstanceProperties.java index befb2eac799..2b65ef3a93f 100644 --- a/java/core/src/main/java/sleeper/core/deploy/PopulateInstanceProperties.java +++ b/java/core/src/main/java/sleeper/core/deploy/PopulateInstanceProperties.java @@ -28,8 +28,8 @@ import java.util.function.Supplier; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; import static sleeper.core.properties.instance.CommonProperty.ID; -import static sleeper.core.properties.instance.CommonProperty.REGION; import static sleeper.core.properties.instance.CommonProperty.SUBNETS; import static sleeper.core.properties.instance.CommonProperty.VPC_ID; diff --git a/java/core/src/main/java/sleeper/core/properties/instance/CommonProperty.java b/java/core/src/main/java/sleeper/core/properties/instance/CommonProperty.java index e9913536ed0..8ade9400801 100644 --- a/java/core/src/main/java/sleeper/core/properties/instance/CommonProperty.java +++ b/java/core/src/main/java/sleeper/core/properties/instance/CommonProperty.java @@ -115,11 +115,6 @@ public interface CommonProperty { .propertyGroup(InstancePropertyGroup.COMMON) .runCdkDeployWhenChanged(true) .build(); - UserDefinedInstanceProperty REGION = Index.propertyBuilder("sleeper.region") - .description("The AWS region to deploy to.") - .validationPredicate(Objects::nonNull) - .propertyGroup(InstancePropertyGroup.COMMON) - .editable(false).build(); UserDefinedInstanceProperty ENDPOINT_URL = Index.propertyBuilder("sleeper.endpoint.url") .description("The AWS endpoint URL. This should only be set for a non-standard service endpoint. Usually " + "this is used to set the URL to LocalStack for a locally deployed instance.") diff --git a/java/core/src/test/java/sleeper/core/deploy/DockerDeploymentTest.java b/java/core/src/test/java/sleeper/core/deploy/DockerDeploymentTest.java index a7b4703f21a..b3fd947de67 100644 --- a/java/core/src/test/java/sleeper/core/deploy/DockerDeploymentTest.java +++ b/java/core/src/test/java/sleeper/core/deploy/DockerDeploymentTest.java @@ -21,10 +21,10 @@ import static org.assertj.core.api.Assertions.assertThat; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.VERSION; import static sleeper.core.properties.instance.CommonProperty.ECR_REPOSITORY_PREFIX; import static sleeper.core.properties.instance.CommonProperty.ID; -import static sleeper.core.properties.instance.CommonProperty.REGION; public class DockerDeploymentTest { diff --git a/java/core/src/test/java/sleeper/core/deploy/PopulateInstancePropertiesTest.java b/java/core/src/test/java/sleeper/core/deploy/PopulateInstancePropertiesTest.java index 32a67344ad8..af3aa21beec 100644 --- a/java/core/src/test/java/sleeper/core/deploy/PopulateInstancePropertiesTest.java +++ b/java/core/src/test/java/sleeper/core/deploy/PopulateInstancePropertiesTest.java @@ -25,8 +25,8 @@ import static sleeper.core.deploy.PopulatePropertiesTestHelper.createTestPopulateInstanceProperties; import static sleeper.core.deploy.PopulatePropertiesTestHelper.testPopulateInstancePropertiesBuilder; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; import static sleeper.core.properties.instance.CommonProperty.ID; -import static sleeper.core.properties.instance.CommonProperty.REGION; import static sleeper.core.properties.instance.CommonProperty.SUBNETS; import static sleeper.core.properties.instance.CommonProperty.VPC_ID; import static sleeper.core.properties.instance.PartitionSplittingProperty.DEFAULT_PARTITION_SPLIT_THRESHOLD; diff --git a/java/core/src/test/java/sleeper/core/properties/SleeperPropertiesValidationTest.java b/java/core/src/test/java/sleeper/core/properties/SleeperPropertiesValidationTest.java index b678a58c5a3..4214c9100e1 100644 --- a/java/core/src/test/java/sleeper/core/properties/SleeperPropertiesValidationTest.java +++ b/java/core/src/test/java/sleeper/core/properties/SleeperPropertiesValidationTest.java @@ -30,12 +30,12 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; import static sleeper.core.properties.PropertiesUtils.loadProperties; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.VERSION; import static sleeper.core.properties.instance.CommonProperty.ID; import static sleeper.core.properties.instance.CommonProperty.JARS_BUCKET; import static sleeper.core.properties.instance.CommonProperty.LOG_RETENTION_IN_DAYS; import static sleeper.core.properties.instance.CommonProperty.MAXIMUM_CONNECTIONS_TO_S3; -import static sleeper.core.properties.instance.CommonProperty.REGION; import static sleeper.core.properties.instance.CommonProperty.SUBNETS; import static sleeper.core.properties.instance.CommonProperty.VPC_ID; import static sleeper.core.properties.table.TableProperty.COMPACTION_JOB_SEND_TIMEOUT_SECS; diff --git a/java/core/src/test/java/sleeper/core/properties/instance/InstancePropertiesTest.java b/java/core/src/test/java/sleeper/core/properties/instance/InstancePropertiesTest.java index 0be826a3070..693fa417d6a 100644 --- a/java/core/src/test/java/sleeper/core/properties/instance/InstancePropertiesTest.java +++ b/java/core/src/test/java/sleeper/core/properties/instance/InstancePropertiesTest.java @@ -47,6 +47,7 @@ import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.QUERY_LAMBDA_ROLE; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.QUERY_QUEUE_URL; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.QUERY_RESULTS_QUEUE_URL; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.VERSION; import static sleeper.core.properties.instance.CommonProperty.DEFAULT_LAMBDA_CONCURRENCY_RESERVED; import static sleeper.core.properties.instance.CommonProperty.EMAIL_ADDRESS_FOR_ERROR_NOTIFICATION; @@ -56,7 +57,6 @@ import static sleeper.core.properties.instance.CommonProperty.JARS_BUCKET; import static sleeper.core.properties.instance.CommonProperty.LOG_RETENTION_IN_DAYS; import static sleeper.core.properties.instance.CommonProperty.MAXIMUM_CONNECTIONS_TO_S3; -import static sleeper.core.properties.instance.CommonProperty.REGION; import static sleeper.core.properties.instance.CommonProperty.SUBNETS; import static sleeper.core.properties.instance.CommonProperty.TASK_RUNNER_LAMBDA_MEMORY_IN_MB; import static sleeper.core.properties.instance.CommonProperty.TASK_RUNNER_LAMBDA_TIMEOUT_IN_SECONDS; diff --git a/java/core/src/test/java/sleeper/core/properties/testutils/InstancePropertiesTestHelper.java b/java/core/src/test/java/sleeper/core/properties/testutils/InstancePropertiesTestHelper.java index 2b90e10698a..e6d108bcbd9 100644 --- a/java/core/src/test/java/sleeper/core/properties/testutils/InstancePropertiesTestHelper.java +++ b/java/core/src/test/java/sleeper/core/properties/testutils/InstancePropertiesTestHelper.java @@ -37,6 +37,7 @@ import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.QUERY_RESULTS_BUCKET; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.QUERY_TRACKER_TABLE_NAME; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.QUERY_WEBSOCKET_API_URL; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.TABLE_ID_INDEX_DYNAMO_TABLENAME; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.TABLE_NAME_INDEX_DYNAMO_TABLENAME; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.TABLE_ONLINE_INDEX_DYNAMO_TABLENAME; @@ -48,7 +49,6 @@ import static sleeper.core.properties.instance.CommonProperty.ID; import static sleeper.core.properties.instance.CommonProperty.JARS_BUCKET; import static sleeper.core.properties.instance.CommonProperty.MAXIMUM_CONNECTIONS_TO_S3; -import static sleeper.core.properties.instance.CommonProperty.REGION; import static sleeper.core.properties.instance.CommonProperty.SUBNETS; import static sleeper.core.properties.instance.CommonProperty.VPC_ID; import static sleeper.core.properties.instance.IngestProperty.INGEST_PARTITION_REFRESH_PERIOD_IN_SECONDS; From e8aab6d7c9d1ce2dab5e6a1abd65842f74824223 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Fri, 14 Nov 2025 12:04:23 +0000 Subject: [PATCH 06/65] 5793: Update documentation --- docs/usage/properties/instance/cdk/common.md | 2 ++ docs/usage/properties/instance/user/common.md | 2 -- example/basic/instance.properties | 6 ------ example/full/instance.properties | 6 ------ .../src/main/java/sleeper/systemtest/cdk/SystemTestApp.java | 2 +- scripts/templates/instanceproperties.template | 6 ------ 6 files changed, 3 insertions(+), 21 deletions(-) diff --git a/docs/usage/properties/instance/cdk/common.md b/docs/usage/properties/instance/cdk/common.md index b9a108613b0..33dc75949d7 100644 --- a/docs/usage/properties/instance/cdk/common.md +++ b/docs/usage/properties/instance/cdk/common.md @@ -7,6 +7,8 @@ The following instance properties are commonly used throughout Sleeper. | sleeper.version | The version of Sleeper that is being used. This property is used to identify the correct jars in the S3 jars bucket and to select the correct tag in the ECR repositories. | | sleeper.admin.role.arn | The ARN of a role that has permissions to administer the instance. | | sleeper.config.bucket | The S3 bucket name used to store configuration files. | +| sleeper.account | The AWS account number. This is the AWS account that the instance will be deployed to. | +| sleeper.region | The AWS region to deploy to. | | sleeper.data.bucket | The S3 bucket name used to store table data. | | sleeper.tables.name.index.dynamo.table | The name of the DynamoDB table indexing tables by their name. | | sleeper.tables.id.index.dynamo.table | The name of the DynamoDB table indexing tables by their ID. | diff --git a/docs/usage/properties/instance/user/common.md b/docs/usage/properties/instance/user/common.md index 1293f3c8aae..6ea0a41c7d9 100644 --- a/docs/usage/properties/instance/user/common.md +++ b/docs/usage/properties/instance/user/common.md @@ -15,8 +15,6 @@ The following instance properties are commonly used throughout Sleeper. | sleeper.retain.logs.after.destroy | Whether to keep the sleeper log groups when the instance is destroyed. | true | true | | sleeper.optional.stacks | The optional stacks to deploy. Not case sensitive.
Valid values: [IngestStack, IngestBatcherStack, EmrServerlessBulkImportStack, EmrBulkImportStack, PersistentEmrBulkImportStack, EksBulkImportStack, EmrStudioStack, BulkExportStack, QueryStack, WebSocketQueryStack, AthenaStack, KeepLambdaWarmStack, CompactionStack, GarbageCollectorStack, PartitionSplittingStack, DashboardStack, TableMetricsStack] | IngestStack,IngestBatcherStack,EmrServerlessBulkImportStack,EmrStudioStack,QueryStack,AthenaStack,CompactionStack,GarbageCollectorStack,PartitionSplittingStack,DashboardStack,TableMetricsStack | true | | sleeper.lambda.deploy.type | The deployment type for AWS Lambda. Not case sensitive.
There are two types of Lambda deployments, jar and container.
If the size of the jar file is too large, it will always be deployed as a container.
Valid values: [jar, container] | jar | true | -| sleeper.account | The AWS account number. This is the AWS account that the instance will be deployed to. | | false | -| sleeper.region | The AWS region to deploy to. | | false | | sleeper.endpoint.url | The AWS endpoint URL. This should only be set for a non-standard service endpoint. Usually this is used to set the URL to LocalStack for a locally deployed instance. | | false | | sleeper.vpc | The id of the VPC to deploy to. | | false | | sleeper.vpc.endpoint.check | Whether to check that the VPC that the instance is deployed to has an S3 endpoint. If there is no S3 endpoint then the NAT costs can be very significant. | true | false | diff --git a/example/basic/instance.properties b/example/basic/instance.properties index bef1178267b..3f6b591db30 100644 --- a/example/basic/instance.properties +++ b/example/basic/instance.properties @@ -25,12 +25,6 @@ sleeper.retain.logs.after.destroy=true # PartitionSplittingStack, DashboardStack, TableMetricsStack] sleeper.optional.stacks=IngestStack,IngestBatcherStack,EmrServerlessBulkImportStack,EmrStudioStack,QueryStack,AthenaStack,CompactionStack,GarbageCollectorStack,PartitionSplittingStack,DashboardStack,TableMetricsStack -# The AWS account number. This is the AWS account that the instance will be deployed to. -sleeper.account=1234567890 - -# The AWS region to deploy to. -sleeper.region=eu-west-2 - # The id of the VPC to deploy to. sleeper.vpc=1234567890 diff --git a/example/full/instance.properties b/example/full/instance.properties index 8ab12c80ad2..e3b1cd91553 100644 --- a/example/full/instance.properties +++ b/example/full/instance.properties @@ -51,12 +51,6 @@ sleeper.optional.stacks=IngestStack,IngestBatcherStack,EmrServerlessBulkImportSt # Valid values: [jar, container] sleeper.lambda.deploy.type=jar -# The AWS account number. This is the AWS account that the instance will be deployed to. -sleeper.account=1234567890 - -# The AWS region to deploy to. -sleeper.region=eu-west-2 - # The AWS endpoint URL. This should only be set for a non-standard service endpoint. Usually this is # used to set the URL to LocalStack for a locally deployed instance. # sleeper.endpoint.url= diff --git a/java/system-test/system-test-cdk/src/main/java/sleeper/systemtest/cdk/SystemTestApp.java b/java/system-test/system-test-cdk/src/main/java/sleeper/systemtest/cdk/SystemTestApp.java index 413e93d82a3..5ca23015bfd 100644 --- a/java/system-test/system-test-cdk/src/main/java/sleeper/systemtest/cdk/SystemTestApp.java +++ b/java/system-test/system-test-cdk/src/main/java/sleeper/systemtest/cdk/SystemTestApp.java @@ -34,8 +34,8 @@ import sleeper.systemtest.configuration.SystemTestProperties; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; import static sleeper.core.properties.instance.CommonProperty.ID; -import static sleeper.core.properties.instance.CommonProperty.REGION; import static sleeper.systemtest.configuration.SystemTestProperty.SYSTEM_TEST_CLUSTER_ENABLED; /** diff --git a/scripts/templates/instanceproperties.template b/scripts/templates/instanceproperties.template index c207e6657b8..d8d80915765 100644 --- a/scripts/templates/instanceproperties.template +++ b/scripts/templates/instanceproperties.template @@ -27,12 +27,6 @@ sleeper.jars.bucket=changeme # ECR repository names are generated in the format `/`. sleeper.ecr.repository.prefix=changeme -# The AWS account number. This is the AWS account that the instance will be deployed to. -sleeper.account=changeme - -# The AWS region to deploy to. -sleeper.region=changeme - # The id of the VPC to deploy to. sleeper.vpc=changeme From 2df45457cf73a68f5788f4e0858f9dbd89dd6095 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Mon, 17 Nov 2025 09:49:19 +0000 Subject: [PATCH 07/65] 5793: Test updates for property changes --- .../core/properties/SleeperPropertiesPrettyPrinterTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/java/core/src/test/java/sleeper/core/properties/SleeperPropertiesPrettyPrinterTest.java b/java/core/src/test/java/sleeper/core/properties/SleeperPropertiesPrettyPrinterTest.java index 87b823cbd74..8bf063296a7 100644 --- a/java/core/src/test/java/sleeper/core/properties/SleeperPropertiesPrettyPrinterTest.java +++ b/java/core/src/test/java/sleeper/core/properties/SleeperPropertiesPrettyPrinterTest.java @@ -102,7 +102,7 @@ void shouldPrintPropertiesInTheCorrectOrder() { String output = printEmptyInstanceProperties(); // Then - assertThat(output.indexOf("sleeper.account")) + assertThat(output.indexOf("sleeper.endpoint.url")) .isLessThan(output.indexOf("sleeper.log.retention.days")) .isLessThan(output.indexOf("sleeper.vpc")); assertThat(output.indexOf("sleeper.ingest")) @@ -118,6 +118,7 @@ void shouldPrintPropertyValueWithDescription() { // When / Then assertThat(printInstanceProperties("sleeper.account=1234567890")) .contains("# The AWS account number. This is the AWS account that the instance will be deployed to.\n" + + "# (this property is system-defined and may not be edited)\n" + "sleeper.account=1234567890\n"); } From f1896ce059615b90fb38c141b75751b50b27e178 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Mon, 17 Nov 2025 13:02:35 +0000 Subject: [PATCH 08/65] 5793: Update to SleeperPropertiesValidationTest --- .../core/properties/SleeperPropertiesValidationTest.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/java/core/src/test/java/sleeper/core/properties/SleeperPropertiesValidationTest.java b/java/core/src/test/java/sleeper/core/properties/SleeperPropertiesValidationTest.java index 4214c9100e1..d47d1591308 100644 --- a/java/core/src/test/java/sleeper/core/properties/SleeperPropertiesValidationTest.java +++ b/java/core/src/test/java/sleeper/core/properties/SleeperPropertiesValidationTest.java @@ -29,8 +29,6 @@ import static org.assertj.core.api.Assertions.assertThatCode; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static sleeper.core.properties.PropertiesUtils.loadProperties; -import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; -import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.VERSION; import static sleeper.core.properties.instance.CommonProperty.ID; import static sleeper.core.properties.instance.CommonProperty.JARS_BUCKET; @@ -114,16 +112,14 @@ class ValidateInstanceProperties { void shouldFailValidationIfRequiredPropertyIsMissing() { // Given - no account set InstanceProperties instanceProperties = new InstanceProperties(); - instanceProperties.set(REGION, "eu-west-2"); instanceProperties.set(JARS_BUCKET, "jars"); instanceProperties.set(VERSION, "0.1"); - instanceProperties.set(ID, "test"); instanceProperties.set(VPC_ID, "aVPC"); instanceProperties.set(SUBNETS, "subnet1"); // When / Then assertThatThrownBy(instanceProperties::validate) - .hasMessageContaining(ACCOUNT.getPropertyName()); + .hasMessageContaining(ID.getPropertyName()); } @Test From a47612a28608260d5ad25eb53cab3dbe2a90b74f Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Mon, 17 Nov 2025 14:19:22 +0000 Subject: [PATCH 09/65] 5793: Update mandatory details for Account and Region --- .../documentation/GeneratePropertiesTemplatesTest.java | 4 ---- java/clients/src/test/resources/reports/table/property.txt | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/java/clients/src/test/java/sleeper/clients/documentation/GeneratePropertiesTemplatesTest.java b/java/clients/src/test/java/sleeper/clients/documentation/GeneratePropertiesTemplatesTest.java index 73aee0e6a01..2523bf8fd8d 100644 --- a/java/clients/src/test/java/sleeper/clients/documentation/GeneratePropertiesTemplatesTest.java +++ b/java/clients/src/test/java/sleeper/clients/documentation/GeneratePropertiesTemplatesTest.java @@ -42,8 +42,6 @@ import static java.util.regex.Pattern.DOTALL; import static org.assertj.core.api.Assertions.assertThat; import static sleeper.core.properties.PropertiesUtils.loadProperties; -import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; -import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; import static sleeper.core.properties.instance.CommonProperty.ID; import static sleeper.core.properties.instance.CommonProperty.SUBNETS; import static sleeper.core.properties.instance.CommonProperty.VPC_ID; @@ -66,8 +64,6 @@ static class MandatoryInstancePropertyTemplateValues implements ArgumentsProvide public Stream provideArguments(ExtensionContext context) { return Stream.of( Arguments.of(ID, "full-example"), - Arguments.of(ACCOUNT, "1234567890"), - Arguments.of(REGION, "eu-west-2"), Arguments.of(VPC_ID, "1234567890"), Arguments.of(SUBNETS, "subnet-abcdefgh")); } diff --git a/java/clients/src/test/resources/reports/table/property.txt b/java/clients/src/test/resources/reports/table/property.txt index 83d001b81bb..f73e316658d 100644 --- a/java/clients/src/test/resources/reports/table/property.txt +++ b/java/clients/src/test/resources/reports/table/property.txt @@ -1,4 +1,4 @@ | Property Name | Description | Default Value | Run CDK Deploy When Changed | |--------------------|----------------------------------------------|---------------|-----------------------------| -| sleeper.region | The AWS region to deploy to. | | false | +| sleeper.region | The AWS region to deploy to. | | true | | sleeper.filesystem | The Hadoop filesystem used to connect to S3. | s3a:// | false | From 52fb4db67a31431318d02381f24d3f5a3d02a011 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Mon, 17 Nov 2025 15:38:46 +0000 Subject: [PATCH 10/65] 5793: Remove properties not UserDefined --- .../java/sleeper/cdk/stack/SleeperInstanceStacksPropsIT.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/java/cdk/src/test/java/sleeper/cdk/stack/SleeperInstanceStacksPropsIT.java b/java/cdk/src/test/java/sleeper/cdk/stack/SleeperInstanceStacksPropsIT.java index 8a0f170a76e..5a40ef2bf05 100644 --- a/java/cdk/src/test/java/sleeper/cdk/stack/SleeperInstanceStacksPropsIT.java +++ b/java/cdk/src/test/java/sleeper/cdk/stack/SleeperInstanceStacksPropsIT.java @@ -36,9 +36,7 @@ import static org.assertj.core.api.Assertions.assertThatCode; import static org.assertj.core.api.Assertions.assertThatThrownBy; import static sleeper.core.SleeperVersion.getVersion; -import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.BULK_IMPORT_BUCKET; -import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.VERSION; import static sleeper.core.properties.instance.CommonProperty.ID; import static sleeper.core.properties.instance.CommonProperty.JARS_BUCKET; @@ -197,8 +195,6 @@ private static InstanceProperties createUserDefinedInstanceProperties() { InstanceProperties instanceProperties = new InstanceProperties(); instanceProperties.set(ID, id); instanceProperties.set(JARS_BUCKET, "test-bucket"); - instanceProperties.set(ACCOUNT, "test-account"); - instanceProperties.set(REGION, "test-region"); instanceProperties.set(VPC_ID, "test-vpc"); instanceProperties.set(SUBNETS, "test-subnet"); return instanceProperties; From aae952b59fd9ee799f8661e600921124b9e676e7 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Tue, 18 Nov 2025 09:29:10 +0000 Subject: [PATCH 11/65] 5793: Update change for ID tag property --- java/cdk/src/main/java/sleeper/cdk/SleeperCdkApp.java | 8 +++++++- .../sleeper/core/deploy/PopulateInstanceProperties.java | 4 ---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/java/cdk/src/main/java/sleeper/cdk/SleeperCdkApp.java b/java/cdk/src/main/java/sleeper/cdk/SleeperCdkApp.java index fca1df718b5..48ca076cc1a 100644 --- a/java/cdk/src/main/java/sleeper/cdk/SleeperCdkApp.java +++ b/java/cdk/src/main/java/sleeper/cdk/SleeperCdkApp.java @@ -25,6 +25,8 @@ import sleeper.core.properties.instance.InstanceProperties; +import java.util.Properties; + import static sleeper.core.properties.instance.CommonProperty.ID; /** @@ -44,8 +46,12 @@ public static void main(String[] args) { DynamoDbClient dynamoClient = DynamoDbClient.create()) { SleeperInstanceProps props = SleeperInstanceProps.fromContext(app, s3Client, dynamoClient); InstanceProperties instanceProperties = props.getInstanceProperties(); - String id = instanceProperties.get(ID); + + Properties tagsProperties = instanceProperties.getTagsProperties(); + tagsProperties.setProperty("InstanceID", id); + instanceProperties.loadTags(tagsProperties); + Environment environment = Environment.builder() .account(System.getenv("CDK_DEFAULT_ACCOUNT")) .region(System.getenv("CDK_DEFAULT_REGION")) diff --git a/java/core/src/main/java/sleeper/core/deploy/PopulateInstanceProperties.java b/java/core/src/main/java/sleeper/core/deploy/PopulateInstanceProperties.java index 2b65ef3a93f..07b255e1653 100644 --- a/java/core/src/main/java/sleeper/core/deploy/PopulateInstanceProperties.java +++ b/java/core/src/main/java/sleeper/core/deploy/PopulateInstanceProperties.java @@ -23,7 +23,6 @@ import sleeper.core.properties.instance.UserDefinedInstanceProperty; import java.util.Objects; -import java.util.Properties; import java.util.function.Consumer; import java.util.function.Supplier; @@ -67,9 +66,6 @@ public static Builder builder() { */ public InstanceProperties populate(InstanceProperties properties) { properties.set(ID, instanceId); - Properties tagsProperties = properties.getTagsProperties(); - tagsProperties.setProperty("InstanceID", instanceId); - properties.loadTags(tagsProperties); properties.set(ACCOUNT, accountSupplier.get()); properties.set(REGION, regionIdSupplier.get()); properties.set(VPC_ID, vpcId); From 838ef0da43c718c4724f11c4615f017b8b42cbb3 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Wed, 19 Nov 2025 13:26:12 +0000 Subject: [PATCH 12/65] 5793: Update to remove populate for production code --- .../clients/deploy/DeployNewInstance.java | 23 +++++++++++-------- .../drivers/cdk/DeployNewTestInstance.java | 3 +-- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java index 2285e07cb5a..2ccd67d7e42 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java @@ -17,8 +17,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import software.amazon.awssdk.regions.providers.AwsRegionProvider; -import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.ecr.EcrClient; import software.amazon.awssdk.services.s3.S3Client; @@ -29,7 +27,6 @@ import sleeper.clients.deploy.container.UploadDockerImages; import sleeper.clients.deploy.container.UploadDockerImagesToEcr; import sleeper.clients.deploy.jar.SyncJars; -import sleeper.clients.deploy.properties.PopulateInstancePropertiesAws; import sleeper.clients.table.AddTable; import sleeper.clients.util.cdk.CdkCommand; import sleeper.clients.util.cdk.InvokeCdk; @@ -38,7 +35,6 @@ import sleeper.configuration.properties.S3InstanceProperties; import sleeper.configuration.properties.S3TableProperties; import sleeper.core.deploy.DeployInstanceConfiguration; -import sleeper.core.deploy.PopulateInstanceProperties; import sleeper.core.properties.instance.InstanceProperties; import sleeper.core.properties.table.TableProperties; import sleeper.statestore.StateStoreFactory; @@ -48,6 +44,9 @@ import java.util.List; import static sleeper.clients.util.ClientUtils.optionalArgument; +import static sleeper.core.properties.instance.CommonProperty.ID; +import static sleeper.core.properties.instance.CommonProperty.SUBNETS; +import static sleeper.core.properties.instance.CommonProperty.VPC_ID; public class DeployNewInstance { private static final Logger LOGGER = LoggerFactory.getLogger(DeployNewInstance.class); @@ -83,20 +82,24 @@ public static void main(String[] args) throws IOException, InterruptedException throw new IllegalArgumentException("Usage: " + " "); } - AwsRegionProvider regionProvider = DefaultAwsRegionProviderChain.builder().build(); try (S3Client s3Client = S3Client.create(); DynamoDbClient dynamoClient = DynamoDbClient.create(); StsClient stsClient = StsClient.create(); EcrClient ecrClient = EcrClient.create()) { Path scriptsDirectory = Path.of(args[0]); - PopulateInstanceProperties populateInstanceProperties = PopulateInstancePropertiesAws.builder(stsClient, regionProvider) - .instanceId(args[1]).vpcId(args[2]).subnetIds(args[3]) - .build(); + Path instancePropertiesFile = optionalArgument(args, 4).map(Path::of).orElse(null); boolean deployPaused = "true".equalsIgnoreCase(optionalArgument(args, 5).orElse("false")); + + DeployInstanceConfiguration config = DeployInstanceConfiguration.forNewInstanceDefaultingInstance( + instancePropertiesFile, scriptsDirectory.resolve("templates")); + + config.getInstanceProperties().set(ID, args[1]); + config.getInstanceProperties().set(VPC_ID, args[2]); + config.getInstanceProperties().set(SUBNETS, args[3]); + builder().scriptsDirectory(scriptsDirectory) - .deployInstanceConfiguration(DeployInstanceConfiguration.forNewInstanceDefaultingInstance( - instancePropertiesFile, populateInstanceProperties, scriptsDirectory.resolve("templates"))) + .deployInstanceConfiguration(config) .deployPaused(deployPaused) .instanceType(InvokeCdk.Type.STANDARD) .deployWithClients(s3Client, dynamoClient, ecrClient); diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java index c60322d5f37..574c89b6d23 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java @@ -65,8 +65,7 @@ public static void main(String[] args) throws IOException, InterruptedException Path splitPointsFileForTemplate = optionalArgument(args, 6).map(Path::of).orElse(null); DeployNewInstance.builder().scriptsDirectory(scriptsDirectory) .deployInstanceConfiguration(DeployInstanceConfiguration.forNewInstanceDefaultingTables( - propertiesFile, populateInstanceProperties, - templates(scriptsDirectory, splitPointsFileForTemplate))) + propertiesFile, templates(scriptsDirectory, splitPointsFileForTemplate))) .extraDockerImages(List.of(SYSTEM_TEST_IMAGE)) .deployPaused(deployPaused) .instanceType(InvokeCdk.Type.SYSTEM_TEST) From cff70890c200458a5b479d6471b9fc804fff8811 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Wed, 19 Nov 2025 13:26:46 +0000 Subject: [PATCH 13/65] 5793: Add missing configuration file --- .../deploy/DeployInstanceConfiguration.java | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/java/core/src/main/java/sleeper/core/deploy/DeployInstanceConfiguration.java b/java/core/src/main/java/sleeper/core/deploy/DeployInstanceConfiguration.java index 5c3dfc95e7c..ceb648a2c9b 100644 --- a/java/core/src/main/java/sleeper/core/deploy/DeployInstanceConfiguration.java +++ b/java/core/src/main/java/sleeper/core/deploy/DeployInstanceConfiguration.java @@ -73,9 +73,8 @@ public static DeployInstanceConfiguration withNoTables(InstanceProperties instan * @return the instance configuration */ public static DeployInstanceConfiguration forNewInstanceDefaultingTables( - Path instancePropertiesPath, PopulateInstanceProperties populateInstanceProperties, - DeployInstanceConfigurationFromTemplates fromTemplates) { - DeployInstanceConfiguration configuration = fromLocalConfiguration(instancePropertiesPath, populateInstanceProperties); + Path instancePropertiesPath, DeployInstanceConfigurationFromTemplates fromTemplates) { + DeployInstanceConfiguration configuration = fromLocalConfiguration(instancePropertiesPath); if (configuration.getTableProperties().isEmpty()) { configuration = configuration.withTableProperties(instanceProperties -> List.of( fromTemplates.loadTableProperties(instanceProperties))); @@ -94,13 +93,11 @@ public static DeployInstanceConfiguration forNewInstanceDefaultingTables( * @return the instance configuration */ public static DeployInstanceConfiguration forNewInstanceDefaultingInstance( - Path instancePropertiesPath, PopulateInstanceProperties populateInstanceProperties, - Path templatesDir) { + Path instancePropertiesPath, Path templatesDir) { if (instancePropertiesPath != null) { - return fromLocalConfiguration(instancePropertiesPath, populateInstanceProperties); + return fromLocalConfiguration(instancePropertiesPath); } else { InstanceProperties instanceProperties = DeployInstanceConfigurationFromTemplates.loadInstanceProperties(templatesDir); - populateInstanceProperties.populate(instanceProperties); return new DeployInstanceConfiguration(instanceProperties, List.of()); } } @@ -121,13 +118,6 @@ public static DeployInstanceConfiguration fromLocalConfiguration(Path instancePr .tableProperties(tableProperties).build(); } - private static DeployInstanceConfiguration fromLocalConfiguration( - Path instancePropertiesPath, PopulateInstanceProperties populateInstanceProperties) { - DeployInstanceConfiguration configuration = fromLocalConfiguration(instancePropertiesPath); - populateInstanceProperties.populate(configuration.getInstanceProperties()); - return configuration; - } - /** * Checks that the configuration is valid and throws an exception otherwise. * From 262238c399026c74fd418ac3c9bee3d852ede0e9 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Wed, 19 Nov 2025 13:31:34 +0000 Subject: [PATCH 14/65] 5793: Refactor DeployInstanceConfigurationIT --- .../deploy/DeployInstanceConfigurationIT.java | 23 ++++++++----------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/java/core/src/test/java/sleeper/core/deploy/DeployInstanceConfigurationIT.java b/java/core/src/test/java/sleeper/core/deploy/DeployInstanceConfigurationIT.java index d00b8e76921..d3d82075da4 100644 --- a/java/core/src/test/java/sleeper/core/deploy/DeployInstanceConfigurationIT.java +++ b/java/core/src/test/java/sleeper/core/deploy/DeployInstanceConfigurationIT.java @@ -34,7 +34,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -import static sleeper.core.deploy.PopulatePropertiesTestHelper.createTestPopulateInstanceProperties; import static sleeper.core.properties.instance.CommonProperty.FILE_SYSTEM; import static sleeper.core.properties.instance.CommonProperty.ID; import static sleeper.core.properties.table.TableProperty.ROW_GROUP_SIZE; @@ -163,14 +162,13 @@ void shouldPopulateInstancePropertiesFromLocalConfig() throws Exception { Path instancePropertiesPath = Files.writeString( propertiesDir.resolve("instance.properties"), "sleeper.filesystem=test://"); - PopulateInstanceProperties populateProperties = createTestPopulateInstanceProperties(); // When DeployInstanceConfiguration config = DeployInstanceConfiguration.forNewInstanceDefaultingInstance( - instancePropertiesPath, populateProperties, templatesDir); + instancePropertiesPath, templatesDir); // Then - InstanceProperties expectedInstanceProperties = populateProperties.populate(new InstanceProperties()); + InstanceProperties expectedInstanceProperties = new InstanceProperties(); expectedInstanceProperties.set(FILE_SYSTEM, "test://"); assertThat(config).isEqualTo(new DeployInstanceConfiguration(expectedInstanceProperties, List.of())); } @@ -179,14 +177,13 @@ void shouldPopulateInstancePropertiesFromLocalConfig() throws Exception { void shouldPopulateInstancePropertiesFromTemplate() throws Exception { // Given writeTemplates(); - PopulateInstanceProperties populateProperties = createTestPopulateInstanceProperties(); - // When DeployInstanceConfiguration config = DeployInstanceConfiguration.forNewInstanceDefaultingInstance( - null, populateProperties, templatesDir); + null, templatesDir); // Then - InstanceProperties expectedInstanceProperties = populateProperties.populate(new InstanceProperties()); + InstanceProperties expectedInstanceProperties = new InstanceProperties(); + Map tags = new HashMap<>(expectedInstanceProperties.getTags()); tags.put("Project", "TemplateProject"); expectedInstanceProperties.setTags(tags); @@ -209,17 +206,16 @@ void shouldPopulateTablePropertiesFromLocalConfig() throws Exception { Files.writeString( propertiesDir.resolve("table.properties"), "sleeper.table.name=some-table"); - PopulateInstanceProperties populateProperties = createTestPopulateInstanceProperties(); DeployInstanceConfigurationFromTemplates fromTemplates = DeployInstanceConfigurationFromTemplates.builder() .templatesDir(templatesDir) .tableNameForTemplate("template-table").build(); // When DeployInstanceConfiguration config = DeployInstanceConfiguration.forNewInstanceDefaultingTables( - instancePropertiesPath, populateProperties, fromTemplates); + instancePropertiesPath, fromTemplates); // Then - InstanceProperties expectedInstanceProperties = populateProperties.populate(new InstanceProperties()); + InstanceProperties expectedInstanceProperties = new InstanceProperties(); expectedInstanceProperties.set(FILE_SYSTEM, "test://"); TableProperties expectedTableProperties = new TableProperties(expectedInstanceProperties); expectedTableProperties.set(TABLE_NAME, "some-table"); @@ -233,17 +229,16 @@ void shouldPopulateTablePropertiesFromTemplate() throws Exception { Path instancePropertiesPath = Files.writeString( propertiesDir.resolve("instance.properties"), "sleeper.filesystem=test://"); - PopulateInstanceProperties populateProperties = createTestPopulateInstanceProperties(); DeployInstanceConfigurationFromTemplates fromTemplates = DeployInstanceConfigurationFromTemplates.builder() .templatesDir(templatesDir) .tableNameForTemplate("template-table").build(); // When DeployInstanceConfiguration config = DeployInstanceConfiguration.forNewInstanceDefaultingTables( - instancePropertiesPath, populateProperties, fromTemplates); + instancePropertiesPath, fromTemplates); // Then - InstanceProperties expectedInstanceProperties = populateProperties.populate(new InstanceProperties()); + InstanceProperties expectedInstanceProperties = new InstanceProperties(); expectedInstanceProperties.set(FILE_SYSTEM, "test://"); TableProperties expectedTableProperties = new TableProperties(expectedInstanceProperties); expectedTableProperties.set(TABLE_NAME, "template-table"); From 27983bace7b65377bec4256ef746097ff797992b Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Wed, 19 Nov 2025 14:46:48 +0000 Subject: [PATCH 15/65] 5793: Correct class usage --- .../sleeper/core/deploy/SleeperInstanceConfigurationIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/core/src/test/java/sleeper/core/deploy/SleeperInstanceConfigurationIT.java b/java/core/src/test/java/sleeper/core/deploy/SleeperInstanceConfigurationIT.java index 71d364fda65..90e3c83ccad 100644 --- a/java/core/src/test/java/sleeper/core/deploy/SleeperInstanceConfigurationIT.java +++ b/java/core/src/test/java/sleeper/core/deploy/SleeperInstanceConfigurationIT.java @@ -206,7 +206,7 @@ void shouldPopulateTablePropertiesFromLocalConfig() throws Exception { Files.writeString( propertiesDir.resolve("table.properties"), "sleeper.table.name=some-table"); - SleeperInstanceConfiguration fromTemplates = SleeperInstanceConfiguration.builder() + SleeperInstanceConfigurationFromTemplates fromTemplates = SleeperInstanceConfigurationFromTemplates.builder() .templatesDir(templatesDir) .tableNameForTemplate("template-table").build(); @@ -229,7 +229,7 @@ void shouldPopulateTablePropertiesFromTemplate() throws Exception { Path instancePropertiesPath = Files.writeString( propertiesDir.resolve("instance.properties"), "sleeper.filesystem=test://"); - SleeperInstanceConfiguration fromTemplates = SleeperInstanceConfiguration.builder() + SleeperInstanceConfigurationFromTemplates fromTemplates = SleeperInstanceConfigurationFromTemplates.builder() .templatesDir(templatesDir) .tableNameForTemplate("template-table").build(); From 319baadbea474340658d699436ce872376098b78 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Fri, 21 Nov 2025 13:35:14 +0000 Subject: [PATCH 16/65] 5793: Remove remaining populate usage in DeployInstance --- .../java/sleeper/clients/deploy/DeployInstance.java | 12 +++--------- .../core/deploy/PopulateInstancePropertiesTest.java | 2 +- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployInstance.java index 20d88363d6f..8908d244b7f 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployInstance.java @@ -17,8 +17,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import software.amazon.awssdk.regions.providers.AwsRegionProvider; -import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain; import software.amazon.awssdk.services.ecr.EcrClient; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.sts.StsClient; @@ -29,7 +27,6 @@ import sleeper.clients.deploy.container.UploadDockerImagesToEcrRequest; import sleeper.clients.deploy.jar.SyncJars; import sleeper.clients.deploy.jar.SyncJarsRequest; -import sleeper.clients.deploy.properties.PopulateInstancePropertiesAws; import sleeper.clients.util.ClientUtils; import sleeper.clients.util.cdk.CdkCommand; import sleeper.clients.util.cdk.InvokeCdk; @@ -79,17 +76,14 @@ public static void main(String[] args) throws IOException, InterruptedException Set options = Stream.of(args).skip(5).map(String::toLowerCase).collect(toUnmodifiableSet()); boolean deployPaused = options.contains("--deploy-paused"); - AwsRegionProvider regionProvider = DefaultAwsRegionProviderChain.builder().build(); try (S3Client s3Client = S3Client.create(); EcrClient ecrClient = EcrClient.create(); StsClient stsClient = StsClient.create()) { SleeperInstanceConfiguration instanceConfiguration = SleeperInstanceConfiguration.fromLocalConfiguration(propertiesFile); - PopulateInstancePropertiesAws.builder(stsClient, regionProvider) - .instanceId(instanceId) - .vpcId(vpcId) - .subnetIds(subnetIds) - .build().populate(instanceConfiguration.getInstanceProperties()); + instanceConfiguration.getInstanceProperties().set(ID, instanceId); + instanceConfiguration.getInstanceProperties().set(VPC_ID, vpcId); + instanceConfiguration.getInstanceProperties().set(SUBNETS, subnetIds); PopulateInstanceProperties.setFromEnvironmentVariables(instanceConfiguration.getInstanceProperties()); instanceConfiguration.validate(); diff --git a/java/core/src/test/java/sleeper/core/deploy/PopulateInstancePropertiesTest.java b/java/core/src/test/java/sleeper/core/deploy/PopulateInstancePropertiesTest.java index af3aa21beec..9ca781f38c1 100644 --- a/java/core/src/test/java/sleeper/core/deploy/PopulateInstancePropertiesTest.java +++ b/java/core/src/test/java/sleeper/core/deploy/PopulateInstancePropertiesTest.java @@ -35,7 +35,7 @@ public class PopulateInstancePropertiesTest { private InstanceProperties expectedInstanceProperties() { InstanceProperties expected = new InstanceProperties(); - expected.setTags(Map.of("InstanceID", "test-instance")); + //expected.setTags(Map.of("InstanceID", "test-instance")); expected.set(ID, "test-instance"); expected.set(VPC_ID, "some-vpc"); expected.set(SUBNETS, "some-subnet"); From ac65c53ae7d1c0b668df71017010446fa7f0bd14 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Fri, 21 Nov 2025 13:58:18 +0000 Subject: [PATCH 17/65] 5793: Update property definitions --- docs/usage/properties/instance/user/common.md | 58 +++++++++---------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/docs/usage/properties/instance/user/common.md b/docs/usage/properties/instance/user/common.md index 6ea0a41c7d9..967cf4a69e0 100644 --- a/docs/usage/properties/instance/user/common.md +++ b/docs/usage/properties/instance/user/common.md @@ -2,32 +2,32 @@ The following instance properties are commonly used throughout Sleeper. -| Property Name | Description | Default Value | Run CDK Deploy When Changed | -|---------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------| -| sleeper.id | A string to uniquely identify this deployment. This should be no longer than 20 chars. It should be globally unique as it will be used to name AWS resources such as S3 buckets. | | false | -| sleeper.artefacts.deployment | The ID of the artefacts deployment to use to deploy the Sleeper instance. By default we assume an artefacts deployment with the same ID as the Sleeper instance. This property is used to compute the default values of `sleeper.jars.bucket` and `sleeper.ecr.repository.prefix`. | | true | -| sleeper.jars.bucket | The S3 bucket containing the jar files of the Sleeper components. If unset, a default name is computed from `sleeper.artefacts.deployment` if it is set, or `sleeper.id` if it is not. | | true | -| sleeper.ecr.repository.prefix | If set, this property will be used as a prefix for the names of ECR repositories. If unset, a default prefix is computed from `sleeper.artefacts.deployment` if it is set, or `sleeper.id` if it is not.
ECR repository names are generated in the format `/`. | | true | -| sleeper.userjars | A comma-separated list of the jars containing application specific iterator code. These jars are assumed to be in the bucket given by `sleeper.jars.bucket`. For example, if that bucket contains two iterator jars called iterator1.jar and iterator2.jar then the property should be 'sleeper.userjars=iterator1.jar,iterator2.jar'. | | false | -| sleeper.tags | A list of tags that will automatically be applied to all the resources in this deployment of Sleeper. The list should be in the form "key1,value1,key2,value2,key3,value3,...".
For example if you want to add tags of "user=some-user" and "project-name=sleeper-test", then the list should be "user,some-user,project-name,sleeper-test". | | true | -| sleeper.stack.tag.name | A name for a tag to identify the stack that deployed a resource. This will be set for all AWS resources, to the ID of the CDK stack that they are deployed under. This can be used to organise the cost explorer for billing. | DeploymentStack | true | -| sleeper.retain.infra.after.destroy | Whether to keep the sleeper table bucket, Dynamo tables, query results bucket, etc., when the instance is destroyed. | true | true | -| sleeper.retain.logs.after.destroy | Whether to keep the sleeper log groups when the instance is destroyed. | true | true | -| sleeper.optional.stacks | The optional stacks to deploy. Not case sensitive.
Valid values: [IngestStack, IngestBatcherStack, EmrServerlessBulkImportStack, EmrBulkImportStack, PersistentEmrBulkImportStack, EksBulkImportStack, EmrStudioStack, BulkExportStack, QueryStack, WebSocketQueryStack, AthenaStack, KeepLambdaWarmStack, CompactionStack, GarbageCollectorStack, PartitionSplittingStack, DashboardStack, TableMetricsStack] | IngestStack,IngestBatcherStack,EmrServerlessBulkImportStack,EmrStudioStack,QueryStack,AthenaStack,CompactionStack,GarbageCollectorStack,PartitionSplittingStack,DashboardStack,TableMetricsStack | true | -| sleeper.lambda.deploy.type | The deployment type for AWS Lambda. Not case sensitive.
There are two types of Lambda deployments, jar and container.
If the size of the jar file is too large, it will always be deployed as a container.
Valid values: [jar, container] | jar | true | -| sleeper.endpoint.url | The AWS endpoint URL. This should only be set for a non-standard service endpoint. Usually this is used to set the URL to LocalStack for a locally deployed instance. | | false | -| sleeper.vpc | The id of the VPC to deploy to. | | false | -| sleeper.vpc.endpoint.check | Whether to check that the VPC that the instance is deployed to has an S3 endpoint. If there is no S3 endpoint then the NAT costs can be very significant. | true | false | -| sleeper.subnets | A comma separated list of subnets to deploy to. ECS tasks will be run across multiple subnets. EMR clusters will be deployed in a subnet chosen when the cluster is created. | | false | -| sleeper.filesystem | The Hadoop filesystem used to connect to S3. | s3a:// | false | -| sleeper.errors.email | An email address used by the TopicStack to publish SNS notifications of errors. | | true | -| sleeper.log.retention.days | The length of time in days that CloudWatch logs from lambda functions, ECS containers, etc., are retained.
See https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-logs-loggroup.html for valid options.
Use -1 to indicate infinite retention. | 30 | true | -| sleeper.fs.s3a.max-connections | Used to set the value of fs.s3a.connection.maximum on the Hadoop configuration. This controls the maximum number of http connections to S3.
See https://hadoop.apache.org/docs/stable/hadoop-aws/tools/hadoop-aws/performance.html | 100 | false | -| sleeper.fs.s3a.upload.block.size | Used to set the value of fs.s3a.block.size on the Hadoop configuration. Uploads to S3 happen in blocks, and this sets the size of blocks. If a larger value is used, then more data is buffered before the upload begins.
See https://hadoop.apache.org/docs/stable/hadoop-aws/tools/hadoop-aws/performance.html | 32M | false | -| sleeper.fargate.version | The version of Fargate to use. | 1.4.0 | false | -| sleeper.task.runner.memory.mb | The amount of memory in MB for the lambda that creates ECS tasks to execute compaction and ingest jobs. | 1024 | true | -| sleeper.task.runner.timeout.seconds | The timeout in seconds for the lambda that creates ECS tasks to execute compaction jobs and ingest jobs.
This must be >0 and <= 900. | 900 | true | -| sleeper.properties.force.reload | If true, properties will be reloaded every time a long running job is started or a lambda is run. This will mainly be used in test scenarios to ensure properties are up to date. | false | false | -| sleeper.ecs.security.groups | A comma-separated list of up to 5 security group IDs to be used when running ECS tasks. | | true | -| sleeper.default.lambda.concurrency.reserved | Default value for the reserved concurrency for each lambda in the Sleeper instance that scales according to the number of Sleeper tables.
The state store committer lambda is an exception to this, as it has reserved concurrency by default. This is set in the property sleeper.statestore.committer.concurrency.reserved. Other lambdas are present that do not scale by the number of Sleeper tables, and are not set from this property.
By default no concurrency is reserved for the lambdas. Each lambda also has its own property that overrides the value found here.
See reserved concurrency overview at: https://docs.aws.amazon.com/lambda/latest/dg/configuration-concurrency.html | | false | -| sleeper.default.lambda.concurrency.max | Default value for the maximum concurrency for each lambda in the Sleeper instance that scales according to the number of Sleeper tables.
Other lambdas are present that do not scale by the number of Sleeper tables, and are not set from this property.
By default the maximum concurrency is set to 10, which is enough for 10 online tables. If there are more online tables, this number may need to be increased. Each lambda also has its own property that overrides the value found here.
See maximum concurrency overview at: https://aws.amazon.com/blogs/compute/introducing-maximum-concurrency-of-aws-lambda-functions-when-using-amazon-sqs-as-an-event-source/ | 10 | false | +| Property Name | Description | Default Value | Run CDK Deploy When Changed | +|---------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------| +| sleeper.id | A string to uniquely identify this deployment. This should be no longer than 20 chars. It should be globally unique as it will be used to name AWS resources such as S3 buckets. | | false | +| sleeper.artefacts.deployment | The ID of the artefacts deployment to use to deploy the Sleeper instance. By default we assume an artefacts deployment with the same ID as the Sleeper instance. This property is used to compute the default values of `sleeper.jars.bucket` and `sleeper.ecr.repository.prefix`. | | true | +| sleeper.jars.bucket | The S3 bucket containing the jar files of the Sleeper components. If unset, a default name is computed from `sleeper.artefacts.deployment` if it is set, or `sleeper.id` if it is not. | | true | +| sleeper.ecr.repository.prefix | If set, this property will be used as a prefix for the names of ECR repositories. If unset, a default prefix is computed from `sleeper.artefacts.deployment` if it is set, or `sleeper.id` if it is not.
ECR repository names are generated in the format `/`. | | true | +| sleeper.userjars | A comma-separated list of the jars containing application specific iterator code. These jars are assumed to be in the bucket given by `sleeper.jars.bucket`. For example, if that bucket contains two iterator jars called iterator1.jar and iterator2.jar then the property should be 'sleeper.userjars=iterator1.jar,iterator2.jar'. | | false | +| sleeper.tags | A list of tags that will automatically be applied to all the resources in this deployment of Sleeper. The list should be in the form "key1,value1,key2,value2,key3,value3,...".
For example if you want to add tags of "user=some-user" and "project-name=sleeper-test", then the list should be "user,some-user,project-name,sleeper-test". | | true | +| sleeper.stack.tag.name | A name for a tag to identify the stack that deployed a resource. This will be set for all AWS resources, to the ID of the CDK stack that they are deployed under. This can be used to organise the cost explorer for billing. | DeploymentStack | true | +| sleeper.retain.infra.after.destroy | Whether to keep the sleeper table bucket, Dynamo tables, query results bucket, etc., when the instance is destroyed. | true | true | +| sleeper.retain.logs.after.destroy | Whether to keep the sleeper log groups when the instance is destroyed. | true | true | +| sleeper.optional.stacks | The optional stacks to deploy. Not case sensitive.
Valid values: [IngestStack, IngestBatcherStack, EmrServerlessBulkImportStack, EmrBulkImportStack, PersistentEmrBulkImportStack, EksBulkImportStack, EmrStudioStack, BulkExportStack, QueryStack, WebSocketQueryStack, AthenaStack, KeepLambdaWarmStack, CompactionStack, GarbageCollectorStack, PartitionSplittingStack, DashboardStack, TableMetricsStack] | IngestStack,IngestBatcherStack,EmrServerlessBulkImportStack,EmrStudioStack,QueryStack,CompactionStack,GarbageCollectorStack,PartitionSplittingStack,DashboardStack,TableMetricsStack | true | +| sleeper.lambda.deploy.type | The deployment type for AWS Lambda. Not case sensitive.
There are two types of Lambda deployments, jar and container.
If the size of the jar file is too large, it will always be deployed as a container.
Valid values: [jar, container] | jar | true | +| sleeper.endpoint.url | The AWS endpoint URL. This should only be set for a non-standard service endpoint. Usually this is used to set the URL to LocalStack for a locally deployed instance. | | false | +| sleeper.vpc | The id of the VPC to deploy to. | | false | +| sleeper.vpc.endpoint.check | Whether to check that the VPC that the instance is deployed to has an S3 endpoint. If there is no S3 endpoint then the NAT costs can be very significant. | true | false | +| sleeper.subnets | A comma separated list of subnets to deploy to. ECS tasks will be run across multiple subnets. EMR clusters will be deployed in a subnet chosen when the cluster is created. | | false | +| sleeper.filesystem | The Hadoop filesystem used to connect to S3. | s3a:// | false | +| sleeper.errors.email | An email address used by the TopicStack to publish SNS notifications of errors. | | true | +| sleeper.log.retention.days | The length of time in days that CloudWatch logs from lambda functions, ECS containers, etc., are retained.
See https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-logs-loggroup.html for valid options.
Use -1 to indicate infinite retention. | 30 | true | +| sleeper.fs.s3a.max-connections | Used to set the value of fs.s3a.connection.maximum on the Hadoop configuration. This controls the maximum number of http connections to S3.
See https://hadoop.apache.org/docs/stable/hadoop-aws/tools/hadoop-aws/performance.html | 100 | false | +| sleeper.fs.s3a.upload.block.size | Used to set the value of fs.s3a.block.size on the Hadoop configuration. Uploads to S3 happen in blocks, and this sets the size of blocks. If a larger value is used, then more data is buffered before the upload begins.
See https://hadoop.apache.org/docs/stable/hadoop-aws/tools/hadoop-aws/performance.html | 32M | false | +| sleeper.fargate.version | The version of Fargate to use. | 1.4.0 | false | +| sleeper.task.runner.memory.mb | The amount of memory in MB for the lambda that creates ECS tasks to execute compaction and ingest jobs. | 1024 | true | +| sleeper.task.runner.timeout.seconds | The timeout in seconds for the lambda that creates ECS tasks to execute compaction jobs and ingest jobs.
This must be >0 and <= 900. | 900 | true | +| sleeper.properties.force.reload | If true, properties will be reloaded every time a long running job is started or a lambda is run. This will mainly be used in test scenarios to ensure properties are up to date. | false | false | +| sleeper.ecs.security.groups | A comma-separated list of up to 5 security group IDs to be used when running ECS tasks. | | true | +| sleeper.default.lambda.concurrency.reserved | Default value for the reserved concurrency for each lambda in the Sleeper instance that scales according to the number of Sleeper tables.
The state store committer lambda is an exception to this, as it has reserved concurrency by default. This is set in the property sleeper.statestore.committer.concurrency.reserved. Other lambdas are present that do not scale by the number of Sleeper tables, and are not set from this property.
By default no concurrency is reserved for the lambdas. Each lambda also has its own property that overrides the value found here.
See reserved concurrency overview at: https://docs.aws.amazon.com/lambda/latest/dg/configuration-concurrency.html | | false | +| sleeper.default.lambda.concurrency.max | Default value for the maximum concurrency for each lambda in the Sleeper instance that scales according to the number of Sleeper tables.
Other lambdas are present that do not scale by the number of Sleeper tables, and are not set from this property.
By default the maximum concurrency is set to 10, which is enough for 10 online tables. If there are more online tables, this number may need to be increased. Each lambda also has its own property that overrides the value found here.
See maximum concurrency overview at: https://aws.amazon.com/blogs/compute/introducing-maximum-concurrency-of-aws-lambda-functions-when-using-amazon-sqs-as-an-event-source/ | 10 | false | From 30dfc5c07e84b5f46794b8a5dee87ced03533f77 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Fri, 21 Nov 2025 14:26:58 +0000 Subject: [PATCH 18/65] 5793: Update to replace populate within tests --- .../admin/properties/PropertiesDiffTest.java | 26 ++++++++++--------- .../instance/AwsSleeperInstanceDriver.java | 9 ++++--- .../DeployedSleeperInstanceRedeployTest.java | 20 +++++++------- 3 files changed, 30 insertions(+), 25 deletions(-) diff --git a/java/clients/src/test/java/sleeper/clients/admin/properties/PropertiesDiffTest.java b/java/clients/src/test/java/sleeper/clients/admin/properties/PropertiesDiffTest.java index dd65ced0329..0f4e42cc226 100644 --- a/java/clients/src/test/java/sleeper/clients/admin/properties/PropertiesDiffTest.java +++ b/java/clients/src/test/java/sleeper/clients/admin/properties/PropertiesDiffTest.java @@ -25,6 +25,7 @@ import sleeper.core.properties.instance.InstanceProperties; import sleeper.core.properties.instance.InstanceProperty; import sleeper.core.properties.table.TableProperties; +import sleeper.core.properties.testutils.InstancePropertiesTestHelper; import java.util.List; @@ -48,9 +49,10 @@ class CompareInstanceProperties { @Test void shouldDetectNoChanges() { + // Given - InstanceProperties before = generateTestInstanceProperties(); - InstanceProperties after = generateTestInstanceProperties(); + InstanceProperties before = InstancePropertiesTestHelper.createTestInstanceProperties(); + InstanceProperties after = InstancePropertiesTestHelper.createTestInstanceProperties(); // When / Then assertThat(getChanges(before, after)).isEmpty(); @@ -59,9 +61,9 @@ void shouldDetectNoChanges() { @Test void shouldDetectPropertyHasBeenUpdated() { // Given - InstanceProperties before = generateTestInstanceProperties(); + InstanceProperties before = InstancePropertiesTestHelper.createTestInstanceProperties(); before.set(MAXIMUM_CONNECTIONS_TO_S3, "30"); - InstanceProperties after = generateTestInstanceProperties(); + InstanceProperties after = InstancePropertiesTestHelper.createTestInstanceProperties(); after.set(MAXIMUM_CONNECTIONS_TO_S3, "50"); // When / Then @@ -72,8 +74,8 @@ void shouldDetectPropertyHasBeenUpdated() { @Test void shouldDetectPropertyIsNewlySet() { // Given - InstanceProperties before = generateTestInstanceProperties(); - InstanceProperties after = generateTestInstanceProperties(); + InstanceProperties before = InstancePropertiesTestHelper.createTestInstanceProperties(); + InstanceProperties after = InstancePropertiesTestHelper.createTestInstanceProperties(); after.set(INGEST_SOURCE_BUCKET, "some-bucket"); // When / Then @@ -84,9 +86,9 @@ void shouldDetectPropertyIsNewlySet() { @Test void shouldDetectPropertyIsUnset() { // Given - InstanceProperties before = generateTestInstanceProperties(); + InstanceProperties before = InstancePropertiesTestHelper.createTestInstanceProperties(); before.set(INGEST_SOURCE_BUCKET, "some-bucket"); - InstanceProperties after = generateTestInstanceProperties(); + InstanceProperties after = InstancePropertiesTestHelper.createTestInstanceProperties(); // When / Then assertThat(getChanges(before, after)) @@ -96,8 +98,8 @@ void shouldDetectPropertyIsUnset() { @Test void shouldDetectDefaultedPropertyIsNewlySet() { // Given - InstanceProperties before = generateTestInstanceProperties(); - InstanceProperties after = generateTestInstanceProperties(); + InstanceProperties before = InstancePropertiesTestHelper.createTestInstanceProperties(); + InstanceProperties after = InstancePropertiesTestHelper.createTestInstanceProperties(); after.set(MAXIMUM_CONNECTIONS_TO_S3, "50"); // When / Then @@ -108,9 +110,9 @@ void shouldDetectDefaultedPropertyIsNewlySet() { @Test void shouldDetectDefaultedPropertyIsUnset() { // Given - InstanceProperties before = generateTestInstanceProperties(); + InstanceProperties before = InstancePropertiesTestHelper.createTestInstanceProperties(); before.set(MAXIMUM_CONNECTIONS_TO_S3, "50"); - InstanceProperties after = generateTestInstanceProperties(); + InstanceProperties after = InstancePropertiesTestHelper.createTestInstanceProperties(); // When / Then assertThat(getChanges(before, after)) diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java index 78aa4a3fedf..0740508888c 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java @@ -29,7 +29,6 @@ import sleeper.clients.deploy.DeployExistingInstance; import sleeper.clients.deploy.DeployNewInstance; -import sleeper.clients.deploy.properties.PopulateInstancePropertiesAws; import sleeper.clients.util.cdk.InvokeCdk; import sleeper.clients.util.command.CommandUtils; import sleeper.configuration.properties.S3InstanceProperties; @@ -46,6 +45,8 @@ import java.util.Set; import static sleeper.core.properties.instance.CommonProperty.ID; +import static sleeper.core.properties.instance.CommonProperty.SUBNETS; +import static sleeper.core.properties.instance.CommonProperty.VPC_ID; import static software.amazon.awssdk.services.cloudformation.model.StackStatus.CREATE_FAILED; import static software.amazon.awssdk.services.cloudformation.model.StackStatus.ROLLBACK_COMPLETE; @@ -86,9 +87,9 @@ public boolean deployInstanceIfNotPresent(String instanceId, SleeperInstanceConf return false; } LOGGER.info("Deploying instance: {}", instanceId); - PopulateInstancePropertiesAws.builder(sts, regionProvider) - .instanceId(instanceId).vpcId(parameters.getVpcId()).subnetIds(parameters.getSubnetIds()) - .build().populate(deployConfig.getInstanceProperties()); + deployConfig.getInstanceProperties().set(ID, instanceId); + deployConfig.getInstanceProperties().set(VPC_ID, parameters.getVpcId()); + deployConfig.getInstanceProperties().set(SUBNETS, parameters.getSubnetIds()); try { DeployNewInstance.builder().scriptsDirectory(parameters.getScriptsDirectory()) .deployInstanceConfiguration(deployConfig) diff --git a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/instance/DeployedSleeperInstanceRedeployTest.java b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/instance/DeployedSleeperInstanceRedeployTest.java index 732013010bf..1e787e0a027 100644 --- a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/instance/DeployedSleeperInstanceRedeployTest.java +++ b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/instance/DeployedSleeperInstanceRedeployTest.java @@ -18,7 +18,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import sleeper.core.deploy.PopulateInstanceProperties; import sleeper.core.deploy.SleeperInstanceConfiguration; import sleeper.core.properties.instance.InstanceProperties; import sleeper.systemtest.configuration.SystemTestStandaloneProperties; @@ -28,6 +27,10 @@ import java.util.function.Consumer; import static org.assertj.core.api.Assertions.assertThat; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; +import static sleeper.core.properties.instance.CommonProperty.ID; +import static sleeper.core.properties.instance.CommonProperty.SUBNETS; import static sleeper.core.properties.instance.CommonProperty.USER_JARS; import static sleeper.core.properties.instance.CommonProperty.VPC_ID; import static sleeper.core.properties.instance.CompactionProperty.COMPACTION_COMMIT_BATCHING_WINDOW_IN_SECONDS; @@ -121,14 +124,13 @@ private InstanceProperties fakeDeploy() { // Replicate system test behaviour InstanceProperties properties = buildDeployConfig().getInstanceProperties(); // Replicate behaviour in DeployNewInstance - return PopulateInstanceProperties.builder() - .accountSupplier(() -> "test-account") - .regionIdSupplier(() -> "test-region") - .instanceId("test-instance") - .vpcId("test-vpc") - .subnetIds("test-subnet") - .build() - .populate(properties); + properties.set(ACCOUNT, "test-account"); + properties.set(REGION, "test-region"); + properties.set(ID, "test-instacne"); + properties.set(VPC_ID, "test-vpc"); + properties.set(SUBNETS, "test-subnet"); + + return properties; } private SleeperInstanceConfiguration buildDeployConfig() { From d90949125148b596e4ddf09926ddf3f257cbdb27 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Fri, 21 Nov 2025 14:57:21 +0000 Subject: [PATCH 19/65] 5793: Update more testhelper usages --- .../admin/properties/PropertiesDiffTest.java | 5 +- .../UpdatePropertiesWithTextEditorIT.java | 42 ++++---- .../PopulateInstancePropertiesTest.java | 98 +++++++++---------- 3 files changed, 69 insertions(+), 76 deletions(-) diff --git a/java/clients/src/test/java/sleeper/clients/admin/properties/PropertiesDiffTest.java b/java/clients/src/test/java/sleeper/clients/admin/properties/PropertiesDiffTest.java index 0f4e42cc226..ce2b7fa7e4f 100644 --- a/java/clients/src/test/java/sleeper/clients/admin/properties/PropertiesDiffTest.java +++ b/java/clients/src/test/java/sleeper/clients/admin/properties/PropertiesDiffTest.java @@ -33,7 +33,6 @@ import static sleeper.clients.admin.properties.PropertiesDiffTestHelper.newValue; import static sleeper.clients.admin.properties.PropertiesDiffTestHelper.valueChanged; import static sleeper.clients.admin.properties.PropertiesDiffTestHelper.valueDeleted; -import static sleeper.core.deploy.PopulatePropertiesTestHelper.generateTestInstanceProperties; import static sleeper.core.deploy.PopulatePropertiesTestHelper.generateTestTableProperties; import static sleeper.core.properties.PropertiesUtils.loadProperties; import static sleeper.core.properties.instance.CommonProperty.LOG_RETENTION_IN_DAYS; @@ -238,9 +237,9 @@ void shouldNotCombineDiffsWhenPropertyIsDifferent() { } private PropertiesDiff generateSingleDiff(InstanceProperty property, String oldValue, String newValue) { - InstanceProperties before = generateTestInstanceProperties(); + InstanceProperties before = InstancePropertiesTestHelper.createTestInstanceProperties(); before.set(property, oldValue); - InstanceProperties after = generateTestInstanceProperties(); + InstanceProperties after = InstancePropertiesTestHelper.createTestInstanceProperties(); after.set(property, newValue); return new PropertiesDiff(before, after); } diff --git a/java/clients/src/test/java/sleeper/clients/admin/properties/UpdatePropertiesWithTextEditorIT.java b/java/clients/src/test/java/sleeper/clients/admin/properties/UpdatePropertiesWithTextEditorIT.java index b665abe0fe3..cb40f46f5b1 100644 --- a/java/clients/src/test/java/sleeper/clients/admin/properties/UpdatePropertiesWithTextEditorIT.java +++ b/java/clients/src/test/java/sleeper/clients/admin/properties/UpdatePropertiesWithTextEditorIT.java @@ -25,6 +25,7 @@ import sleeper.core.properties.instance.InstancePropertyGroup; import sleeper.core.properties.table.TableProperties; import sleeper.core.properties.table.TablePropertyGroup; +import sleeper.core.properties.testutils.InstancePropertiesTestHelper; import java.io.PrintWriter; import java.io.StringWriter; @@ -34,7 +35,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.InstanceOfAssertFactories.LIST; import static sleeper.clients.admin.properties.PropertiesDiffTestHelper.valueChanged; -import static sleeper.core.deploy.PopulatePropertiesTestHelper.generateTestInstanceProperties; import static sleeper.core.deploy.PopulatePropertiesTestHelper.generateTestTableProperties; import static sleeper.core.properties.PropertiesUtils.loadProperties; import static sleeper.core.properties.instance.CommonProperty.MAXIMUM_CONNECTIONS_TO_S3; @@ -61,7 +61,7 @@ class OpenInstanceProperties { @Test void shouldInvokeNanoOnInstancePropertiesFile() throws Exception { // Given - InstanceProperties properties = generateTestInstanceProperties(); + InstanceProperties properties = InstancePropertiesTestHelper.createTestInstanceProperties(); // When / Then assertThat(helper.openInstancePropertiesGetCommandRun(properties)) @@ -71,7 +71,7 @@ void shouldInvokeNanoOnInstancePropertiesFile() throws Exception { @Test void shouldWriteInstancePropertiesFile() throws Exception { // Given - InstanceProperties properties = generateTestInstanceProperties(); + InstanceProperties properties = InstancePropertiesTestHelper.createTestInstanceProperties(); // When / Then assertThat(helper.openInstancePropertiesGetPropertiesWritten(properties)) @@ -81,9 +81,9 @@ void shouldWriteInstancePropertiesFile() throws Exception { @Test void shouldGetDiffAfterPropertiesChanged() throws Exception { // Given - InstanceProperties before = generateTestInstanceProperties(); + InstanceProperties before = InstancePropertiesTestHelper.createTestInstanceProperties(); before.set(INGEST_SOURCE_BUCKET, "bucket-before"); - InstanceProperties after = generateTestInstanceProperties(); + InstanceProperties after = InstancePropertiesTestHelper.createTestInstanceProperties(); after.set(INGEST_SOURCE_BUCKET, "bucket-after"); // When / Then @@ -95,8 +95,8 @@ void shouldGetDiffAfterPropertiesChanged() throws Exception { @Test void shouldRetrievePropertiesAfterChange() throws Exception { // Given - InstanceProperties before = generateTestInstanceProperties(); - InstanceProperties after = generateTestInstanceProperties(); + InstanceProperties before = InstancePropertiesTestHelper.createTestInstanceProperties(); + InstanceProperties after = InstancePropertiesTestHelper.createTestInstanceProperties(); after.set(MAXIMUM_CONNECTIONS_TO_S3, "abc"); // When @@ -109,7 +109,7 @@ void shouldRetrievePropertiesAfterChange() throws Exception { @Test void shouldFormatPropertiesUsingPrettyPrinter() throws Exception { // Given - InstanceProperties properties = generateTestInstanceProperties(); + InstanceProperties properties = InstancePropertiesTestHelper.createTestInstanceProperties(); // When String tempFileString = Files.readString(helper.openInstancePropertiesGetPathToFile(properties)); @@ -164,7 +164,7 @@ class FilterByGroup { @Test void shouldWriteSingleInstancePropertyGroupToFile() throws Exception { // Given - InstanceProperties properties = generateTestInstanceProperties(); + InstanceProperties properties = InstancePropertiesTestHelper.createTestInstanceProperties(); properties.set(LOGGING_LEVEL, "ERROR"); // When / Then @@ -176,7 +176,7 @@ void shouldWriteSingleInstancePropertyGroupToFile() throws Exception { @Test void shouldFormatPropertiesUsingPrettyPrinter() throws Exception { // Given - InstanceProperties properties = generateTestInstanceProperties(); + InstanceProperties properties = InstancePropertiesTestHelper.createTestInstanceProperties(); properties.set(LOGGING_LEVEL, "ERROR"); // When @@ -193,9 +193,9 @@ void shouldFormatPropertiesUsingPrettyPrinter() throws Exception { @Test void shouldCreateUpdateRequestWithInstanceProperties() throws Exception { // Given - InstanceProperties before = generateTestInstanceProperties(); + InstanceProperties before = InstancePropertiesTestHelper.createTestInstanceProperties(); before.set(LOGGING_LEVEL, "ERROR"); - InstanceProperties after = generateTestInstanceProperties(); + InstanceProperties after = InstancePropertiesTestHelper.createTestInstanceProperties(); after.set(LOGGING_LEVEL, "INFO"); // When @@ -231,9 +231,9 @@ void shouldCreateUpdateRequestWithTableProperties() throws Exception { @Test void shouldUnsetPropertyWhenRemovedInEditor() throws Exception { // Given - InstanceProperties before = generateTestInstanceProperties(); + InstanceProperties before = InstancePropertiesTestHelper.createTestInstanceProperties(); before.set(LOGGING_LEVEL, "ERROR"); - InstanceProperties after = generateTestInstanceProperties(); + InstanceProperties after = InstancePropertiesTestHelper.createTestInstanceProperties(); after.unset(LOGGING_LEVEL); // When @@ -250,7 +250,7 @@ void shouldUnsetPropertyWhenRemovedInEditor() throws Exception { @Test void shouldNotShowUnknownProperties() throws Exception { // Given - InstanceProperties properties = generateTestInstanceProperties(); + InstanceProperties properties = InstancePropertiesTestHelper.createTestInstanceProperties(); properties.getProperties().setProperty("unknown.property", "some-value"); // When @@ -261,9 +261,9 @@ void shouldNotShowUnknownProperties() throws Exception { @Test void shouldUpdateAnUnknownProperty() throws Exception { // Given - InstanceProperties before = generateTestInstanceProperties(); + InstanceProperties before = InstancePropertiesTestHelper.createTestInstanceProperties(); before.getProperties().setProperty("unknown.property", "value-before"); - InstanceProperties after = generateTestInstanceProperties(); + InstanceProperties after = InstancePropertiesTestHelper.createTestInstanceProperties(); after.getProperties().setProperty("unknown.property", "value-after"); // When @@ -280,9 +280,9 @@ void shouldUpdateAnUnknownProperty() throws Exception { @Test void shouldUpdateAPropertyOutsideTheSpecifiedGroup() throws Exception { // Given - InstanceProperties before = generateTestInstanceProperties(); + InstanceProperties before = InstancePropertiesTestHelper.createTestInstanceProperties(); before.set(INGEST_SOURCE_BUCKET, "bucket-before"); - InstanceProperties after = generateTestInstanceProperties(); + InstanceProperties after = InstancePropertiesTestHelper.createTestInstanceProperties(); after.set(INGEST_SOURCE_BUCKET, "bucket-after"); // When @@ -299,9 +299,9 @@ void shouldUpdateAPropertyOutsideTheSpecifiedGroup() throws Exception { @Test void shouldLeaveUnknownPropertyUnchangedWhenEditingAnotherProperty() throws Exception { // Given - InstanceProperties before = generateTestInstanceProperties(); + InstanceProperties before = InstancePropertiesTestHelper.createTestInstanceProperties(); before.getProperties().setProperty("unknown.property", "test-value"); - InstanceProperties after = generateTestInstanceProperties(); + InstanceProperties after = InstancePropertiesTestHelper.createTestInstanceProperties(); after.getProperties().setProperty("unknown.property", "test-value"); after.set(LOGGING_LEVEL, "TRACE"); diff --git a/java/core/src/test/java/sleeper/core/deploy/PopulateInstancePropertiesTest.java b/java/core/src/test/java/sleeper/core/deploy/PopulateInstancePropertiesTest.java index 9ca781f38c1..b29fb3d21b6 100644 --- a/java/core/src/test/java/sleeper/core/deploy/PopulateInstancePropertiesTest.java +++ b/java/core/src/test/java/sleeper/core/deploy/PopulateInstancePropertiesTest.java @@ -15,21 +15,13 @@ */ package sleeper.core.deploy; -import org.junit.jupiter.api.Test; - import sleeper.core.properties.instance.InstanceProperties; -import java.util.Map; - -import static org.assertj.core.api.Assertions.assertThat; -import static sleeper.core.deploy.PopulatePropertiesTestHelper.createTestPopulateInstanceProperties; -import static sleeper.core.deploy.PopulatePropertiesTestHelper.testPopulateInstancePropertiesBuilder; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; import static sleeper.core.properties.instance.CommonProperty.ID; import static sleeper.core.properties.instance.CommonProperty.SUBNETS; import static sleeper.core.properties.instance.CommonProperty.VPC_ID; -import static sleeper.core.properties.instance.PartitionSplittingProperty.DEFAULT_PARTITION_SPLIT_THRESHOLD; public class PopulateInstancePropertiesTest { @@ -44,49 +36,51 @@ private InstanceProperties expectedInstanceProperties() { return expected; } - @Test - void shouldPopulateInstanceProperties() { - // Given/When - InstanceProperties properties = createTestPopulateInstanceProperties().populate(new InstanceProperties()); - - // Then - assertThat(properties).isEqualTo(expectedInstanceProperties()); - } - - @Test - void shouldGetDefaultTagsWhenNotProvidedAndNotSetInInstanceProperties() { - // Given/When - InstanceProperties properties = createTestPopulateInstanceProperties().populate(new InstanceProperties()); - - // Then - assertThat(properties.getTags()) - .isEqualTo(Map.of("InstanceID", "test-instance")); - } - - @Test - void shouldAddToExistingTagsWhenSetInInstanceProperties() { - // Given/When - InstanceProperties properties = new InstanceProperties(); - properties.setTags(Map.of("TestTag", "TestValue")); - createTestPopulateInstanceProperties().populate(properties); - - // Then - assertThat(properties.getTags()) - .isEqualTo(Map.of("TestTag", "TestValue", - "InstanceID", "test-instance")); - } - - @Test - void shouldSetExtraProperties() { - // Given/When - InstanceProperties properties = new InstanceProperties(); - testPopulateInstancePropertiesBuilder() - .extraInstanceProperties(p -> p.setNumber(DEFAULT_PARTITION_SPLIT_THRESHOLD, 1000)) - .build().populate(properties); - - // Then - assertThat(properties.getInt(DEFAULT_PARTITION_SPLIT_THRESHOLD)) - .isEqualTo(1000); - } + /* + * @Test + * void shouldPopulateInstanceProperties() { + * // Given/When + * InstanceProperties properties = createTestPopulateInstanceProperties().populate(new InstanceProperties()); + * + * // Then + * assertThat(properties).isEqualTo(expectedInstanceProperties()); + * } + * + * @Test + * void shouldGetDefaultTagsWhenNotProvidedAndNotSetInInstanceProperties() { + * // Given/When + * InstanceProperties properties = createTestPopulateInstanceProperties().populate(new InstanceProperties()); + * + * // Then + * assertThat(properties.getTags()) + * .isEqualTo(Map.of("InstanceID", "test-instance")); + * } + * + * @Test + * void shouldAddToExistingTagsWhenSetInInstanceProperties() { + * // Given/When + * InstanceProperties properties = new InstanceProperties(); + * properties.setTags(Map.of("TestTag", "TestValue")); + * createTestPopulateInstanceProperties().populate(properties); + * + * // Then + * assertThat(properties.getTags()) + * .isEqualTo(Map.of("TestTag", "TestValue", + * "InstanceID", "test-instance")); + * } + * + * @Test + * void shouldSetExtraProperties() { + * // Given/When + * InstanceProperties properties = new InstanceProperties(); + * testPopulateInstancePropertiesBuilder() + * .extraInstanceProperties(p -> p.setNumber(DEFAULT_PARTITION_SPLIT_THRESHOLD, 1000)) + * .build().populate(properties); + * + * // Then + * assertThat(properties.getInt(DEFAULT_PARTITION_SPLIT_THRESHOLD)) + * .isEqualTo(1000); + * } + */ } From d1c8285beeaebb1db05d08f0fd7f3d0ca9f026db Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Fri, 21 Nov 2025 15:09:28 +0000 Subject: [PATCH 20/65] 5793: Delete test class --- .../PopulateInstancePropertiesTest.java | 86 ------------------- 1 file changed, 86 deletions(-) delete mode 100644 java/core/src/test/java/sleeper/core/deploy/PopulateInstancePropertiesTest.java diff --git a/java/core/src/test/java/sleeper/core/deploy/PopulateInstancePropertiesTest.java b/java/core/src/test/java/sleeper/core/deploy/PopulateInstancePropertiesTest.java deleted file mode 100644 index b29fb3d21b6..00000000000 --- a/java/core/src/test/java/sleeper/core/deploy/PopulateInstancePropertiesTest.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright 2022-2025 Crown Copyright - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package sleeper.core.deploy; - -import sleeper.core.properties.instance.InstanceProperties; - -import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; -import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; -import static sleeper.core.properties.instance.CommonProperty.ID; -import static sleeper.core.properties.instance.CommonProperty.SUBNETS; -import static sleeper.core.properties.instance.CommonProperty.VPC_ID; - -public class PopulateInstancePropertiesTest { - - private InstanceProperties expectedInstanceProperties() { - InstanceProperties expected = new InstanceProperties(); - //expected.setTags(Map.of("InstanceID", "test-instance")); - expected.set(ID, "test-instance"); - expected.set(VPC_ID, "some-vpc"); - expected.set(SUBNETS, "some-subnet"); - expected.set(ACCOUNT, "test-account-id"); - expected.set(REGION, "test-region"); - return expected; - } - - /* - * @Test - * void shouldPopulateInstanceProperties() { - * // Given/When - * InstanceProperties properties = createTestPopulateInstanceProperties().populate(new InstanceProperties()); - * - * // Then - * assertThat(properties).isEqualTo(expectedInstanceProperties()); - * } - * - * @Test - * void shouldGetDefaultTagsWhenNotProvidedAndNotSetInInstanceProperties() { - * // Given/When - * InstanceProperties properties = createTestPopulateInstanceProperties().populate(new InstanceProperties()); - * - * // Then - * assertThat(properties.getTags()) - * .isEqualTo(Map.of("InstanceID", "test-instance")); - * } - * - * @Test - * void shouldAddToExistingTagsWhenSetInInstanceProperties() { - * // Given/When - * InstanceProperties properties = new InstanceProperties(); - * properties.setTags(Map.of("TestTag", "TestValue")); - * createTestPopulateInstanceProperties().populate(properties); - * - * // Then - * assertThat(properties.getTags()) - * .isEqualTo(Map.of("TestTag", "TestValue", - * "InstanceID", "test-instance")); - * } - * - * @Test - * void shouldSetExtraProperties() { - * // Given/When - * InstanceProperties properties = new InstanceProperties(); - * testPopulateInstancePropertiesBuilder() - * .extraInstanceProperties(p -> p.setNumber(DEFAULT_PARTITION_SPLIT_THRESHOLD, 1000)) - * .build().populate(properties); - * - * // Then - * assertThat(properties.getInt(DEFAULT_PARTITION_SPLIT_THRESHOLD)) - * .isEqualTo(1000); - * } - */ - -} From 1aba20c34382529c0a84487ee4f02f3b0c0a4f7d Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Mon, 24 Nov 2025 08:21:41 +0000 Subject: [PATCH 21/65] 5793: Refactor location of setting of test instance properties --- .../admin/properties/PropertiesDiffTest.java | 31 +++++++++-------- .../InstancePropertiesTestHelper.java | 34 +++++++++++++++---- 2 files changed, 44 insertions(+), 21 deletions(-) diff --git a/java/clients/src/test/java/sleeper/clients/admin/properties/PropertiesDiffTest.java b/java/clients/src/test/java/sleeper/clients/admin/properties/PropertiesDiffTest.java index ce2b7fa7e4f..26d31b94e50 100644 --- a/java/clients/src/test/java/sleeper/clients/admin/properties/PropertiesDiffTest.java +++ b/java/clients/src/test/java/sleeper/clients/admin/properties/PropertiesDiffTest.java @@ -25,7 +25,6 @@ import sleeper.core.properties.instance.InstanceProperties; import sleeper.core.properties.instance.InstanceProperty; import sleeper.core.properties.table.TableProperties; -import sleeper.core.properties.testutils.InstancePropertiesTestHelper; import java.util.List; @@ -39,6 +38,8 @@ import static sleeper.core.properties.instance.CommonProperty.MAXIMUM_CONNECTIONS_TO_S3; import static sleeper.core.properties.instance.IngestProperty.INGEST_SOURCE_BUCKET; import static sleeper.core.properties.table.TableProperty.ITERATOR_CONFIG; +import static sleeper.core.properties.testutils.InstancePropertiesTestHelper.createSimpleTestInstanceProperties; +import static sleeper.core.properties.testutils.InstancePropertiesTestHelper.createTestInstanceProperties; public class PropertiesDiffTest { @@ -50,8 +51,8 @@ class CompareInstanceProperties { void shouldDetectNoChanges() { // Given - InstanceProperties before = InstancePropertiesTestHelper.createTestInstanceProperties(); - InstanceProperties after = InstancePropertiesTestHelper.createTestInstanceProperties(); + InstanceProperties before = createTestInstanceProperties(); + InstanceProperties after = createTestInstanceProperties(); // When / Then assertThat(getChanges(before, after)).isEmpty(); @@ -60,9 +61,9 @@ void shouldDetectNoChanges() { @Test void shouldDetectPropertyHasBeenUpdated() { // Given - InstanceProperties before = InstancePropertiesTestHelper.createTestInstanceProperties(); + InstanceProperties before = createTestInstanceProperties(); before.set(MAXIMUM_CONNECTIONS_TO_S3, "30"); - InstanceProperties after = InstancePropertiesTestHelper.createTestInstanceProperties(); + InstanceProperties after = createTestInstanceProperties(); after.set(MAXIMUM_CONNECTIONS_TO_S3, "50"); // When / Then @@ -73,8 +74,8 @@ void shouldDetectPropertyHasBeenUpdated() { @Test void shouldDetectPropertyIsNewlySet() { // Given - InstanceProperties before = InstancePropertiesTestHelper.createTestInstanceProperties(); - InstanceProperties after = InstancePropertiesTestHelper.createTestInstanceProperties(); + InstanceProperties before = createTestInstanceProperties(); + InstanceProperties after = createTestInstanceProperties(); after.set(INGEST_SOURCE_BUCKET, "some-bucket"); // When / Then @@ -85,9 +86,9 @@ void shouldDetectPropertyIsNewlySet() { @Test void shouldDetectPropertyIsUnset() { // Given - InstanceProperties before = InstancePropertiesTestHelper.createTestInstanceProperties(); + InstanceProperties before = createTestInstanceProperties(); before.set(INGEST_SOURCE_BUCKET, "some-bucket"); - InstanceProperties after = InstancePropertiesTestHelper.createTestInstanceProperties(); + InstanceProperties after = createTestInstanceProperties(); // When / Then assertThat(getChanges(before, after)) @@ -97,8 +98,8 @@ void shouldDetectPropertyIsUnset() { @Test void shouldDetectDefaultedPropertyIsNewlySet() { // Given - InstanceProperties before = InstancePropertiesTestHelper.createTestInstanceProperties(); - InstanceProperties after = InstancePropertiesTestHelper.createTestInstanceProperties(); + InstanceProperties before = createTestInstanceProperties(); + InstanceProperties after = createTestInstanceProperties(); after.set(MAXIMUM_CONNECTIONS_TO_S3, "50"); // When / Then @@ -109,9 +110,9 @@ void shouldDetectDefaultedPropertyIsNewlySet() { @Test void shouldDetectDefaultedPropertyIsUnset() { // Given - InstanceProperties before = InstancePropertiesTestHelper.createTestInstanceProperties(); + InstanceProperties before = createTestInstanceProperties(); before.set(MAXIMUM_CONNECTIONS_TO_S3, "50"); - InstanceProperties after = InstancePropertiesTestHelper.createTestInstanceProperties(); + InstanceProperties after = createTestInstanceProperties(); // When / Then assertThat(getChanges(before, after)) @@ -237,9 +238,9 @@ void shouldNotCombineDiffsWhenPropertyIsDifferent() { } private PropertiesDiff generateSingleDiff(InstanceProperty property, String oldValue, String newValue) { - InstanceProperties before = InstancePropertiesTestHelper.createTestInstanceProperties(); + InstanceProperties before = createSimpleTestInstanceProperties(); before.set(property, oldValue); - InstanceProperties after = InstancePropertiesTestHelper.createTestInstanceProperties(); + InstanceProperties after = createSimpleTestInstanceProperties(); after.set(property, newValue); return new PropertiesDiff(before, after); } diff --git a/java/core/src/test/java/sleeper/core/properties/testutils/InstancePropertiesTestHelper.java b/java/core/src/test/java/sleeper/core/properties/testutils/InstancePropertiesTestHelper.java index e6d108bcbd9..32d274c1ec5 100644 --- a/java/core/src/test/java/sleeper/core/properties/testutils/InstancePropertiesTestHelper.java +++ b/java/core/src/test/java/sleeper/core/properties/testutils/InstancePropertiesTestHelper.java @@ -82,17 +82,12 @@ public static InstanceProperties createTestInstanceProperties() { * @return the instance properties */ public static InstanceProperties createTestInstancePropertiesWithId(String id) { - InstanceProperties instanceProperties = new InstanceProperties(); - instanceProperties.set(ID, id); + InstanceProperties instanceProperties = createSimpleTestInstanceProperties(); instanceProperties.set(CONFIG_BUCKET, InstanceProperties.getConfigBucketFromInstanceId(id)); instanceProperties.set(DATA_BUCKET, "sleeper-" + id + "-table-data"); instanceProperties.set(JARS_BUCKET, "sleeper-" + id + "-jars"); instanceProperties.set(QUERY_RESULTS_BUCKET, "sleeper-" + id + "-query-results"); - instanceProperties.set(ACCOUNT, "test-account"); - instanceProperties.set(REGION, "test-region"); instanceProperties.set(VERSION, "1.2.3"); - instanceProperties.set(VPC_ID, "test-vpc"); - instanceProperties.set(SUBNETS, "test-subnet"); instanceProperties.set(TRANSACTION_LOG_FILES_TABLENAME, "sleeper-" + id + "-file-transaction-log"); instanceProperties.set(TRANSACTION_LOG_PARTITIONS_TABLENAME, "sleeper-" + id + "-partition-transaction-log"); instanceProperties.set(TRANSACTION_LOG_ALL_SNAPSHOTS_TABLENAME, "sleeper-" + id + "-transaction-log-all-snapshots"); @@ -120,6 +115,33 @@ public static InstanceProperties createTestInstancePropertiesWithId(String id) { return instanceProperties; } + /** + * Creates properties for a Sleeper instance. Generates a random instance ID and pre-populates minimal properties + * set during deployment. + * + * @return the instance properties + */ + public static InstanceProperties createSimpleTestInstanceProperties() { + String id = UUID.randomUUID().toString().toLowerCase(Locale.ROOT).substring(0, 18); + return createTestInstancePropertiesWithId(id); + } + + /** + * Creates properties for a Sleeper instance. Sets miminal properties for simpler testing parameters. + * + * @param id the instance ID + * @return the instance properties + */ + public static InstanceProperties createSimpleTestInstanceProperties(String id) { + InstanceProperties properties = new InstanceProperties(); + properties.set(ID, id); + properties.set(ACCOUNT, "test-account"); + properties.set(REGION, "test-region"); + properties.set(VPC_ID, "test-vpc"); + properties.set(SUBNETS, "test-subnet"); + return properties; + } + /** * Writes a properties file as a string. * From b599681f598e32bfdefcbe80bb524657adca2149 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Mon, 24 Nov 2025 08:29:19 +0000 Subject: [PATCH 22/65] 5793: Break unintentional loop --- .../properties/testutils/InstancePropertiesTestHelper.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/java/core/src/test/java/sleeper/core/properties/testutils/InstancePropertiesTestHelper.java b/java/core/src/test/java/sleeper/core/properties/testutils/InstancePropertiesTestHelper.java index 32d274c1ec5..3ca002478ad 100644 --- a/java/core/src/test/java/sleeper/core/properties/testutils/InstancePropertiesTestHelper.java +++ b/java/core/src/test/java/sleeper/core/properties/testutils/InstancePropertiesTestHelper.java @@ -82,7 +82,7 @@ public static InstanceProperties createTestInstanceProperties() { * @return the instance properties */ public static InstanceProperties createTestInstancePropertiesWithId(String id) { - InstanceProperties instanceProperties = createSimpleTestInstanceProperties(); + InstanceProperties instanceProperties = createSimpleTestInstanceProperties(id); instanceProperties.set(CONFIG_BUCKET, InstanceProperties.getConfigBucketFromInstanceId(id)); instanceProperties.set(DATA_BUCKET, "sleeper-" + id + "-table-data"); instanceProperties.set(JARS_BUCKET, "sleeper-" + id + "-jars"); @@ -123,7 +123,7 @@ public static InstanceProperties createTestInstancePropertiesWithId(String id) { */ public static InstanceProperties createSimpleTestInstanceProperties() { String id = UUID.randomUUID().toString().toLowerCase(Locale.ROOT).substring(0, 18); - return createTestInstancePropertiesWithId(id); + return createSimpleTestInstancePropertiesWithId(id); } /** @@ -132,7 +132,7 @@ public static InstanceProperties createSimpleTestInstanceProperties() { * @param id the instance ID * @return the instance properties */ - public static InstanceProperties createSimpleTestInstanceProperties(String id) { + public static InstanceProperties createSimpleTestInstancePropertiesWithId(String id) { InstanceProperties properties = new InstanceProperties(); properties.set(ID, id); properties.set(ACCOUNT, "test-account"); From e8294d99667291b84087e96df048b0196428e20a Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Mon, 24 Nov 2025 08:31:40 +0000 Subject: [PATCH 23/65] 5793: Fix method signature --- .../core/properties/testutils/InstancePropertiesTestHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/core/src/test/java/sleeper/core/properties/testutils/InstancePropertiesTestHelper.java b/java/core/src/test/java/sleeper/core/properties/testutils/InstancePropertiesTestHelper.java index 3ca002478ad..22c9c286302 100644 --- a/java/core/src/test/java/sleeper/core/properties/testutils/InstancePropertiesTestHelper.java +++ b/java/core/src/test/java/sleeper/core/properties/testutils/InstancePropertiesTestHelper.java @@ -82,7 +82,7 @@ public static InstanceProperties createTestInstanceProperties() { * @return the instance properties */ public static InstanceProperties createTestInstancePropertiesWithId(String id) { - InstanceProperties instanceProperties = createSimpleTestInstanceProperties(id); + InstanceProperties instanceProperties = createSimpleTestInstancePropertiesWithId(id); instanceProperties.set(CONFIG_BUCKET, InstanceProperties.getConfigBucketFromInstanceId(id)); instanceProperties.set(DATA_BUCKET, "sleeper-" + id + "-table-data"); instanceProperties.set(JARS_BUCKET, "sleeper-" + id + "-jars"); From 9c59978e6211a5482b2565c2894df24ca5abd22b Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Mon, 24 Nov 2025 09:01:25 +0000 Subject: [PATCH 24/65] 5793: Update method signature --- .../clients/admin/properties/PropertiesDiffTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/java/clients/src/test/java/sleeper/clients/admin/properties/PropertiesDiffTest.java b/java/clients/src/test/java/sleeper/clients/admin/properties/PropertiesDiffTest.java index 26d31b94e50..b299d0c8e2b 100644 --- a/java/clients/src/test/java/sleeper/clients/admin/properties/PropertiesDiffTest.java +++ b/java/clients/src/test/java/sleeper/clients/admin/properties/PropertiesDiffTest.java @@ -38,7 +38,7 @@ import static sleeper.core.properties.instance.CommonProperty.MAXIMUM_CONNECTIONS_TO_S3; import static sleeper.core.properties.instance.IngestProperty.INGEST_SOURCE_BUCKET; import static sleeper.core.properties.table.TableProperty.ITERATOR_CONFIG; -import static sleeper.core.properties.testutils.InstancePropertiesTestHelper.createSimpleTestInstanceProperties; +import static sleeper.core.properties.testutils.InstancePropertiesTestHelper.createSimpleTestInstancePropertiesWithId; import static sleeper.core.properties.testutils.InstancePropertiesTestHelper.createTestInstanceProperties; public class PropertiesDiffTest { @@ -238,9 +238,9 @@ void shouldNotCombineDiffsWhenPropertyIsDifferent() { } private PropertiesDiff generateSingleDiff(InstanceProperty property, String oldValue, String newValue) { - InstanceProperties before = createSimpleTestInstanceProperties(); + InstanceProperties before = createSimpleTestInstancePropertiesWithId("simple-id"); before.set(property, oldValue); - InstanceProperties after = createSimpleTestInstanceProperties(); + InstanceProperties after = createSimpleTestInstancePropertiesWithId("simple-id"); after.set(property, newValue); return new PropertiesDiff(before, after); } From 8b2c37106bb6bd14e9db871e7d1624d26bd969c0 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Mon, 24 Nov 2025 09:51:10 +0000 Subject: [PATCH 25/65] 5793: Correctt static method usage --- .../admin/properties/PropertiesDiffTest.java | 27 ++++++------ .../UpdatePropertiesWithTextEditorIT.java | 42 +++++++++---------- 2 files changed, 35 insertions(+), 34 deletions(-) diff --git a/java/clients/src/test/java/sleeper/clients/admin/properties/PropertiesDiffTest.java b/java/clients/src/test/java/sleeper/clients/admin/properties/PropertiesDiffTest.java index b299d0c8e2b..37532853327 100644 --- a/java/clients/src/test/java/sleeper/clients/admin/properties/PropertiesDiffTest.java +++ b/java/clients/src/test/java/sleeper/clients/admin/properties/PropertiesDiffTest.java @@ -27,6 +27,7 @@ import sleeper.core.properties.table.TableProperties; import java.util.List; +import java.util.UUID; import static org.assertj.core.api.Assertions.assertThat; import static sleeper.clients.admin.properties.PropertiesDiffTestHelper.newValue; @@ -39,20 +40,20 @@ import static sleeper.core.properties.instance.IngestProperty.INGEST_SOURCE_BUCKET; import static sleeper.core.properties.table.TableProperty.ITERATOR_CONFIG; import static sleeper.core.properties.testutils.InstancePropertiesTestHelper.createSimpleTestInstancePropertiesWithId; -import static sleeper.core.properties.testutils.InstancePropertiesTestHelper.createTestInstanceProperties; public class PropertiesDiffTest { @DisplayName("Compare instance properties") @Nested class CompareInstanceProperties { + String id = UUID.randomUUID().toString(); @Test void shouldDetectNoChanges() { // Given - InstanceProperties before = createTestInstanceProperties(); - InstanceProperties after = createTestInstanceProperties(); + InstanceProperties before = createSimpleTestInstancePropertiesWithId(id); + InstanceProperties after = createSimpleTestInstancePropertiesWithId(id); // When / Then assertThat(getChanges(before, after)).isEmpty(); @@ -61,9 +62,9 @@ void shouldDetectNoChanges() { @Test void shouldDetectPropertyHasBeenUpdated() { // Given - InstanceProperties before = createTestInstanceProperties(); + InstanceProperties before = createSimpleTestInstancePropertiesWithId(id); before.set(MAXIMUM_CONNECTIONS_TO_S3, "30"); - InstanceProperties after = createTestInstanceProperties(); + InstanceProperties after = createSimpleTestInstancePropertiesWithId(id); after.set(MAXIMUM_CONNECTIONS_TO_S3, "50"); // When / Then @@ -74,8 +75,8 @@ void shouldDetectPropertyHasBeenUpdated() { @Test void shouldDetectPropertyIsNewlySet() { // Given - InstanceProperties before = createTestInstanceProperties(); - InstanceProperties after = createTestInstanceProperties(); + InstanceProperties before = createSimpleTestInstancePropertiesWithId(id); + InstanceProperties after = createSimpleTestInstancePropertiesWithId(id); after.set(INGEST_SOURCE_BUCKET, "some-bucket"); // When / Then @@ -86,9 +87,9 @@ void shouldDetectPropertyIsNewlySet() { @Test void shouldDetectPropertyIsUnset() { // Given - InstanceProperties before = createTestInstanceProperties(); + InstanceProperties before = createSimpleTestInstancePropertiesWithId(id); before.set(INGEST_SOURCE_BUCKET, "some-bucket"); - InstanceProperties after = createTestInstanceProperties(); + InstanceProperties after = createSimpleTestInstancePropertiesWithId(id); // When / Then assertThat(getChanges(before, after)) @@ -98,8 +99,8 @@ void shouldDetectPropertyIsUnset() { @Test void shouldDetectDefaultedPropertyIsNewlySet() { // Given - InstanceProperties before = createTestInstanceProperties(); - InstanceProperties after = createTestInstanceProperties(); + InstanceProperties before = createSimpleTestInstancePropertiesWithId(id); + InstanceProperties after = createSimpleTestInstancePropertiesWithId(id); after.set(MAXIMUM_CONNECTIONS_TO_S3, "50"); // When / Then @@ -110,9 +111,9 @@ void shouldDetectDefaultedPropertyIsNewlySet() { @Test void shouldDetectDefaultedPropertyIsUnset() { // Given - InstanceProperties before = createTestInstanceProperties(); + InstanceProperties before = createSimpleTestInstancePropertiesWithId(id); before.set(MAXIMUM_CONNECTIONS_TO_S3, "50"); - InstanceProperties after = createTestInstanceProperties(); + InstanceProperties after = createSimpleTestInstancePropertiesWithId(id); // When / Then assertThat(getChanges(before, after)) diff --git a/java/clients/src/test/java/sleeper/clients/admin/properties/UpdatePropertiesWithTextEditorIT.java b/java/clients/src/test/java/sleeper/clients/admin/properties/UpdatePropertiesWithTextEditorIT.java index cb40f46f5b1..e9674425c2e 100644 --- a/java/clients/src/test/java/sleeper/clients/admin/properties/UpdatePropertiesWithTextEditorIT.java +++ b/java/clients/src/test/java/sleeper/clients/admin/properties/UpdatePropertiesWithTextEditorIT.java @@ -25,7 +25,6 @@ import sleeper.core.properties.instance.InstancePropertyGroup; import sleeper.core.properties.table.TableProperties; import sleeper.core.properties.table.TablePropertyGroup; -import sleeper.core.properties.testutils.InstancePropertiesTestHelper; import java.io.PrintWriter; import java.io.StringWriter; @@ -42,6 +41,7 @@ import static sleeper.core.properties.instance.LoggingLevelsProperty.LOGGING_LEVEL; import static sleeper.core.properties.table.TableProperty.DYNAMODB_STRONGLY_CONSISTENT_READS; import static sleeper.core.properties.table.TableProperty.ROW_GROUP_SIZE; +import static sleeper.core.properties.testutils.InstancePropertiesTestHelper.createTestInstanceProperties; class UpdatePropertiesWithTextEditorIT { @@ -61,7 +61,7 @@ class OpenInstanceProperties { @Test void shouldInvokeNanoOnInstancePropertiesFile() throws Exception { // Given - InstanceProperties properties = InstancePropertiesTestHelper.createTestInstanceProperties(); + InstanceProperties properties = createTestInstanceProperties(); // When / Then assertThat(helper.openInstancePropertiesGetCommandRun(properties)) @@ -71,7 +71,7 @@ void shouldInvokeNanoOnInstancePropertiesFile() throws Exception { @Test void shouldWriteInstancePropertiesFile() throws Exception { // Given - InstanceProperties properties = InstancePropertiesTestHelper.createTestInstanceProperties(); + InstanceProperties properties = createTestInstanceProperties(); // When / Then assertThat(helper.openInstancePropertiesGetPropertiesWritten(properties)) @@ -81,9 +81,9 @@ void shouldWriteInstancePropertiesFile() throws Exception { @Test void shouldGetDiffAfterPropertiesChanged() throws Exception { // Given - InstanceProperties before = InstancePropertiesTestHelper.createTestInstanceProperties(); + InstanceProperties before = createTestInstanceProperties(); before.set(INGEST_SOURCE_BUCKET, "bucket-before"); - InstanceProperties after = InstancePropertiesTestHelper.createTestInstanceProperties(); + InstanceProperties after = createTestInstanceProperties(); after.set(INGEST_SOURCE_BUCKET, "bucket-after"); // When / Then @@ -95,8 +95,8 @@ void shouldGetDiffAfterPropertiesChanged() throws Exception { @Test void shouldRetrievePropertiesAfterChange() throws Exception { // Given - InstanceProperties before = InstancePropertiesTestHelper.createTestInstanceProperties(); - InstanceProperties after = InstancePropertiesTestHelper.createTestInstanceProperties(); + InstanceProperties before = createTestInstanceProperties(); + InstanceProperties after = createTestInstanceProperties(); after.set(MAXIMUM_CONNECTIONS_TO_S3, "abc"); // When @@ -109,7 +109,7 @@ void shouldRetrievePropertiesAfterChange() throws Exception { @Test void shouldFormatPropertiesUsingPrettyPrinter() throws Exception { // Given - InstanceProperties properties = InstancePropertiesTestHelper.createTestInstanceProperties(); + InstanceProperties properties = createTestInstanceProperties(); // When String tempFileString = Files.readString(helper.openInstancePropertiesGetPathToFile(properties)); @@ -164,7 +164,7 @@ class FilterByGroup { @Test void shouldWriteSingleInstancePropertyGroupToFile() throws Exception { // Given - InstanceProperties properties = InstancePropertiesTestHelper.createTestInstanceProperties(); + InstanceProperties properties = createTestInstanceProperties(); properties.set(LOGGING_LEVEL, "ERROR"); // When / Then @@ -176,7 +176,7 @@ void shouldWriteSingleInstancePropertyGroupToFile() throws Exception { @Test void shouldFormatPropertiesUsingPrettyPrinter() throws Exception { // Given - InstanceProperties properties = InstancePropertiesTestHelper.createTestInstanceProperties(); + InstanceProperties properties = createTestInstanceProperties(); properties.set(LOGGING_LEVEL, "ERROR"); // When @@ -193,9 +193,9 @@ void shouldFormatPropertiesUsingPrettyPrinter() throws Exception { @Test void shouldCreateUpdateRequestWithInstanceProperties() throws Exception { // Given - InstanceProperties before = InstancePropertiesTestHelper.createTestInstanceProperties(); + InstanceProperties before = createTestInstanceProperties(); before.set(LOGGING_LEVEL, "ERROR"); - InstanceProperties after = InstancePropertiesTestHelper.createTestInstanceProperties(); + InstanceProperties after = createTestInstanceProperties(); after.set(LOGGING_LEVEL, "INFO"); // When @@ -231,9 +231,9 @@ void shouldCreateUpdateRequestWithTableProperties() throws Exception { @Test void shouldUnsetPropertyWhenRemovedInEditor() throws Exception { // Given - InstanceProperties before = InstancePropertiesTestHelper.createTestInstanceProperties(); + InstanceProperties before = createTestInstanceProperties(); before.set(LOGGING_LEVEL, "ERROR"); - InstanceProperties after = InstancePropertiesTestHelper.createTestInstanceProperties(); + InstanceProperties after = createTestInstanceProperties(); after.unset(LOGGING_LEVEL); // When @@ -250,7 +250,7 @@ void shouldUnsetPropertyWhenRemovedInEditor() throws Exception { @Test void shouldNotShowUnknownProperties() throws Exception { // Given - InstanceProperties properties = InstancePropertiesTestHelper.createTestInstanceProperties(); + InstanceProperties properties = createTestInstanceProperties(); properties.getProperties().setProperty("unknown.property", "some-value"); // When @@ -261,9 +261,9 @@ void shouldNotShowUnknownProperties() throws Exception { @Test void shouldUpdateAnUnknownProperty() throws Exception { // Given - InstanceProperties before = InstancePropertiesTestHelper.createTestInstanceProperties(); + InstanceProperties before = createTestInstanceProperties(); before.getProperties().setProperty("unknown.property", "value-before"); - InstanceProperties after = InstancePropertiesTestHelper.createTestInstanceProperties(); + InstanceProperties after = createTestInstanceProperties(); after.getProperties().setProperty("unknown.property", "value-after"); // When @@ -280,9 +280,9 @@ void shouldUpdateAnUnknownProperty() throws Exception { @Test void shouldUpdateAPropertyOutsideTheSpecifiedGroup() throws Exception { // Given - InstanceProperties before = InstancePropertiesTestHelper.createTestInstanceProperties(); + InstanceProperties before = createTestInstanceProperties(); before.set(INGEST_SOURCE_BUCKET, "bucket-before"); - InstanceProperties after = InstancePropertiesTestHelper.createTestInstanceProperties(); + InstanceProperties after = createTestInstanceProperties(); after.set(INGEST_SOURCE_BUCKET, "bucket-after"); // When @@ -299,9 +299,9 @@ void shouldUpdateAPropertyOutsideTheSpecifiedGroup() throws Exception { @Test void shouldLeaveUnknownPropertyUnchangedWhenEditingAnotherProperty() throws Exception { // Given - InstanceProperties before = InstancePropertiesTestHelper.createTestInstanceProperties(); + InstanceProperties before = createTestInstanceProperties(); before.getProperties().setProperty("unknown.property", "test-value"); - InstanceProperties after = InstancePropertiesTestHelper.createTestInstanceProperties(); + InstanceProperties after = createTestInstanceProperties(); after.getProperties().setProperty("unknown.property", "test-value"); after.set(LOGGING_LEVEL, "TRACE"); From e4b91e7a74ae35ab6f150a960ed704edc5435a36 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Mon, 24 Nov 2025 10:00:45 +0000 Subject: [PATCH 26/65] 5793: Refactor remaining populate usages out of testing --- .../admin/properties/PropertiesDiffTest.java | 6 +- .../UpdatePropertiesWithTextEditorIT.java | 14 ++-- .../PopulateInstancePropertiesAwsIT.java | 58 --------------- .../deploy/PopulatePropertiesTestHelper.java | 71 ------------------- .../testutils/TablePropertiesTestHelper.java | 14 ++++ 5 files changed, 24 insertions(+), 139 deletions(-) delete mode 100644 java/clients/src/test/java/sleeper/clients/deploy/properties/PopulateInstancePropertiesAwsIT.java delete mode 100644 java/core/src/test/java/sleeper/core/deploy/PopulatePropertiesTestHelper.java diff --git a/java/clients/src/test/java/sleeper/clients/admin/properties/PropertiesDiffTest.java b/java/clients/src/test/java/sleeper/clients/admin/properties/PropertiesDiffTest.java index 37532853327..bf88a379b8c 100644 --- a/java/clients/src/test/java/sleeper/clients/admin/properties/PropertiesDiffTest.java +++ b/java/clients/src/test/java/sleeper/clients/admin/properties/PropertiesDiffTest.java @@ -33,13 +33,13 @@ import static sleeper.clients.admin.properties.PropertiesDiffTestHelper.newValue; import static sleeper.clients.admin.properties.PropertiesDiffTestHelper.valueChanged; import static sleeper.clients.admin.properties.PropertiesDiffTestHelper.valueDeleted; -import static sleeper.core.deploy.PopulatePropertiesTestHelper.generateTestTableProperties; import static sleeper.core.properties.PropertiesUtils.loadProperties; import static sleeper.core.properties.instance.CommonProperty.LOG_RETENTION_IN_DAYS; import static sleeper.core.properties.instance.CommonProperty.MAXIMUM_CONNECTIONS_TO_S3; import static sleeper.core.properties.instance.IngestProperty.INGEST_SOURCE_BUCKET; import static sleeper.core.properties.table.TableProperty.ITERATOR_CONFIG; import static sleeper.core.properties.testutils.InstancePropertiesTestHelper.createSimpleTestInstancePropertiesWithId; +import static sleeper.core.properties.testutils.TablePropertiesTestHelper.generateSimpleTestTableProperties; public class PropertiesDiffTest { @@ -181,9 +181,9 @@ class CompareTableProperties { @Test void shouldDetectPropertyHasBeenUpdated() { // Given - TableProperties before = generateTestTableProperties(); + TableProperties before = generateSimpleTestTableProperties(); before.set(ITERATOR_CONFIG, "config-before"); - TableProperties after = generateTestTableProperties(); + TableProperties after = generateSimpleTestTableProperties(); after.set(ITERATOR_CONFIG, "config-after"); // When / Then diff --git a/java/clients/src/test/java/sleeper/clients/admin/properties/UpdatePropertiesWithTextEditorIT.java b/java/clients/src/test/java/sleeper/clients/admin/properties/UpdatePropertiesWithTextEditorIT.java index e9674425c2e..81ef1ab4060 100644 --- a/java/clients/src/test/java/sleeper/clients/admin/properties/UpdatePropertiesWithTextEditorIT.java +++ b/java/clients/src/test/java/sleeper/clients/admin/properties/UpdatePropertiesWithTextEditorIT.java @@ -34,7 +34,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.InstanceOfAssertFactories.LIST; import static sleeper.clients.admin.properties.PropertiesDiffTestHelper.valueChanged; -import static sleeper.core.deploy.PopulatePropertiesTestHelper.generateTestTableProperties; import static sleeper.core.properties.PropertiesUtils.loadProperties; import static sleeper.core.properties.instance.CommonProperty.MAXIMUM_CONNECTIONS_TO_S3; import static sleeper.core.properties.instance.IngestProperty.INGEST_SOURCE_BUCKET; @@ -42,6 +41,7 @@ import static sleeper.core.properties.table.TableProperty.DYNAMODB_STRONGLY_CONSISTENT_READS; import static sleeper.core.properties.table.TableProperty.ROW_GROUP_SIZE; import static sleeper.core.properties.testutils.InstancePropertiesTestHelper.createTestInstanceProperties; +import static sleeper.core.properties.testutils.TablePropertiesTestHelper.generateSimpleTestTableProperties; class UpdatePropertiesWithTextEditorIT { @@ -128,9 +128,9 @@ class OpenTableProperties { @Test void shouldUpdateTableProperties() throws Exception { // Given - TableProperties before = generateTestTableProperties(); + TableProperties before = generateSimpleTestTableProperties(); before.set(ROW_GROUP_SIZE, "123"); - TableProperties after = generateTestTableProperties(); + TableProperties after = generateSimpleTestTableProperties(); after.set(ROW_GROUP_SIZE, "456"); // When @@ -145,8 +145,8 @@ void shouldUpdateTableProperties() throws Exception { @Test void shouldRetrieveTablePropertiesAfterChange() throws Exception { // Given - TableProperties before = generateTestTableProperties(); - TableProperties after = generateTestTableProperties(); + TableProperties before = generateSimpleTestTableProperties(); + TableProperties after = generateSimpleTestTableProperties(); after.set(ROW_GROUP_SIZE, "456"); // When @@ -212,9 +212,9 @@ void shouldCreateUpdateRequestWithInstanceProperties() throws Exception { @Test void shouldCreateUpdateRequestWithTableProperties() throws Exception { // Given - TableProperties before = generateTestTableProperties(); + TableProperties before = generateSimpleTestTableProperties(); before.set(DYNAMODB_STRONGLY_CONSISTENT_READS, "false"); - TableProperties after = generateTestTableProperties(); + TableProperties after = generateSimpleTestTableProperties(); after.set(DYNAMODB_STRONGLY_CONSISTENT_READS, "true"); // When diff --git a/java/clients/src/test/java/sleeper/clients/deploy/properties/PopulateInstancePropertiesAwsIT.java b/java/clients/src/test/java/sleeper/clients/deploy/properties/PopulateInstancePropertiesAwsIT.java deleted file mode 100644 index 2ac06995f35..00000000000 --- a/java/clients/src/test/java/sleeper/clients/deploy/properties/PopulateInstancePropertiesAwsIT.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright 2022-2025 Crown Copyright - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package sleeper.clients.deploy.properties; - -import org.junit.jupiter.api.Test; -import software.amazon.awssdk.regions.Region; - -import sleeper.core.deploy.PopulateInstanceProperties; -import sleeper.core.properties.instance.InstanceProperties; -import sleeper.localstack.test.LocalStackTestBase; - -import java.util.Map; - -import static org.assertj.core.api.Assertions.assertThat; -import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; -import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; -import static sleeper.core.properties.instance.CommonProperty.ID; -import static sleeper.core.properties.instance.CommonProperty.SUBNETS; -import static sleeper.core.properties.instance.CommonProperty.VPC_ID; - -public class PopulateInstancePropertiesAwsIT extends LocalStackTestBase { - - @Test - void shouldPopulateInstancePropertiesCorrectly() { - // Given/When - InstanceProperties properties = populateInstancePropertiesBuilder() - .build().populate(new InstanceProperties()); - - // Then - InstanceProperties expected = new InstanceProperties(); - expected.setTags(Map.of("InstanceID", "test-instance")); - expected.set(ID, "test-instance"); - expected.set(VPC_ID, "some-vpc"); - expected.set(SUBNETS, "some-subnet"); - expected.set(ACCOUNT, stsClient.getCallerIdentity().account()); - expected.set(REGION, localStackContainer.getRegion()); - - assertThat(properties).isEqualTo(expected); - } - - private PopulateInstanceProperties.Builder populateInstancePropertiesBuilder() { - return PopulateInstancePropertiesAws.builder(stsClient, () -> Region.of(localStackContainer.getRegion())) - .instanceId("test-instance").vpcId("some-vpc").subnetIds("some-subnet"); - } -} diff --git a/java/core/src/test/java/sleeper/core/deploy/PopulatePropertiesTestHelper.java b/java/core/src/test/java/sleeper/core/deploy/PopulatePropertiesTestHelper.java deleted file mode 100644 index 86bf3e0021f..00000000000 --- a/java/core/src/test/java/sleeper/core/deploy/PopulatePropertiesTestHelper.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright 2022-2025 Crown Copyright - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package sleeper.core.deploy; - -import sleeper.core.properties.instance.InstanceProperties; -import sleeper.core.properties.table.TableProperties; - -import static sleeper.core.properties.table.TableProperty.TABLE_NAME; -import static sleeper.core.schema.SchemaTestHelper.createSchemaWithKey; - -/** - * Test helpers to generate properties using PopulateInstanceProperties. - */ -public class PopulatePropertiesTestHelper { - private PopulatePropertiesTestHelper() { - } - - /** - * Generates test instance properties using PopulateInstanceProperties. - * - * @return the generated properties - */ - public static InstanceProperties generateTestInstanceProperties() { - return createTestPopulateInstanceProperties().populate(new InstanceProperties()); - } - - /** - * Creates a test instance of PopulateInstanceProperties with dummy values. - * - * @return the object - */ - public static PopulateInstanceProperties createTestPopulateInstanceProperties() { - return testPopulateInstancePropertiesBuilder().build(); - } - - /** - * Creates a test builder for PopulateInstanceProperties with dummy values. - * - * @return the builder - */ - public static PopulateInstanceProperties.Builder testPopulateInstancePropertiesBuilder() { - return PopulateInstanceProperties.builder() - .accountSupplier(() -> "test-account-id").regionIdSupplier(() -> "test-region") - .instanceId("test-instance").vpcId("some-vpc").subnetIds("some-subnet"); - } - - /** - * Generates test table properties using PopulateInstanceProperties. - * - * @return the generated properties - */ - public static TableProperties generateTestTableProperties() { - TableProperties tableProperties = new TableProperties(generateTestInstanceProperties()); - tableProperties.set(TABLE_NAME, "test-table"); - tableProperties.setSchema(createSchemaWithKey("key")); - return tableProperties; - } -} diff --git a/java/core/src/test/java/sleeper/core/properties/testutils/TablePropertiesTestHelper.java b/java/core/src/test/java/sleeper/core/properties/testutils/TablePropertiesTestHelper.java index 41557dff24e..c4f16100ff6 100644 --- a/java/core/src/test/java/sleeper/core/properties/testutils/TablePropertiesTestHelper.java +++ b/java/core/src/test/java/sleeper/core/properties/testutils/TablePropertiesTestHelper.java @@ -24,6 +24,8 @@ import static sleeper.core.properties.table.TableProperty.TABLE_ID; import static sleeper.core.properties.table.TableProperty.TABLE_NAME; +import static sleeper.core.properties.testutils.InstancePropertiesTestHelper.createSimpleTestInstanceProperties; +import static sleeper.core.schema.SchemaTestHelper.createSchemaWithKey; /** * Helpers to create table properties. @@ -63,4 +65,16 @@ public static TableProperties createTestTablePropertiesWithNoSchema(InstanceProp tableProperties.set(TABLE_ID, tableId); return tableProperties; } + + /** + * Creates simple properties for a Sleeper table. + * + * @return the table properties + */ + public static TableProperties generateSimpleTestTableProperties() { + TableProperties tableProperties = new TableProperties(createSimpleTestInstanceProperties()); + tableProperties.set(TABLE_NAME, "test-table"); + tableProperties.setSchema(createSchemaWithKey("key")); + return tableProperties; + } } From 81b92bb9bf4c0a999becd31e9717ffc8ec9f7866 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Mon, 24 Nov 2025 10:03:55 +0000 Subject: [PATCH 27/65] 5793: Refactor out the populateInstaceProperties class and last remaining method --- .../clients/deploy/DeployInstance.java | 13 +- .../PopulateInstancePropertiesAws.java | 34 ---- .../deploy/PopulateInstanceProperties.java | 176 ------------------ 3 files changed, 11 insertions(+), 212 deletions(-) delete mode 100644 java/clients/src/main/java/sleeper/clients/deploy/properties/PopulateInstancePropertiesAws.java delete mode 100644 java/core/src/main/java/sleeper/core/deploy/PopulateInstanceProperties.java diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployInstance.java index 8908d244b7f..409659a07fb 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployInstance.java @@ -30,9 +30,9 @@ import sleeper.clients.util.ClientUtils; import sleeper.clients.util.cdk.CdkCommand; import sleeper.clients.util.cdk.InvokeCdk; -import sleeper.core.deploy.PopulateInstanceProperties; import sleeper.core.deploy.SleeperInstanceConfiguration; import sleeper.core.properties.instance.InstanceProperties; +import sleeper.core.properties.instance.UserDefinedInstanceProperty; import sleeper.core.properties.local.SaveLocalProperties; import java.io.IOException; @@ -84,7 +84,7 @@ public static void main(String[] args) throws IOException, InterruptedException instanceConfiguration.getInstanceProperties().set(ID, instanceId); instanceConfiguration.getInstanceProperties().set(VPC_ID, vpcId); instanceConfiguration.getInstanceProperties().set(SUBNETS, subnetIds); - PopulateInstanceProperties.setFromEnvironmentVariables(instanceConfiguration.getInstanceProperties()); + setFromEnvironmentVariables(instanceConfiguration.getInstanceProperties()); instanceConfiguration.validate(); DeployInstance deployInstance = new DeployInstance( @@ -149,4 +149,13 @@ static WriteLocalProperties toDirectory(Path directory) { }; } } + + private static void setFromEnvironmentVariables(InstanceProperties instanceProperties) { + for (UserDefinedInstanceProperty property : UserDefinedInstanceProperty.getAll()) { + String value = System.getenv(property.toEnvironmentVariable()); + if (value != null) { + instanceProperties.set(property, value); + } + } + } } diff --git a/java/clients/src/main/java/sleeper/clients/deploy/properties/PopulateInstancePropertiesAws.java b/java/clients/src/main/java/sleeper/clients/deploy/properties/PopulateInstancePropertiesAws.java deleted file mode 100644 index acd966499bd..00000000000 --- a/java/clients/src/main/java/sleeper/clients/deploy/properties/PopulateInstancePropertiesAws.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2022-2025 Crown Copyright - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package sleeper.clients.deploy.properties; - -import software.amazon.awssdk.regions.providers.AwsRegionProvider; -import software.amazon.awssdk.services.sts.StsClient; - -import sleeper.core.deploy.PopulateInstanceProperties; - -public class PopulateInstancePropertiesAws { - - private PopulateInstancePropertiesAws() { - } - - public static PopulateInstanceProperties.Builder builder(StsClient stsClient, AwsRegionProvider regionProvider) { - return PopulateInstanceProperties.builder() - .accountSupplier(stsClient.getCallerIdentity()::account) - .regionIdSupplier(() -> regionProvider.getRegion().id()); - } -} diff --git a/java/core/src/main/java/sleeper/core/deploy/PopulateInstanceProperties.java b/java/core/src/main/java/sleeper/core/deploy/PopulateInstanceProperties.java deleted file mode 100644 index 07b255e1653..00000000000 --- a/java/core/src/main/java/sleeper/core/deploy/PopulateInstanceProperties.java +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Copyright 2022-2025 Crown Copyright - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package sleeper.core.deploy; - -import org.apache.commons.lang3.ObjectUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import sleeper.core.properties.instance.InstanceProperties; -import sleeper.core.properties.instance.UserDefinedInstanceProperty; - -import java.util.Objects; -import java.util.function.Consumer; -import java.util.function.Supplier; - -import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; -import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; -import static sleeper.core.properties.instance.CommonProperty.ID; -import static sleeper.core.properties.instance.CommonProperty.SUBNETS; -import static sleeper.core.properties.instance.CommonProperty.VPC_ID; - -/** - * Populates instance properties when deploying a new instance or when tearing down an instance without the properties. - */ -public class PopulateInstanceProperties { - public static final Logger LOGGER = LoggerFactory.getLogger(PopulateInstanceProperties.class); - - private final Supplier accountSupplier; - private final Supplier regionIdSupplier; - private final String instanceId; - private final String vpcId; - private final String subnetIds; - private final Consumer extraInstanceProperties; - - private PopulateInstanceProperties(Builder builder) { - accountSupplier = Objects.requireNonNull(builder.accountSupplier, "accountSupplier must not be null"); - regionIdSupplier = Objects.requireNonNull(builder.regionIdSupplier, "regionIdSupplier must not be null"); - instanceId = ObjectUtils.requireNonEmpty(builder.instanceId, "instanceId must not be empty"); - vpcId = ObjectUtils.requireNonEmpty(builder.vpcId, "vpcId must not be empty"); - subnetIds = ObjectUtils.requireNonEmpty(builder.subnetIds, "subnetIds must not be empty"); - extraInstanceProperties = Objects.requireNonNull(builder.extraInstanceProperties, "extraInstanceProperties must not be null"); - } - - public static Builder builder() { - return new Builder(); - } - - /** - * Sets instance properties when deploying a new instance. - * - * @param properties the properties specified by the user - * @return the populated properties - */ - public InstanceProperties populate(InstanceProperties properties) { - properties.set(ID, instanceId); - properties.set(ACCOUNT, accountSupplier.get()); - properties.set(REGION, regionIdSupplier.get()); - properties.set(VPC_ID, vpcId); - properties.set(SUBNETS, subnetIds); - extraInstanceProperties.accept(properties); - return properties; - } - - /** - * Sets the values of properties that have been set in environment variables. - * - * @param instanceProperties the instance properties - */ - public static void setFromEnvironmentVariables(InstanceProperties instanceProperties) { - for (UserDefinedInstanceProperty property : UserDefinedInstanceProperty.getAll()) { - String value = System.getenv(property.toEnvironmentVariable()); - if (value != null) { - instanceProperties.set(property, value); - } - } - } - - /** - * Builds instances of this class. - */ - public static final class Builder { - private Supplier accountSupplier; - private Supplier regionIdSupplier; - private String instanceId; - private String vpcId; - private String subnetIds; - private Consumer extraInstanceProperties = properties -> { - }; - - private Builder() { - } - - /** - * Sets the AWS code to retrieve the account to deploy to. - * - * @param accountSupplier the code - * @return this builder - */ - public Builder accountSupplier(Supplier accountSupplier) { - this.accountSupplier = accountSupplier; - return this; - } - - /** - * Sets the AWS code to retrieve the ID of the region to deploy to. - * - * @param regionIdSupplier the code - * @return this builder - */ - public Builder regionIdSupplier(Supplier regionIdSupplier) { - this.regionIdSupplier = regionIdSupplier; - return this; - } - - /** - * Sets the instance ID of the Sleeper instance to deploy. - * - * @param instanceId the instance ID - * @return this builder - */ - public Builder instanceId(String instanceId) { - this.instanceId = instanceId; - return this; - } - - /** - * Sets the ID of the VPC to deploy to. - * - * @param vpcId the VPC ID - * @return this builder - */ - public Builder vpcId(String vpcId) { - this.vpcId = vpcId; - return this; - } - - /** - * Sets the comma-separated IDs of the subnets to deploy to. - * - * @param subnetIds the subnet IDs (comma-separated) - * @return this builder - */ - public Builder subnetIds(String subnetIds) { - this.subnetIds = subnetIds; - return this; - } - - /** - * Sets any extra instance properties that should be set for the instance. - * - * @param extraInstanceProperties the function to set the extra properties - * @return this builder - */ - public Builder extraInstanceProperties(Consumer extraInstanceProperties) { - this.extraInstanceProperties = extraInstanceProperties; - return this; - } - - public PopulateInstanceProperties build() { - return new PopulateInstanceProperties(this); - } - } -} From 0bba933e5097be9f4f6c67b88f53b61ae33f56fe Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Mon, 24 Nov 2025 10:58:05 +0000 Subject: [PATCH 28/65] 5793: Typo fix --- .../dsl/instance/DeployedSleeperInstanceRedeployTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/instance/DeployedSleeperInstanceRedeployTest.java b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/instance/DeployedSleeperInstanceRedeployTest.java index 1e787e0a027..6f4607a8387 100644 --- a/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/instance/DeployedSleeperInstanceRedeployTest.java +++ b/java/system-test/system-test-dsl/src/test/java/sleeper/systemtest/dsl/instance/DeployedSleeperInstanceRedeployTest.java @@ -126,7 +126,7 @@ private InstanceProperties fakeDeploy() { // Replicate behaviour in DeployNewInstance properties.set(ACCOUNT, "test-account"); properties.set(REGION, "test-region"); - properties.set(ID, "test-instacne"); + properties.set(ID, "test-instance"); properties.set(VPC_ID, "test-vpc"); properties.set(SUBNETS, "test-subnet"); From 405d36aef110782b665a9e22841b73b65ec3a7d2 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Mon, 24 Nov 2025 11:00:35 +0000 Subject: [PATCH 29/65] 5793: Refactor location of loadTags --- java/cdk/src/main/java/sleeper/cdk/SleeperCdkApp.java | 6 ------ .../cdk/src/main/java/sleeper/cdk/SleeperInstanceProps.java | 5 +++++ 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/java/cdk/src/main/java/sleeper/cdk/SleeperCdkApp.java b/java/cdk/src/main/java/sleeper/cdk/SleeperCdkApp.java index 48ca076cc1a..073252ee1c5 100644 --- a/java/cdk/src/main/java/sleeper/cdk/SleeperCdkApp.java +++ b/java/cdk/src/main/java/sleeper/cdk/SleeperCdkApp.java @@ -25,8 +25,6 @@ import sleeper.core.properties.instance.InstanceProperties; -import java.util.Properties; - import static sleeper.core.properties.instance.CommonProperty.ID; /** @@ -48,10 +46,6 @@ public static void main(String[] args) { InstanceProperties instanceProperties = props.getInstanceProperties(); String id = instanceProperties.get(ID); - Properties tagsProperties = instanceProperties.getTagsProperties(); - tagsProperties.setProperty("InstanceID", id); - instanceProperties.loadTags(tagsProperties); - Environment environment = Environment.builder() .account(System.getenv("CDK_DEFAULT_ACCOUNT")) .region(System.getenv("CDK_DEFAULT_REGION")) diff --git a/java/cdk/src/main/java/sleeper/cdk/SleeperInstanceProps.java b/java/cdk/src/main/java/sleeper/cdk/SleeperInstanceProps.java index 4a372c62c21..b42e2f9de0c 100644 --- a/java/cdk/src/main/java/sleeper/cdk/SleeperInstanceProps.java +++ b/java/cdk/src/main/java/sleeper/cdk/SleeperInstanceProps.java @@ -33,6 +33,7 @@ import java.nio.file.Path; import java.util.List; import java.util.Objects; +import java.util.Properties; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.VERSION; import static sleeper.core.properties.instance.CommonProperty.ID; @@ -260,6 +261,10 @@ public SleeperInstanceProps build() { Objects.requireNonNull(tableProperties, "tableProperties must not be null"); Objects.requireNonNull(jars, "jars must not be null"); + Properties tagsProperties = instanceProperties.getTagsProperties(); + tagsProperties.setProperty("InstanceID", instanceProperties.get(ID)); + instanceProperties.loadTags(tagsProperties); + if (validateProperties) { instanceProperties.validate(); try { From 475ed847a518cf0da0439beaea2416fca8ae1031 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Mon, 24 Nov 2025 11:02:44 +0000 Subject: [PATCH 30/65] 5793: Update local variable definition --- .../sleeper/clients/admin/properties/PropertiesDiffTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/java/clients/src/test/java/sleeper/clients/admin/properties/PropertiesDiffTest.java b/java/clients/src/test/java/sleeper/clients/admin/properties/PropertiesDiffTest.java index bf88a379b8c..143ed8f1621 100644 --- a/java/clients/src/test/java/sleeper/clients/admin/properties/PropertiesDiffTest.java +++ b/java/clients/src/test/java/sleeper/clients/admin/properties/PropertiesDiffTest.java @@ -27,7 +27,6 @@ import sleeper.core.properties.table.TableProperties; import java.util.List; -import java.util.UUID; import static org.assertj.core.api.Assertions.assertThat; import static sleeper.clients.admin.properties.PropertiesDiffTestHelper.newValue; @@ -46,7 +45,7 @@ public class PropertiesDiffTest { @DisplayName("Compare instance properties") @Nested class CompareInstanceProperties { - String id = UUID.randomUUID().toString(); + String id = "test-id"; @Test void shouldDetectNoChanges() { From 435c227b8e6e650a048cd9a2d1a17360baf0635c Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Mon, 24 Nov 2025 11:03:12 +0000 Subject: [PATCH 31/65] 5793: Update wording --- .../sleeper/clients/admin/properties/PropertiesDiffTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/clients/src/test/java/sleeper/clients/admin/properties/PropertiesDiffTest.java b/java/clients/src/test/java/sleeper/clients/admin/properties/PropertiesDiffTest.java index 143ed8f1621..8137ac1d6e0 100644 --- a/java/clients/src/test/java/sleeper/clients/admin/properties/PropertiesDiffTest.java +++ b/java/clients/src/test/java/sleeper/clients/admin/properties/PropertiesDiffTest.java @@ -45,7 +45,7 @@ public class PropertiesDiffTest { @DisplayName("Compare instance properties") @Nested class CompareInstanceProperties { - String id = "test-id"; + String id = "test-instance"; @Test void shouldDetectNoChanges() { From 2ddd95b754ab4aaabec8356b70cf38a779abf55d Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Mon, 24 Nov 2025 11:04:30 +0000 Subject: [PATCH 32/65] 5793: Remove properties not required --- .../deploy/container/UploadDockerImagesToEcrTestBase.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/java/clients/src/test/java/sleeper/clients/deploy/container/UploadDockerImagesToEcrTestBase.java b/java/clients/src/test/java/sleeper/clients/deploy/container/UploadDockerImagesToEcrTestBase.java index 6e2e1121634..173aea2169b 100644 --- a/java/clients/src/test/java/sleeper/clients/deploy/container/UploadDockerImagesToEcrTestBase.java +++ b/java/clients/src/test/java/sleeper/clients/deploy/container/UploadDockerImagesToEcrTestBase.java @@ -26,8 +26,6 @@ import static sleeper.clients.util.command.Command.command; import static sleeper.clients.util.command.CommandPipeline.pipeline; -import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; -import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.VERSION; import static sleeper.core.properties.instance.CommonProperty.ID; import static sleeper.core.properties.testutils.InstancePropertiesTestHelper.createTestInstanceProperties; @@ -39,8 +37,6 @@ public abstract class UploadDockerImagesToEcrTestBase extends DockerImagesTestBa @BeforeEach void setUpBase() { properties.set(ID, "test-instance"); - properties.set(ACCOUNT, "123"); - properties.set(REGION, "test-region"); properties.set(VERSION, "1.0.0"); } From 0b995c043c37146b72b92b064cad9104c879ccf8 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Mon, 24 Nov 2025 11:04:53 +0000 Subject: [PATCH 33/65] 5793: Update templates definiton --- .../deploy/documentation/GeneratePropertiesTemplates.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/java/clients/src/main/java/sleeper/clients/deploy/documentation/GeneratePropertiesTemplates.java b/java/clients/src/main/java/sleeper/clients/deploy/documentation/GeneratePropertiesTemplates.java index 48ad5c1dddb..6695577c5f1 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/documentation/GeneratePropertiesTemplates.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/documentation/GeneratePropertiesTemplates.java @@ -39,8 +39,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; -import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; import static sleeper.core.properties.instance.CommonProperty.ID; import static sleeper.core.properties.instance.CommonProperty.SUBNETS; import static sleeper.core.properties.instance.CommonProperty.VPC_ID; @@ -68,8 +66,6 @@ public class GeneratePropertiesTemplates { private static final Map BASIC_INSTANCE_EXAMPLE_VALUES = Map.of( ID, "basic-example", - ACCOUNT, "1234567890", - REGION, "eu-west-2", VPC_ID, "1234567890", SUBNETS, "subnet-abcdefgh"); From 159bc0c39c6a2a087fd9cefc5e10fdad3145067a Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Mon, 24 Nov 2025 11:12:30 +0000 Subject: [PATCH 34/65] 5793: Update for a common property usage --- .../properties/SleeperPropertyMarkdownTableTest.java | 2 +- .../clients/src/test/resources/reports/table/property.txt | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/java/clients/src/test/java/sleeper/clients/deploy/properties/SleeperPropertyMarkdownTableTest.java b/java/clients/src/test/java/sleeper/clients/deploy/properties/SleeperPropertyMarkdownTableTest.java index 5b86f22486f..83474e01d1c 100644 --- a/java/clients/src/test/java/sleeper/clients/deploy/properties/SleeperPropertyMarkdownTableTest.java +++ b/java/clients/src/test/java/sleeper/clients/deploy/properties/SleeperPropertyMarkdownTableTest.java @@ -36,7 +36,7 @@ public class SleeperPropertyMarkdownTableTest { void shouldWriteTable() throws IOException { // Given TableWriter writer = SleeperPropertyMarkdownTable.createTableWriterForUserDefinedProperties(List.of( - CdkDefinedInstanceProperty.REGION, + CommonProperty.RETAIN_LOGS_AFTER_DESTROY, CommonProperty.FILE_SYSTEM)); // When diff --git a/java/clients/src/test/resources/reports/table/property.txt b/java/clients/src/test/resources/reports/table/property.txt index f73e316658d..624a9aefa64 100644 --- a/java/clients/src/test/resources/reports/table/property.txt +++ b/java/clients/src/test/resources/reports/table/property.txt @@ -1,4 +1,4 @@ -| Property Name | Description | Default Value | Run CDK Deploy When Changed | -|--------------------|----------------------------------------------|---------------|-----------------------------| -| sleeper.region | The AWS region to deploy to. | | true | -| sleeper.filesystem | The Hadoop filesystem used to connect to S3. | s3a:// | false | +| Property Name | Description | Default Value | Run CDK Deploy When Changed | +|---------------------------------------|------------------------------------------------------------------------|---------------|-----------------------------| +| sleeper.retain.logs.after.destroy | Whether to keep the sleeper log groups when the instance is destroyed. | | false | +| sleeper.filesystem | The Hadoop filesystem used to connect to S3. | s3a:// | false | From 9c332cff90acb2cac4116f9c162b5856e33c3c48 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Mon, 24 Nov 2025 14:00:48 +0000 Subject: [PATCH 35/65] 5793: Update property.txt --- .../clients/src/test/resources/reports/table/property.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/java/clients/src/test/resources/reports/table/property.txt b/java/clients/src/test/resources/reports/table/property.txt index 624a9aefa64..ceacf7f6b94 100644 --- a/java/clients/src/test/resources/reports/table/property.txt +++ b/java/clients/src/test/resources/reports/table/property.txt @@ -1,4 +1,4 @@ -| Property Name | Description | Default Value | Run CDK Deploy When Changed | -|---------------------------------------|------------------------------------------------------------------------|---------------|-----------------------------| -| sleeper.retain.logs.after.destroy | Whether to keep the sleeper log groups when the instance is destroyed. | | false | -| sleeper.filesystem | The Hadoop filesystem used to connect to S3. | s3a:// | false | +| Property Name | Description | Default Value | Run CDK Deploy When Changed | +|-----------------------------------|------------------------------------------------------------------------|---------------|-----------------------------| +| sleeper.retain.logs.after.destroy | Whether to keep the sleeper log groups when the instance is destroyed. | true | true | +| sleeper.filesystem | The Hadoop filesystem used to connect to S3. | s3a:// | false | From 364690a7c038bf56ba2aadc13dcb640285ea17af Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Mon, 24 Nov 2025 14:05:17 +0000 Subject: [PATCH 36/65] 5793: Update region and account setting --- java/cdk/src/main/java/sleeper/cdk/SleeperInstance.java | 2 ++ .../src/main/java/sleeper/systemtest/cdk/SystemTestApp.java | 3 +++ 2 files changed, 5 insertions(+) diff --git a/java/cdk/src/main/java/sleeper/cdk/SleeperInstance.java b/java/cdk/src/main/java/sleeper/cdk/SleeperInstance.java index 450235220f6..6c79570bc1b 100644 --- a/java/cdk/src/main/java/sleeper/cdk/SleeperInstance.java +++ b/java/cdk/src/main/java/sleeper/cdk/SleeperInstance.java @@ -65,6 +65,8 @@ public SleeperInstanceRoles getRoles() { */ public static SleeperInstance createAsNestedStack(Construct scope, String id, NestedStackProps stackProps, SleeperInstanceProps sleeperProps) { NestedStack stack = new NestedStack(scope, id, stackProps); + sleeperProps.getInstanceProperties().set(ACCOUNT, stack.getAccount()); + sleeperProps.getInstanceProperties().set(REGION, stack.getRegion()); return addNestedStacks(stack, sleeperProps); } diff --git a/java/system-test/system-test-cdk/src/main/java/sleeper/systemtest/cdk/SystemTestApp.java b/java/system-test/system-test-cdk/src/main/java/sleeper/systemtest/cdk/SystemTestApp.java index 5ca23015bfd..abd67ab906d 100644 --- a/java/system-test/system-test-cdk/src/main/java/sleeper/systemtest/cdk/SystemTestApp.java +++ b/java/system-test/system-test-cdk/src/main/java/sleeper/systemtest/cdk/SystemTestApp.java @@ -49,6 +49,7 @@ public class SystemTestApp extends Stack { public SystemTestApp(App app, String id, StackProps props, SleeperInstanceProps sleeperProps) { super(app, id, props); this.props = sleeperProps; + this.jars = sleeperProps.getJars(); this.instanceProperties = SystemTestProperties.from(sleeperProps.getInstanceProperties()); } @@ -65,6 +66,8 @@ public void create() { // Sleeper instance SleeperCoreStacks coreStacks = SleeperCoreStacks.create(this, props, loggingStack, autoDeleteS3Stack); SleeperOptionalStacks.create(this, props, coreStacks); + props.getInstanceProperties().set(ACCOUNT, this.getAccount()); + props.getInstanceProperties().set(REGION, this.getRegion()); coreStacks.createRoles(); // Stack for writing random data From 56497c5b0b6347462e9f9a687586492501c2a8b0 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Mon, 24 Nov 2025 14:11:46 +0000 Subject: [PATCH 37/65] 5793: Removal of unused Region from SyncJarsRequest --- .../sleeper/clients/deploy/UploadArtefacts.java | 1 - .../sleeper/clients/deploy/jar/SyncJars.java | 8 +++----- .../clients/deploy/jar/SyncJarsRequest.java | 16 +--------------- .../clients/testutil/JarsBucketITBase.java | 1 - .../instance/AwsSystemTestDeploymentDriver.java | 1 - scripts/deploy/syncJars.sh | 2 +- 6 files changed, 5 insertions(+), 24 deletions(-) diff --git a/java/clients/src/main/java/sleeper/clients/deploy/UploadArtefacts.java b/java/clients/src/main/java/sleeper/clients/deploy/UploadArtefacts.java index 2afa79e0672..68ad7375bfa 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/UploadArtefacts.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/UploadArtefacts.java @@ -137,7 +137,6 @@ public static void main(String[] rawArgs) throws IOException, InterruptedExcepti } syncJars.sync(SyncJarsRequest.builder() .bucketName(jarsBucket) - .region(region) .build()); uploadImages.upload(UploadDockerImagesToEcrRequest.builder() .ecrPrefix(ecrPrefix) diff --git a/java/clients/src/main/java/sleeper/clients/deploy/jar/SyncJars.java b/java/clients/src/main/java/sleeper/clients/deploy/jar/SyncJars.java index aca6ca04b1f..fc9d6137e55 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/jar/SyncJars.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/jar/SyncJars.java @@ -46,20 +46,18 @@ public static SyncJars fromScriptsDirectory(S3Client s3, Path scriptsDirectory) } public static void main(String[] args) throws IOException { - if (args.length < 3 || args.length > 4) { - throw new IllegalArgumentException("Usage: "); + if (args.length < 2 || args.length > 3) { + throw new IllegalArgumentException("Usage: "); } Path jarsDirectory = Path.of(args[0]); String bucketName = args[1]; - String region = args[2]; - boolean deleteOldJars = optionalArgument(args, 3) + boolean deleteOldJars = optionalArgument(args, 2) .map(Boolean::parseBoolean) .orElse(false); try (S3Client s3 = S3Client.create()) { new SyncJars(s3, jarsDirectory) .sync(SyncJarsRequest.builder() .bucketName(bucketName) - .region(region) .deleteOldJars(deleteOldJars) .build()); } diff --git a/java/clients/src/main/java/sleeper/clients/deploy/jar/SyncJarsRequest.java b/java/clients/src/main/java/sleeper/clients/deploy/jar/SyncJarsRequest.java index 8a56df32fd1..d61117c490c 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/jar/SyncJarsRequest.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/jar/SyncJarsRequest.java @@ -21,19 +21,16 @@ import java.util.Objects; import java.util.function.Predicate; -import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; import static sleeper.core.properties.instance.CommonProperty.JARS_BUCKET; public class SyncJarsRequest { private final String bucketName; - private final String region; private final Predicate uploadFilter; private final boolean deleteOldJars; private SyncJarsRequest(Builder builder) { bucketName = Objects.requireNonNull(builder.bucketName, "bucketName must not be null"); - region = Objects.requireNonNull(builder.region, "region must not be null"); uploadFilter = Objects.requireNonNull(builder.uploadFilter, "uploadFilter must not be null"); deleteOldJars = builder.deleteOldJars; } @@ -50,10 +47,6 @@ public String getBucketName() { return bucketName; } - public String getRegion() { - return region; - } - public Predicate getUploadFilter() { return uploadFilter; } @@ -64,7 +57,6 @@ public boolean isDeleteOldJars() { public static class Builder { private String bucketName; - private String region; private Predicate uploadFilter = jar -> true; private boolean deleteOldJars = false; @@ -73,11 +65,6 @@ public Builder bucketName(String bucketName) { return this; } - public Builder region(String region) { - this.region = region; - return this; - } - public Builder uploadFilter(Predicate uploadFilter) { this.uploadFilter = uploadFilter; return this; @@ -89,8 +76,7 @@ public Builder deleteOldJars(boolean deleteOldJars) { } public Builder instanceProperties(InstanceProperties instanceProperties) { - return bucketName(instanceProperties.get(JARS_BUCKET)) - .region(instanceProperties.get(REGION)); + return bucketName(instanceProperties.get(JARS_BUCKET)); } public SyncJarsRequest build() { diff --git a/java/clients/src/test/java/sleeper/clients/testutil/JarsBucketITBase.java b/java/clients/src/test/java/sleeper/clients/testutil/JarsBucketITBase.java index 7ed0bf621ad..29f41aaca4f 100644 --- a/java/clients/src/test/java/sleeper/clients/testutil/JarsBucketITBase.java +++ b/java/clients/src/test/java/sleeper/clients/testutil/JarsBucketITBase.java @@ -60,7 +60,6 @@ protected boolean syncJarsToBucket(String bucketName, boolean deleteOld) throws return new SyncJars(s3Client, tempDir) .sync(SyncJarsRequest.builder() .bucketName(bucketName) - .region(localStackContainer.getRegion()) .deleteOldJars(deleteOld) .build()); } diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSystemTestDeploymentDriver.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSystemTestDeploymentDriver.java index 0690e59654d..fe554383898 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSystemTestDeploymentDriver.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSystemTestDeploymentDriver.java @@ -120,7 +120,6 @@ private void uploadJarsAndDockerImages() throws IOException, InterruptedExceptio new SyncJars(s3, parameters.getJarsDirectory()) .sync(SyncJarsRequest.builder() .bucketName(SleeperArtefactsLocation.getDefaultJarsBucketName(parameters.getArtefactsDeploymentId())) - .region(parameters.getRegion()) .uploadFilter(jar -> LambdaJar.isFileJar(jar, CUSTOM_RESOURCES)) .build()); if (!parameters.isSystemTestClusterEnabled()) { diff --git a/scripts/deploy/syncJars.sh b/scripts/deploy/syncJars.sh index 5b7894f1d22..2dfecac30dd 100755 --- a/scripts/deploy/syncJars.sh +++ b/scripts/deploy/syncJars.sh @@ -17,7 +17,7 @@ set -e unset CDPATH if [ "$#" -lt 2 ] || [ "$#" -gt 3 ]; then - echo "Usage: $0 " + echo "Usage: $0 " exit 1 fi From 029b5ae116bf1f0a8642df6cdc1cabe3eef4e946 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Mon, 24 Nov 2025 14:20:00 +0000 Subject: [PATCH 38/65] 5793: Update wording --- docs/usage/properties/instance/cdk/common.md | 4 ++-- .../core/properties/instance/CdkDefinedInstanceProperty.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/usage/properties/instance/cdk/common.md b/docs/usage/properties/instance/cdk/common.md index 33dc75949d7..d6b87deb779 100644 --- a/docs/usage/properties/instance/cdk/common.md +++ b/docs/usage/properties/instance/cdk/common.md @@ -7,8 +7,8 @@ The following instance properties are commonly used throughout Sleeper. | sleeper.version | The version of Sleeper that is being used. This property is used to identify the correct jars in the S3 jars bucket and to select the correct tag in the ECR repositories. | | sleeper.admin.role.arn | The ARN of a role that has permissions to administer the instance. | | sleeper.config.bucket | The S3 bucket name used to store configuration files. | -| sleeper.account | The AWS account number. This is the AWS account that the instance will be deployed to. | -| sleeper.region | The AWS region to deploy to. | +| sleeper.account | The AWS account number. This is the AWS account that the instance is be deployed within. | +| sleeper.region | The AWS region the sleeper is deploy within. | | sleeper.data.bucket | The S3 bucket name used to store table data. | | sleeper.tables.name.index.dynamo.table | The name of the DynamoDB table indexing tables by their name. | | sleeper.tables.id.index.dynamo.table | The name of the DynamoDB table indexing tables by their ID. | diff --git a/java/core/src/main/java/sleeper/core/properties/instance/CdkDefinedInstanceProperty.java b/java/core/src/main/java/sleeper/core/properties/instance/CdkDefinedInstanceProperty.java index 313d0af2c5c..4736b9cbda6 100644 --- a/java/core/src/main/java/sleeper/core/properties/instance/CdkDefinedInstanceProperty.java +++ b/java/core/src/main/java/sleeper/core/properties/instance/CdkDefinedInstanceProperty.java @@ -65,11 +65,11 @@ static List getAllInGroup(PropertyGroup group) { //AWS Config CdkDefinedInstanceProperty ACCOUNT = Index.propertyBuilder("sleeper.account") - .description("The AWS account number. This is the AWS account that the instance will be deployed to.") + .description("The AWS account number. This is the AWS account that the instance is be deployed within.") .propertyGroup(InstancePropertyGroup.COMMON) .build(); CdkDefinedInstanceProperty REGION = Index.propertyBuilder("sleeper.region") - .description("The AWS region to deploy to.") + .description("The AWS region the sleeper is deploy within.") .propertyGroup(InstancePropertyGroup.COMMON) .build(); From b334c6054d194b99ae997e24cac88dc22e4ce94c Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Mon, 24 Nov 2025 14:31:01 +0000 Subject: [PATCH 39/65] 5793: Refactor TablePropertiesTestHelper to within relevant tests --- .../admin/properties/PropertiesDiffTest.java | 16 ++++++++++--- .../UpdatePropertiesWithTextEditorIT.java | 23 +++++++++++++------ .../testutils/TablePropertiesTestHelper.java | 13 ----------- 3 files changed, 29 insertions(+), 23 deletions(-) diff --git a/java/clients/src/test/java/sleeper/clients/admin/properties/PropertiesDiffTest.java b/java/clients/src/test/java/sleeper/clients/admin/properties/PropertiesDiffTest.java index 8137ac1d6e0..83f28ffc47a 100644 --- a/java/clients/src/test/java/sleeper/clients/admin/properties/PropertiesDiffTest.java +++ b/java/clients/src/test/java/sleeper/clients/admin/properties/PropertiesDiffTest.java @@ -37,8 +37,10 @@ import static sleeper.core.properties.instance.CommonProperty.MAXIMUM_CONNECTIONS_TO_S3; import static sleeper.core.properties.instance.IngestProperty.INGEST_SOURCE_BUCKET; import static sleeper.core.properties.table.TableProperty.ITERATOR_CONFIG; +import static sleeper.core.properties.table.TableProperty.TABLE_NAME; +import static sleeper.core.properties.testutils.InstancePropertiesTestHelper.createSimpleTestInstanceProperties; import static sleeper.core.properties.testutils.InstancePropertiesTestHelper.createSimpleTestInstancePropertiesWithId; -import static sleeper.core.properties.testutils.TablePropertiesTestHelper.generateSimpleTestTableProperties; +import static sleeper.core.schema.SchemaTestHelper.createSchemaWithKey; public class PropertiesDiffTest { @@ -180,15 +182,22 @@ class CompareTableProperties { @Test void shouldDetectPropertyHasBeenUpdated() { // Given - TableProperties before = generateSimpleTestTableProperties(); + TableProperties before = generateCompareTestTableProperties(); before.set(ITERATOR_CONFIG, "config-before"); - TableProperties after = generateSimpleTestTableProperties(); + TableProperties after = generateCompareTestTableProperties(); after.set(ITERATOR_CONFIG, "config-after"); // When / Then assertThat(getChanges(before, after)) .containsExactly(valueChanged(ITERATOR_CONFIG, "config-before", "config-after")); } + + public static TableProperties generateCompareTestTableProperties() { + TableProperties tableProperties = new TableProperties(createSimpleTestInstanceProperties()); + tableProperties.set(TABLE_NAME, "test-table"); + tableProperties.setSchema(createSchemaWithKey("key")); + return tableProperties; + } } @DisplayName("Combine multiple diffs") @@ -244,6 +253,7 @@ private PropertiesDiff generateSingleDiff(InstanceProperty property, String oldV after.set(property, newValue); return new PropertiesDiff(before, after); } + } private List getChanges(SleeperProperties before, SleeperProperties after) { diff --git a/java/clients/src/test/java/sleeper/clients/admin/properties/UpdatePropertiesWithTextEditorIT.java b/java/clients/src/test/java/sleeper/clients/admin/properties/UpdatePropertiesWithTextEditorIT.java index 81ef1ab4060..d4d23438a82 100644 --- a/java/clients/src/test/java/sleeper/clients/admin/properties/UpdatePropertiesWithTextEditorIT.java +++ b/java/clients/src/test/java/sleeper/clients/admin/properties/UpdatePropertiesWithTextEditorIT.java @@ -40,8 +40,10 @@ import static sleeper.core.properties.instance.LoggingLevelsProperty.LOGGING_LEVEL; import static sleeper.core.properties.table.TableProperty.DYNAMODB_STRONGLY_CONSISTENT_READS; import static sleeper.core.properties.table.TableProperty.ROW_GROUP_SIZE; +import static sleeper.core.properties.table.TableProperty.TABLE_NAME; +import static sleeper.core.properties.testutils.InstancePropertiesTestHelper.createSimpleTestInstanceProperties; import static sleeper.core.properties.testutils.InstancePropertiesTestHelper.createTestInstanceProperties; -import static sleeper.core.properties.testutils.TablePropertiesTestHelper.generateSimpleTestTableProperties; +import static sleeper.core.schema.SchemaTestHelper.createSchemaWithKey; class UpdatePropertiesWithTextEditorIT { @@ -128,9 +130,9 @@ class OpenTableProperties { @Test void shouldUpdateTableProperties() throws Exception { // Given - TableProperties before = generateSimpleTestTableProperties(); + TableProperties before = generateCompareTestTableProperties(); before.set(ROW_GROUP_SIZE, "123"); - TableProperties after = generateSimpleTestTableProperties(); + TableProperties after = generateCompareTestTableProperties(); after.set(ROW_GROUP_SIZE, "456"); // When @@ -145,8 +147,8 @@ void shouldUpdateTableProperties() throws Exception { @Test void shouldRetrieveTablePropertiesAfterChange() throws Exception { // Given - TableProperties before = generateSimpleTestTableProperties(); - TableProperties after = generateSimpleTestTableProperties(); + TableProperties before = generateCompareTestTableProperties(); + TableProperties after = generateCompareTestTableProperties(); after.set(ROW_GROUP_SIZE, "456"); // When @@ -212,9 +214,9 @@ void shouldCreateUpdateRequestWithInstanceProperties() throws Exception { @Test void shouldCreateUpdateRequestWithTableProperties() throws Exception { // Given - TableProperties before = generateSimpleTestTableProperties(); + TableProperties before = generateCompareTestTableProperties(); before.set(DYNAMODB_STRONGLY_CONSISTENT_READS, "false"); - TableProperties after = generateSimpleTestTableProperties(); + TableProperties after = generateCompareTestTableProperties(); after.set(DYNAMODB_STRONGLY_CONSISTENT_READS, "true"); // When @@ -316,4 +318,11 @@ void shouldLeaveUnknownPropertyUnchangedWhenEditingAnotherProperty() throws Exce .isEqualTo(new PropertiesDiff(before, after)); } } + + public static TableProperties generateCompareTestTableProperties() { + TableProperties tableProperties = new TableProperties(createSimpleTestInstanceProperties()); + tableProperties.set(TABLE_NAME, "test-table"); + tableProperties.setSchema(createSchemaWithKey("key")); + return tableProperties; + } } diff --git a/java/core/src/test/java/sleeper/core/properties/testutils/TablePropertiesTestHelper.java b/java/core/src/test/java/sleeper/core/properties/testutils/TablePropertiesTestHelper.java index c4f16100ff6..88ef6da3879 100644 --- a/java/core/src/test/java/sleeper/core/properties/testutils/TablePropertiesTestHelper.java +++ b/java/core/src/test/java/sleeper/core/properties/testutils/TablePropertiesTestHelper.java @@ -24,8 +24,6 @@ import static sleeper.core.properties.table.TableProperty.TABLE_ID; import static sleeper.core.properties.table.TableProperty.TABLE_NAME; -import static sleeper.core.properties.testutils.InstancePropertiesTestHelper.createSimpleTestInstanceProperties; -import static sleeper.core.schema.SchemaTestHelper.createSchemaWithKey; /** * Helpers to create table properties. @@ -66,15 +64,4 @@ public static TableProperties createTestTablePropertiesWithNoSchema(InstanceProp return tableProperties; } - /** - * Creates simple properties for a Sleeper table. - * - * @return the table properties - */ - public static TableProperties generateSimpleTestTableProperties() { - TableProperties tableProperties = new TableProperties(createSimpleTestInstanceProperties()); - tableProperties.set(TABLE_NAME, "test-table"); - tableProperties.setSchema(createSchemaWithKey("key")); - return tableProperties; - } } From 8887d579733268129480d42d6e7d0a80e10f0559 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Mon, 24 Nov 2025 14:40:12 +0000 Subject: [PATCH 40/65] 5793: Update for a different CommonProperty --- .../properties/SleeperPropertiesPrettyPrinterTest.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/java/core/src/test/java/sleeper/core/properties/SleeperPropertiesPrettyPrinterTest.java b/java/core/src/test/java/sleeper/core/properties/SleeperPropertiesPrettyPrinterTest.java index 8bf063296a7..bfe7f5c9acc 100644 --- a/java/core/src/test/java/sleeper/core/properties/SleeperPropertiesPrettyPrinterTest.java +++ b/java/core/src/test/java/sleeper/core/properties/SleeperPropertiesPrettyPrinterTest.java @@ -116,10 +116,12 @@ class PrintValues { @Test void shouldPrintPropertyValueWithDescription() { // When / Then - assertThat(printInstanceProperties("sleeper.account=1234567890")) - .contains("# The AWS account number. This is the AWS account that the instance will be deployed to.\n" + - "# (this property is system-defined and may not be edited)\n" + - "sleeper.account=1234567890\n"); + assertThat(printInstanceProperties("sleeper.log.retention.days=30")) + .contains("# The length of time in days that CloudWatch logs from lambda functions, ECS containers, etc., are\n" + + "# retained.\n" + + "# See https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-logs-loggroup.html\n" + + "# for valid options.\n" + + "# Use -1 to indicate infinite retention."); } @Test From 87e6d9bf4cf167044dfab0cb2efb9c0c648fc034 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Mon, 24 Nov 2025 14:49:18 +0000 Subject: [PATCH 41/65] 5793: Update for determination of Account and Region --- .../src/main/java/sleeper/systemtest/cdk/SystemTestApp.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/system-test/system-test-cdk/src/main/java/sleeper/systemtest/cdk/SystemTestApp.java b/java/system-test/system-test-cdk/src/main/java/sleeper/systemtest/cdk/SystemTestApp.java index abd67ab906d..10ef2bfe056 100644 --- a/java/system-test/system-test-cdk/src/main/java/sleeper/systemtest/cdk/SystemTestApp.java +++ b/java/system-test/system-test-cdk/src/main/java/sleeper/systemtest/cdk/SystemTestApp.java @@ -91,8 +91,8 @@ public static void main(String[] args) { String id = instanceProperties.get(ID); Environment environment = Environment.builder() - .account(instanceProperties.get(ACCOUNT)) - .region(instanceProperties.get(REGION)) + .account(System.getenv("CDK_DEFAULT_ACCOUNT")) + .region(System.getenv("CDK_DEFAULT_REGION")) .build(); new SystemTestApp(app, id, StackProps.builder() From d4905df615221d8b98734fdd9588211c198bb171 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Mon, 24 Nov 2025 15:29:05 +0000 Subject: [PATCH 42/65] 5793: Simply InstanePropertiesTestHelper --- .../admin/properties/PropertiesDiffTest.java | 43 ++++++++++--------- .../UpdatePropertiesWithTextEditorIT.java | 32 ++++++++------ .../InstancePropertiesTestHelper.java | 34 +++------------ 3 files changed, 46 insertions(+), 63 deletions(-) diff --git a/java/clients/src/test/java/sleeper/clients/admin/properties/PropertiesDiffTest.java b/java/clients/src/test/java/sleeper/clients/admin/properties/PropertiesDiffTest.java index 83f28ffc47a..cec77890eac 100644 --- a/java/clients/src/test/java/sleeper/clients/admin/properties/PropertiesDiffTest.java +++ b/java/clients/src/test/java/sleeper/clients/admin/properties/PropertiesDiffTest.java @@ -35,11 +35,12 @@ import static sleeper.core.properties.PropertiesUtils.loadProperties; import static sleeper.core.properties.instance.CommonProperty.LOG_RETENTION_IN_DAYS; import static sleeper.core.properties.instance.CommonProperty.MAXIMUM_CONNECTIONS_TO_S3; +import static sleeper.core.properties.instance.CommonProperty.S3_UPLOAD_BLOCK_SIZE; import static sleeper.core.properties.instance.IngestProperty.INGEST_SOURCE_BUCKET; import static sleeper.core.properties.table.TableProperty.ITERATOR_CONFIG; import static sleeper.core.properties.table.TableProperty.TABLE_NAME; -import static sleeper.core.properties.testutils.InstancePropertiesTestHelper.createSimpleTestInstanceProperties; -import static sleeper.core.properties.testutils.InstancePropertiesTestHelper.createSimpleTestInstancePropertiesWithId; +import static sleeper.core.properties.testutils.InstancePropertiesTestHelper.createTestInstanceProperties; +import static sleeper.core.properties.testutils.InstancePropertiesTestHelper.createTestInstancePropertiesWithId; import static sleeper.core.schema.SchemaTestHelper.createSchemaWithKey; public class PropertiesDiffTest { @@ -53,8 +54,8 @@ class CompareInstanceProperties { void shouldDetectNoChanges() { // Given - InstanceProperties before = createSimpleTestInstancePropertiesWithId(id); - InstanceProperties after = createSimpleTestInstancePropertiesWithId(id); + InstanceProperties before = createTestInstancePropertiesWithId(id); + InstanceProperties after = createTestInstancePropertiesWithId(id); // When / Then assertThat(getChanges(before, after)).isEmpty(); @@ -63,9 +64,9 @@ void shouldDetectNoChanges() { @Test void shouldDetectPropertyHasBeenUpdated() { // Given - InstanceProperties before = createSimpleTestInstancePropertiesWithId(id); + InstanceProperties before = createTestInstancePropertiesWithId(id); before.set(MAXIMUM_CONNECTIONS_TO_S3, "30"); - InstanceProperties after = createSimpleTestInstancePropertiesWithId(id); + InstanceProperties after = createTestInstancePropertiesWithId(id); after.set(MAXIMUM_CONNECTIONS_TO_S3, "50"); // When / Then @@ -76,8 +77,8 @@ void shouldDetectPropertyHasBeenUpdated() { @Test void shouldDetectPropertyIsNewlySet() { // Given - InstanceProperties before = createSimpleTestInstancePropertiesWithId(id); - InstanceProperties after = createSimpleTestInstancePropertiesWithId(id); + InstanceProperties before = createTestInstancePropertiesWithId(id); + InstanceProperties after = createTestInstancePropertiesWithId(id); after.set(INGEST_SOURCE_BUCKET, "some-bucket"); // When / Then @@ -88,9 +89,9 @@ void shouldDetectPropertyIsNewlySet() { @Test void shouldDetectPropertyIsUnset() { // Given - InstanceProperties before = createSimpleTestInstancePropertiesWithId(id); + InstanceProperties before = createTestInstancePropertiesWithId(id); before.set(INGEST_SOURCE_BUCKET, "some-bucket"); - InstanceProperties after = createSimpleTestInstancePropertiesWithId(id); + InstanceProperties after = createTestInstancePropertiesWithId(id); // When / Then assertThat(getChanges(before, after)) @@ -100,25 +101,25 @@ void shouldDetectPropertyIsUnset() { @Test void shouldDetectDefaultedPropertyIsNewlySet() { // Given - InstanceProperties before = createSimpleTestInstancePropertiesWithId(id); - InstanceProperties after = createSimpleTestInstancePropertiesWithId(id); - after.set(MAXIMUM_CONNECTIONS_TO_S3, "50"); + InstanceProperties before = createTestInstancePropertiesWithId(id); + InstanceProperties after = createTestInstancePropertiesWithId(id); + after.set(S3_UPLOAD_BLOCK_SIZE, "64M"); // When / Then assertThat(getChanges(before, after)) - .containsExactly(newValue(MAXIMUM_CONNECTIONS_TO_S3, "50")); + .containsExactly(newValue(S3_UPLOAD_BLOCK_SIZE, "64M")); } @Test void shouldDetectDefaultedPropertyIsUnset() { // Given - InstanceProperties before = createSimpleTestInstancePropertiesWithId(id); - before.set(MAXIMUM_CONNECTIONS_TO_S3, "50"); - InstanceProperties after = createSimpleTestInstancePropertiesWithId(id); + InstanceProperties before = createTestInstancePropertiesWithId(id); + before.set(S3_UPLOAD_BLOCK_SIZE, "64M"); + InstanceProperties after = createTestInstancePropertiesWithId(id); // When / Then assertThat(getChanges(before, after)) - .containsExactly(valueDeleted(MAXIMUM_CONNECTIONS_TO_S3, "50")); + .containsExactly(valueDeleted(S3_UPLOAD_BLOCK_SIZE, "64M")); } } @@ -193,7 +194,7 @@ void shouldDetectPropertyHasBeenUpdated() { } public static TableProperties generateCompareTestTableProperties() { - TableProperties tableProperties = new TableProperties(createSimpleTestInstanceProperties()); + TableProperties tableProperties = new TableProperties(createTestInstanceProperties()); tableProperties.set(TABLE_NAME, "test-table"); tableProperties.setSchema(createSchemaWithKey("key")); return tableProperties; @@ -247,9 +248,9 @@ void shouldNotCombineDiffsWhenPropertyIsDifferent() { } private PropertiesDiff generateSingleDiff(InstanceProperty property, String oldValue, String newValue) { - InstanceProperties before = createSimpleTestInstancePropertiesWithId("simple-id"); + InstanceProperties before = createTestInstancePropertiesWithId("simple-id"); before.set(property, oldValue); - InstanceProperties after = createSimpleTestInstancePropertiesWithId("simple-id"); + InstanceProperties after = createTestInstancePropertiesWithId("simple-id"); after.set(property, newValue); return new PropertiesDiff(before, after); } diff --git a/java/clients/src/test/java/sleeper/clients/admin/properties/UpdatePropertiesWithTextEditorIT.java b/java/clients/src/test/java/sleeper/clients/admin/properties/UpdatePropertiesWithTextEditorIT.java index d4d23438a82..587dfccc17d 100644 --- a/java/clients/src/test/java/sleeper/clients/admin/properties/UpdatePropertiesWithTextEditorIT.java +++ b/java/clients/src/test/java/sleeper/clients/admin/properties/UpdatePropertiesWithTextEditorIT.java @@ -35,14 +35,15 @@ import static org.assertj.core.api.InstanceOfAssertFactories.LIST; import static sleeper.clients.admin.properties.PropertiesDiffTestHelper.valueChanged; import static sleeper.core.properties.PropertiesUtils.loadProperties; +import static sleeper.core.properties.instance.CommonProperty.ID; import static sleeper.core.properties.instance.CommonProperty.MAXIMUM_CONNECTIONS_TO_S3; import static sleeper.core.properties.instance.IngestProperty.INGEST_SOURCE_BUCKET; import static sleeper.core.properties.instance.LoggingLevelsProperty.LOGGING_LEVEL; import static sleeper.core.properties.table.TableProperty.DYNAMODB_STRONGLY_CONSISTENT_READS; import static sleeper.core.properties.table.TableProperty.ROW_GROUP_SIZE; import static sleeper.core.properties.table.TableProperty.TABLE_NAME; -import static sleeper.core.properties.testutils.InstancePropertiesTestHelper.createSimpleTestInstanceProperties; import static sleeper.core.properties.testutils.InstancePropertiesTestHelper.createTestInstanceProperties; +import static sleeper.core.properties.testutils.InstancePropertiesTestHelper.createTestInstancePropertiesWithId; import static sleeper.core.schema.SchemaTestHelper.createSchemaWithKey; class UpdatePropertiesWithTextEditorIT { @@ -50,6 +51,7 @@ class UpdatePropertiesWithTextEditorIT { @TempDir private Path tempDir; private UpdatePropertiesWithTextEditorTestHelper helper; + private String id = "test-instance"; @BeforeEach void setUp() { @@ -83,9 +85,9 @@ void shouldWriteInstancePropertiesFile() throws Exception { @Test void shouldGetDiffAfterPropertiesChanged() throws Exception { // Given - InstanceProperties before = createTestInstanceProperties(); + InstanceProperties before = createTestInstancePropertiesWithId(id); before.set(INGEST_SOURCE_BUCKET, "bucket-before"); - InstanceProperties after = createTestInstanceProperties(); + InstanceProperties after = createTestInstancePropertiesWithId(id); after.set(INGEST_SOURCE_BUCKET, "bucket-after"); // When / Then @@ -195,9 +197,9 @@ void shouldFormatPropertiesUsingPrettyPrinter() throws Exception { @Test void shouldCreateUpdateRequestWithInstanceProperties() throws Exception { // Given - InstanceProperties before = createTestInstanceProperties(); + InstanceProperties before = createTestInstancePropertiesWithId(id); before.set(LOGGING_LEVEL, "ERROR"); - InstanceProperties after = createTestInstanceProperties(); + InstanceProperties after = createTestInstancePropertiesWithId(id); after.set(LOGGING_LEVEL, "INFO"); // When @@ -233,9 +235,9 @@ void shouldCreateUpdateRequestWithTableProperties() throws Exception { @Test void shouldUnsetPropertyWhenRemovedInEditor() throws Exception { // Given - InstanceProperties before = createTestInstanceProperties(); + InstanceProperties before = createTestInstancePropertiesWithId(id); before.set(LOGGING_LEVEL, "ERROR"); - InstanceProperties after = createTestInstanceProperties(); + InstanceProperties after = createTestInstancePropertiesWithId(id); after.unset(LOGGING_LEVEL); // When @@ -263,9 +265,9 @@ void shouldNotShowUnknownProperties() throws Exception { @Test void shouldUpdateAnUnknownProperty() throws Exception { // Given - InstanceProperties before = createTestInstanceProperties(); + InstanceProperties before = createTestInstancePropertiesWithId(id); before.getProperties().setProperty("unknown.property", "value-before"); - InstanceProperties after = createTestInstanceProperties(); + InstanceProperties after = createTestInstancePropertiesWithId(id); after.getProperties().setProperty("unknown.property", "value-after"); // When @@ -282,9 +284,9 @@ void shouldUpdateAnUnknownProperty() throws Exception { @Test void shouldUpdateAPropertyOutsideTheSpecifiedGroup() throws Exception { // Given - InstanceProperties before = createTestInstanceProperties(); + InstanceProperties before = createTestInstancePropertiesWithId(id); before.set(INGEST_SOURCE_BUCKET, "bucket-before"); - InstanceProperties after = createTestInstanceProperties(); + InstanceProperties after = createTestInstancePropertiesWithId(id); after.set(INGEST_SOURCE_BUCKET, "bucket-after"); // When @@ -301,9 +303,9 @@ void shouldUpdateAPropertyOutsideTheSpecifiedGroup() throws Exception { @Test void shouldLeaveUnknownPropertyUnchangedWhenEditingAnotherProperty() throws Exception { // Given - InstanceProperties before = createTestInstanceProperties(); + InstanceProperties before = createTestInstancePropertiesWithId(id); before.getProperties().setProperty("unknown.property", "test-value"); - InstanceProperties after = createTestInstanceProperties(); + InstanceProperties after = createTestInstancePropertiesWithId(id); after.getProperties().setProperty("unknown.property", "test-value"); after.set(LOGGING_LEVEL, "TRACE"); @@ -320,7 +322,9 @@ void shouldLeaveUnknownPropertyUnchangedWhenEditingAnotherProperty() throws Exce } public static TableProperties generateCompareTestTableProperties() { - TableProperties tableProperties = new TableProperties(createSimpleTestInstanceProperties()); + InstanceProperties instanceProperties = new InstanceProperties(); + instanceProperties.set(ID, "test-instance"); + TableProperties tableProperties = new TableProperties(instanceProperties); tableProperties.set(TABLE_NAME, "test-table"); tableProperties.setSchema(createSchemaWithKey("key")); return tableProperties; diff --git a/java/core/src/test/java/sleeper/core/properties/testutils/InstancePropertiesTestHelper.java b/java/core/src/test/java/sleeper/core/properties/testutils/InstancePropertiesTestHelper.java index 22c9c286302..9bbe177cb3f 100644 --- a/java/core/src/test/java/sleeper/core/properties/testutils/InstancePropertiesTestHelper.java +++ b/java/core/src/test/java/sleeper/core/properties/testutils/InstancePropertiesTestHelper.java @@ -82,7 +82,12 @@ public static InstanceProperties createTestInstanceProperties() { * @return the instance properties */ public static InstanceProperties createTestInstancePropertiesWithId(String id) { - InstanceProperties instanceProperties = createSimpleTestInstancePropertiesWithId(id); + InstanceProperties instanceProperties = new InstanceProperties(); + instanceProperties.set(ID, id); + instanceProperties.set(ACCOUNT, "test-account"); + instanceProperties.set(REGION, "test-region"); + instanceProperties.set(VPC_ID, "test-vpc"); + instanceProperties.set(SUBNETS, "test-subnet"); instanceProperties.set(CONFIG_BUCKET, InstanceProperties.getConfigBucketFromInstanceId(id)); instanceProperties.set(DATA_BUCKET, "sleeper-" + id + "-table-data"); instanceProperties.set(JARS_BUCKET, "sleeper-" + id + "-jars"); @@ -115,33 +120,6 @@ public static InstanceProperties createTestInstancePropertiesWithId(String id) { return instanceProperties; } - /** - * Creates properties for a Sleeper instance. Generates a random instance ID and pre-populates minimal properties - * set during deployment. - * - * @return the instance properties - */ - public static InstanceProperties createSimpleTestInstanceProperties() { - String id = UUID.randomUUID().toString().toLowerCase(Locale.ROOT).substring(0, 18); - return createSimpleTestInstancePropertiesWithId(id); - } - - /** - * Creates properties for a Sleeper instance. Sets miminal properties for simpler testing parameters. - * - * @param id the instance ID - * @return the instance properties - */ - public static InstanceProperties createSimpleTestInstancePropertiesWithId(String id) { - InstanceProperties properties = new InstanceProperties(); - properties.set(ID, id); - properties.set(ACCOUNT, "test-account"); - properties.set(REGION, "test-region"); - properties.set(VPC_ID, "test-vpc"); - properties.set(SUBNETS, "test-subnet"); - return properties; - } - /** * Writes a properties file as a string. * From 1e909b02b25036f10d097d418c65092e2d8c2fd0 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Tue, 25 Nov 2025 12:02:30 +0000 Subject: [PATCH 43/65] 5793: Temp change to pass tests --- .../deploy/container/UploadDockerImagesToEcrTestBase.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/java/clients/src/test/java/sleeper/clients/deploy/container/UploadDockerImagesToEcrTestBase.java b/java/clients/src/test/java/sleeper/clients/deploy/container/UploadDockerImagesToEcrTestBase.java index 173aea2169b..d2d2a02efee 100644 --- a/java/clients/src/test/java/sleeper/clients/deploy/container/UploadDockerImagesToEcrTestBase.java +++ b/java/clients/src/test/java/sleeper/clients/deploy/container/UploadDockerImagesToEcrTestBase.java @@ -26,6 +26,8 @@ import static sleeper.clients.util.command.Command.command; import static sleeper.clients.util.command.CommandPipeline.pipeline; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; +import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.VERSION; import static sleeper.core.properties.instance.CommonProperty.ID; import static sleeper.core.properties.testutils.InstancePropertiesTestHelper.createTestInstanceProperties; @@ -37,6 +39,10 @@ public abstract class UploadDockerImagesToEcrTestBase extends DockerImagesTestBa @BeforeEach void setUpBase() { properties.set(ID, "test-instance"); + //TODO REMOVE THIS 2 + properties.set(ACCOUNT, "123"); + properties.set(REGION, "test-region"); + properties.set(VERSION, "1.0.0"); } From 542b4b4382f920a3598b6d47884b1a3503668be7 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Tue, 25 Nov 2025 14:48:48 +0000 Subject: [PATCH 44/65] 5793: Update UploadDockerImagesToEcr to use Account and Region --- .../sleeper/clients/admin/AdminClient.java | 3 +- .../deploy/DeployExistingInstance.java | 2 +- .../clients/deploy/DeployInstance.java | 5 ++- .../clients/deploy/DeployNewInstance.java | 2 +- .../clients/deploy/UploadArtefacts.java | 3 +- .../container/UploadDockerImagesToEcr.java | 14 +++++-- .../UploadDockerImagesToEcrRequest.java | 42 ++----------------- .../UploadDockerImagesToEcrFileIT.java | 2 +- .../UploadDockerImagesToEcrTest.java | 2 +- .../AwsSystemTestDeploymentDriver.java | 4 +- 10 files changed, 26 insertions(+), 53 deletions(-) diff --git a/java/clients/src/main/java/sleeper/clients/admin/AdminClient.java b/java/clients/src/main/java/sleeper/clients/admin/AdminClient.java index 5cb70b7ee89..55a413824ed 100644 --- a/java/clients/src/main/java/sleeper/clients/admin/AdminClient.java +++ b/java/clients/src/main/java/sleeper/clients/admin/AdminClient.java @@ -97,7 +97,8 @@ public static void main(String[] args) throws IOException, InterruptedException EmrClient emrClient = AwsV2ClientHelper.buildAwsV2Client(EmrClient.builder())) { UploadDockerImagesToEcr uploadDockerImages = new UploadDockerImagesToEcr( UploadDockerImages.fromScriptsDirectory(scriptsDir), - CheckVersionExistsInEcr.withEcrClient(ecrClient)); + CheckVersionExistsInEcr.withEcrClient(ecrClient), + "", ""); //TODO errorCode = start(instanceId, s3Client, dynamoClient, cdk, generatedDir, uploadDockerImages, out, in, new UpdatePropertiesWithTextEditor(Path.of("/tmp")), diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java index f19bb2d4884..4c59863f9c3 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java @@ -101,7 +101,7 @@ public void update() throws IOException, InterruptedException { .commandRunner(runCommand) .createMultiplatformBuilder(createMultiPlatformBuilder) .build(), - CheckVersionExistsInEcr.withEcrClient(ecr)), + CheckVersionExistsInEcr.withEcrClient(ecr), "", ""), //TODO DeployInstance.WriteLocalProperties.underScriptsDirectory(scriptsDirectory), InvokeCdk.builder().scriptsDirectory(scriptsDirectory).runCommand(runCommand).build()); diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployInstance.java index 409659a07fb..6cedd71f885 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployInstance.java @@ -17,6 +17,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import software.amazon.awssdk.regions.providers.AwsRegionProvider; +import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain; import software.amazon.awssdk.services.ecr.EcrClient; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.sts.StsClient; @@ -76,6 +78,7 @@ public static void main(String[] args) throws IOException, InterruptedException Set options = Stream.of(args).skip(5).map(String::toLowerCase).collect(toUnmodifiableSet()); boolean deployPaused = options.contains("--deploy-paused"); + AwsRegionProvider regionProvider = DefaultAwsRegionProviderChain.builder().build(); try (S3Client s3Client = S3Client.create(); EcrClient ecrClient = EcrClient.create(); StsClient stsClient = StsClient.create()) { @@ -91,7 +94,7 @@ public static void main(String[] args) throws IOException, InterruptedException SyncJars.fromScriptsDirectory(s3Client, scriptsDirectory), new UploadDockerImagesToEcr( UploadDockerImages.fromScriptsDirectory(scriptsDirectory), - CheckVersionExistsInEcr.withEcrClient(ecrClient)), + CheckVersionExistsInEcr.withEcrClient(ecrClient), stsClient.getCallerIdentity().account(), regionProvider.getRegion().id()), // TODO WriteLocalProperties.underScriptsDirectory(scriptsDirectory), InvokeCdk.fromScriptsDirectory(scriptsDirectory)); diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java index d5a716be594..a15f12c6827 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java @@ -122,7 +122,7 @@ public void deploy() throws IOException, InterruptedException { .commandRunner(runCommand) .createMultiplatformBuilder(createMultiPlatformBuilder) .build(), - CheckVersionExistsInEcr.withEcrClient(ecrClient)), + CheckVersionExistsInEcr.withEcrClient(ecrClient), "", ""), //TODO DeployInstance.WriteLocalProperties.underScriptsDirectory(scriptsDirectory), InvokeCdk.builder().scriptsDirectory(scriptsDirectory).runCommand(runCommand).build()); diff --git a/java/clients/src/main/java/sleeper/clients/deploy/UploadArtefacts.java b/java/clients/src/main/java/sleeper/clients/deploy/UploadArtefacts.java index 68ad7375bfa..1de67f25e5e 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/UploadArtefacts.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/UploadArtefacts.java @@ -129,7 +129,7 @@ public static void main(String[] rawArgs) throws IOException, InterruptedExcepti .deployConfig(DeployConfiguration.fromScriptsDirectory(args.scriptsDir())) .createMultiplatformBuilder(args.createMultiplatformBuilder()) .build(), - CheckVersionExistsInEcr.withEcrClient(ecrClient)); + CheckVersionExistsInEcr.withEcrClient(ecrClient), account, region); if (args.createDeployment()) { InvokeCdk.fromScriptsDirectory(args.scriptsDir()) @@ -140,7 +140,6 @@ public static void main(String[] rawArgs) throws IOException, InterruptedExcepti .build()); uploadImages.upload(UploadDockerImagesToEcrRequest.builder() .ecrPrefix(ecrPrefix) - .account(account).region(region) .images(images) .build()); } diff --git a/java/clients/src/main/java/sleeper/clients/deploy/container/UploadDockerImagesToEcr.java b/java/clients/src/main/java/sleeper/clients/deploy/container/UploadDockerImagesToEcr.java index 8cfcd2a04de..b6413ff9877 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/container/UploadDockerImagesToEcr.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/container/UploadDockerImagesToEcr.java @@ -34,10 +34,16 @@ public class UploadDockerImagesToEcr { private final UploadDockerImages uploader; private final CheckVersionExistsInEcr repositoryChecker; + private final String account; + private final String region; + private final String repositoryHost; - public UploadDockerImagesToEcr(UploadDockerImages uploader, CheckVersionExistsInEcr repositoryChecker) { + public UploadDockerImagesToEcr(UploadDockerImages uploader, CheckVersionExistsInEcr repositoryChecker, String accountIn, String regionIn) { this.uploader = uploader; this.repositoryChecker = repositoryChecker; + this.account = accountIn; + this.region = regionIn; + this.repositoryHost = String.format("%s.dkr.ecr.%s.amazonaws.com", account, region); } public void upload(UploadDockerImagesToEcrRequest request) throws IOException, InterruptedException { @@ -46,11 +52,11 @@ public void upload(UploadDockerImagesToEcrRequest request) throws IOException, I List imagesToUpload = requestedImages.stream() .filter(image -> imageDoesNotExistInRepositoryWithVersion(image, request)) .collect(Collectors.toUnmodifiableList()); - String repositoryPrefix = request.getRepositoryHost() + "/" + request.getEcrPrefix(); + String repositoryPrefix = repositoryHost + "/" + request.getEcrPrefix(); if (!imagesToUpload.isEmpty()) { uploader.getCommandRunner().runOrThrow(pipeline( - command("aws", "ecr", "get-login-password", "--region", request.getRegion()), - command("docker", "login", "--username", "AWS", "--password-stdin", request.getRepositoryHost()))); + command("aws", "ecr", "get-login-password", "--region", region), + command("docker", "login", "--username", "AWS", "--password-stdin", repositoryHost))); } uploader.upload(repositoryPrefix, imagesToUpload); } diff --git a/java/clients/src/main/java/sleeper/clients/deploy/container/UploadDockerImagesToEcrRequest.java b/java/clients/src/main/java/sleeper/clients/deploy/container/UploadDockerImagesToEcrRequest.java index 47caadd6eaa..f4be0392575 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/container/UploadDockerImagesToEcrRequest.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/container/UploadDockerImagesToEcrRequest.java @@ -25,24 +25,16 @@ import java.util.Optional; import static java.util.Objects.requireNonNull; -import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; -import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.VERSION; import static sleeper.core.properties.instance.CommonProperty.ECR_REPOSITORY_PREFIX; public class UploadDockerImagesToEcrRequest { private final String ecrPrefix; - private final String account; - private final String region; private final List images; - private final String repositoryHost; private UploadDockerImagesToEcrRequest(Builder builder) { ecrPrefix = requireNonNull(builder.ecrPrefix, "ecrPrefix must not be null"); - account = requireNonNull(builder.account, "account must not be null"); - region = requireNonNull(builder.region, "region must not be null"); images = requireNonNull(builder.images, "images must not be null"); - repositoryHost = String.format("%s.dkr.ecr.%s.amazonaws.com", account, region); } public static Builder builder() { @@ -67,7 +59,7 @@ public static Optional forUpdateIfNeeded(Instanc } public Builder toBuilder() { - return builder().ecrPrefix(ecrPrefix).account(account).region(region).images(images); + return builder().ecrPrefix(ecrPrefix).images(images); } public UploadDockerImagesToEcrRequest withExtraImages(List extraImages) { @@ -84,22 +76,10 @@ public String getEcrPrefix() { return ecrPrefix; } - public String getAccount() { - return account; - } - - public String getRegion() { - return region; - } - public List getImages() { return images; } - public String getRepositoryHost() { - return repositoryHost; - } - @Override public boolean equals(Object obj) { if (this == obj) { @@ -109,23 +89,21 @@ public boolean equals(Object obj) { return false; } UploadDockerImagesToEcrRequest other = (UploadDockerImagesToEcrRequest) obj; - return Objects.equals(ecrPrefix, other.ecrPrefix) && Objects.equals(account, other.account) && Objects.equals(region, other.region) && Objects.equals(images, other.images); + return Objects.equals(ecrPrefix, other.ecrPrefix) && Objects.equals(images, other.images); } @Override public int hashCode() { - return Objects.hash(ecrPrefix, account, region, images); + return Objects.hash(ecrPrefix, images); } @Override public String toString() { - return "UploadDockerImagesToEcrRequest{ecrPrefix=" + ecrPrefix + ", account=" + account + ", region=" + region + ", images=" + images + "}"; + return "UploadDockerImagesToEcrRequest{ecrPrefix=" + ecrPrefix + ", images=" + images + "}"; } public static final class Builder { private String ecrPrefix; - private String account; - private String region; private List images; private Builder() { @@ -133,8 +111,6 @@ private Builder() { public Builder properties(InstanceProperties properties) { return ecrPrefix(properties.get(ECR_REPOSITORY_PREFIX)) - .account(properties.get(ACCOUNT)) - .region(properties.get(REGION)) .version(properties.get(VERSION)); } @@ -143,16 +119,6 @@ public Builder ecrPrefix(String ecrPrefix) { return this; } - public Builder account(String account) { - this.account = account; - return this; - } - - public Builder region(String region) { - this.region = region; - return this; - } - public Builder version(String version) { return this; } diff --git a/java/clients/src/test/java/sleeper/clients/deploy/container/UploadDockerImagesToEcrFileIT.java b/java/clients/src/test/java/sleeper/clients/deploy/container/UploadDockerImagesToEcrFileIT.java index 5a004832d0b..9168c9fc23e 100644 --- a/java/clients/src/test/java/sleeper/clients/deploy/container/UploadDockerImagesToEcrFileIT.java +++ b/java/clients/src/test/java/sleeper/clients/deploy/container/UploadDockerImagesToEcrFileIT.java @@ -88,7 +88,7 @@ protected UploadDockerImagesToEcr uploader() { .baseDockerDirectory(dockerDir).jarsDirectory(jarsDir) .version("1.0.0") .build(), - ecrClient); + ecrClient, "123", "test-region"); } private static Map fileToContentUnder(Path directory) throws Exception { diff --git a/java/clients/src/test/java/sleeper/clients/deploy/container/UploadDockerImagesToEcrTest.java b/java/clients/src/test/java/sleeper/clients/deploy/container/UploadDockerImagesToEcrTest.java index 715e2befc0a..1bb6a88cf7a 100644 --- a/java/clients/src/test/java/sleeper/clients/deploy/container/UploadDockerImagesToEcrTest.java +++ b/java/clients/src/test/java/sleeper/clients/deploy/container/UploadDockerImagesToEcrTest.java @@ -622,6 +622,6 @@ protected UploadDockerImagesToEcr uploader() { .baseDockerDirectory(Path.of("./docker")).jarsDirectory(Path.of("./jars")) .version("1.0.0") .build(), - ecrClient); + ecrClient, "123", "test-region"); } } diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSystemTestDeploymentDriver.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSystemTestDeploymentDriver.java index fe554383898..e068a823074 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSystemTestDeploymentDriver.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSystemTestDeploymentDriver.java @@ -131,11 +131,9 @@ private void uploadJarsAndDockerImages() throws IOException, InterruptedExceptio .deployConfig(DeployConfiguration.fromScriptsDirectory(parameters.getScriptsDirectory())) .commandRunner(CommandUtils::runCommandLogOutput) .build(), - CheckVersionExistsInEcr.withEcrClient(ecr)); + CheckVersionExistsInEcr.withEcrClient(ecr), parameters.getAccount(), parameters.getRegion()); dockerUploader.upload(UploadDockerImagesToEcrRequest.builder() .ecrPrefix(SleeperArtefactsLocation.getDefaultEcrRepositoryPrefix(parameters.getArtefactsDeploymentId())) - .account(parameters.getAccount()) - .region(parameters.getRegion()) .version(SleeperVersion.getVersion()) .images(List.of(SYSTEM_TEST_IMAGE)) .build()); From 1d8685fac351b46e525fc956d11c837158630cb4 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Tue, 25 Nov 2025 14:51:49 +0000 Subject: [PATCH 45/65] 5793: Update admin Client --- .../src/main/java/sleeper/clients/admin/AdminClient.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/java/clients/src/main/java/sleeper/clients/admin/AdminClient.java b/java/clients/src/main/java/sleeper/clients/admin/AdminClient.java index 55a413824ed..937b139b6ee 100644 --- a/java/clients/src/main/java/sleeper/clients/admin/AdminClient.java +++ b/java/clients/src/main/java/sleeper/clients/admin/AdminClient.java @@ -15,11 +15,14 @@ */ package sleeper.clients.admin; +import software.amazon.awssdk.regions.providers.AwsRegionProvider; +import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.ecr.EcrClient; import software.amazon.awssdk.services.emr.EmrClient; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.sqs.SqsClient; +import software.amazon.awssdk.services.sts.StsClient; import sleeper.clients.admin.properties.AdminClientPropertiesStore; import sleeper.clients.admin.properties.UpdatePropertiesWithTextEditor; @@ -94,11 +97,13 @@ public static void main(String[] args) throws IOException, InterruptedException DynamoDbClient dynamoClient = AwsV2ClientHelper.buildAwsV2Client(DynamoDbClient.builder()); SqsClient sqsClient = AwsV2ClientHelper.buildAwsV2Client(SqsClient.builder()); EcrClient ecrClient = AwsV2ClientHelper.buildAwsV2Client(EcrClient.builder()); - EmrClient emrClient = AwsV2ClientHelper.buildAwsV2Client(EmrClient.builder())) { + EmrClient emrClient = AwsV2ClientHelper.buildAwsV2Client(EmrClient.builder()); + StsClient stsClient = AwsV2ClientHelper.buildAwsV2Client(StsClient.builder())) { + AwsRegionProvider regionProvider = DefaultAwsRegionProviderChain.builder().build(); UploadDockerImagesToEcr uploadDockerImages = new UploadDockerImagesToEcr( UploadDockerImages.fromScriptsDirectory(scriptsDir), CheckVersionExistsInEcr.withEcrClient(ecrClient), - "", ""); //TODO + stsClient.getCallerIdentity().account(), regionProvider.getRegion().id()); errorCode = start(instanceId, s3Client, dynamoClient, cdk, generatedDir, uploadDockerImages, out, in, new UpdatePropertiesWithTextEditor(Path.of("/tmp")), From c4d7c3fb5ee319534b47bc66004f800e0416e5a0 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Tue, 25 Nov 2025 15:10:37 +0000 Subject: [PATCH 46/65] 5793: Refactor for StsClients --- .../clients/deploy/DeployExistingInstance.java | 17 +++++++++++++---- .../clients/deploy/DeployNewInstance.java | 18 +++++++++++++++--- .../drivers/cdk/DeployNewTestInstance.java | 2 +- .../instance/AwsSleeperInstanceDriver.java | 4 ++-- 4 files changed, 31 insertions(+), 10 deletions(-) diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java index 4c59863f9c3..38dacacfe66 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java @@ -18,9 +18,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import software.amazon.awssdk.regions.providers.AwsRegionProvider; +import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.ecr.EcrClient; import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.sts.StsClient; import sleeper.clients.deploy.container.CheckVersionExistsInEcr; import sleeper.clients.deploy.container.UploadDockerImages; @@ -51,6 +54,7 @@ public class DeployExistingInstance { private final List tablePropertiesList; private final S3Client s3; private final EcrClient ecr; + private final StsClient sts; private final boolean deployPaused; private final CommandPipelineRunner runCommand; private final boolean createMultiPlatformBuilder; @@ -61,6 +65,7 @@ private DeployExistingInstance(Builder builder) { tablePropertiesList = builder.tablePropertiesList; s3 = builder.s3; ecr = builder.ecr; + sts = builder.sts; deployPaused = builder.deployPaused; runCommand = builder.runCommand; createMultiPlatformBuilder = builder.createMultiPlatformBuilder; @@ -81,8 +86,9 @@ public static void main(String[] args) throws IOException, InterruptedException try (S3Client s3Client = S3Client.create(); DynamoDbClient dynamoClient = DynamoDbClient.create(); - EcrClient ecrClient = EcrClient.create()) { - builder().clients(s3Client, ecrClient) + EcrClient ecrClient = EcrClient.create(); + StsClient stsClient = StsClient.create()) { + builder().clients(s3Client, ecrClient, stsClient) .scriptsDirectory(Path.of(args[0])) .instanceId(args[1]) .deployPaused(deployPaused) @@ -92,6 +98,7 @@ public static void main(String[] args) throws IOException, InterruptedException } public void update() throws IOException, InterruptedException { + AwsRegionProvider regionProvider = DefaultAwsRegionProviderChain.builder().build(); DeployInstance deployInstance = new DeployInstance( SyncJars.fromScriptsDirectory(s3, scriptsDirectory), new UploadDockerImagesToEcr( @@ -101,7 +108,7 @@ public void update() throws IOException, InterruptedException { .commandRunner(runCommand) .createMultiplatformBuilder(createMultiPlatformBuilder) .build(), - CheckVersionExistsInEcr.withEcrClient(ecr), "", ""), //TODO + CheckVersionExistsInEcr.withEcrClient(ecr), sts.getCallerIdentity().account(), regionProvider.getRegion().id()), //TODO DeployInstance.WriteLocalProperties.underScriptsDirectory(scriptsDirectory), InvokeCdk.builder().scriptsDirectory(scriptsDirectory).runCommand(runCommand).build()); @@ -121,6 +128,7 @@ public static final class Builder { private List tablePropertiesList; private S3Client s3; private EcrClient ecr; + private StsClient sts; private boolean deployPaused; private CommandPipelineRunner runCommand = CommandUtils::runCommandInheritIO; private boolean createMultiPlatformBuilder = true; @@ -152,9 +160,10 @@ public Builder tablePropertiesList(List tablePropertiesList) { return this; } - public Builder clients(S3Client s3, EcrClient ecr) { + public Builder clients(S3Client s3, EcrClient ecr, StsClient sts) { this.s3 = s3; this.ecr = ecr; + this.sts = sts; return this; } diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java index a15f12c6827..4974697bad9 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java @@ -17,6 +17,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import software.amazon.awssdk.regions.providers.AwsRegionProvider; +import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.ecr.EcrClient; import software.amazon.awssdk.services.s3.S3Client; @@ -54,6 +56,7 @@ public class DeployNewInstance { private final S3Client s3Client; private final DynamoDbClient dynamoClient; private final EcrClient ecrClient; + private final StsClient stsClient; private final Path scriptsDirectory; private final SleeperInstanceConfiguration deployInstanceConfiguration; private final List extraDockerImages; @@ -66,6 +69,7 @@ private DeployNewInstance(Builder builder) { s3Client = builder.s3Client; dynamoClient = builder.dynamoClient; ecrClient = builder.ecrClient; + stsClient = builder.stsClient; scriptsDirectory = builder.scriptsDirectory; deployInstanceConfiguration = builder.deployInstanceConfiguration; extraDockerImages = builder.extraDockerImages; @@ -104,7 +108,7 @@ public static void main(String[] args) throws IOException, InterruptedException .deployInstanceConfiguration(config) .deployPaused(deployPaused) .instanceType(InvokeCdk.Type.STANDARD) - .deployWithClients(s3Client, dynamoClient, ecrClient); + .deployWithClients(s3Client, dynamoClient, ecrClient, stsClient); } } @@ -113,6 +117,7 @@ public void deploy() throws IOException, InterruptedException { LOGGER.info("Running Deployment"); LOGGER.info("-------------------------------------------------------"); deployInstanceConfiguration.validate(); + AwsRegionProvider regionProvider = DefaultAwsRegionProviderChain.builder().build(); DeployInstance deployInstance = new DeployInstance( SyncJars.fromScriptsDirectory(s3Client, scriptsDirectory), new UploadDockerImagesToEcr( @@ -122,7 +127,7 @@ public void deploy() throws IOException, InterruptedException { .commandRunner(runCommand) .createMultiplatformBuilder(createMultiPlatformBuilder) .build(), - CheckVersionExistsInEcr.withEcrClient(ecrClient), "", ""), //TODO + CheckVersionExistsInEcr.withEcrClient(ecrClient), stsClient.getCallerIdentity().account(), regionProvider.getRegion().id()), DeployInstance.WriteLocalProperties.underScriptsDirectory(scriptsDirectory), InvokeCdk.builder().scriptsDirectory(scriptsDirectory).runCommand(runCommand).build()); @@ -146,6 +151,7 @@ public void deploy() throws IOException, InterruptedException { public static final class Builder { private S3Client s3Client; + private StsClient stsClient; private DynamoDbClient dynamoClient; private EcrClient ecrClient; private Path scriptsDirectory; @@ -164,6 +170,11 @@ public Builder s3Client(S3Client s3Client) { return this; } + public Builder stsClient(StsClient stsClient) { + this.stsClient = stsClient; + return this; + } + public Builder dynamoClient(DynamoDbClient dynamoClient) { this.dynamoClient = dynamoClient; return this; @@ -214,10 +225,11 @@ public DeployNewInstance build() { } public void deployWithClients( - S3Client s3Client, DynamoDbClient dynamoClient, EcrClient ecrClient) throws IOException, InterruptedException { + S3Client s3Client, DynamoDbClient dynamoClient, EcrClient ecrClient, StsClient stsClient) throws IOException, InterruptedException { s3Client(s3Client) .dynamoClient(dynamoClient) .ecrClient(ecrClient) + .stsClient(stsClient) .build().deploy(); } } diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java index 487143e55c5..61fb66e4772 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java @@ -69,7 +69,7 @@ public static void main(String[] args) throws IOException, InterruptedException .extraDockerImages(List.of(SYSTEM_TEST_IMAGE)) .deployPaused(deployPaused) .instanceType(InvokeCdk.Type.SYSTEM_TEST) - .deployWithClients(s3Client, dynamoClient, ecrClient); + .deployWithClients(s3Client, dynamoClient, ecrClient, stsClient); } } diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java index 0740508888c..081acb956d8 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java @@ -96,7 +96,7 @@ public boolean deployInstanceIfNotPresent(String instanceId, SleeperInstanceConf .instanceType(InvokeCdk.Type.STANDARD) .runCommand(CommandUtils::runCommandLogOutput) .createMultiPlatformBuilder(parameters.isCreateMultiPlatformBuilder()) - .deployWithClients(s3, dynamoDB, ecr); + .deployWithClients(s3, dynamoDB, ecr, sts); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RuntimeException(e); @@ -125,7 +125,7 @@ private boolean deployedStackIsPresent(String instanceId) { public void redeploy(InstanceProperties instanceProperties, List tableProperties) { try { DeployExistingInstance.builder() - .clients(s3, ecr) + .clients(s3, ecr, sts) .properties(instanceProperties) .tablePropertiesList(tableProperties) .scriptsDirectory(parameters.getScriptsDirectory()) From f14148e0b1b8472ece707e2031c39ed1f24420c0 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Tue, 25 Nov 2025 15:13:50 +0000 Subject: [PATCH 47/65] 5793: Remove TODO's --- .../java/sleeper/clients/deploy/DeployExistingInstance.java | 2 +- .../deploy/container/UploadDockerImagesToEcrTestBase.java | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java index 38dacacfe66..a8b2c0cb535 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java @@ -108,7 +108,7 @@ public void update() throws IOException, InterruptedException { .commandRunner(runCommand) .createMultiplatformBuilder(createMultiPlatformBuilder) .build(), - CheckVersionExistsInEcr.withEcrClient(ecr), sts.getCallerIdentity().account(), regionProvider.getRegion().id()), //TODO + CheckVersionExistsInEcr.withEcrClient(ecr), sts.getCallerIdentity().account(), regionProvider.getRegion().id()), DeployInstance.WriteLocalProperties.underScriptsDirectory(scriptsDirectory), InvokeCdk.builder().scriptsDirectory(scriptsDirectory).runCommand(runCommand).build()); diff --git a/java/clients/src/test/java/sleeper/clients/deploy/container/UploadDockerImagesToEcrTestBase.java b/java/clients/src/test/java/sleeper/clients/deploy/container/UploadDockerImagesToEcrTestBase.java index d2d2a02efee..173aea2169b 100644 --- a/java/clients/src/test/java/sleeper/clients/deploy/container/UploadDockerImagesToEcrTestBase.java +++ b/java/clients/src/test/java/sleeper/clients/deploy/container/UploadDockerImagesToEcrTestBase.java @@ -26,8 +26,6 @@ import static sleeper.clients.util.command.Command.command; import static sleeper.clients.util.command.CommandPipeline.pipeline; -import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; -import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.VERSION; import static sleeper.core.properties.instance.CommonProperty.ID; import static sleeper.core.properties.testutils.InstancePropertiesTestHelper.createTestInstanceProperties; @@ -39,10 +37,6 @@ public abstract class UploadDockerImagesToEcrTestBase extends DockerImagesTestBa @BeforeEach void setUpBase() { properties.set(ID, "test-instance"); - //TODO REMOVE THIS 2 - properties.set(ACCOUNT, "123"); - properties.set(REGION, "test-region"); - properties.set(VERSION, "1.0.0"); } From 65cf6085fd26b2665dec7c8132bf0543f354ce82 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Wed, 26 Nov 2025 07:49:59 +0000 Subject: [PATCH 48/65] 5793: Correct missing method usage --- .../admin/properties/AdminClientPropertiesStoreIT.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/java/clients/src/test/java/sleeper/clients/admin/properties/AdminClientPropertiesStoreIT.java b/java/clients/src/test/java/sleeper/clients/admin/properties/AdminClientPropertiesStoreIT.java index e9647c94775..69a8b7bf102 100644 --- a/java/clients/src/test/java/sleeper/clients/admin/properties/AdminClientPropertiesStoreIT.java +++ b/java/clients/src/test/java/sleeper/clients/admin/properties/AdminClientPropertiesStoreIT.java @@ -50,8 +50,6 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; import static sleeper.clients.deploy.container.StackDockerImage.dockerBuildImage; -import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.ACCOUNT; -import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.REGION; import static sleeper.core.properties.instance.CdkDefinedInstanceProperty.VERSION; import static sleeper.core.properties.instance.CommonProperty.FARGATE_VERSION; import static sleeper.core.properties.instance.CommonProperty.FORCE_RELOAD_PROPERTIES; @@ -395,8 +393,6 @@ private void updateInstanceProperty(String instanceId, InstanceProperty property private UploadDockerImagesToEcrRequest withImages(StackDockerImage... images) { return UploadDockerImagesToEcrRequest.builder() .ecrPrefix(instanceProperties.get(ID)) - .account(instanceProperties.get(ACCOUNT)) - .region(instanceProperties.get(REGION)) .version(instanceProperties.get(VERSION)) .images(List.of(images)) .build(); From ec77e8208bb0207d79719a02ee8f59604723b6fe Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Wed, 26 Nov 2025 08:15:13 +0000 Subject: [PATCH 49/65] 5793: Update missing tags validation --- .../sleeper/cdk/stack/SleeperInstanceStacksPropsIT.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/java/cdk/src/test/java/sleeper/cdk/stack/SleeperInstanceStacksPropsIT.java b/java/cdk/src/test/java/sleeper/cdk/stack/SleeperInstanceStacksPropsIT.java index 5a40ef2bf05..c80b37c9424 100644 --- a/java/cdk/src/test/java/sleeper/cdk/stack/SleeperInstanceStacksPropsIT.java +++ b/java/cdk/src/test/java/sleeper/cdk/stack/SleeperInstanceStacksPropsIT.java @@ -29,6 +29,7 @@ import java.io.IOException; import java.nio.file.Path; import java.util.Map; +import java.util.Properties; import java.util.UUID; import java.util.stream.Stream; @@ -197,6 +198,11 @@ private static InstanceProperties createUserDefinedInstanceProperties() { instanceProperties.set(JARS_BUCKET, "test-bucket"); instanceProperties.set(VPC_ID, "test-vpc"); instanceProperties.set(SUBNETS, "test-subnet"); + + Properties tagsProperties = instanceProperties.getTagsProperties(); + tagsProperties.setProperty("InstanceID", id); + instanceProperties.loadTags(tagsProperties); + return instanceProperties; } } From cf71cdc3bb1c922adebece16bc8e65aefceb6897 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Wed, 26 Nov 2025 08:19:36 +0000 Subject: [PATCH 50/65] 5793: Update javadoc --- .../core/src/main/java/sleeper/core/deploy/DockerDeployment.java | 1 + 1 file changed, 1 insertion(+) diff --git a/java/core/src/main/java/sleeper/core/deploy/DockerDeployment.java b/java/core/src/main/java/sleeper/core/deploy/DockerDeployment.java index b7e6361a298..39f8f6d30d1 100644 --- a/java/core/src/main/java/sleeper/core/deploy/DockerDeployment.java +++ b/java/core/src/main/java/sleeper/core/deploy/DockerDeployment.java @@ -116,6 +116,7 @@ public boolean isCreateEmrServerlessPolicy() { /** * Retrieves the Docker image name for this deployment. Includes the repository URL and the tag. + * This method requires that CDK defined properties are set due to requiring Account and Region. * * @param properties the instance properties * @return the Docker image name From ebdc2986ab671d6238584dbd4ee0bf7f6db7be46 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Wed, 26 Nov 2025 13:20:14 +0000 Subject: [PATCH 51/65] 5793: Update Region and Account descriptions --- docs/usage/properties/instance/cdk/common.md | 4 ++-- .../core/properties/instance/CdkDefinedInstanceProperty.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/usage/properties/instance/cdk/common.md b/docs/usage/properties/instance/cdk/common.md index d6b87deb779..0d8b8883e22 100644 --- a/docs/usage/properties/instance/cdk/common.md +++ b/docs/usage/properties/instance/cdk/common.md @@ -7,8 +7,8 @@ The following instance properties are commonly used throughout Sleeper. | sleeper.version | The version of Sleeper that is being used. This property is used to identify the correct jars in the S3 jars bucket and to select the correct tag in the ECR repositories. | | sleeper.admin.role.arn | The ARN of a role that has permissions to administer the instance. | | sleeper.config.bucket | The S3 bucket name used to store configuration files. | -| sleeper.account | The AWS account number. This is the AWS account that the instance is be deployed within. | -| sleeper.region | The AWS region the sleeper is deploy within. | +| sleeper.account | The AWS account number. This is the AWS account that the instance is deployed in. | +| sleeper.region | The AWS region the instance is deployed in. | | sleeper.data.bucket | The S3 bucket name used to store table data. | | sleeper.tables.name.index.dynamo.table | The name of the DynamoDB table indexing tables by their name. | | sleeper.tables.id.index.dynamo.table | The name of the DynamoDB table indexing tables by their ID. | diff --git a/java/core/src/main/java/sleeper/core/properties/instance/CdkDefinedInstanceProperty.java b/java/core/src/main/java/sleeper/core/properties/instance/CdkDefinedInstanceProperty.java index 4736b9cbda6..7c6fcc33832 100644 --- a/java/core/src/main/java/sleeper/core/properties/instance/CdkDefinedInstanceProperty.java +++ b/java/core/src/main/java/sleeper/core/properties/instance/CdkDefinedInstanceProperty.java @@ -65,11 +65,11 @@ static List getAllInGroup(PropertyGroup group) { //AWS Config CdkDefinedInstanceProperty ACCOUNT = Index.propertyBuilder("sleeper.account") - .description("The AWS account number. This is the AWS account that the instance is be deployed within.") + .description("The AWS account number. This is the AWS account that the instance is deployed in.") .propertyGroup(InstancePropertyGroup.COMMON) .build(); CdkDefinedInstanceProperty REGION = Index.propertyBuilder("sleeper.region") - .description("The AWS region the sleeper is deploy within.") + .description("The AWS region the instance is deployed in.") .propertyGroup(InstancePropertyGroup.COMMON) .build(); From 19f0098886cecdc06b437c16f8e75206d1af81bf Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Wed, 26 Nov 2025 13:30:40 +0000 Subject: [PATCH 52/65] 5793: Update SystemTestApp --- .../main/java/sleeper/systemtest/cdk/SystemTestApp.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/java/system-test/system-test-cdk/src/main/java/sleeper/systemtest/cdk/SystemTestApp.java b/java/system-test/system-test-cdk/src/main/java/sleeper/systemtest/cdk/SystemTestApp.java index 10ef2bfe056..701b5fde11f 100644 --- a/java/system-test/system-test-cdk/src/main/java/sleeper/systemtest/cdk/SystemTestApp.java +++ b/java/system-test/system-test-cdk/src/main/java/sleeper/systemtest/cdk/SystemTestApp.java @@ -46,10 +46,12 @@ public class SystemTestApp extends Stack { private final SleeperJarsInBucket jars; private final SystemTestProperties instanceProperties; - public SystemTestApp(App app, String id, StackProps props, SleeperInstanceProps sleeperProps) { - super(app, id, props); + public SystemTestApp(App app, String id, StackProps stackProps, SleeperInstanceProps sleeperProps) { + super(app, id, stackProps); this.props = sleeperProps; + props.getInstanceProperties().set(ACCOUNT, getAccount()); + props.getInstanceProperties().set(REGION, getRegion()); this.jars = sleeperProps.getJars(); this.instanceProperties = SystemTestProperties.from(sleeperProps.getInstanceProperties()); } @@ -66,8 +68,7 @@ public void create() { // Sleeper instance SleeperCoreStacks coreStacks = SleeperCoreStacks.create(this, props, loggingStack, autoDeleteS3Stack); SleeperOptionalStacks.create(this, props, coreStacks); - props.getInstanceProperties().set(ACCOUNT, this.getAccount()); - props.getInstanceProperties().set(REGION, this.getRegion()); + coreStacks.createRoles(); // Stack for writing random data From e9845e5af6bf377832fb8aae3156aeef10b15317 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Wed, 26 Nov 2025 13:31:24 +0000 Subject: [PATCH 53/65] 5793: Remove redundant TODO --- .../src/main/java/sleeper/clients/deploy/DeployInstance.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployInstance.java index 6cedd71f885..8f90a20d860 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployInstance.java @@ -94,7 +94,7 @@ public static void main(String[] args) throws IOException, InterruptedException SyncJars.fromScriptsDirectory(s3Client, scriptsDirectory), new UploadDockerImagesToEcr( UploadDockerImages.fromScriptsDirectory(scriptsDirectory), - CheckVersionExistsInEcr.withEcrClient(ecrClient), stsClient.getCallerIdentity().account(), regionProvider.getRegion().id()), // TODO + CheckVersionExistsInEcr.withEcrClient(ecrClient), stsClient.getCallerIdentity().account(), regionProvider.getRegion().id()), WriteLocalProperties.underScriptsDirectory(scriptsDirectory), InvokeCdk.fromScriptsDirectory(scriptsDirectory)); From ef1393ee0ed94f407c1dcd978aab89a28d8c1ced Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Wed, 26 Nov 2025 13:32:38 +0000 Subject: [PATCH 54/65] 5793: Rename method arguments --- .../clients/deploy/container/UploadDockerImagesToEcr.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/java/clients/src/main/java/sleeper/clients/deploy/container/UploadDockerImagesToEcr.java b/java/clients/src/main/java/sleeper/clients/deploy/container/UploadDockerImagesToEcr.java index b6413ff9877..90d9fe8e4b0 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/container/UploadDockerImagesToEcr.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/container/UploadDockerImagesToEcr.java @@ -38,12 +38,12 @@ public class UploadDockerImagesToEcr { private final String region; private final String repositoryHost; - public UploadDockerImagesToEcr(UploadDockerImages uploader, CheckVersionExistsInEcr repositoryChecker, String accountIn, String regionIn) { + public UploadDockerImagesToEcr(UploadDockerImages uploader, CheckVersionExistsInEcr repositoryChecker, String account, String region) { this.uploader = uploader; this.repositoryChecker = repositoryChecker; - this.account = accountIn; - this.region = regionIn; - this.repositoryHost = String.format("%s.dkr.ecr.%s.amazonaws.com", account, region); + this.account = account; + this.region = region; + this.repositoryHost = String.format("%s.dkr.ecr.%s.amazonaws.com", this.account, this.region); } public void upload(UploadDockerImagesToEcrRequest request) throws IOException, InterruptedException { From 692482986a380a66b90744eb7896ea4ab2171704 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Wed, 26 Nov 2025 13:33:17 +0000 Subject: [PATCH 55/65] 5793: Update argument verification --- scripts/deploy/syncJars.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/deploy/syncJars.sh b/scripts/deploy/syncJars.sh index 2dfecac30dd..aea12f8bbdf 100755 --- a/scripts/deploy/syncJars.sh +++ b/scripts/deploy/syncJars.sh @@ -16,7 +16,7 @@ set -e unset CDPATH -if [ "$#" -lt 2 ] || [ "$#" -gt 3 ]; then +if [ "$#" -lt 1 ] || [ "$#" -gt 2 ]; then echo "Usage: $0 " exit 1 fi From aea3b69411a370c26fb2597cde751f9bed672d87 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Wed, 26 Nov 2025 13:33:54 +0000 Subject: [PATCH 56/65] 5793: Update captilisation on JavaDoc --- .../src/main/java/sleeper/core/deploy/DockerDeployment.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/core/src/main/java/sleeper/core/deploy/DockerDeployment.java b/java/core/src/main/java/sleeper/core/deploy/DockerDeployment.java index 39f8f6d30d1..d87abe2ab73 100644 --- a/java/core/src/main/java/sleeper/core/deploy/DockerDeployment.java +++ b/java/core/src/main/java/sleeper/core/deploy/DockerDeployment.java @@ -116,7 +116,7 @@ public boolean isCreateEmrServerlessPolicy() { /** * Retrieves the Docker image name for this deployment. Includes the repository URL and the tag. - * This method requires that CDK defined properties are set due to requiring Account and Region. + * This method requires that CDK defined properties are set due to requiring account and region. * * @param properties the instance properties * @return the Docker image name From eee31b7f802fd61359343821bbd3c32516c5b6a7 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Wed, 26 Nov 2025 13:50:55 +0000 Subject: [PATCH 57/65] 5793: New test for tagProperties --- .../cdk/stack/SleeperInstanceStacksPropsIT.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/java/cdk/src/test/java/sleeper/cdk/stack/SleeperInstanceStacksPropsIT.java b/java/cdk/src/test/java/sleeper/cdk/stack/SleeperInstanceStacksPropsIT.java index c80b37c9424..8902d17caeb 100644 --- a/java/cdk/src/test/java/sleeper/cdk/stack/SleeperInstanceStacksPropsIT.java +++ b/java/cdk/src/test/java/sleeper/cdk/stack/SleeperInstanceStacksPropsIT.java @@ -92,6 +92,20 @@ void shouldSetVersionWhenInstancePropertiesAreLoaded() throws IOException { assertThat(loaded.get(VERSION)) .matches("\\d+\\.\\d+\\.\\d+(-SNAPSHOT)?"); } + + @Test + void shouldSetTagPropertiesWhenInstancePropertiesAreLoaded() throws IOException { + // Given + InstanceProperties properties = createUserDefinedInstanceProperties(); + SaveLocalProperties.saveToDirectory(tempDir, properties, Stream.empty()); + + // When + InstanceProperties loaded = loadInstanceProperties(cdkContextWithPropertiesFile(tempDir)); + + // Then + assertThat(loaded.getTags().get("InstanceID")) + .matches("^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}$"); + } } @Nested From 331eecd47587595826ea55685585554d301b46d3 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Wed, 26 Nov 2025 13:53:18 +0000 Subject: [PATCH 58/65] 5793: Include value as part of validation --- .../core/properties/SleeperPropertiesPrettyPrinterTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/java/core/src/test/java/sleeper/core/properties/SleeperPropertiesPrettyPrinterTest.java b/java/core/src/test/java/sleeper/core/properties/SleeperPropertiesPrettyPrinterTest.java index bfe7f5c9acc..9616c447fc5 100644 --- a/java/core/src/test/java/sleeper/core/properties/SleeperPropertiesPrettyPrinterTest.java +++ b/java/core/src/test/java/sleeper/core/properties/SleeperPropertiesPrettyPrinterTest.java @@ -121,7 +121,8 @@ void shouldPrintPropertyValueWithDescription() { "# retained.\n" + "# See https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-logs-loggroup.html\n" + "# for valid options.\n" + - "# Use -1 to indicate infinite retention."); + "# Use -1 to indicate infinite retention.\n" + + "sleeper.log.retention.days=30"); } @Test From 73ea10419b33e00647811300a221df91f2aa57fd Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Wed, 26 Nov 2025 14:01:29 +0000 Subject: [PATCH 59/65] 5793: Change RegionProvider to field over local variable --- .../clients/deploy/DeployExistingInstance.java | 10 +++++++++- .../sleeper/clients/deploy/DeployNewInstance.java | 15 ++++++++++++--- .../drivers/cdk/DeployNewTestInstance.java | 3 ++- .../instance/AwsSleeperInstanceDriver.java | 4 +++- 4 files changed, 26 insertions(+), 6 deletions(-) diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java index a8b2c0cb535..c51653fd865 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployExistingInstance.java @@ -55,6 +55,7 @@ public class DeployExistingInstance { private final S3Client s3; private final EcrClient ecr; private final StsClient sts; + private final AwsRegionProvider regionProvider; private final boolean deployPaused; private final CommandPipelineRunner runCommand; private final boolean createMultiPlatformBuilder; @@ -66,6 +67,7 @@ private DeployExistingInstance(Builder builder) { s3 = builder.s3; ecr = builder.ecr; sts = builder.sts; + regionProvider = builder.regionProvider; deployPaused = builder.deployPaused; runCommand = builder.runCommand; createMultiPlatformBuilder = builder.createMultiPlatformBuilder; @@ -89,6 +91,7 @@ public static void main(String[] args) throws IOException, InterruptedException EcrClient ecrClient = EcrClient.create(); StsClient stsClient = StsClient.create()) { builder().clients(s3Client, ecrClient, stsClient) + .regionProvider(DefaultAwsRegionProviderChain.builder().build()) .scriptsDirectory(Path.of(args[0])) .instanceId(args[1]) .deployPaused(deployPaused) @@ -98,7 +101,6 @@ public static void main(String[] args) throws IOException, InterruptedException } public void update() throws IOException, InterruptedException { - AwsRegionProvider regionProvider = DefaultAwsRegionProviderChain.builder().build(); DeployInstance deployInstance = new DeployInstance( SyncJars.fromScriptsDirectory(s3, scriptsDirectory), new UploadDockerImagesToEcr( @@ -129,6 +131,7 @@ public static final class Builder { private S3Client s3; private EcrClient ecr; private StsClient sts; + private AwsRegionProvider regionProvider; private boolean deployPaused; private CommandPipelineRunner runCommand = CommandUtils::runCommandInheritIO; private boolean createMultiPlatformBuilder = true; @@ -167,6 +170,11 @@ public Builder clients(S3Client s3, EcrClient ecr, StsClient sts) { return this; } + public Builder regionProvider(AwsRegionProvider regionProvider) { + this.regionProvider = regionProvider; + return this; + } + public Builder deployPaused(boolean deployPaused) { this.deployPaused = deployPaused; return this; diff --git a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java index 4974697bad9..80f1adf88be 100644 --- a/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java +++ b/java/clients/src/main/java/sleeper/clients/deploy/DeployNewInstance.java @@ -57,6 +57,7 @@ public class DeployNewInstance { private final DynamoDbClient dynamoClient; private final EcrClient ecrClient; private final StsClient stsClient; + private final AwsRegionProvider regionProvider; private final Path scriptsDirectory; private final SleeperInstanceConfiguration deployInstanceConfiguration; private final List extraDockerImages; @@ -70,6 +71,7 @@ private DeployNewInstance(Builder builder) { dynamoClient = builder.dynamoClient; ecrClient = builder.ecrClient; stsClient = builder.stsClient; + regionProvider = builder.regionProvider; scriptsDirectory = builder.scriptsDirectory; deployInstanceConfiguration = builder.deployInstanceConfiguration; extraDockerImages = builder.extraDockerImages; @@ -108,7 +110,7 @@ public static void main(String[] args) throws IOException, InterruptedException .deployInstanceConfiguration(config) .deployPaused(deployPaused) .instanceType(InvokeCdk.Type.STANDARD) - .deployWithClients(s3Client, dynamoClient, ecrClient, stsClient); + .deployWithClients(s3Client, dynamoClient, ecrClient, stsClient, DefaultAwsRegionProviderChain.builder().build()); } } @@ -117,7 +119,6 @@ public void deploy() throws IOException, InterruptedException { LOGGER.info("Running Deployment"); LOGGER.info("-------------------------------------------------------"); deployInstanceConfiguration.validate(); - AwsRegionProvider regionProvider = DefaultAwsRegionProviderChain.builder().build(); DeployInstance deployInstance = new DeployInstance( SyncJars.fromScriptsDirectory(s3Client, scriptsDirectory), new UploadDockerImagesToEcr( @@ -154,6 +155,7 @@ public static final class Builder { private StsClient stsClient; private DynamoDbClient dynamoClient; private EcrClient ecrClient; + private AwsRegionProvider regionProvider; private Path scriptsDirectory; private SleeperInstanceConfiguration deployInstanceConfiguration; private List extraDockerImages = List.of(); @@ -185,6 +187,11 @@ public Builder ecrClient(EcrClient ecrClient) { return this; } + public Builder regionProvider(AwsRegionProvider regionProvider) { + this.regionProvider = regionProvider; + return this; + } + public Builder scriptsDirectory(Path scriptsDirectory) { this.scriptsDirectory = scriptsDirectory; return this; @@ -225,11 +232,13 @@ public DeployNewInstance build() { } public void deployWithClients( - S3Client s3Client, DynamoDbClient dynamoClient, EcrClient ecrClient, StsClient stsClient) throws IOException, InterruptedException { + S3Client s3Client, DynamoDbClient dynamoClient, EcrClient ecrClient, StsClient stsClient, + AwsRegionProvider regionProvider) throws IOException, InterruptedException { s3Client(s3Client) .dynamoClient(dynamoClient) .ecrClient(ecrClient) .stsClient(stsClient) + .regionProvider(regionProvider) .build().deploy(); } } diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java index 61fb66e4772..ea78b13e2e4 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/cdk/DeployNewTestInstance.java @@ -15,6 +15,7 @@ */ package sleeper.systemtest.drivers.cdk; +import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; import software.amazon.awssdk.services.ecr.EcrClient; import software.amazon.awssdk.services.s3.S3Client; @@ -69,7 +70,7 @@ public static void main(String[] args) throws IOException, InterruptedException .extraDockerImages(List.of(SYSTEM_TEST_IMAGE)) .deployPaused(deployPaused) .instanceType(InvokeCdk.Type.SYSTEM_TEST) - .deployWithClients(s3Client, dynamoClient, ecrClient, stsClient); + .deployWithClients(s3Client, dynamoClient, ecrClient, stsClient, DefaultAwsRegionProviderChain.builder().build()); } } diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java index 081acb956d8..1badd6c760e 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java @@ -19,6 +19,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.regions.providers.AwsRegionProvider; +import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain; import software.amazon.awssdk.services.cloudformation.CloudFormationClient; import software.amazon.awssdk.services.cloudformation.model.CloudFormationException; import software.amazon.awssdk.services.cloudformation.model.Stack; @@ -96,7 +97,7 @@ public boolean deployInstanceIfNotPresent(String instanceId, SleeperInstanceConf .instanceType(InvokeCdk.Type.STANDARD) .runCommand(CommandUtils::runCommandLogOutput) .createMultiPlatformBuilder(parameters.isCreateMultiPlatformBuilder()) - .deployWithClients(s3, dynamoDB, ecr, sts); + .deployWithClients(s3, dynamoDB, ecr, sts, DefaultAwsRegionProviderChain.builder().build()); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RuntimeException(e); @@ -126,6 +127,7 @@ public void redeploy(InstanceProperties instanceProperties, List Date: Wed, 26 Nov 2025 14:41:24 +0000 Subject: [PATCH 60/65] 5793: Update approval for default-instance --- .../test/java/sleeper/cdk/default-instance.approved.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/java/cdk/src/test/java/sleeper/cdk/default-instance.approved.json b/java/cdk/src/test/java/sleeper/cdk/default-instance.approved.json index 0e42bc54862..1b630e08ded 100644 --- a/java/cdk/src/test/java/sleeper/cdk/default-instance.approved.json +++ b/java/cdk/src/test/java/sleeper/cdk/default-instance.approved.json @@ -12839,7 +12839,9 @@ { "Action": "elasticmapreduce:RunJobFlow", "Condition": { - "StringEquals": {} + "StringEquals": { + "elasticmapreduce:RequestTag/InstanceID": "test-instance" + } }, "Effect": "Allow", "Resource": "*" @@ -22099,7 +22101,7 @@ { "Ref": "referencetoTestInstanceCompactionNestedStackCompactionNestedStackResource654ED576OutputsTestInstanceCompactionPendingCompactionJobBatchDLQE42B71ECRef" }, - "\nsleeper.ingest.arrow.max.local.store.bytes\u003d536870912\nsleeper.query.leaf.partition.queue.arn\u003d", + "\nsleeper.tags\u003dInstanceID,test-instance\nsleeper.ingest.arrow.max.local.store.bytes\u003d536870912\nsleeper.query.leaf.partition.queue.arn\u003d", { "Ref": "referencetoTestInstanceQueryNestedStackQueryNestedStackResource79C6CBEEOutputsTestInstanceQueryLeafPartitionQueryQueue64615BA9Arn" }, From 2427c5fca948cc375fd515e2486b175db4c1e513 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Wed, 26 Nov 2025 15:32:31 +0000 Subject: [PATCH 61/65] 5793: Fix for spot bugs issue --- .../src/main/java/sleeper/systemtest/cdk/SystemTestApp.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/system-test/system-test-cdk/src/main/java/sleeper/systemtest/cdk/SystemTestApp.java b/java/system-test/system-test-cdk/src/main/java/sleeper/systemtest/cdk/SystemTestApp.java index 701b5fde11f..2cd1c592e7e 100644 --- a/java/system-test/system-test-cdk/src/main/java/sleeper/systemtest/cdk/SystemTestApp.java +++ b/java/system-test/system-test-cdk/src/main/java/sleeper/systemtest/cdk/SystemTestApp.java @@ -50,8 +50,8 @@ public SystemTestApp(App app, String id, StackProps stackProps, SleeperInstanceP super(app, id, stackProps); this.props = sleeperProps; - props.getInstanceProperties().set(ACCOUNT, getAccount()); - props.getInstanceProperties().set(REGION, getRegion()); + props.getInstanceProperties().set(ACCOUNT, this.getAccount()); + props.getInstanceProperties().set(REGION, this.getRegion()); this.jars = sleeperProps.getJars(); this.instanceProperties = SystemTestProperties.from(sleeperProps.getInstanceProperties()); } From 9c99c37a9439fc890d5a0d82b90bea319222e69e Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Thu, 27 Nov 2025 08:31:52 +0000 Subject: [PATCH 62/65] 5793: System test adjust for spot bugs --- .../java/sleeper/systemtest/cdk/SystemTestApp.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/java/system-test/system-test-cdk/src/main/java/sleeper/systemtest/cdk/SystemTestApp.java b/java/system-test/system-test-cdk/src/main/java/sleeper/systemtest/cdk/SystemTestApp.java index 2cd1c592e7e..e89235e0cb5 100644 --- a/java/system-test/system-test-cdk/src/main/java/sleeper/systemtest/cdk/SystemTestApp.java +++ b/java/system-test/system-test-cdk/src/main/java/sleeper/systemtest/cdk/SystemTestApp.java @@ -48,10 +48,7 @@ public class SystemTestApp extends Stack { public SystemTestApp(App app, String id, StackProps stackProps, SleeperInstanceProps sleeperProps) { super(app, id, stackProps); - this.props = sleeperProps; - - props.getInstanceProperties().set(ACCOUNT, this.getAccount()); - props.getInstanceProperties().set(REGION, this.getRegion()); + this.props = setProps(sleeperProps); this.jars = sleeperProps.getJars(); this.instanceProperties = SystemTestProperties.from(sleeperProps.getInstanceProperties()); } @@ -105,4 +102,10 @@ public static void main(String[] args) { app.synth(); } } + + private final SleeperInstanceProps setProps(SleeperInstanceProps props) { + props.getInstanceProperties().set(ACCOUNT, getAccount()); + props.getInstanceProperties().set(REGION, getRegion()); + return props; + } } From fed550f2cd1af5fb59b3f74fb6cfb1d74e6bc344 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Thu, 27 Nov 2025 08:53:00 +0000 Subject: [PATCH 63/65] 5793: Remove invalid modifier --- .../src/main/java/sleeper/systemtest/cdk/SystemTestApp.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/system-test/system-test-cdk/src/main/java/sleeper/systemtest/cdk/SystemTestApp.java b/java/system-test/system-test-cdk/src/main/java/sleeper/systemtest/cdk/SystemTestApp.java index e89235e0cb5..583ef05bece 100644 --- a/java/system-test/system-test-cdk/src/main/java/sleeper/systemtest/cdk/SystemTestApp.java +++ b/java/system-test/system-test-cdk/src/main/java/sleeper/systemtest/cdk/SystemTestApp.java @@ -54,7 +54,7 @@ public SystemTestApp(App app, String id, StackProps stackProps, SleeperInstanceP } public void create() { - + // LoggingStack loggingStack = new LoggingStack(this, "Logging", instanceProperties); AutoDeleteS3ObjectsStack autoDeleteS3Stack = new AutoDeleteS3ObjectsStack(this, "AutoDeleteS3Objects", instanceProperties, jars, loggingStack); @@ -103,7 +103,7 @@ public static void main(String[] args) { } } - private final SleeperInstanceProps setProps(SleeperInstanceProps props) { + private SleeperInstanceProps setProps(SleeperInstanceProps props) { props.getInstanceProperties().set(ACCOUNT, getAccount()); props.getInstanceProperties().set(REGION, getRegion()); return props; From cd297bdfb40f5ba75be6e356b46943b52a490424 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Thu, 27 Nov 2025 09:32:18 +0000 Subject: [PATCH 64/65] 5793: Update from review comments --- .../src/main/java/sleeper/systemtest/cdk/SystemTestApp.java | 1 - .../systemtest/drivers/instance/AwsSleeperInstanceDriver.java | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/java/system-test/system-test-cdk/src/main/java/sleeper/systemtest/cdk/SystemTestApp.java b/java/system-test/system-test-cdk/src/main/java/sleeper/systemtest/cdk/SystemTestApp.java index 583ef05bece..e7ed93db5cd 100644 --- a/java/system-test/system-test-cdk/src/main/java/sleeper/systemtest/cdk/SystemTestApp.java +++ b/java/system-test/system-test-cdk/src/main/java/sleeper/systemtest/cdk/SystemTestApp.java @@ -54,7 +54,6 @@ public SystemTestApp(App app, String id, StackProps stackProps, SleeperInstanceP } public void create() { - // LoggingStack loggingStack = new LoggingStack(this, "Logging", instanceProperties); AutoDeleteS3ObjectsStack autoDeleteS3Stack = new AutoDeleteS3ObjectsStack(this, "AutoDeleteS3Objects", instanceProperties, jars, loggingStack); diff --git a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java index 1badd6c760e..df4b7c1300d 100644 --- a/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java +++ b/java/system-test/system-test-drivers/src/main/java/sleeper/systemtest/drivers/instance/AwsSleeperInstanceDriver.java @@ -19,7 +19,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.regions.providers.AwsRegionProvider; -import software.amazon.awssdk.regions.providers.DefaultAwsRegionProviderChain; import software.amazon.awssdk.services.cloudformation.CloudFormationClient; import software.amazon.awssdk.services.cloudformation.model.CloudFormationException; import software.amazon.awssdk.services.cloudformation.model.Stack; @@ -97,7 +96,7 @@ public boolean deployInstanceIfNotPresent(String instanceId, SleeperInstanceConf .instanceType(InvokeCdk.Type.STANDARD) .runCommand(CommandUtils::runCommandLogOutput) .createMultiPlatformBuilder(parameters.isCreateMultiPlatformBuilder()) - .deployWithClients(s3, dynamoDB, ecr, sts, DefaultAwsRegionProviderChain.builder().build()); + .deployWithClients(s3, dynamoDB, ecr, sts, regionProvider); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new RuntimeException(e); From 77ef9639efc7891e1e066d044949f14d42e18b65 Mon Sep 17 00:00:00 2001 From: rtjd6554 Date: Thu, 27 Nov 2025 10:00:42 +0000 Subject: [PATCH 65/65] 5793: Update logic surrounding unit tests --- .../cdk/stack/SleeperInstanceStacksPropsIT.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/java/cdk/src/test/java/sleeper/cdk/stack/SleeperInstanceStacksPropsIT.java b/java/cdk/src/test/java/sleeper/cdk/stack/SleeperInstanceStacksPropsIT.java index 8902d17caeb..feb655a29f5 100644 --- a/java/cdk/src/test/java/sleeper/cdk/stack/SleeperInstanceStacksPropsIT.java +++ b/java/cdk/src/test/java/sleeper/cdk/stack/SleeperInstanceStacksPropsIT.java @@ -59,10 +59,16 @@ void shouldLoadValidInstancePropertiesFromFile() throws IOException { InstanceProperties properties = createUserDefinedInstanceProperties(); SaveLocalProperties.saveToDirectory(tempDir, properties, Stream.empty()); - // When / Then + // When + InstanceProperties loaded = loadInstanceProperties(cdkContextWithPropertiesFile(tempDir)); + + // Then + Properties tagsProperties = properties.getTagsProperties(); + tagsProperties.setProperty("InstanceID", properties.get(ID)); + properties.loadTags(tagsProperties); properties.set(VERSION, getVersion()); - assertThat(loadInstanceProperties(cdkContextWithPropertiesFile(tempDir))) - .isEqualTo(properties); + + assertThat(loaded).isEqualTo(properties); } @Test @@ -213,10 +219,6 @@ private static InstanceProperties createUserDefinedInstanceProperties() { instanceProperties.set(VPC_ID, "test-vpc"); instanceProperties.set(SUBNETS, "test-subnet"); - Properties tagsProperties = instanceProperties.getTagsProperties(); - tagsProperties.setProperty("InstanceID", id); - instanceProperties.loadTags(tagsProperties); - return instanceProperties; } }