diff --git a/localstack-core/localstack/services/cloudformation/api_utils.py b/localstack-core/localstack/services/cloudformation/api_utils.py index 556435ed699a7..7eda69fefdedf 100644 --- a/localstack-core/localstack/services/cloudformation/api_utils.py +++ b/localstack-core/localstack/services/cloudformation/api_utils.py @@ -4,6 +4,7 @@ from localstack import config, constants from localstack.aws.connect import connect_to +from localstack.services.cloudformation.engine.entities import Stack from localstack.services.s3.utils import ( extract_bucket_name_and_key_from_headers_and_path, normalize_bucket_name, @@ -16,7 +17,12 @@ LOG = logging.getLogger(__name__) -def prepare_template_body(req_data: dict) -> str | bytes | None: # TODO: mutating and returning +# TODO: remove optional stack and make it required + + +def prepare_template_body( + req_data: dict, stack: Stack | None = None +) -> str | bytes | None: # TODO: mutating and returning template_url = req_data.get("TemplateURL") if template_url: req_data["TemplateURL"] = convert_s3_to_local_url(template_url) @@ -26,13 +32,13 @@ def prepare_template_body(req_data: dict) -> str | bytes | None: # TODO: mutati if modified_template_body: req_data.pop("TemplateURL", None) req_data["TemplateBody"] = modified_template_body - modified_template_body = get_template_body(req_data) + modified_template_body = get_template_body(req_data, stack) if modified_template_body: req_data["TemplateBody"] = modified_template_body return modified_template_body -def get_template_body(req_data: dict) -> str: +def get_template_body(req_data: dict, stack: Stack | None = None) -> str: body = req_data.get("TemplateBody") if body: return body @@ -60,6 +66,9 @@ def get_template_body(req_data: dict) -> str: "Unable to fetch template body (code %s) from URL %s" % (status_code, url) ) return to_str(response.content) + if req_data.get("UsePreviousTemplate") and stack: + return stack.template_body + raise Exception("Unable to get template body from input: %s" % req_data) diff --git a/localstack-core/localstack/services/cloudformation/engine/entities.py b/localstack-core/localstack/services/cloudformation/engine/entities.py index e1cac8d6aeda1..d01648beb3c1d 100644 --- a/localstack-core/localstack/services/cloudformation/engine/entities.py +++ b/localstack-core/localstack/services/cloudformation/engine/entities.py @@ -1,7 +1,12 @@ import logging from typing import Optional, TypedDict -from localstack.aws.api.cloudformation import Capability, ChangeSetType, Parameter +from localstack.aws.api.cloudformation import ( + Capability, + ChangeSetType, + CreateStackSetInput, + Parameter, +) from localstack.services.cloudformation.engine.parameters import ( StackParameter, convert_stack_parameters_to_list, @@ -17,30 +22,62 @@ LOG = logging.getLogger(__name__) -class StackSet: - """A stack set contains multiple stack instances.""" +class StackInstance: + """A stack instance belongs to a stack set and is specific to a region / account ID.""" # FIXME: confusing name. metadata is the complete incoming request object def __init__(self, metadata: dict): self.metadata = metadata + # reference to the deployed stack belonging to this stack instance + self.stack = None + + @property + def account(self) -> str: + return self.metadata["Account"] + + @property + def region(self) -> str: + return self.metadata["Region"] + + +class StackSet: + """A stack set contains multiple stack instances.""" + + def __init__(self, request: CreateStackSetInput): + self.request = request # list of stack instances - self.stack_instances = [] + self.stack_instances: list[StackInstance] = [] # maps operation ID to stack set operation details self.operations = {} + # compatibility with old API + @property + def metadata(self) -> CreateStackSetInput: + return self.request + @property def stack_set_name(self): - return self.metadata.get("StackSetName") + return self.request.get("StackSetName") + @property + def template_url(self) -> str | None: + return self.request.get("TemplateURL") -class StackInstance: - """A stack instance belongs to a stack set and is specific to a region / account ID.""" + @property + def template_body(self) -> str | None: + return self.request.get("TemplateBody") - # FIXME: confusing name. metadata is the complete incoming request object - def __init__(self, metadata: dict): - self.metadata = metadata - # reference to the deployed stack belonging to this stack instance - self.stack = None + def get_template(self): + if body := self.template_body: + return body + + raise NotImplementedError("template URL") + + def get_instance(self, account: str, region: str) -> StackInstance | None: + for instance in self.stack_instances: + if instance.metadata["Account"] == account and instance.metadata["Region"] == region: + return instance + return None class StackMetadata(TypedDict): diff --git a/localstack-core/localstack/services/cloudformation/engine/parameters.py b/localstack-core/localstack/services/cloudformation/engine/parameters.py index d84f4a7751f52..c2a20ad630686 100644 --- a/localstack-core/localstack/services/cloudformation/engine/parameters.py +++ b/localstack-core/localstack/services/cloudformation/engine/parameters.py @@ -26,7 +26,7 @@ def extract_stack_parameter_declarations(template: dict) -> dict[str, ParameterDeclaration]: """ - Extract and build a dict of stack parameter declarations from a CloudFormation stack templatef + Extract and build a dict of stack parameter declarations from a CloudFormation stack template :param template: the parsed CloudFormation stack template :return: a dictionary of declared parameters, mapping logical IDs to the corresponding parameter declaration diff --git a/localstack-core/localstack/services/cloudformation/provider.py b/localstack-core/localstack/services/cloudformation/provider.py index ddef294e7cd22..c6a35dd565d56 100644 --- a/localstack-core/localstack/services/cloudformation/provider.py +++ b/localstack-core/localstack/services/cloudformation/provider.py @@ -1,4 +1,5 @@ import copy +import itertools import json import logging import re @@ -110,6 +111,7 @@ find_change_set, find_stack, find_stack_by_id, + find_stack_set, get_cloudformation_store, ) from localstack.state import StateVisitor @@ -331,7 +333,7 @@ def update_stack( if not stack: return not_found_error(f'Unable to update non-existing stack "{stack_name}"') - api_utils.prepare_template_body(request) + api_utils.prepare_template_body(request, stack) template = template_preparer.parse_template(request["TemplateBody"]) if ( @@ -530,8 +532,18 @@ def get_template_summary( request: GetTemplateSummaryInput, ) -> GetTemplateSummaryOutput: stack_name = request.get("StackName") + stack_set_name = request.get("StackSetName") - if stack_name: + if stack_set_name: + stack_set = find_stack_set(context.account_id, context.region, stack_set_name) + if not stack_set: + # TODO: different error? + return stack_not_found_error(stack_set_name) + + template = template_preparer.parse_template(stack_set.template_body) + request["StackName"] = "tmp-stack" + stack = Stack(context.account_id, context.region, request, template) + elif stack_name: stack = find_stack(context.account_id, context.region, stack_name) if not stack: return stack_not_found_error(stack_name) @@ -555,10 +567,12 @@ def get_template_summary( id_summaries[res_type].append(resource_id) result["ResourceTypes"] = list(id_summaries.keys()) - result["ResourceIdentifierSummaries"] = [ - {"ResourceType": key, "LogicalResourceIds": values} - for key, values in id_summaries.items() - ] + if not stack_set_name: + # this property is only available for stacks, not stack sets + result["ResourceIdentifierSummaries"] = [ + {"ResourceType": key, "LogicalResourceIds": values} + for key, values in id_summaries.items() + ] result["Metadata"] = stack.template.get("Metadata") result["Version"] = stack.template.get("AWSTemplateFormatVersion", "2010-09-09") # these do not appear in the output @@ -1163,6 +1177,45 @@ def create_stack_instances( accounts = request["Accounts"] regions = request["Regions"] + # check if any stacks exist already + for account, region, instance in itertools.product( + accounts, regions, stack_set.stack_instances + ): + if instance.account == account and instance.region == region: + # record an operation + operation = { + "OperationId": op_id, + "StackSetId": stack_set.metadata["StackSetId"], + "Action": "CREATE", + "Status": "SUCCEEDED", + } + stack_set.operations[op_id] = operation + return CreateStackInstancesOutput(OperationId=op_id) + + # temp: find a better home for this + def merge_parameters(*parameters_lists: list) -> list: + """ + Merge multiple lists of parameters + + >>> ps1 = [{"ParameterKey": "a", "ParameterValue": "b"}] + >>> ps2 = [{"ParameterKey": "c", "ParameterValue": "d"}] + >>> ps3 = [{"ParameterKey": "a", "ParameterValue": "1"}] + >>> ps = merge_parameters(ps1, ps2, ps3) + >>> assert ps == [{"ParameterKey": "a", "ParameterValue": "1"}, + ... {"ParameterKey": "c", "ParameterValue": "d"}] + """ + from functools import reduce + + def _merge(acc: list, new: list) -> list: + acc_d = {every["ParameterKey"]: every["ParameterValue"] for every in acc} + new_d = {every["ParameterKey"]: every["ParameterValue"] for every in new} + acc_d.update(**new_d) + return [ + {"ParameterKey": key, "ParameterValue": value} for (key, value) in acc_d.items() + ] + + return reduce(_merge, parameters_lists, []) + stacks_to_await = [] for account in accounts: for region in regions: @@ -1177,6 +1230,10 @@ def create_stack_instances( kwargs = select_attributes(sset_meta, ["TemplateBody"]) or select_attributes( sset_meta, ["TemplateURL"] ) + kwargs["Parameters"] = merge_parameters( + sset_meta.get("Parameters", []), request.get("ParameterOverrides", []) + ) + # allow overrides from the request stack_name = f"sset-{set_name}-{account}" # skip creation of existing stacks diff --git a/localstack-core/localstack/services/cloudformation/stores.py b/localstack-core/localstack/services/cloudformation/stores.py index e14a911183e54..4749755e20cd8 100644 --- a/localstack-core/localstack/services/cloudformation/stores.py +++ b/localstack-core/localstack/services/cloudformation/stores.py @@ -61,6 +61,14 @@ def find_stack(account_id: str, region_name: str, stack_name: str) -> Stack | No )[0] +def find_stack_set(account_id: str, region_name: str, stack_set_name: str) -> StackSet | None: + state = get_cloudformation_store(account_id, region_name) + for name, stack_set in state.stack_sets.items(): + # TODO: stack set id? + if stack_set_name in [name, stack_set.stack_set_name]: + return stack_set + + def find_stack_by_id(account_id: str, region_name: str, stack_id: str) -> Stack | None: """ Find the stack by id. diff --git a/localstack-core/localstack/utils/collections.py b/localstack-core/localstack/utils/collections.py index c036bdc6b2bcd..595c4f921698e 100644 --- a/localstack-core/localstack/utils/collections.py +++ b/localstack-core/localstack/utils/collections.py @@ -26,6 +26,7 @@ ) import cachetools +import jsonpath_ng LOG = logging.getLogger(__name__) @@ -533,3 +534,15 @@ def is_comma_delimited_list(string: str, item_regex: Optional[str] = None) -> bo if pattern.match(string) is None: return False return True + + +def convert_in_place_at_jsonpath(params: dict, jsonpath: str, conversion_fn: Callable[[Any], Any]): + """ + Invokes a conversion function on a dictionary nested entry at a specific jsonpath with `conversion_fn` + """ + jp = jsonpath_ng.parse(jsonpath) + old_value = jp.find(params)[0].value + if not old_value: + return + new_value = conversion_fn(old_value) + jp.update(params, new_value) diff --git a/tests/aws/services/cloudformation/resources/test_stack_sets.py b/tests/aws/services/cloudformation/resources/test_stack_sets.py index f35fc30023d91..f510f2ae315a3 100644 --- a/tests/aws/services/cloudformation/resources/test_stack_sets.py +++ b/tests/aws/services/cloudformation/resources/test_stack_sets.py @@ -24,9 +24,15 @@ def _operation_is_ready(): @markers.aws.validated +@markers.snapshot.skip_snapshot_verify( + paths=[ + "$..Parameters..NoEcho", + "$..Parameters..ParameterConstraints", + "$..Parameters..DefaultValue", + ], +) def test_create_stack_set_with_stack_instances( account_id, - region_name, aws_client, snapshot, wait_stack_set_operation, @@ -36,20 +42,29 @@ def test_create_stack_set_with_stack_instances( stack_set_name = f"StackSet-{short_uid()}" template_body = load_file( - os.path.join(os.path.dirname(__file__), "../../../templates/s3_cors_bucket.yaml") + os.path.join(os.path.dirname(__file__), "../../../templates/sns_topic_parameter.yml") ) + topic_name = f"topic-{short_uid()}" result = aws_client.cloudformation.create_stack_set( StackSetName=stack_set_name, TemplateBody=template_body, + Parameters=[{"ParameterKey": "TopicName", "ParameterValue": topic_name}], ) snapshot.match("create_stack_set", result) + template_summary = aws_client.cloudformation.get_template_summary( + StackSetName=stack_set_name, + ) + snapshot.match("template-summary", template_summary) + + regions = ["us-west-2", "eu-north-1"] + create_instances_result = aws_client.cloudformation.create_stack_instances( StackSetName=stack_set_name, Accounts=[account_id], - Regions=[region_name], + Regions=regions, ) snapshot.match("create_stack_instances", create_instances_result) @@ -58,20 +73,20 @@ def test_create_stack_set_with_stack_instances( # make sure additional calls do not result in errors # even the stack already exists, but returns operation id instead - create_instances_result = aws_client.cloudformation.create_stack_instances( + recreate_instances_result = aws_client.cloudformation.create_stack_instances( StackSetName=stack_set_name, Accounts=[account_id], - Regions=[region_name], + Regions=regions, ) - assert "OperationId" in create_instances_result + snapshot.match("recreate_stack_instances", recreate_instances_result) - wait_stack_set_operation(stack_set_name, create_instances_result["OperationId"]) + wait_stack_set_operation(stack_set_name, recreate_instances_result["OperationId"]) delete_instances_result = aws_client.cloudformation.delete_stack_instances( StackSetName=stack_set_name, Accounts=[account_id], - Regions=[region_name], + Regions=regions, RetainStacks=False, ) wait_stack_set_operation(stack_set_name, delete_instances_result["OperationId"]) diff --git a/tests/aws/services/cloudformation/resources/test_stack_sets.snapshot.json b/tests/aws/services/cloudformation/resources/test_stack_sets.snapshot.json index 3585f6e07d3c7..40a900563f062 100644 --- a/tests/aws/services/cloudformation/resources/test_stack_sets.snapshot.json +++ b/tests/aws/services/cloudformation/resources/test_stack_sets.snapshot.json @@ -1,6 +1,6 @@ { "tests/aws/services/cloudformation/resources/test_stack_sets.py::test_create_stack_set_with_stack_instances": { - "recorded-date": "24-05-2023, 15:32:47", + "recorded-date": "03-05-2024, 16:06:54", "recorded-content": { "create_stack_set": { "StackSetId": "", @@ -9,12 +9,40 @@ "HTTPStatusCode": 200 } }, + "template-summary": { + "Parameters": [ + { + "DefaultValue": "sns-topic-simple", + "NoEcho": false, + "ParameterConstraints": { + "AllowedValues": [] + }, + "ParameterKey": "TopicName", + "ParameterType": "String" + } + ], + "ResourceTypes": [ + "AWS::SNS::Topic" + ], + "Version": "2010-09-09", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, "create_stack_instances": { "OperationId": "", "ResponseMetadata": { "HTTPHeaders": {}, "HTTPStatusCode": 200 } + }, + "recreate_stack_instances": { + "OperationId": "", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } } } } diff --git a/tests/aws/services/cloudformation/resources/test_stack_sets.validation.json b/tests/aws/services/cloudformation/resources/test_stack_sets.validation.json index f7406f8c55f29..24d53f4442b5d 100644 --- a/tests/aws/services/cloudformation/resources/test_stack_sets.validation.json +++ b/tests/aws/services/cloudformation/resources/test_stack_sets.validation.json @@ -1,5 +1,5 @@ { "tests/aws/services/cloudformation/resources/test_stack_sets.py::test_create_stack_set_with_stack_instances": { - "last_validated_date": "2023-05-24T13:32:47+00:00" + "last_validated_date": "2024-05-03T16:06:54+00:00" } } diff --git a/tests/unit/utils/test_collections.py b/tests/unit/utils/test_collections.py index adb2581e77460..6f07a8d3c3755 100644 --- a/tests/unit/utils/test_collections.py +++ b/tests/unit/utils/test_collections.py @@ -7,6 +7,7 @@ HashableList, ImmutableDict, ImmutableList, + convert_in_place_at_jsonpath, convert_to_typed_dict, is_comma_delimited_list, select_from_typed_dict, @@ -193,3 +194,21 @@ def test_is_comma_limited_list(): assert not is_comma_delimited_list("foo, bar baz") assert not is_comma_delimited_list("foo,") assert not is_comma_delimited_list("") + + +@pytest.mark.parametrize( + "input,jsonpath,fn,expected", + [ + # examples taken from ECS + ({"desiredCount": "1"}, "desiredCount", int, {"desiredCount": 1}), + ( + {"serviceConnectConfiguration": {"services": [{"clientAliases": [{"port": "80"}]}]}}, + "serviceConnectConfiguration.services[*].clientAliases[*].port", + int, + {"serviceConnectConfiguration": {"services": [{"clientAliases": [{"port": 80}]}]}}, + ), + ], +) +def test_convert_in_place_at_jsonpath(input, jsonpath, fn, expected): + convert_in_place_at_jsonpath(input, jsonpath, fn) + assert input == expected