From 3f40cdc1381ab08d64681dbbf780c36219755ef4 Mon Sep 17 00:00:00 2001 From: Simon Walker Date: Fri, 3 May 2024 16:22:30 +0100 Subject: [PATCH 1/8] Implement stack set summary --- .../cloudformation/engine/entities.py | 81 ++++++++++++++++--- .../cloudformation/engine/parameters.py | 2 +- .../services/cloudformation/provider.py | 9 +++ .../services/cloudformation/stores.py | 8 ++ .../resources/test_stack_sets.py | 68 +++++++++------- .../resources/test_stack_sets.snapshot.json | 23 +++++- .../resources/test_stack_sets.validation.json | 2 +- tests/aws/templates/s3_cors_bucket.yaml | 5 ++ 8 files changed, 154 insertions(+), 44 deletions(-) diff --git a/localstack-core/localstack/services/cloudformation/engine/entities.py b/localstack-core/localstack/services/cloudformation/engine/entities.py index e1cac8d6aeda1..d3fec79d78e7b 100644 --- a/localstack-core/localstack/services/cloudformation/engine/entities.py +++ b/localstack-core/localstack/services/cloudformation/engine/entities.py @@ -1,10 +1,17 @@ import logging from typing import Optional, TypedDict -from localstack.aws.api.cloudformation import Capability, ChangeSetType, Parameter +from localstack.aws.api.cloudformation import ( + Capability, + ChangeSetType, + CreateStackSetInput, + GetTemplateSummaryOutput, + Parameter, +) from localstack.services.cloudformation.engine.parameters import ( StackParameter, convert_stack_parameters_to_list, + extract_stack_parameter_declarations, strip_parameter_type, ) from localstack.utils.aws import arns @@ -17,30 +24,80 @@ LOG = logging.getLogger(__name__) -class StackSet: - """A stack set contains multiple stack instances.""" +class StackInstance: + """A stack instance belongs to a stack set and is specific to a region / account ID.""" # FIXME: confusing name. metadata is the complete incoming request object def __init__(self, metadata: dict): self.metadata = metadata + # reference to the deployed stack belonging to this stack instance + self.stack = None + + +class StackSet: + """A stack set contains multiple stack instances.""" + + def __init__(self, request: CreateStackSetInput): + self.request = request # list of stack instances - self.stack_instances = [] + self.stack_instances: list[StackInstance] = [] # maps operation ID to stack set operation details self.operations = {} + # compatibility with old API + @property + def metadata(self) -> CreateStackSetInput: + return self.request + @property def stack_set_name(self): - return self.metadata.get("StackSetName") + return self.request.get("StackSetName") + @property + def template_url(self) -> str | None: + return self.request.get("TemplateURL") -class StackInstance: - """A stack instance belongs to a stack set and is specific to a region / account ID.""" + @property + def template_body(self) -> str | None: + return self.request.get("TemplateBody") + + def get_template(self): + if body := self.template_body: + return body + + raise NotImplementedError("template URL") + + def get_template_summary(self) -> GetTemplateSummaryOutput: + # id_summaries = defaultdict(list) + # for resource_id, resource in stack.template_resources.items(): + # res_type = resource["Type"] + # id_summaries[res_type].append(resource_id) + + # hack to use this helper function + parameters = { + every["ParameterKey"]: { + "Type": "String", + } + for every in self.metadata["Parameters"] + } + result = { + "Version": "2010-09-09", + "ResourceTypes": ["AWS::S3::Bucket", "AWS::S3::Bucket"], + "Parameters": list( + extract_stack_parameter_declarations({"Parameters": parameters}).values() + ), + } + # result["ResourceIdentifierSummaries"] = [ + # {"ResourceType": key, "LogicalResourceIds": values} + # for key, values in id_summaries.items() + # ] + return result - # FIXME: confusing name. metadata is the complete incoming request object - def __init__(self, metadata: dict): - self.metadata = metadata - # reference to the deployed stack belonging to this stack instance - self.stack = None + def get_instance(self, account: str, region: str) -> StackInstance | None: + for instance in self.stack_instances: + if instance.metadata["Account"] == account and instance.metadata["Region"] == region: + return instance + return None class StackMetadata(TypedDict): diff --git a/localstack-core/localstack/services/cloudformation/engine/parameters.py b/localstack-core/localstack/services/cloudformation/engine/parameters.py index d84f4a7751f52..c2a20ad630686 100644 --- a/localstack-core/localstack/services/cloudformation/engine/parameters.py +++ b/localstack-core/localstack/services/cloudformation/engine/parameters.py @@ -26,7 +26,7 @@ def extract_stack_parameter_declarations(template: dict) -> dict[str, ParameterDeclaration]: """ - Extract and build a dict of stack parameter declarations from a CloudFormation stack templatef + Extract and build a dict of stack parameter declarations from a CloudFormation stack template :param template: the parsed CloudFormation stack template :return: a dictionary of declared parameters, mapping logical IDs to the corresponding parameter declaration diff --git a/localstack-core/localstack/services/cloudformation/provider.py b/localstack-core/localstack/services/cloudformation/provider.py index ddef294e7cd22..b759693bcd84c 100644 --- a/localstack-core/localstack/services/cloudformation/provider.py +++ b/localstack-core/localstack/services/cloudformation/provider.py @@ -110,6 +110,7 @@ find_change_set, find_stack, find_stack_by_id, + find_stack_set, get_cloudformation_store, ) from localstack.state import StateVisitor @@ -530,6 +531,14 @@ def get_template_summary( request: GetTemplateSummaryInput, ) -> GetTemplateSummaryOutput: stack_name = request.get("StackName") + stack_set_name = request.get("StackSetName") + + if stack_set_name: + stack_set = find_stack_set(context.account_id, context.region, stack_set_name) + if not stack_set: + # TODO: different error? + return stack_not_found_error(stack_set_name) + return stack_set.get_template_summary() if stack_name: stack = find_stack(context.account_id, context.region, stack_name) diff --git a/localstack-core/localstack/services/cloudformation/stores.py b/localstack-core/localstack/services/cloudformation/stores.py index e14a911183e54..4749755e20cd8 100644 --- a/localstack-core/localstack/services/cloudformation/stores.py +++ b/localstack-core/localstack/services/cloudformation/stores.py @@ -61,6 +61,14 @@ def find_stack(account_id: str, region_name: str, stack_name: str) -> Stack | No )[0] +def find_stack_set(account_id: str, region_name: str, stack_set_name: str) -> StackSet | None: + state = get_cloudformation_store(account_id, region_name) + for name, stack_set in state.stack_sets.items(): + # TODO: stack set id? + if stack_set_name in [name, stack_set.stack_set_name]: + return stack_set + + def find_stack_by_id(account_id: str, region_name: str, stack_id: str) -> Stack | None: """ Find the stack by id. diff --git a/tests/aws/services/cloudformation/resources/test_stack_sets.py b/tests/aws/services/cloudformation/resources/test_stack_sets.py index f35fc30023d91..6cf3f6a49c0f0 100644 --- a/tests/aws/services/cloudformation/resources/test_stack_sets.py +++ b/tests/aws/services/cloudformation/resources/test_stack_sets.py @@ -24,6 +24,9 @@ def _operation_is_ready(): @markers.aws.validated +@markers.snapshot.skip_snapshot_verify( + paths=["$..Parameters..NoEcho", "$..Parameters..ParameterConstraints"], +) def test_create_stack_set_with_stack_instances( account_id, region_name, @@ -39,41 +42,48 @@ def test_create_stack_set_with_stack_instances( os.path.join(os.path.dirname(__file__), "../../../templates/s3_cors_bucket.yaml") ) + bucket_name = f"bucket-{short_uid()}" result = aws_client.cloudformation.create_stack_set( StackSetName=stack_set_name, TemplateBody=template_body, + Parameters=[{"ParameterKey": "BucketName", "ParameterValue": bucket_name}], ) snapshot.match("create_stack_set", result) - create_instances_result = aws_client.cloudformation.create_stack_instances( + template_summary = aws_client.cloudformation.get_template_summary( StackSetName=stack_set_name, - Accounts=[account_id], - Regions=[region_name], ) - - snapshot.match("create_stack_instances", create_instances_result) - - wait_stack_set_operation(stack_set_name, create_instances_result["OperationId"]) - - # make sure additional calls do not result in errors - # even the stack already exists, but returns operation id instead - create_instances_result = aws_client.cloudformation.create_stack_instances( - StackSetName=stack_set_name, - Accounts=[account_id], - Regions=[region_name], - ) - - assert "OperationId" in create_instances_result - - wait_stack_set_operation(stack_set_name, create_instances_result["OperationId"]) - - delete_instances_result = aws_client.cloudformation.delete_stack_instances( - StackSetName=stack_set_name, - Accounts=[account_id], - Regions=[region_name], - RetainStacks=False, - ) - wait_stack_set_operation(stack_set_name, delete_instances_result["OperationId"]) - - aws_client.cloudformation.delete_stack_set(StackSetName=stack_set_name) + snapshot.match("template-summary", template_summary) + + # create_instances_result = aws_client.cloudformation.create_stack_instances( + # StackSetName=stack_set_name, + # Accounts=[account_id], + # Regions=[region_name], + # ) + # + # snapshot.match("create_stack_instances", create_instances_result) + # + # wait_stack_set_operation(stack_set_name, create_instances_result["OperationId"]) + # + # # make sure additional calls do not result in errors + # # even the stack already exists, but returns operation id instead + # create_instances_result = aws_client.cloudformation.create_stack_instances( + # StackSetName=stack_set_name, + # Accounts=[account_id], + # Regions=[region_name], + # ) + # + # assert "OperationId" in create_instances_result + # + # wait_stack_set_operation(stack_set_name, create_instances_result["OperationId"]) + # + # delete_instances_result = aws_client.cloudformation.delete_stack_instances( + # StackSetName=stack_set_name, + # Accounts=[account_id], + # Regions=[region_name], + # RetainStacks=False, + # ) + # wait_stack_set_operation(stack_set_name, delete_instances_result["OperationId"]) + # + # aws_client.cloudformation.delete_stack_set(StackSetName=stack_set_name) diff --git a/tests/aws/services/cloudformation/resources/test_stack_sets.snapshot.json b/tests/aws/services/cloudformation/resources/test_stack_sets.snapshot.json index 3585f6e07d3c7..3c1385f9ca5d8 100644 --- a/tests/aws/services/cloudformation/resources/test_stack_sets.snapshot.json +++ b/tests/aws/services/cloudformation/resources/test_stack_sets.snapshot.json @@ -1,6 +1,6 @@ { "tests/aws/services/cloudformation/resources/test_stack_sets.py::test_create_stack_set_with_stack_instances": { - "recorded-date": "24-05-2023, 15:32:47", + "recorded-date": "01-05-2024, 16:29:26", "recorded-content": { "create_stack_set": { "StackSetId": "", @@ -9,6 +9,27 @@ "HTTPStatusCode": 200 } }, + "template-summary": { + "Parameters": [ + { + "NoEcho": false, + "ParameterConstraints": { + "AllowedValues": [] + }, + "ParameterKey": "BucketName", + "ParameterType": "String" + } + ], + "ResourceTypes": [ + "AWS::S3::Bucket", + "AWS::S3::Bucket" + ], + "Version": "2010-09-09", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, "create_stack_instances": { "OperationId": "", "ResponseMetadata": { diff --git a/tests/aws/services/cloudformation/resources/test_stack_sets.validation.json b/tests/aws/services/cloudformation/resources/test_stack_sets.validation.json index f7406f8c55f29..815ec67aaaf1e 100644 --- a/tests/aws/services/cloudformation/resources/test_stack_sets.validation.json +++ b/tests/aws/services/cloudformation/resources/test_stack_sets.validation.json @@ -1,5 +1,5 @@ { "tests/aws/services/cloudformation/resources/test_stack_sets.py::test_create_stack_set_with_stack_instances": { - "last_validated_date": "2023-05-24T13:32:47+00:00" + "last_validated_date": "2024-05-01T16:29:26+00:00" } } diff --git a/tests/aws/templates/s3_cors_bucket.yaml b/tests/aws/templates/s3_cors_bucket.yaml index 5e7a120ba676b..cdeafa2a3e6dd 100644 --- a/tests/aws/templates/s3_cors_bucket.yaml +++ b/tests/aws/templates/s3_cors_bucket.yaml @@ -1,7 +1,12 @@ +Parameters: + BucketName: + Type: String + Resources: LocalBucket: Type: AWS::S3::Bucket Properties: + BucketName: !Ref BucketName CorsConfiguration: CorsRules: - AllowedHeaders: From 12e1a881cb5297385d7009f105e5c3b275589bcb Mon Sep 17 00:00:00 2001 From: Simon Walker Date: Fri, 3 May 2024 17:19:38 +0100 Subject: [PATCH 2/8] Support deployment of stack sets --- .../cloudformation/engine/entities.py | 10 ++- .../services/cloudformation/provider.py | 44 +++++++++++ .../resources/test_stack_sets.py | 77 ++++++++++--------- .../resources/test_stack_sets.snapshot.json | 15 +++- .../resources/test_stack_sets.validation.json | 2 +- 5 files changed, 106 insertions(+), 42 deletions(-) diff --git a/localstack-core/localstack/services/cloudformation/engine/entities.py b/localstack-core/localstack/services/cloudformation/engine/entities.py index d3fec79d78e7b..a6702dd243b1a 100644 --- a/localstack-core/localstack/services/cloudformation/engine/entities.py +++ b/localstack-core/localstack/services/cloudformation/engine/entities.py @@ -33,6 +33,14 @@ def __init__(self, metadata: dict): # reference to the deployed stack belonging to this stack instance self.stack = None + @property + def account(self) -> str: + return self.metadata["Account"] + + @property + def region(self) -> str: + return self.metadata["Region"] + class StackSet: """A stack set contains multiple stack instances.""" @@ -82,7 +90,7 @@ def get_template_summary(self) -> GetTemplateSummaryOutput: } result = { "Version": "2010-09-09", - "ResourceTypes": ["AWS::S3::Bucket", "AWS::S3::Bucket"], + "ResourceTypes": ["AWS::SNS::Topic"], "Parameters": list( extract_stack_parameter_declarations({"Parameters": parameters}).values() ), diff --git a/localstack-core/localstack/services/cloudformation/provider.py b/localstack-core/localstack/services/cloudformation/provider.py index b759693bcd84c..852ad2579297e 100644 --- a/localstack-core/localstack/services/cloudformation/provider.py +++ b/localstack-core/localstack/services/cloudformation/provider.py @@ -1,4 +1,5 @@ import copy +import itertools import json import logging import re @@ -1172,6 +1173,45 @@ def create_stack_instances( accounts = request["Accounts"] regions = request["Regions"] + # check if any stacks exist already + for account, region, instance in itertools.product( + accounts, regions, stack_set.stack_instances + ): + if instance.account == account and instance.region == region: + # record an operation + operation = { + "OperationId": op_id, + "StackSetId": stack_set.metadata["StackSetId"], + "Action": "CREATE", + "Status": "SUCCEEDED", + } + stack_set.operations[op_id] = operation + return CreateStackInstancesOutput(OperationId=op_id) + + # temp: find a better home for this + def merge_parameters(*parameters_lists: list) -> list: + """ + Merge multiple lists of parameters + + >>> ps1 = [{"ParameterKey": "a", "ParameterValue": "b"}] + >>> ps2 = [{"ParameterKey": "c", "ParameterValue": "d"}] + >>> ps3 = [{"ParameterKey": "a", "ParameterValue": "1"}] + >>> ps = merge_parameters(ps1, ps2, ps3) + >>> assert ps == [{"ParameterKey": "a", "ParameterValue": "1"}, + ... {"ParameterKey": "c", "ParameterValue": "d"}] + """ + from functools import reduce + + def _merge(acc: list, new: list) -> list: + acc_d = {every["ParameterKey"]: every["ParameterValue"] for every in acc} + new_d = {every["ParameterKey"]: every["ParameterValue"] for every in new} + acc_d.update(**new_d) + return [ + {"ParameterKey": key, "ParameterValue": value} for (key, value) in acc_d.items() + ] + + return reduce(_merge, parameters_lists, []) + stacks_to_await = [] for account in accounts: for region in regions: @@ -1186,6 +1226,10 @@ def create_stack_instances( kwargs = select_attributes(sset_meta, ["TemplateBody"]) or select_attributes( sset_meta, ["TemplateURL"] ) + kwargs["Parameters"] = merge_parameters( + sset_meta["Parameters"], request.get("ParameterOverrides", []) + ) + # allow overrides from the request stack_name = f"sset-{set_name}-{account}" # skip creation of existing stacks diff --git a/tests/aws/services/cloudformation/resources/test_stack_sets.py b/tests/aws/services/cloudformation/resources/test_stack_sets.py index 6cf3f6a49c0f0..f510f2ae315a3 100644 --- a/tests/aws/services/cloudformation/resources/test_stack_sets.py +++ b/tests/aws/services/cloudformation/resources/test_stack_sets.py @@ -25,11 +25,14 @@ def _operation_is_ready(): @markers.aws.validated @markers.snapshot.skip_snapshot_verify( - paths=["$..Parameters..NoEcho", "$..Parameters..ParameterConstraints"], + paths=[ + "$..Parameters..NoEcho", + "$..Parameters..ParameterConstraints", + "$..Parameters..DefaultValue", + ], ) def test_create_stack_set_with_stack_instances( account_id, - region_name, aws_client, snapshot, wait_stack_set_operation, @@ -39,14 +42,14 @@ def test_create_stack_set_with_stack_instances( stack_set_name = f"StackSet-{short_uid()}" template_body = load_file( - os.path.join(os.path.dirname(__file__), "../../../templates/s3_cors_bucket.yaml") + os.path.join(os.path.dirname(__file__), "../../../templates/sns_topic_parameter.yml") ) - bucket_name = f"bucket-{short_uid()}" + topic_name = f"topic-{short_uid()}" result = aws_client.cloudformation.create_stack_set( StackSetName=stack_set_name, TemplateBody=template_body, - Parameters=[{"ParameterKey": "BucketName", "ParameterValue": bucket_name}], + Parameters=[{"ParameterKey": "TopicName", "ParameterValue": topic_name}], ) snapshot.match("create_stack_set", result) @@ -56,34 +59,36 @@ def test_create_stack_set_with_stack_instances( ) snapshot.match("template-summary", template_summary) - # create_instances_result = aws_client.cloudformation.create_stack_instances( - # StackSetName=stack_set_name, - # Accounts=[account_id], - # Regions=[region_name], - # ) - # - # snapshot.match("create_stack_instances", create_instances_result) - # - # wait_stack_set_operation(stack_set_name, create_instances_result["OperationId"]) - # - # # make sure additional calls do not result in errors - # # even the stack already exists, but returns operation id instead - # create_instances_result = aws_client.cloudformation.create_stack_instances( - # StackSetName=stack_set_name, - # Accounts=[account_id], - # Regions=[region_name], - # ) - # - # assert "OperationId" in create_instances_result - # - # wait_stack_set_operation(stack_set_name, create_instances_result["OperationId"]) - # - # delete_instances_result = aws_client.cloudformation.delete_stack_instances( - # StackSetName=stack_set_name, - # Accounts=[account_id], - # Regions=[region_name], - # RetainStacks=False, - # ) - # wait_stack_set_operation(stack_set_name, delete_instances_result["OperationId"]) - # - # aws_client.cloudformation.delete_stack_set(StackSetName=stack_set_name) + regions = ["us-west-2", "eu-north-1"] + + create_instances_result = aws_client.cloudformation.create_stack_instances( + StackSetName=stack_set_name, + Accounts=[account_id], + Regions=regions, + ) + + snapshot.match("create_stack_instances", create_instances_result) + + wait_stack_set_operation(stack_set_name, create_instances_result["OperationId"]) + + # make sure additional calls do not result in errors + # even the stack already exists, but returns operation id instead + recreate_instances_result = aws_client.cloudformation.create_stack_instances( + StackSetName=stack_set_name, + Accounts=[account_id], + Regions=regions, + ) + + snapshot.match("recreate_stack_instances", recreate_instances_result) + + wait_stack_set_operation(stack_set_name, recreate_instances_result["OperationId"]) + + delete_instances_result = aws_client.cloudformation.delete_stack_instances( + StackSetName=stack_set_name, + Accounts=[account_id], + Regions=regions, + RetainStacks=False, + ) + wait_stack_set_operation(stack_set_name, delete_instances_result["OperationId"]) + + aws_client.cloudformation.delete_stack_set(StackSetName=stack_set_name) diff --git a/tests/aws/services/cloudformation/resources/test_stack_sets.snapshot.json b/tests/aws/services/cloudformation/resources/test_stack_sets.snapshot.json index 3c1385f9ca5d8..40a900563f062 100644 --- a/tests/aws/services/cloudformation/resources/test_stack_sets.snapshot.json +++ b/tests/aws/services/cloudformation/resources/test_stack_sets.snapshot.json @@ -1,6 +1,6 @@ { "tests/aws/services/cloudformation/resources/test_stack_sets.py::test_create_stack_set_with_stack_instances": { - "recorded-date": "01-05-2024, 16:29:26", + "recorded-date": "03-05-2024, 16:06:54", "recorded-content": { "create_stack_set": { "StackSetId": "", @@ -12,17 +12,17 @@ "template-summary": { "Parameters": [ { + "DefaultValue": "sns-topic-simple", "NoEcho": false, "ParameterConstraints": { "AllowedValues": [] }, - "ParameterKey": "BucketName", + "ParameterKey": "TopicName", "ParameterType": "String" } ], "ResourceTypes": [ - "AWS::S3::Bucket", - "AWS::S3::Bucket" + "AWS::SNS::Topic" ], "Version": "2010-09-09", "ResponseMetadata": { @@ -36,6 +36,13 @@ "HTTPHeaders": {}, "HTTPStatusCode": 200 } + }, + "recreate_stack_instances": { + "OperationId": "", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } } } } diff --git a/tests/aws/services/cloudformation/resources/test_stack_sets.validation.json b/tests/aws/services/cloudformation/resources/test_stack_sets.validation.json index 815ec67aaaf1e..24d53f4442b5d 100644 --- a/tests/aws/services/cloudformation/resources/test_stack_sets.validation.json +++ b/tests/aws/services/cloudformation/resources/test_stack_sets.validation.json @@ -1,5 +1,5 @@ { "tests/aws/services/cloudformation/resources/test_stack_sets.py::test_create_stack_set_with_stack_instances": { - "last_validated_date": "2024-05-01T16:29:26+00:00" + "last_validated_date": "2024-05-03T16:06:54+00:00" } } From ec8b9d21761d044e497ee25b58cf18dee35e3633 Mon Sep 17 00:00:00 2001 From: Simon Walker Date: Fri, 3 May 2024 17:30:16 +0100 Subject: [PATCH 3/8] Hacky support for get_template_summary for stack sets --- .../cloudformation/engine/entities.py | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/localstack-core/localstack/services/cloudformation/engine/entities.py b/localstack-core/localstack/services/cloudformation/engine/entities.py index a6702dd243b1a..eadfe44d30b29 100644 --- a/localstack-core/localstack/services/cloudformation/engine/entities.py +++ b/localstack-core/localstack/services/cloudformation/engine/entities.py @@ -1,4 +1,6 @@ +import copy import logging +from collections import defaultdict from typing import Optional, TypedDict from localstack.aws.api.cloudformation import ( @@ -8,6 +10,8 @@ GetTemplateSummaryOutput, Parameter, ) +from localstack.constants import AWS_REGION_US_EAST_1, DEFAULT_AWS_ACCOUNT_ID +from localstack.services.cloudformation import api_utils from localstack.services.cloudformation.engine.parameters import ( StackParameter, convert_stack_parameters_to_list, @@ -76,29 +80,26 @@ def get_template(self): raise NotImplementedError("template URL") def get_template_summary(self) -> GetTemplateSummaryOutput: - # id_summaries = defaultdict(list) - # for resource_id, resource in stack.template_resources.items(): - # res_type = resource["Type"] - # id_summaries[res_type].append(resource_id) + # TEMP: prevent circular imports + from localstack.services.cloudformation.engine import template_preparer + + request = copy.deepcopy(self.metadata) + api_utils.prepare_template_body(request) + template = template_preparer.parse_template(request["TemplateBody"]) + request["StackName"] = "tmp-stack" + stack = Stack(DEFAULT_AWS_ACCOUNT_ID, AWS_REGION_US_EAST_1, request, template) + + id_summaries = defaultdict(list) + for resource_id, resource in stack.template_resources.items(): + res_type = resource["Type"] + id_summaries[res_type].append(resource_id) # hack to use this helper function - parameters = { - every["ParameterKey"]: { - "Type": "String", - } - for every in self.metadata["Parameters"] - } result = { "Version": "2010-09-09", - "ResourceTypes": ["AWS::SNS::Topic"], - "Parameters": list( - extract_stack_parameter_declarations({"Parameters": parameters}).values() - ), + "ResourceTypes": list(id_summaries.keys()), + "Parameters": list(extract_stack_parameter_declarations(template).values()), } - # result["ResourceIdentifierSummaries"] = [ - # {"ResourceType": key, "LogicalResourceIds": values} - # for key, values in id_summaries.items() - # ] return result def get_instance(self, account: str, region: str) -> StackInstance | None: From ba985270415968cd04c4dc13a1ea262e9751d6e3 Mon Sep 17 00:00:00 2001 From: Simon Walker Date: Fri, 3 May 2024 17:35:08 +0100 Subject: [PATCH 4/8] Unify logic of get_template_summaries --- .../cloudformation/engine/entities.py | 29 ------------------- .../services/cloudformation/provider.py | 16 ++++++---- 2 files changed, 10 insertions(+), 35 deletions(-) diff --git a/localstack-core/localstack/services/cloudformation/engine/entities.py b/localstack-core/localstack/services/cloudformation/engine/entities.py index eadfe44d30b29..d01648beb3c1d 100644 --- a/localstack-core/localstack/services/cloudformation/engine/entities.py +++ b/localstack-core/localstack/services/cloudformation/engine/entities.py @@ -1,21 +1,15 @@ -import copy import logging -from collections import defaultdict from typing import Optional, TypedDict from localstack.aws.api.cloudformation import ( Capability, ChangeSetType, CreateStackSetInput, - GetTemplateSummaryOutput, Parameter, ) -from localstack.constants import AWS_REGION_US_EAST_1, DEFAULT_AWS_ACCOUNT_ID -from localstack.services.cloudformation import api_utils from localstack.services.cloudformation.engine.parameters import ( StackParameter, convert_stack_parameters_to_list, - extract_stack_parameter_declarations, strip_parameter_type, ) from localstack.utils.aws import arns @@ -79,29 +73,6 @@ def get_template(self): raise NotImplementedError("template URL") - def get_template_summary(self) -> GetTemplateSummaryOutput: - # TEMP: prevent circular imports - from localstack.services.cloudformation.engine import template_preparer - - request = copy.deepcopy(self.metadata) - api_utils.prepare_template_body(request) - template = template_preparer.parse_template(request["TemplateBody"]) - request["StackName"] = "tmp-stack" - stack = Stack(DEFAULT_AWS_ACCOUNT_ID, AWS_REGION_US_EAST_1, request, template) - - id_summaries = defaultdict(list) - for resource_id, resource in stack.template_resources.items(): - res_type = resource["Type"] - id_summaries[res_type].append(resource_id) - - # hack to use this helper function - result = { - "Version": "2010-09-09", - "ResourceTypes": list(id_summaries.keys()), - "Parameters": list(extract_stack_parameter_declarations(template).values()), - } - return result - def get_instance(self, account: str, region: str) -> StackInstance | None: for instance in self.stack_instances: if instance.metadata["Account"] == account and instance.metadata["Region"] == region: diff --git a/localstack-core/localstack/services/cloudformation/provider.py b/localstack-core/localstack/services/cloudformation/provider.py index 852ad2579297e..694ee5a8ab562 100644 --- a/localstack-core/localstack/services/cloudformation/provider.py +++ b/localstack-core/localstack/services/cloudformation/provider.py @@ -539,9 +539,11 @@ def get_template_summary( if not stack_set: # TODO: different error? return stack_not_found_error(stack_set_name) - return stack_set.get_template_summary() - if stack_name: + template = template_preparer.parse_template(stack_set.template_body) + request["StackName"] = "tmp-stack" + stack = Stack(context.account_id, context.region, request, template) + elif stack_name: stack = find_stack(context.account_id, context.region, stack_name) if not stack: return stack_not_found_error(stack_name) @@ -565,10 +567,12 @@ def get_template_summary( id_summaries[res_type].append(resource_id) result["ResourceTypes"] = list(id_summaries.keys()) - result["ResourceIdentifierSummaries"] = [ - {"ResourceType": key, "LogicalResourceIds": values} - for key, values in id_summaries.items() - ] + if not stack_set_name: + # this property is only available for stacks, not stack sets + result["ResourceIdentifierSummaries"] = [ + {"ResourceType": key, "LogicalResourceIds": values} + for key, values in id_summaries.items() + ] result["Metadata"] = stack.template.get("Metadata") result["Version"] = stack.template.get("AWSTemplateFormatVersion", "2010-09-09") # these do not appear in the output From 205cd6484226a9b6f351fdd596cd5e29f9056592 Mon Sep 17 00:00:00 2001 From: Simon Walker Date: Fri, 3 May 2024 17:40:18 +0100 Subject: [PATCH 5/8] Handle absent parameters --- localstack-core/localstack/services/cloudformation/provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/localstack-core/localstack/services/cloudformation/provider.py b/localstack-core/localstack/services/cloudformation/provider.py index 694ee5a8ab562..e7b419af3cc73 100644 --- a/localstack-core/localstack/services/cloudformation/provider.py +++ b/localstack-core/localstack/services/cloudformation/provider.py @@ -1231,7 +1231,7 @@ def _merge(acc: list, new: list) -> list: sset_meta, ["TemplateURL"] ) kwargs["Parameters"] = merge_parameters( - sset_meta["Parameters"], request.get("ParameterOverrides", []) + sset_meta.get("Parameters", []), request.get("ParameterOverrides", []) ) # allow overrides from the request stack_name = f"sset-{set_name}-{account}" From 3f6054c99a5ecc189bae646c73af29d1a67ebdbe Mon Sep 17 00:00:00 2001 From: Simon Walker Date: Fri, 3 May 2024 23:56:34 +0100 Subject: [PATCH 6/8] Handle using previous stack template --- .../services/cloudformation/api_utils.py | 15 ++++++++++++--- .../services/cloudformation/provider.py | 2 +- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/localstack-core/localstack/services/cloudformation/api_utils.py b/localstack-core/localstack/services/cloudformation/api_utils.py index 556435ed699a7..7eda69fefdedf 100644 --- a/localstack-core/localstack/services/cloudformation/api_utils.py +++ b/localstack-core/localstack/services/cloudformation/api_utils.py @@ -4,6 +4,7 @@ from localstack import config, constants from localstack.aws.connect import connect_to +from localstack.services.cloudformation.engine.entities import Stack from localstack.services.s3.utils import ( extract_bucket_name_and_key_from_headers_and_path, normalize_bucket_name, @@ -16,7 +17,12 @@ LOG = logging.getLogger(__name__) -def prepare_template_body(req_data: dict) -> str | bytes | None: # TODO: mutating and returning +# TODO: remove optional stack and make it required + + +def prepare_template_body( + req_data: dict, stack: Stack | None = None +) -> str | bytes | None: # TODO: mutating and returning template_url = req_data.get("TemplateURL") if template_url: req_data["TemplateURL"] = convert_s3_to_local_url(template_url) @@ -26,13 +32,13 @@ def prepare_template_body(req_data: dict) -> str | bytes | None: # TODO: mutati if modified_template_body: req_data.pop("TemplateURL", None) req_data["TemplateBody"] = modified_template_body - modified_template_body = get_template_body(req_data) + modified_template_body = get_template_body(req_data, stack) if modified_template_body: req_data["TemplateBody"] = modified_template_body return modified_template_body -def get_template_body(req_data: dict) -> str: +def get_template_body(req_data: dict, stack: Stack | None = None) -> str: body = req_data.get("TemplateBody") if body: return body @@ -60,6 +66,9 @@ def get_template_body(req_data: dict) -> str: "Unable to fetch template body (code %s) from URL %s" % (status_code, url) ) return to_str(response.content) + if req_data.get("UsePreviousTemplate") and stack: + return stack.template_body + raise Exception("Unable to get template body from input: %s" % req_data) diff --git a/localstack-core/localstack/services/cloudformation/provider.py b/localstack-core/localstack/services/cloudformation/provider.py index e7b419af3cc73..c6a35dd565d56 100644 --- a/localstack-core/localstack/services/cloudformation/provider.py +++ b/localstack-core/localstack/services/cloudformation/provider.py @@ -333,7 +333,7 @@ def update_stack( if not stack: return not_found_error(f'Unable to update non-existing stack "{stack_name}"') - api_utils.prepare_template_body(request) + api_utils.prepare_template_body(request, stack) template = template_preparer.parse_template(request["TemplateBody"]) if ( From 0fd6b8bd0a41b5d4c14b4dc5f6f91972240ca029 Mon Sep 17 00:00:00 2001 From: Simon Walker Date: Sat, 4 May 2024 13:57:16 +0100 Subject: [PATCH 7/8] Reset previous test stack back to former glory --- tests/aws/templates/s3_cors_bucket.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/aws/templates/s3_cors_bucket.yaml b/tests/aws/templates/s3_cors_bucket.yaml index cdeafa2a3e6dd..5e7a120ba676b 100644 --- a/tests/aws/templates/s3_cors_bucket.yaml +++ b/tests/aws/templates/s3_cors_bucket.yaml @@ -1,12 +1,7 @@ -Parameters: - BucketName: - Type: String - Resources: LocalBucket: Type: AWS::S3::Bucket Properties: - BucketName: !Ref BucketName CorsConfiguration: CorsRules: - AllowedHeaders: From a2f88f17fdfd1bf2b1d92d128a360645f371ae2c Mon Sep 17 00:00:00 2001 From: Simon Walker Date: Wed, 8 May 2024 15:38:04 +0100 Subject: [PATCH 8/8] Add helper function for type conversions based on jsonpath --- .../localstack/utils/collections.py | 13 +++++++++++++ tests/unit/utils/test_collections.py | 19 +++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/localstack-core/localstack/utils/collections.py b/localstack-core/localstack/utils/collections.py index c036bdc6b2bcd..595c4f921698e 100644 --- a/localstack-core/localstack/utils/collections.py +++ b/localstack-core/localstack/utils/collections.py @@ -26,6 +26,7 @@ ) import cachetools +import jsonpath_ng LOG = logging.getLogger(__name__) @@ -533,3 +534,15 @@ def is_comma_delimited_list(string: str, item_regex: Optional[str] = None) -> bo if pattern.match(string) is None: return False return True + + +def convert_in_place_at_jsonpath(params: dict, jsonpath: str, conversion_fn: Callable[[Any], Any]): + """ + Invokes a conversion function on a dictionary nested entry at a specific jsonpath with `conversion_fn` + """ + jp = jsonpath_ng.parse(jsonpath) + old_value = jp.find(params)[0].value + if not old_value: + return + new_value = conversion_fn(old_value) + jp.update(params, new_value) diff --git a/tests/unit/utils/test_collections.py b/tests/unit/utils/test_collections.py index adb2581e77460..6f07a8d3c3755 100644 --- a/tests/unit/utils/test_collections.py +++ b/tests/unit/utils/test_collections.py @@ -7,6 +7,7 @@ HashableList, ImmutableDict, ImmutableList, + convert_in_place_at_jsonpath, convert_to_typed_dict, is_comma_delimited_list, select_from_typed_dict, @@ -193,3 +194,21 @@ def test_is_comma_limited_list(): assert not is_comma_delimited_list("foo, bar baz") assert not is_comma_delimited_list("foo,") assert not is_comma_delimited_list("") + + +@pytest.mark.parametrize( + "input,jsonpath,fn,expected", + [ + # examples taken from ECS + ({"desiredCount": "1"}, "desiredCount", int, {"desiredCount": 1}), + ( + {"serviceConnectConfiguration": {"services": [{"clientAliases": [{"port": "80"}]}]}}, + "serviceConnectConfiguration.services[*].clientAliases[*].port", + int, + {"serviceConnectConfiguration": {"services": [{"clientAliases": [{"port": 80}]}]}}, + ), + ], +) +def test_convert_in_place_at_jsonpath(input, jsonpath, fn, expected): + convert_in_place_at_jsonpath(input, jsonpath, fn) + assert input == expected