From 38f6557dffa70cd62fc80b95b198e16a1b015959 Mon Sep 17 00:00:00 2001 From: Renato Valenzuela Date: Fri, 3 Oct 2025 19:25:20 +0000 Subject: [PATCH 1/4] test: disable MSK events in integration tests MSK tests are created more resources when they start polling (and the Topic we're connecting from doesn't even exist), so let's create the events with `Enabled: false`, so the polling doesn't actually start. --- .../combination/test_function_with_msk.py | 28 +++++++++---------- .../combination/function_with_msk.yaml | 1 + ...trigger_and_confluent_schema_registry.yaml | 1 + ..._and_s3_onfailure_events_destinations.yaml | 7 +++++ ...unction_with_msk_using_managed_policy.yaml | 1 + .../schema_source/aws_serverless_function.py | 1 + samtranslator/schema/schema.json | 3 ++ schema_source/sam.schema.json | 3 ++ 8 files changed, 31 insertions(+), 14 deletions(-) diff --git a/integration/combination/test_function_with_msk.py b/integration/combination/test_function_with_msk.py index 472a128f4e..98709ac8b8 100644 --- a/integration/combination/test_function_with_msk.py +++ b/integration/combination/test_function_with_msk.py @@ -25,12 +25,12 @@ def test_function_with_msk_trigger(self): parameters.append(self.generate_parameter("MskClusterName", cluster_name)) self._common_validations_for_MSK("combination/function_with_msk", parameters) - def test_function_with_msk_trigger_using_manage_policy(self): - companion_stack_outputs = self.companion_stack_outputs - parameters = self.get_parameters(companion_stack_outputs) - cluster_name = "MskCluster2-" + generate_suffix() - parameters.append(self.generate_parameter("MskClusterName2", cluster_name)) - self._common_validations_for_MSK("combination/function_with_msk_using_managed_policy", parameters) + # def test_function_with_msk_trigger_using_manage_policy(self): + # companion_stack_outputs = self.companion_stack_outputs + # parameters = self.get_parameters(companion_stack_outputs) + # cluster_name = "MskCluster2-" + generate_suffix() + # parameters.append(self.generate_parameter("MskClusterName2", cluster_name)) + # self._common_validations_for_MSK("combination/function_with_msk_using_managed_policy", parameters) def test_function_with_msk_trigger_and_s3_onfailure_events_destinations(self): companion_stack_outputs = self.companion_stack_outputs @@ -41,14 +41,14 @@ def test_function_with_msk_trigger_and_s3_onfailure_events_destinations(self): "combination/function_with_msk_trigger_and_s3_onfailure_events_destinations", parameters ) - def test_function_with_msk_trigger_and_confluent_schema_registry(self): - companion_stack_outputs = self.companion_stack_outputs - parameters = self.get_parameters(companion_stack_outputs) - cluster_name = "MskCluster4-" + generate_suffix() - parameters.append(self.generate_parameter("MskClusterName4", cluster_name)) - self._common_validations_for_MSK( - "combination/function_with_msk_trigger_and_confluent_schema_registry", parameters - ) + # def test_function_with_msk_trigger_and_confluent_schema_registry(self): + # companion_stack_outputs = self.companion_stack_outputs + # parameters = self.get_parameters(companion_stack_outputs) + # cluster_name = "MskCluster4-" + generate_suffix() + # parameters.append(self.generate_parameter("MskClusterName4", cluster_name)) + # self._common_validations_for_MSK( + # "combination/function_with_msk_trigger_and_confluent_schema_registry", parameters + # ) def _common_validations_for_MSK(self, file_name, parameters): self.create_and_verify_stack(file_name, parameters) diff --git a/integration/resources/templates/combination/function_with_msk.yaml b/integration/resources/templates/combination/function_with_msk.yaml index 6b72c36e16..91f3a0992d 100644 --- a/integration/resources/templates/combination/function_with_msk.yaml +++ b/integration/resources/templates/combination/function_with_msk.yaml @@ -60,6 +60,7 @@ Resources: MyMskEvent: Type: MSK Properties: + Enabled: false StartingPosition: LATEST Stream: Ref: MyMskCluster diff --git a/integration/resources/templates/combination/function_with_msk_trigger_and_confluent_schema_registry.yaml b/integration/resources/templates/combination/function_with_msk_trigger_and_confluent_schema_registry.yaml index dedc27203e..44fad6ec90 100644 --- a/integration/resources/templates/combination/function_with_msk_trigger_and_confluent_schema_registry.yaml +++ b/integration/resources/templates/combination/function_with_msk_trigger_and_confluent_schema_registry.yaml @@ -60,6 +60,7 @@ Resources: MyMskEvent: Type: MSK Properties: + Enabled: false StartingPosition: LATEST Stream: Ref: MyMskCluster diff --git a/integration/resources/templates/combination/function_with_msk_trigger_and_s3_onfailure_events_destinations.yaml b/integration/resources/templates/combination/function_with_msk_trigger_and_s3_onfailure_events_destinations.yaml index c449edbc4f..c6e595263e 100644 --- a/integration/resources/templates/combination/function_with_msk_trigger_and_s3_onfailure_events_destinations.yaml +++ b/integration/resources/templates/combination/function_with_msk_trigger_and_s3_onfailure_events_destinations.yaml @@ -27,6 +27,12 @@ Resources: logs:CreateLogStream, logs:PutLogEvents, s3:ListBucket] Effect: Allow Resource: '*' + - Action: s3:PutObject + Effect: Allow + Resource: + Fn::GetAtt: + - PreCreatedS3Bucket + - Arn ManagedPolicyArns: - !Sub arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole Tags: @@ -60,6 +66,7 @@ Resources: MyMskEvent: Type: MSK Properties: + Enabled: false StartingPosition: LATEST Stream: Ref: MyMskCluster diff --git a/integration/resources/templates/combination/function_with_msk_using_managed_policy.yaml b/integration/resources/templates/combination/function_with_msk_using_managed_policy.yaml index 05b0b3e54d..4c2bcf7e92 100644 --- a/integration/resources/templates/combination/function_with_msk_using_managed_policy.yaml +++ b/integration/resources/templates/combination/function_with_msk_using_managed_policy.yaml @@ -33,6 +33,7 @@ Resources: MyMskEvent: Type: MSK Properties: + Enabled: false StartingPosition: LATEST Stream: Ref: MyMskCluster diff --git a/samtranslator/internal/schema_source/aws_serverless_function.py b/samtranslator/internal/schema_source/aws_serverless_function.py index 260d2eae7b..bd60e740bc 100644 --- a/samtranslator/internal/schema_source/aws_serverless_function.py +++ b/samtranslator/internal/schema_source/aws_serverless_function.py @@ -411,6 +411,7 @@ class HttpApiEvent(BaseModel): class MSKEventProperties(BaseModel): ConsumerGroupId: Optional[PassThroughProp] = mskeventproperties("ConsumerGroupId") + Enabled: Optional[PassThroughProp] # TODO: it doesn't show up in docs yet FilterCriteria: Optional[PassThroughProp] = mskeventproperties("FilterCriteria") KmsKeyArn: Optional[PassThroughProp] # TODO: add documentation MaximumBatchingWindowInSeconds: Optional[PassThroughProp] = mskeventproperties("MaximumBatchingWindowInSeconds") diff --git a/samtranslator/schema/schema.json b/samtranslator/schema/schema.json index b6541cf061..d879a60730 100644 --- a/samtranslator/schema/schema.json +++ b/samtranslator/schema/schema.json @@ -275792,6 +275792,9 @@ "DestinationConfig": { "$ref": "#/definitions/PassThroughProp" }, + "Enabled": { + "$ref": "#/definitions/PassThroughProp" + }, "FilterCriteria": { "allOf": [ { diff --git a/schema_source/sam.schema.json b/schema_source/sam.schema.json index 169a78c7f1..f1d47e9aaf 100644 --- a/schema_source/sam.schema.json +++ b/schema_source/sam.schema.json @@ -2397,6 +2397,9 @@ "DestinationConfig": { "$ref": "#/definitions/PassThroughProp" }, + "Enabled": { + "$ref": "#/definitions/PassThroughProp" + }, "FilterCriteria": { "allOf": [ { From a7d26f709a235ed86e683ff3ee8a95aaa07db659 Mon Sep 17 00:00:00 2001 From: Renato Valenzuela Date: Sat, 4 Oct 2025 01:41:53 +0000 Subject: [PATCH 2/4] uncomment tests --- .../combination/test_function_with_msk.py | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/integration/combination/test_function_with_msk.py b/integration/combination/test_function_with_msk.py index 98709ac8b8..472a128f4e 100644 --- a/integration/combination/test_function_with_msk.py +++ b/integration/combination/test_function_with_msk.py @@ -25,12 +25,12 @@ def test_function_with_msk_trigger(self): parameters.append(self.generate_parameter("MskClusterName", cluster_name)) self._common_validations_for_MSK("combination/function_with_msk", parameters) - # def test_function_with_msk_trigger_using_manage_policy(self): - # companion_stack_outputs = self.companion_stack_outputs - # parameters = self.get_parameters(companion_stack_outputs) - # cluster_name = "MskCluster2-" + generate_suffix() - # parameters.append(self.generate_parameter("MskClusterName2", cluster_name)) - # self._common_validations_for_MSK("combination/function_with_msk_using_managed_policy", parameters) + def test_function_with_msk_trigger_using_manage_policy(self): + companion_stack_outputs = self.companion_stack_outputs + parameters = self.get_parameters(companion_stack_outputs) + cluster_name = "MskCluster2-" + generate_suffix() + parameters.append(self.generate_parameter("MskClusterName2", cluster_name)) + self._common_validations_for_MSK("combination/function_with_msk_using_managed_policy", parameters) def test_function_with_msk_trigger_and_s3_onfailure_events_destinations(self): companion_stack_outputs = self.companion_stack_outputs @@ -41,14 +41,14 @@ def test_function_with_msk_trigger_and_s3_onfailure_events_destinations(self): "combination/function_with_msk_trigger_and_s3_onfailure_events_destinations", parameters ) - # def test_function_with_msk_trigger_and_confluent_schema_registry(self): - # companion_stack_outputs = self.companion_stack_outputs - # parameters = self.get_parameters(companion_stack_outputs) - # cluster_name = "MskCluster4-" + generate_suffix() - # parameters.append(self.generate_parameter("MskClusterName4", cluster_name)) - # self._common_validations_for_MSK( - # "combination/function_with_msk_trigger_and_confluent_schema_registry", parameters - # ) + def test_function_with_msk_trigger_and_confluent_schema_registry(self): + companion_stack_outputs = self.companion_stack_outputs + parameters = self.get_parameters(companion_stack_outputs) + cluster_name = "MskCluster4-" + generate_suffix() + parameters.append(self.generate_parameter("MskClusterName4", cluster_name)) + self._common_validations_for_MSK( + "combination/function_with_msk_trigger_and_confluent_schema_registry", parameters + ) def _common_validations_for_MSK(self, file_name, parameters): self.create_and_verify_stack(file_name, parameters) From 69a6d8743658a91285f9244a02a6bd80a143e23d Mon Sep 17 00:00:00 2001 From: Renato Valenzuela Date: Sat, 4 Oct 2025 02:17:03 +0000 Subject: [PATCH 3/4] fix s3 permissions --- ...h_msk_trigger_and_s3_onfailure_events_destinations.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/integration/resources/templates/combination/function_with_msk_trigger_and_s3_onfailure_events_destinations.yaml b/integration/resources/templates/combination/function_with_msk_trigger_and_s3_onfailure_events_destinations.yaml index c6e595263e..a36b8d36f7 100644 --- a/integration/resources/templates/combination/function_with_msk_trigger_and_s3_onfailure_events_destinations.yaml +++ b/integration/resources/templates/combination/function_with_msk_trigger_and_s3_onfailure_events_destinations.yaml @@ -27,12 +27,11 @@ Resources: logs:CreateLogStream, logs:PutLogEvents, s3:ListBucket] Effect: Allow Resource: '*' - - Action: s3:PutObject + - Action: [s3:PutObject, s3:ListBucket] Effect: Allow Resource: - Fn::GetAtt: - - PreCreatedS3Bucket - - Arn + - "arn:aws:s3:::*/*" + - "arn:aws:s3:::*" ManagedPolicyArns: - !Sub arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole Tags: From 9237b494970db11936c963a357d2e9ae47e8b91f Mon Sep 17 00:00:00 2001 From: Renato Valenzuela Date: Mon, 6 Oct 2025 19:35:19 +0000 Subject: [PATCH 4/4] fix yaml extra indentation --- ...with_msk_trigger_and_s3_onfailure_events_destinations.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration/resources/templates/combination/function_with_msk_trigger_and_s3_onfailure_events_destinations.yaml b/integration/resources/templates/combination/function_with_msk_trigger_and_s3_onfailure_events_destinations.yaml index a36b8d36f7..6e79110756 100644 --- a/integration/resources/templates/combination/function_with_msk_trigger_and_s3_onfailure_events_destinations.yaml +++ b/integration/resources/templates/combination/function_with_msk_trigger_and_s3_onfailure_events_destinations.yaml @@ -30,8 +30,8 @@ Resources: - Action: [s3:PutObject, s3:ListBucket] Effect: Allow Resource: - - "arn:aws:s3:::*/*" - - "arn:aws:s3:::*" + - arn:aws:s3:::*/* + - arn:aws:s3:::* ManagedPolicyArns: - !Sub arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole Tags: