diff --git a/localstack-core/localstack/services/lambda_/event_source_mapping/esm_config_factory.py b/localstack-core/localstack/services/lambda_/event_source_mapping/esm_config_factory.py index d67902721a9a1..e5d720bbafee1 100644 --- a/localstack-core/localstack/services/lambda_/event_source_mapping/esm_config_factory.py +++ b/localstack-core/localstack/services/lambda_/event_source_mapping/esm_config_factory.py @@ -73,9 +73,12 @@ def get_esm_config(self) -> EventSourceMappingConfiguration: # c) Are FilterCriteria.Filters merged or replaced upon update? # TODO: can we ignore extra parameters from the request (e.g., Kinesis params for SQS source)? derived_source_parameters = merge_recursive(default_source_parameters, self.request) - derived_source_parameters["FunctionResponseTypes"] = derived_source_parameters.get( - "FunctionResponseTypes", [] - ) + + # TODO What happens when FunctionResponseTypes value or target service is invalid? + if service in ["sqs", "kinesis", "dynamodbstreams"]: + derived_source_parameters["FunctionResponseTypes"] = derived_source_parameters.get( + "FunctionResponseTypes", [] + ) state = EsmState.CREATING if self.request.get("Enabled", True) else EsmState.DISABLED esm_config = EventSourceMappingConfiguration( diff --git a/localstack-core/localstack/services/lambda_/event_source_mapping/esm_event_processor.py b/localstack-core/localstack/services/lambda_/event_source_mapping/esm_event_processor.py index d02b0a6336a02..fc860ee74abd5 100644 --- a/localstack-core/localstack/services/lambda_/event_source_mapping/esm_event_processor.py +++ b/localstack-core/localstack/services/lambda_/event_source_mapping/esm_event_processor.py @@ -58,7 +58,9 @@ def process_events_batch(self, input_events: list[dict]) -> None: # TODO: check whether partial batch item failures is enabled by default or need to be explicitly enabled # using --function-response-types "ReportBatchItemFailures" # https://docs.aws.amazon.com/lambda/latest/dg/services-sqs-errorhandling.html - raise PartialBatchFailureError from e + raise PartialBatchFailureError( + partial_failure_payload=e.partial_failure_payload, error=e.error + ) from e except SenderError as e: self.logger.log( messageType="ExecutionFailed", diff --git a/localstack-core/localstack/services/lambda_/event_source_mapping/esm_worker_factory.py b/localstack-core/localstack/services/lambda_/event_source_mapping/esm_worker_factory.py index 739d3e200c0f3..81919357c4f37 100644 --- a/localstack-core/localstack/services/lambda_/event_source_mapping/esm_worker_factory.py +++ b/localstack-core/localstack/services/lambda_/event_source_mapping/esm_worker_factory.py @@ -1,5 +1,6 @@ from localstack.aws.api.lambda_ import ( EventSourceMappingConfiguration, + FunctionResponseType, ) from localstack.aws.api.pipes import ( DynamoDBStreamStartPosition, @@ -59,6 +60,8 @@ def get_esm_worker(self) -> EsmWorker: ), target_client=lambda_client, payload_dict=True, + report_batch_item_failures=self.esm_config.get("FunctionResponseTypes") + == [FunctionResponseType.ReportBatchItemFailures], ) # Logger diff --git a/localstack-core/localstack/services/lambda_/event_source_mapping/event_processor.py b/localstack-core/localstack/services/lambda_/event_source_mapping/event_processor.py index 8a014d7d2a676..cccd02e843aec 100644 --- a/localstack-core/localstack/services/lambda_/event_source_mapping/event_processor.py +++ b/localstack-core/localstack/services/lambda_/event_source_mapping/event_processor.py @@ -46,7 +46,9 @@ class PartialBatchFailureError(EventProcessorError): def __init__( self, partial_failure_payload: PartialFailurePayload | None = None, + error=None, ) -> None: + self.error = error self.partial_failure_payload = partial_failure_payload diff --git a/localstack-core/localstack/services/lambda_/event_source_mapping/pollers/poller.py b/localstack-core/localstack/services/lambda_/event_source_mapping/pollers/poller.py index 8bd52c3750d1a..71a50dfab98bb 100644 --- a/localstack-core/localstack/services/lambda_/event_source_mapping/pollers/poller.py +++ b/localstack-core/localstack/services/lambda_/event_source_mapping/pollers/poller.py @@ -121,10 +121,25 @@ def has_batch_item_failures( try: failed_items_ids = parse_batch_item_failures(result, valid_item_ids) return len(failed_items_ids) > 0 - except KeyError: + except (KeyError, ValueError): return True +def get_batch_item_failures( + result: dict | str | None, valid_item_ids: set[str] | None = None +) -> list[str] | None: + """ + Returns a list of failed batch item IDs. If an empty list is returned, then the batch should be considered as a complete success. + + If `None` is returned, the batch should be considered a complete failure. + """ + try: + failed_items_ids = parse_batch_item_failures(result, valid_item_ids) + return failed_items_ids + except (KeyError, ValueError): + return None + + def parse_batch_item_failures( result: dict | str | None, valid_item_ids: set[str] | None = None ) -> list[str]: @@ -178,6 +193,8 @@ def parse_batch_item_failures( raise KeyError(f"missing itemIdentifier in batchItemFailure record {item}") item_identifier = item["itemIdentifier"] + if not item_identifier: + raise ValueError("itemIdentifier cannot be empty or null") # Optionally validate whether the item_identifier is part of the batch if valid_item_ids and item_identifier not in valid_item_ids: diff --git a/localstack-core/localstack/services/lambda_/event_source_mapping/pollers/stream_poller.py b/localstack-core/localstack/services/lambda_/event_source_mapping/pollers/stream_poller.py index 2bbbc0fb8b92f..fc240b29a8274 100644 --- a/localstack-core/localstack/services/lambda_/event_source_mapping/pollers/stream_poller.py +++ b/localstack-core/localstack/services/lambda_/event_source_mapping/pollers/stream_poller.py @@ -22,12 +22,11 @@ get_datetime_from_timestamp, get_internal_client, ) -from localstack.services.lambda_.event_source_mapping.pollers.poller import Poller -from localstack.services.lambda_.event_source_mapping.pollers.sqs_poller import get_queue_url -from localstack.services.lambda_.event_source_mapping.senders.sender import ( - PartialFailureSenderError, - SenderError, +from localstack.services.lambda_.event_source_mapping.pollers.poller import ( + Poller, + get_batch_item_failures, ) +from localstack.services.lambda_.event_source_mapping.pollers.sqs_poller import get_queue_url from localstack.utils.aws.arns import parse_arn LOG = logging.getLogger(__name__) @@ -192,7 +191,7 @@ def poll_events_from_shard(self, shard_id: str, shard_iterator: str): except PartialBatchFailureError as ex: # TODO: add tests for partial batch failure scenarios if ( - self.stream_parameters["OnPartialBatchItemFailure"] + self.stream_parameters.get("OnPartialBatchItemFailure") == OnPartialBatchItemFailureStreams.AUTOMATIC_BISECT ): # TODO: implement and test splitting batches in half until batch size 1 @@ -200,10 +199,32 @@ def poll_events_from_shard(self, shard_id: str, shard_iterator: str): LOG.warning( "AUTOMATIC_BISECT upon partial batch item failure is not yet implemented. Retrying the entire batch." ) - error_payload = ex.partial_failure_payload - # let entire batch fail (ideally raise BatchFailureError) - except (SenderError, PartialFailureSenderError, BatchFailureError, Exception) as ex: - if isinstance(ex, (SenderError, PartialFailureSenderError, BatchFailureError)): + error_payload = ex.error + + # If the batchItemFailures array contains multiple items, Lambda uses the record with the lowest sequence number as the checkpoint. + # Lambda then retries all records starting from that checkpoint. + + failed_sequence_ids: list[int] | None = get_batch_item_failures( + ex.partial_failure_payload + ) + + # If None is returned, consider the entire batch a failure. + if failed_sequence_ids is None: + continue + + # This shouldn't be possible since a PartialBatchFailureError was raised + if len(failed_sequence_ids) == 0: + LOG.warning( + "Invalid state encountered: PartialBatchFailureError raised but no batch item failures found." + ) + return + + lowest_sequence_id: str = min(failed_sequence_ids, key=int) + + # Discard all successful events and re-process from sequence number of failed event + _, events = self.bisect_events(lowest_sequence_id, events) + except (BatchFailureError, Exception) as ex: + if isinstance(ex, BatchFailureError): error_payload = ex.error # FIXME partner_resource_arn is not defined in ESM @@ -213,11 +234,9 @@ def poll_events_from_shard(self, shard_id: str, shard_iterator: str): self.partner_resource_arn or self.source_arn, events, ) - attempts += 1 + finally: # Retry polling until the record expires at the source - if self.stream_parameters.get("MaximumRetryAttempts", -1) == -1: - # TODO: handle iterator expired scenario - return + attempts += 1 # Send failed events to potential DLQ abort_condition = abort_condition or "RetryAttemptsExhausted" @@ -324,3 +343,12 @@ def max_retries_exceeded(self, attempts: int) -> bool: if maximum_retry_attempts == -1: return False return attempts > maximum_retry_attempts + + def bisect_events( + self, sequence_number: str, events: list[dict] + ) -> tuple[list[dict], list[dict]]: + for i, event in enumerate(events): + if self.get_sequence_number(event) == sequence_number: + return events[:i], events[i:] + + return events, [] diff --git a/localstack-core/localstack/services/lambda_/event_source_mapping/senders/lambda_sender.py b/localstack-core/localstack/services/lambda_/event_source_mapping/senders/lambda_sender.py index c5cfb4a233b0d..2287659c4db72 100644 --- a/localstack-core/localstack/services/lambda_/event_source_mapping/senders/lambda_sender.py +++ b/localstack-core/localstack/services/lambda_/event_source_mapping/senders/lambda_sender.py @@ -19,9 +19,20 @@ class LambdaSender(Sender): # Flag to enable the payload dict using the "Records" key used for Lambda event source mapping payload_dict: bool - def __init__(self, target_arn, target_parameters=None, target_client=None, payload_dict=False): + # Flag to enable partial successes/failures when processing batched events through a Lambda event source mapping + report_batch_item_failures: bool + + def __init__( + self, + target_arn, + target_parameters=None, + target_client=None, + payload_dict=False, + report_batch_item_failures=False, + ): super().__init__(target_arn, target_parameters, target_client) self.payload_dict = payload_dict + self.report_batch_item_failures = report_batch_item_failures def send_events(self, events: list[dict]) -> dict: if self.payload_dict: @@ -72,11 +83,9 @@ def send_events(self, events: list[dict]) -> dict: error=error, ) - # TODO: test all success, partial, and failure conditions: - # https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-pipes-dynamodb.html#pipes-ddb-batch-failures # The payload can contain the key "batchItemFailures" with a list of partial batch failures: # https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-pipes-batching-concurrency.html - if has_batch_item_failures(payload): + if self.report_batch_item_failures and has_batch_item_failures(payload): error = { "message": "Target invocation failed partially.", "httpStatusCode": invoke_result["StatusCode"], @@ -84,6 +93,7 @@ def send_events(self, events: list[dict]) -> dict: "requestId": invoke_result["ResponseMetadata"]["RequestId"], "exceptionType": "BadRequest", "resourceArn": self.target_arn, + "executedVersion": invoke_result.get("ExecutedVersion", "$LATEST"), } raise PartialFailureSenderError(error=error, partial_failure_payload=payload) diff --git a/localstack-core/localstack/services/lambda_/event_source_mapping/senders/sender.py b/localstack-core/localstack/services/lambda_/event_source_mapping/senders/sender.py index f5a1e30102e4e..78f656c0e2521 100644 --- a/localstack-core/localstack/services/lambda_/event_source_mapping/senders/sender.py +++ b/localstack-core/localstack/services/lambda_/event_source_mapping/senders/sender.py @@ -7,7 +7,7 @@ class SenderError(Exception): def __init__(self, message=None, error=None) -> None: - self.message = message or "Error during during sending events" + self.message = message or "Error during sending events" self.error = error diff --git a/tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_dynamodbstreams.py b/tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_dynamodbstreams.py index c632a2eac2c45..92770a8bb54c1 100644 --- a/tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_dynamodbstreams.py +++ b/tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_dynamodbstreams.py @@ -19,7 +19,12 @@ from localstack.utils.strings import short_uid from localstack.utils.sync import retry from localstack.utils.testutil import check_expected_lambda_log_events_length, get_lambda_log_events -from tests.aws.services.lambda_.event_source_mapping.utils import is_old_esm, is_v2_esm +from tests.aws.services.lambda_.event_source_mapping.utils import ( + LAMBDA_DYNAMODB_BATCH_ITEM_FAILURE, + create_lambda_with_response, + is_old_esm, + is_v2_esm, +) from tests.aws.services.lambda_.test_lambda import ( TEST_LAMBDA_PYTHON_ECHO, TEST_LAMBDA_PYTHON_UNHANDLED_ERROR, @@ -689,3 +694,308 @@ def test_dynamodb_invalid_event_filter( aws_client.lambda_.create_event_source_mapping(**event_source_mapping_kwargs) snapshot.match("exception_event_source_creation", expected.value.response) expected.match(InvalidParameterValueException.code) + + @pytest.mark.skipif(is_old_esm(), reason="ReportBatchItemFailures: Partial ") + @markers.snapshot.skip_snapshot_verify( + paths=[ + "$..TableDescription.TableId", + "$..Records", # TODO Figure out why there is an extra log record + ], + ) + @markers.aws.validated + def test_dynamodb_report_batch_item_failures( + self, + create_lambda_function, + sqs_get_queue_arn, + sqs_create_queue, + create_iam_role_with_policy, + dynamodb_create_table, + snapshot, + cleanups, + aws_client, + ): + snapshot.add_transformer(snapshot.transform.key_value("MD5OfBody")) + snapshot.add_transformer(snapshot.transform.key_value("ReceiptHandle")) + snapshot.add_transformer(snapshot.transform.key_value("startSequenceNumber")) + + function_name = f"lambda_func-{short_uid()}" + role = f"test-lambda-role-{short_uid()}" + policy_name = f"test-lambda-policy-{short_uid()}" + table_name = f"test-table-{short_uid()}" + partition_key = "my_partition_key" + + # Used in ESM config and assertions + expected_successes = 5 + expected_failures = 1 + + role_arn = create_iam_role_with_policy( + RoleName=role, + PolicyName=policy_name, + RoleDefinition=lambda_role, + PolicyDefinition=s3_lambda_permission, + ) + + create_lambda_function( + handler_file=LAMBDA_DYNAMODB_BATCH_ITEM_FAILURE, + func_name=function_name, + runtime=Runtime.python3_12, + role=role_arn, + ) + create_table_response = dynamodb_create_table( + table_name=table_name, partition_key=partition_key + ) + _await_dynamodb_table_active(aws_client.dynamodb, table_name) + snapshot.match("create_table_response", create_table_response) + + update_table_response = aws_client.dynamodb.update_table( + TableName=table_name, + StreamSpecification={"StreamEnabled": True, "StreamViewType": "NEW_IMAGE"}, + ) + snapshot.match("update_table_response", update_table_response) + stream_arn = update_table_response["TableDescription"]["LatestStreamArn"] + + destination_queue = sqs_create_queue() + queue_failure_event_source_mapping_arn = sqs_get_queue_arn(destination_queue) + destination_config = {"OnFailure": {"Destination": queue_failure_event_source_mapping_arn}} + + create_event_source_mapping_response = aws_client.lambda_.create_event_source_mapping( + FunctionName=function_name, + BatchSize=3, + StartingPosition="TRIM_HORIZON", + EventSourceArn=stream_arn, + MaximumBatchingWindowInSeconds=1, + MaximumRetryAttempts=3, + DestinationConfig=destination_config, + FunctionResponseTypes=["ReportBatchItemFailures"], + ) + + snapshot.match("create_event_source_mapping_response", create_event_source_mapping_response) + event_source_uuid = create_event_source_mapping_response["UUID"] + cleanups.append( + lambda: aws_client.lambda_.delete_event_source_mapping(UUID=event_source_uuid) + ) + _await_event_source_mapping_enabled(aws_client.lambda_, event_source_uuid) + + dynamodb_items = [ + {partition_key: {"S": f"testId{i}"}, "should_fail": {"BOOL": i == 5}} + for i in range(expected_successes + expected_failures) + ] + + # TODO Batching behaviour is flakey since DynamoDB streams are unordered. Look into some patterns for ordering. + for db_item in dynamodb_items: + aws_client.dynamodb.put_item(TableName=table_name, Item=db_item) + time.sleep(0.1) + + def verify_failure_received(): + res = aws_client.sqs.receive_message(QueueUrl=destination_queue) + assert res.get("Messages") + return res + + # It can take ~3 min against AWS until the message is received + sleep = 15 if is_aws_cloud() else 5 + messages = retry(verify_failure_received, retries=15, sleep=sleep, sleep_before=5) + snapshot.match("destination_queue_messages", messages) + + batched_records = get_lambda_log_events(function_name, logs_client=aws_client.logs) + flattened_records = [ + record for batch in batched_records for record in batch.get("Records", []) + ] + + # Although DynamoDB streams doesn't guarantee such ordering, this test is more concerned with whether + # the failed items were repeated. + sorted_records = sorted( + flattened_records, key=lambda item: item["dynamodb"]["Keys"][partition_key]["S"] + ) + + snapshot.match("dynamodb_records", {"Records": sorted_records}) + + @pytest.mark.skipif( + is_old_esm(), reason="ReportBatchItemFailures: Total batch fails not implemented in ESM v1" + ) + @pytest.mark.parametrize( + "set_lambda_response", + [ + # Failures + {"batchItemFailures": [{"itemIdentifier": ""}]}, + {"batchItemFailures": [{"itemIdentifier": None}]}, + {"batchItemFailures": [{"foo": 123}]}, + {"batchItemFailures": [{"foo": None}]}, + # Unhandled Exceptions + "(lambda: 1 / 0)()", # This will (lazily) evaluate, raise an exception, and re-trigger the whole batch + ], + ids=[ + # Failures + "empty_string_item_identifier_failure", + "null_item_identifier_failure", + "invalid_key_foo_failure", + "invalid_key_foo_null_value_failure", + # Unhandled Exceptions + "unhandled_exception_in_function", + ], + ) + @markers.aws.validated + def test_dynamodb_report_batch_item_failure_scenarios( + self, + create_lambda_function, + dynamodb_create_table, + cleanups, + wait_for_dynamodb_stream_ready, + sqs_get_queue_arn, + sqs_create_queue, + snapshot, + aws_client, + set_lambda_response, + lambda_su_role, + ): + snapshot.add_transformer(snapshot.transform.key_value("MD5OfBody")) + snapshot.add_transformer(snapshot.transform.key_value("ReceiptHandle")) + snapshot.add_transformer(snapshot.transform.key_value("startSequenceNumber")) + + function_name = f"lambda_func-{short_uid()}" + table_name = f"test-table-{short_uid()}" + partition_key = "my_partition_key" + db_item = {partition_key: {"S": "hello world"}, "binary_key": {"B": b"foobar"}} + + create_lambda_function( + handler_file=create_lambda_with_response(set_lambda_response), + func_name=function_name, + runtime=Runtime.python3_12, + role=lambda_su_role, + ) + + create_table_result = dynamodb_create_table( + table_name=table_name, partition_key=partition_key + ) + # snapshot create table to get the table name registered as resource + snapshot.match("create-table-result", create_table_result) + _await_dynamodb_table_active(aws_client.dynamodb, table_name) + stream_arn = aws_client.dynamodb.update_table( + TableName=table_name, + StreamSpecification={"StreamEnabled": True, "StreamViewType": "NEW_IMAGE"}, + )["TableDescription"]["LatestStreamArn"] + assert wait_for_dynamodb_stream_ready(stream_arn) + + destination_queue = sqs_create_queue() + queue_failure_event_source_mapping_arn = sqs_get_queue_arn(destination_queue) + destination_config = {"OnFailure": {"Destination": queue_failure_event_source_mapping_arn}} + + create_event_source_mapping_response = aws_client.lambda_.create_event_source_mapping( + FunctionName=function_name, + BatchSize=3, + StartingPosition="TRIM_HORIZON", + EventSourceArn=stream_arn, + MaximumBatchingWindowInSeconds=1, + MaximumRetryAttempts=3, + DestinationConfig=destination_config, + FunctionResponseTypes=["ReportBatchItemFailures"], + ) + + event_source_uuid = create_event_source_mapping_response["UUID"] + cleanups.append( + lambda: aws_client.lambda_.delete_event_source_mapping(UUID=event_source_uuid) + ) + + _await_event_source_mapping_enabled(aws_client.lambda_, event_source_uuid) + aws_client.dynamodb.put_item(TableName=table_name, Item=db_item) + + def verify_failure_received(): + res = aws_client.sqs.receive_message(QueueUrl=destination_queue) + assert res.get("Messages") + return res + + # It can take ~3 min against AWS until the message is received + sleep = 15 if is_aws_cloud() else 5 + messages = retry(verify_failure_received, retries=15, sleep=sleep, sleep_before=5) + snapshot.match("destination_queue_messages", messages) + + events = get_lambda_log_events(function_name, logs_client=aws_client.logs) + + # This will filter out exception messages being added to the log stream + invocation_events = [event for event in events if "Records" in event] + snapshot.match("dynamodb_events", invocation_events) + + @markers.aws.validated + @pytest.mark.parametrize( + "set_lambda_response", + [ + # Successes + [], + None, + {}, + {"batchItemFailures": []}, + {"batchItemFailures": None}, + ], + ids=[ + # Successes + "empty_list_success", + "null_success", + "empty_dict_success", + "empty_batch_item_failure_success", + "null_batch_item_failure_success", + ], + ) + def test_dynamodb_report_batch_item_success_scenarios( + self, + create_lambda_function, + dynamodb_create_table, + cleanups, + wait_for_dynamodb_stream_ready, + snapshot, + aws_client, + set_lambda_response, + lambda_su_role, + ): + function_name = f"lambda_func-{short_uid()}" + table_name = f"test-table-{short_uid()}" + partition_key = "my_partition_key" + db_item = {partition_key: {"S": "hello world"}, "binary_key": {"B": b"foobar"}} + + create_lambda_function( + handler_file=create_lambda_with_response(set_lambda_response), + func_name=function_name, + runtime=Runtime.python3_12, + role=lambda_su_role, + ) + + create_table_result = dynamodb_create_table( + table_name=table_name, partition_key=partition_key + ) + # snapshot create table to get the table name registered as resource + snapshot.match("create-table-result", create_table_result) + _await_dynamodb_table_active(aws_client.dynamodb, table_name) + stream_arn = aws_client.dynamodb.update_table( + TableName=table_name, + StreamSpecification={"StreamEnabled": True, "StreamViewType": "NEW_IMAGE"}, + )["TableDescription"]["LatestStreamArn"] + assert wait_for_dynamodb_stream_ready(stream_arn) + + retry_attempts = 2 + create_event_source_mapping_response = aws_client.lambda_.create_event_source_mapping( + EventSourceArn=stream_arn, + FunctionName=function_name, + StartingPosition="TRIM_HORIZON", + BatchSize=1, + MaximumBatchingWindowInSeconds=0, + FunctionResponseTypes=["ReportBatchItemFailures"], + MaximumRetryAttempts=retry_attempts, + ) + + event_source_uuid = create_event_source_mapping_response["UUID"] + cleanups.append( + lambda: aws_client.lambda_.delete_event_source_mapping(UUID=event_source_uuid) + ) + + _await_event_source_mapping_enabled(aws_client.lambda_, event_source_uuid) + aws_client.dynamodb.put_item(TableName=table_name, Item=db_item) + + def _verify_messages_received(): + events = get_lambda_log_events(function_name, logs_client=aws_client.logs) + + # This will filter out exception messages being added to the log stream + record_events = [event for event in events if "Records" in event] + + assert len(record_events) >= 1 + return record_events + + invocation_events = retry(_verify_messages_received, retries=30, sleep=5) + snapshot.match("dynamodb_events", invocation_events) diff --git a/tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_dynamodbstreams.snapshot.json b/tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_dynamodbstreams.snapshot.json index a7d444e1e83e9..e009657dcaf4f 100644 --- a/tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_dynamodbstreams.snapshot.json +++ b/tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_dynamodbstreams.snapshot.json @@ -1959,5 +1959,1969 @@ } ] } + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_dynamodbstreams.py::TestDynamoDBEventSourceMapping::test_dynamodb_report_batch_item_failures": { + "recorded-date": "11-09-2024, 18:47:32", + "recorded-content": { + "create_table_response": { + "TableDescription": { + "AttributeDefinitions": [ + { + "AttributeName": "my_partition_key", + "AttributeType": "S" + } + ], + "BillingModeSummary": { + "BillingMode": "PAY_PER_REQUEST" + }, + "CreationDateTime": "", + "DeletionProtectionEnabled": false, + "ItemCount": 0, + "KeySchema": [ + { + "AttributeName": "my_partition_key", + "KeyType": "HASH" + } + ], + "ProvisionedThroughput": { + "NumberOfDecreasesToday": 0, + "ReadCapacityUnits": 0, + "WriteCapacityUnits": 0 + }, + "TableArn": "arn::dynamodb::111111111111:table/", + "TableId": "", + "TableName": "", + "TableSizeBytes": 0, + "TableStatus": "CREATING" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "update_table_response": { + "TableDescription": { + "AttributeDefinitions": [ + { + "AttributeName": "my_partition_key", + "AttributeType": "S" + } + ], + "BillingModeSummary": { + "BillingMode": "PAY_PER_REQUEST", + "LastUpdateToPayPerRequestDateTime": "" + }, + "CreationDateTime": "", + "DeletionProtectionEnabled": false, + "ItemCount": 0, + "KeySchema": [ + { + "AttributeName": "my_partition_key", + "KeyType": "HASH" + } + ], + "LatestStreamArn": "arn::dynamodb::111111111111:table//stream/", + "LatestStreamLabel": "", + "ProvisionedThroughput": { + "NumberOfDecreasesToday": 0, + "ReadCapacityUnits": 0, + "WriteCapacityUnits": 0 + }, + "StreamSpecification": { + "StreamEnabled": true, + "StreamViewType": "NEW_IMAGE" + }, + "TableArn": "arn::dynamodb::111111111111:table/", + "TableId": "", + "TableName": "", + "TableSizeBytes": 0, + "TableStatus": "UPDATING" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "create_event_source_mapping_response": { + "BatchSize": 3, + "BisectBatchOnFunctionError": false, + "DestinationConfig": { + "OnFailure": { + "Destination": "arn::sqs::111111111111:" + } + }, + "EventSourceArn": "arn::dynamodb::111111111111:table//stream/", + "FunctionArn": "arn::lambda::111111111111:function:", + "FunctionResponseTypes": [ + "ReportBatchItemFailures" + ], + "LastModified": "", + "LastProcessingResult": "No records processed", + "MaximumBatchingWindowInSeconds": 1, + "MaximumRecordAgeInSeconds": -1, + "MaximumRetryAttempts": 3, + "ParallelizationFactor": 1, + "StartingPosition": "TRIM_HORIZON", + "State": "Creating", + "StateTransitionReason": "User action", + "TumblingWindowInSeconds": 0, + "UUID": "", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 202 + } + }, + "destination_queue_messages": { + "Messages": [ + { + "Body": { + "requestContext": { + "requestId": "", + "functionArn": "arn::lambda::111111111111:function:", + "condition": "RetryAttemptsExhausted", + "approximateInvokeCount": 4 + }, + "responseContext": { + "statusCode": 200, + "executedVersion": "$LATEST", + "functionError": null + }, + "version": "1.0", + "timestamp": "", + "DDBStreamBatchInfo": { + "shardId": "", + "startSequenceNumber": "", + "endSequenceNumber": "", + "approximateArrivalOfFirstRecord": "", + "approximateArrivalOfLastRecord": "", + "batchSize": 1, + "streamArn": "arn::dynamodb::111111111111:table//stream/" + } + }, + "MD5OfBody": "", + "MessageId": "", + "ReceiptHandle": "" + } + ], + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "dynamodb_records": { + "Records": [ + { + "awsRegion": "", + "dynamodb": { + "ApproximateCreationDateTime": "", + "Keys": { + "my_partition_key": { + "S": "testId0" + } + }, + "NewImage": { + "my_partition_key": { + "S": "testId0" + }, + "should_fail": { + "BOOL": false + } + }, + "SequenceNumber": "", + "SizeBytes": 58, + "StreamViewType": "NEW_IMAGE" + }, + "eventID": "", + "eventName": "INSERT", + "eventSource": "aws:dynamodb", + "eventSourceARN": "arn::dynamodb::111111111111:table//stream/", + "eventVersion": "1.1" + }, + { + "awsRegion": "", + "dynamodb": { + "ApproximateCreationDateTime": "", + "Keys": { + "my_partition_key": { + "S": "testId1" + } + }, + "NewImage": { + "my_partition_key": { + "S": "testId1" + }, + "should_fail": { + "BOOL": false + } + }, + "SequenceNumber": "", + "SizeBytes": 58, + "StreamViewType": "NEW_IMAGE" + }, + "eventID": "", + "eventName": "INSERT", + "eventSource": "aws:dynamodb", + "eventSourceARN": "arn::dynamodb::111111111111:table//stream/", + "eventVersion": "1.1" + }, + { + "awsRegion": "", + "dynamodb": { + "ApproximateCreationDateTime": "", + "Keys": { + "my_partition_key": { + "S": "testId2" + } + }, + "NewImage": { + "my_partition_key": { + "S": "testId2" + }, + "should_fail": { + "BOOL": false + } + }, + "SequenceNumber": "", + "SizeBytes": 58, + "StreamViewType": "NEW_IMAGE" + }, + "eventID": "", + "eventName": "INSERT", + "eventSource": "aws:dynamodb", + "eventSourceARN": "arn::dynamodb::111111111111:table//stream/", + "eventVersion": "1.1" + }, + { + "awsRegion": "", + "dynamodb": { + "ApproximateCreationDateTime": "", + "Keys": { + "my_partition_key": { + "S": "testId3" + } + }, + "NewImage": { + "my_partition_key": { + "S": "testId3" + }, + "should_fail": { + "BOOL": false + } + }, + "SequenceNumber": "", + "SizeBytes": 58, + "StreamViewType": "NEW_IMAGE" + }, + "eventID": "", + "eventName": "INSERT", + "eventSource": "aws:dynamodb", + "eventSourceARN": "arn::dynamodb::111111111111:table//stream/", + "eventVersion": "1.1" + }, + { + "awsRegion": "", + "dynamodb": { + "ApproximateCreationDateTime": "", + "Keys": { + "my_partition_key": { + "S": "testId4" + } + }, + "NewImage": { + "my_partition_key": { + "S": "testId4" + }, + "should_fail": { + "BOOL": false + } + }, + "SequenceNumber": "", + "SizeBytes": 58, + "StreamViewType": "NEW_IMAGE" + }, + "eventID": "", + "eventName": "INSERT", + "eventSource": "aws:dynamodb", + "eventSourceARN": "arn::dynamodb::111111111111:table//stream/", + "eventVersion": "1.1" + }, + { + "awsRegion": "", + "dynamodb": { + "ApproximateCreationDateTime": "", + "Keys": { + "my_partition_key": { + "S": "testId5" + } + }, + "NewImage": { + "my_partition_key": { + "S": "testId5" + }, + "should_fail": { + "BOOL": true + } + }, + "SequenceNumber": "", + "SizeBytes": 58, + "StreamViewType": "NEW_IMAGE" + }, + "eventID": "", + "eventName": "INSERT", + "eventSource": "aws:dynamodb", + "eventSourceARN": "arn::dynamodb::111111111111:table//stream/", + "eventVersion": "1.1" + }, + { + "awsRegion": "", + "dynamodb": { + "ApproximateCreationDateTime": "", + "Keys": { + "my_partition_key": { + "S": "testId5" + } + }, + "NewImage": { + "my_partition_key": { + "S": "testId5" + }, + "should_fail": { + "BOOL": true + } + }, + "SequenceNumber": "", + "SizeBytes": 58, + "StreamViewType": "NEW_IMAGE" + }, + "eventID": "", + "eventName": "INSERT", + "eventSource": "aws:dynamodb", + "eventSourceARN": "arn::dynamodb::111111111111:table//stream/", + "eventVersion": "1.1" + }, + { + "awsRegion": "", + "dynamodb": { + "ApproximateCreationDateTime": "", + "Keys": { + "my_partition_key": { + "S": "testId5" + } + }, + "NewImage": { + "my_partition_key": { + "S": "testId5" + }, + "should_fail": { + "BOOL": true + } + }, + "SequenceNumber": "", + "SizeBytes": 58, + "StreamViewType": "NEW_IMAGE" + }, + "eventID": "", + "eventName": "INSERT", + "eventSource": "aws:dynamodb", + "eventSourceARN": "arn::dynamodb::111111111111:table//stream/", + "eventVersion": "1.1" + }, + { + "awsRegion": "", + "dynamodb": { + "ApproximateCreationDateTime": "", + "Keys": { + "my_partition_key": { + "S": "testId5" + } + }, + "NewImage": { + "my_partition_key": { + "S": "testId5" + }, + "should_fail": { + "BOOL": true + } + }, + "SequenceNumber": "", + "SizeBytes": 58, + "StreamViewType": "NEW_IMAGE" + }, + "eventID": "", + "eventName": "INSERT", + "eventSource": "aws:dynamodb", + "eventSourceARN": "arn::dynamodb::111111111111:table//stream/", + "eventVersion": "1.1" + }, + { + "awsRegion": "", + "dynamodb": { + "ApproximateCreationDateTime": "", + "Keys": { + "my_partition_key": { + "S": "testId5" + } + }, + "NewImage": { + "my_partition_key": { + "S": "testId5" + }, + "should_fail": { + "BOOL": true + } + }, + "SequenceNumber": "", + "SizeBytes": 58, + "StreamViewType": "NEW_IMAGE" + }, + "eventID": "", + "eventName": "INSERT", + "eventSource": "aws:dynamodb", + "eventSourceARN": "arn::dynamodb::111111111111:table//stream/", + "eventVersion": "1.1" + } + ] + } + } + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_dynamodbstreams.py::TestDynamoDBEventSourceMapping::test_dynamodb_report_batch_item_failures_with_on_failure_destination_config": { + "recorded-date": "11-09-2024, 11:52:02", + "recorded-content": { + "create_table_response": { + "TableDescription": { + "AttributeDefinitions": [ + { + "AttributeName": "my_partition_key", + "AttributeType": "S" + } + ], + "BillingModeSummary": { + "BillingMode": "PAY_PER_REQUEST" + }, + "CreationDateTime": "", + "DeletionProtectionEnabled": false, + "ItemCount": 0, + "KeySchema": [ + { + "AttributeName": "my_partition_key", + "KeyType": "HASH" + } + ], + "ProvisionedThroughput": { + "NumberOfDecreasesToday": 0, + "ReadCapacityUnits": 0, + "WriteCapacityUnits": 0 + }, + "TableArn": "arn::dynamodb::111111111111:table/", + "TableId": "", + "TableName": "", + "TableSizeBytes": 0, + "TableStatus": "CREATING" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "update_table_response": { + "TableDescription": { + "AttributeDefinitions": [ + { + "AttributeName": "my_partition_key", + "AttributeType": "S" + } + ], + "BillingModeSummary": { + "BillingMode": "PAY_PER_REQUEST", + "LastUpdateToPayPerRequestDateTime": "" + }, + "CreationDateTime": "", + "DeletionProtectionEnabled": false, + "ItemCount": 0, + "KeySchema": [ + { + "AttributeName": "my_partition_key", + "KeyType": "HASH" + } + ], + "LatestStreamArn": "arn::dynamodb::111111111111:table//stream/", + "LatestStreamLabel": "", + "ProvisionedThroughput": { + "NumberOfDecreasesToday": 0, + "ReadCapacityUnits": 0, + "WriteCapacityUnits": 0 + }, + "StreamSpecification": { + "StreamEnabled": true, + "StreamViewType": "NEW_IMAGE" + }, + "TableArn": "arn::dynamodb::111111111111:table/", + "TableId": "", + "TableName": "", + "TableSizeBytes": 0, + "TableStatus": "UPDATING" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "create_event_source_mapping_response": { + "BatchSize": 3, + "BisectBatchOnFunctionError": false, + "DestinationConfig": { + "OnFailure": { + "Destination": "arn::sqs::111111111111:" + } + }, + "EventSourceArn": "arn::dynamodb::111111111111:table//stream/", + "FunctionArn": "arn::lambda::111111111111:function:", + "FunctionResponseTypes": [ + "ReportBatchItemFailures" + ], + "LastModified": "", + "LastProcessingResult": "No records processed", + "MaximumBatchingWindowInSeconds": 1, + "MaximumRecordAgeInSeconds": -1, + "MaximumRetryAttempts": 3, + "ParallelizationFactor": 1, + "StartingPosition": "TRIM_HORIZON", + "State": "Creating", + "StateTransitionReason": "User action", + "TumblingWindowInSeconds": 0, + "UUID": "", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 202 + } + }, + "destination_queue_messages": { + "Messages": [ + { + "Body": { + "requestContext": { + "requestId": "", + "functionArn": "arn::lambda::111111111111:function:", + "condition": "RetryAttemptsExhausted", + "approximateInvokeCount": 4 + }, + "responseContext": { + "statusCode": 200, + "executedVersion": "$LATEST", + "functionError": null + }, + "version": "1.0", + "timestamp": "", + "DDBStreamBatchInfo": { + "shardId": "", + "startSequenceNumber": "", + "endSequenceNumber": "", + "approximateArrivalOfFirstRecord": "", + "approximateArrivalOfLastRecord": "", + "batchSize": 1, + "streamArn": "arn::dynamodb::111111111111:table//stream/" + } + }, + "MD5OfBody": "", + "MessageId": "", + "ReceiptHandle": "" + } + ], + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + } + } + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_dynamodbstreams.py::TestDynamoDBEventSourceMapping::test_dynamodb_report_batch_item_failure_scenarios[empty_string_item_identifier_failure]": { + "recorded-date": "12-09-2024, 21:07:42", + "recorded-content": { + "create-table-result": { + "TableDescription": { + "AttributeDefinitions": [ + { + "AttributeName": "my_partition_key", + "AttributeType": "S" + } + ], + "BillingModeSummary": { + "BillingMode": "PAY_PER_REQUEST" + }, + "CreationDateTime": "", + "DeletionProtectionEnabled": false, + "ItemCount": 0, + "KeySchema": [ + { + "AttributeName": "my_partition_key", + "KeyType": "HASH" + } + ], + "ProvisionedThroughput": { + "NumberOfDecreasesToday": 0, + "ReadCapacityUnits": 0, + "WriteCapacityUnits": 0 + }, + "TableArn": "arn::dynamodb::111111111111:table/", + "TableId": "", + "TableName": "", + "TableSizeBytes": 0, + "TableStatus": "CREATING" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "destination_queue_messages": { + "Messages": [ + { + "Body": { + "requestContext": { + "requestId": "", + "functionArn": "arn::lambda::111111111111:function:", + "condition": "RetryAttemptsExhausted", + "approximateInvokeCount": 4 + }, + "responseContext": { + "statusCode": 200, + "executedVersion": "$LATEST", + "functionError": null + }, + "version": "1.0", + "timestamp": "", + "DDBStreamBatchInfo": { + "shardId": "", + "startSequenceNumber": "", + "endSequenceNumber": "", + "approximateArrivalOfFirstRecord": "", + "approximateArrivalOfLastRecord": "", + "batchSize": 1, + "streamArn": "arn::dynamodb::111111111111:table//stream/" + } + }, + "MD5OfBody": "", + "MessageId": "", + "ReceiptHandle": "" + } + ], + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "dynamodb_events": [ + { + "Records": [ + { + "eventID": "", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "", + "dynamodb": { + "ApproximateCreationDateTime": "", + "Keys": { + "my_partition_key": { + "S": "hello world" + } + }, + "NewImage": { + "binary_key": { + "B": "Zm9vYmFy" + }, + "my_partition_key": { + "S": "hello world" + } + }, + "SequenceNumber": "", + "SizeBytes": 70, + "StreamViewType": "NEW_IMAGE" + }, + "eventSourceARN": "arn::dynamodb::111111111111:table//stream/" + } + ] + }, + { + "Records": [ + { + "eventID": "", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "", + "dynamodb": { + "ApproximateCreationDateTime": "", + "Keys": { + "my_partition_key": { + "S": "hello world" + } + }, + "NewImage": { + "binary_key": { + "B": "Zm9vYmFy" + }, + "my_partition_key": { + "S": "hello world" + } + }, + "SequenceNumber": "", + "SizeBytes": 70, + "StreamViewType": "NEW_IMAGE" + }, + "eventSourceARN": "arn::dynamodb::111111111111:table//stream/" + } + ] + }, + { + "Records": [ + { + "eventID": "", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "", + "dynamodb": { + "ApproximateCreationDateTime": "", + "Keys": { + "my_partition_key": { + "S": "hello world" + } + }, + "NewImage": { + "binary_key": { + "B": "Zm9vYmFy" + }, + "my_partition_key": { + "S": "hello world" + } + }, + "SequenceNumber": "", + "SizeBytes": 70, + "StreamViewType": "NEW_IMAGE" + }, + "eventSourceARN": "arn::dynamodb::111111111111:table//stream/" + } + ] + }, + { + "Records": [ + { + "eventID": "", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "", + "dynamodb": { + "ApproximateCreationDateTime": "", + "Keys": { + "my_partition_key": { + "S": "hello world" + } + }, + "NewImage": { + "binary_key": { + "B": "Zm9vYmFy" + }, + "my_partition_key": { + "S": "hello world" + } + }, + "SequenceNumber": "", + "SizeBytes": 70, + "StreamViewType": "NEW_IMAGE" + }, + "eventSourceARN": "arn::dynamodb::111111111111:table//stream/" + } + ] + } + ] + } + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_dynamodbstreams.py::TestDynamoDBEventSourceMapping::test_dynamodb_report_batch_item_failure_scenarios[null_item_identifier_failure]": { + "recorded-date": "12-09-2024, 21:09:44", + "recorded-content": { + "create-table-result": { + "TableDescription": { + "AttributeDefinitions": [ + { + "AttributeName": "my_partition_key", + "AttributeType": "S" + } + ], + "BillingModeSummary": { + "BillingMode": "PAY_PER_REQUEST" + }, + "CreationDateTime": "", + "DeletionProtectionEnabled": false, + "ItemCount": 0, + "KeySchema": [ + { + "AttributeName": "my_partition_key", + "KeyType": "HASH" + } + ], + "ProvisionedThroughput": { + "NumberOfDecreasesToday": 0, + "ReadCapacityUnits": 0, + "WriteCapacityUnits": 0 + }, + "TableArn": "arn::dynamodb::111111111111:table/", + "TableId": "", + "TableName": "", + "TableSizeBytes": 0, + "TableStatus": "CREATING" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "destination_queue_messages": { + "Messages": [ + { + "Body": { + "requestContext": { + "requestId": "", + "functionArn": "arn::lambda::111111111111:function:", + "condition": "RetryAttemptsExhausted", + "approximateInvokeCount": 4 + }, + "responseContext": { + "statusCode": 200, + "executedVersion": "$LATEST", + "functionError": null + }, + "version": "1.0", + "timestamp": "", + "DDBStreamBatchInfo": { + "shardId": "", + "startSequenceNumber": "", + "endSequenceNumber": "", + "approximateArrivalOfFirstRecord": "", + "approximateArrivalOfLastRecord": "", + "batchSize": 1, + "streamArn": "arn::dynamodb::111111111111:table//stream/" + } + }, + "MD5OfBody": "", + "MessageId": "", + "ReceiptHandle": "" + } + ], + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "dynamodb_events": [ + { + "Records": [ + { + "eventID": "", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "", + "dynamodb": { + "ApproximateCreationDateTime": "", + "Keys": { + "my_partition_key": { + "S": "hello world" + } + }, + "NewImage": { + "binary_key": { + "B": "Zm9vYmFy" + }, + "my_partition_key": { + "S": "hello world" + } + }, + "SequenceNumber": "", + "SizeBytes": 70, + "StreamViewType": "NEW_IMAGE" + }, + "eventSourceARN": "arn::dynamodb::111111111111:table//stream/" + } + ] + }, + { + "Records": [ + { + "eventID": "", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "", + "dynamodb": { + "ApproximateCreationDateTime": "", + "Keys": { + "my_partition_key": { + "S": "hello world" + } + }, + "NewImage": { + "binary_key": { + "B": "Zm9vYmFy" + }, + "my_partition_key": { + "S": "hello world" + } + }, + "SequenceNumber": "", + "SizeBytes": 70, + "StreamViewType": "NEW_IMAGE" + }, + "eventSourceARN": "arn::dynamodb::111111111111:table//stream/" + } + ] + }, + { + "Records": [ + { + "eventID": "", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "", + "dynamodb": { + "ApproximateCreationDateTime": "", + "Keys": { + "my_partition_key": { + "S": "hello world" + } + }, + "NewImage": { + "binary_key": { + "B": "Zm9vYmFy" + }, + "my_partition_key": { + "S": "hello world" + } + }, + "SequenceNumber": "", + "SizeBytes": 70, + "StreamViewType": "NEW_IMAGE" + }, + "eventSourceARN": "arn::dynamodb::111111111111:table//stream/" + } + ] + }, + { + "Records": [ + { + "eventID": "", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "", + "dynamodb": { + "ApproximateCreationDateTime": "", + "Keys": { + "my_partition_key": { + "S": "hello world" + } + }, + "NewImage": { + "binary_key": { + "B": "Zm9vYmFy" + }, + "my_partition_key": { + "S": "hello world" + } + }, + "SequenceNumber": "", + "SizeBytes": 70, + "StreamViewType": "NEW_IMAGE" + }, + "eventSourceARN": "arn::dynamodb::111111111111:table//stream/" + } + ] + } + ] + } + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_dynamodbstreams.py::TestDynamoDBEventSourceMapping::test_dynamodb_report_batch_item_failure_scenarios[invalid_key_foo_failure]": { + "recorded-date": "12-09-2024, 21:12:32", + "recorded-content": { + "create-table-result": { + "TableDescription": { + "AttributeDefinitions": [ + { + "AttributeName": "my_partition_key", + "AttributeType": "S" + } + ], + "BillingModeSummary": { + "BillingMode": "PAY_PER_REQUEST" + }, + "CreationDateTime": "", + "DeletionProtectionEnabled": false, + "ItemCount": 0, + "KeySchema": [ + { + "AttributeName": "my_partition_key", + "KeyType": "HASH" + } + ], + "ProvisionedThroughput": { + "NumberOfDecreasesToday": 0, + "ReadCapacityUnits": 0, + "WriteCapacityUnits": 0 + }, + "TableArn": "arn::dynamodb::111111111111:table/", + "TableId": "", + "TableName": "", + "TableSizeBytes": 0, + "TableStatus": "CREATING" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "destination_queue_messages": { + "Messages": [ + { + "Body": { + "requestContext": { + "requestId": "", + "functionArn": "arn::lambda::111111111111:function:", + "condition": "RetryAttemptsExhausted", + "approximateInvokeCount": 4 + }, + "responseContext": { + "statusCode": 200, + "executedVersion": "$LATEST", + "functionError": null + }, + "version": "1.0", + "timestamp": "", + "DDBStreamBatchInfo": { + "shardId": "", + "startSequenceNumber": "", + "endSequenceNumber": "", + "approximateArrivalOfFirstRecord": "", + "approximateArrivalOfLastRecord": "", + "batchSize": 1, + "streamArn": "arn::dynamodb::111111111111:table//stream/" + } + }, + "MD5OfBody": "", + "MessageId": "", + "ReceiptHandle": "" + } + ], + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "dynamodb_events": [ + { + "Records": [ + { + "eventID": "", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "", + "dynamodb": { + "ApproximateCreationDateTime": "", + "Keys": { + "my_partition_key": { + "S": "hello world" + } + }, + "NewImage": { + "binary_key": { + "B": "Zm9vYmFy" + }, + "my_partition_key": { + "S": "hello world" + } + }, + "SequenceNumber": "", + "SizeBytes": 70, + "StreamViewType": "NEW_IMAGE" + }, + "eventSourceARN": "arn::dynamodb::111111111111:table//stream/" + } + ] + }, + { + "Records": [ + { + "eventID": "", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "", + "dynamodb": { + "ApproximateCreationDateTime": "", + "Keys": { + "my_partition_key": { + "S": "hello world" + } + }, + "NewImage": { + "binary_key": { + "B": "Zm9vYmFy" + }, + "my_partition_key": { + "S": "hello world" + } + }, + "SequenceNumber": "", + "SizeBytes": 70, + "StreamViewType": "NEW_IMAGE" + }, + "eventSourceARN": "arn::dynamodb::111111111111:table//stream/" + } + ] + }, + { + "Records": [ + { + "eventID": "", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "", + "dynamodb": { + "ApproximateCreationDateTime": "", + "Keys": { + "my_partition_key": { + "S": "hello world" + } + }, + "NewImage": { + "binary_key": { + "B": "Zm9vYmFy" + }, + "my_partition_key": { + "S": "hello world" + } + }, + "SequenceNumber": "", + "SizeBytes": 70, + "StreamViewType": "NEW_IMAGE" + }, + "eventSourceARN": "arn::dynamodb::111111111111:table//stream/" + } + ] + }, + { + "Records": [ + { + "eventID": "", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "", + "dynamodb": { + "ApproximateCreationDateTime": "", + "Keys": { + "my_partition_key": { + "S": "hello world" + } + }, + "NewImage": { + "binary_key": { + "B": "Zm9vYmFy" + }, + "my_partition_key": { + "S": "hello world" + } + }, + "SequenceNumber": "", + "SizeBytes": 70, + "StreamViewType": "NEW_IMAGE" + }, + "eventSourceARN": "arn::dynamodb::111111111111:table//stream/" + } + ] + } + ] + } + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_dynamodbstreams.py::TestDynamoDBEventSourceMapping::test_dynamodb_report_batch_item_failure_scenarios[invalid_key_foo_null_value_failure]": { + "recorded-date": "12-09-2024, 21:14:19", + "recorded-content": { + "create-table-result": { + "TableDescription": { + "AttributeDefinitions": [ + { + "AttributeName": "my_partition_key", + "AttributeType": "S" + } + ], + "BillingModeSummary": { + "BillingMode": "PAY_PER_REQUEST" + }, + "CreationDateTime": "", + "DeletionProtectionEnabled": false, + "ItemCount": 0, + "KeySchema": [ + { + "AttributeName": "my_partition_key", + "KeyType": "HASH" + } + ], + "ProvisionedThroughput": { + "NumberOfDecreasesToday": 0, + "ReadCapacityUnits": 0, + "WriteCapacityUnits": 0 + }, + "TableArn": "arn::dynamodb::111111111111:table/", + "TableId": "", + "TableName": "", + "TableSizeBytes": 0, + "TableStatus": "CREATING" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "destination_queue_messages": { + "Messages": [ + { + "Body": { + "requestContext": { + "requestId": "", + "functionArn": "arn::lambda::111111111111:function:", + "condition": "RetryAttemptsExhausted", + "approximateInvokeCount": 4 + }, + "responseContext": { + "statusCode": 200, + "executedVersion": "$LATEST", + "functionError": null + }, + "version": "1.0", + "timestamp": "", + "DDBStreamBatchInfo": { + "shardId": "", + "startSequenceNumber": "", + "endSequenceNumber": "", + "approximateArrivalOfFirstRecord": "", + "approximateArrivalOfLastRecord": "", + "batchSize": 1, + "streamArn": "arn::dynamodb::111111111111:table//stream/" + } + }, + "MD5OfBody": "", + "MessageId": "", + "ReceiptHandle": "" + } + ], + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "dynamodb_events": [ + { + "Records": [ + { + "eventID": "", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "", + "dynamodb": { + "ApproximateCreationDateTime": "", + "Keys": { + "my_partition_key": { + "S": "hello world" + } + }, + "NewImage": { + "binary_key": { + "B": "Zm9vYmFy" + }, + "my_partition_key": { + "S": "hello world" + } + }, + "SequenceNumber": "", + "SizeBytes": 70, + "StreamViewType": "NEW_IMAGE" + }, + "eventSourceARN": "arn::dynamodb::111111111111:table//stream/" + } + ] + }, + { + "Records": [ + { + "eventID": "", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "", + "dynamodb": { + "ApproximateCreationDateTime": "", + "Keys": { + "my_partition_key": { + "S": "hello world" + } + }, + "NewImage": { + "binary_key": { + "B": "Zm9vYmFy" + }, + "my_partition_key": { + "S": "hello world" + } + }, + "SequenceNumber": "", + "SizeBytes": 70, + "StreamViewType": "NEW_IMAGE" + }, + "eventSourceARN": "arn::dynamodb::111111111111:table//stream/" + } + ] + }, + { + "Records": [ + { + "eventID": "", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "", + "dynamodb": { + "ApproximateCreationDateTime": "", + "Keys": { + "my_partition_key": { + "S": "hello world" + } + }, + "NewImage": { + "binary_key": { + "B": "Zm9vYmFy" + }, + "my_partition_key": { + "S": "hello world" + } + }, + "SequenceNumber": "", + "SizeBytes": 70, + "StreamViewType": "NEW_IMAGE" + }, + "eventSourceARN": "arn::dynamodb::111111111111:table//stream/" + } + ] + }, + { + "Records": [ + { + "eventID": "", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "", + "dynamodb": { + "ApproximateCreationDateTime": "", + "Keys": { + "my_partition_key": { + "S": "hello world" + } + }, + "NewImage": { + "binary_key": { + "B": "Zm9vYmFy" + }, + "my_partition_key": { + "S": "hello world" + } + }, + "SequenceNumber": "", + "SizeBytes": 70, + "StreamViewType": "NEW_IMAGE" + }, + "eventSourceARN": "arn::dynamodb::111111111111:table//stream/" + } + ] + } + ] + } + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_dynamodbstreams.py::TestDynamoDBEventSourceMapping::test_dynamodb_report_batch_item_failure_scenarios[unhandled_exception_in_function]": { + "recorded-date": "12-09-2024, 21:18:09", + "recorded-content": { + "create-table-result": { + "TableDescription": { + "AttributeDefinitions": [ + { + "AttributeName": "my_partition_key", + "AttributeType": "S" + } + ], + "BillingModeSummary": { + "BillingMode": "PAY_PER_REQUEST" + }, + "CreationDateTime": "", + "DeletionProtectionEnabled": false, + "ItemCount": 0, + "KeySchema": [ + { + "AttributeName": "my_partition_key", + "KeyType": "HASH" + } + ], + "ProvisionedThroughput": { + "NumberOfDecreasesToday": 0, + "ReadCapacityUnits": 0, + "WriteCapacityUnits": 0 + }, + "TableArn": "arn::dynamodb::111111111111:table/", + "TableId": "", + "TableName": "", + "TableSizeBytes": 0, + "TableStatus": "CREATING" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "destination_queue_messages": { + "Messages": [ + { + "Body": { + "requestContext": { + "requestId": "", + "functionArn": "arn::lambda::111111111111:function:", + "condition": "RetryAttemptsExhausted", + "approximateInvokeCount": 4 + }, + "responseContext": { + "statusCode": 200, + "executedVersion": "$LATEST", + "functionError": "Unhandled" + }, + "version": "1.0", + "timestamp": "", + "DDBStreamBatchInfo": { + "shardId": "", + "startSequenceNumber": "", + "endSequenceNumber": "", + "approximateArrivalOfFirstRecord": "", + "approximateArrivalOfLastRecord": "", + "batchSize": 1, + "streamArn": "arn::dynamodb::111111111111:table//stream/" + } + }, + "MD5OfBody": "", + "MessageId": "", + "ReceiptHandle": "" + } + ], + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "dynamodb_events": [ + { + "Records": [ + { + "eventID": "", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "", + "dynamodb": { + "ApproximateCreationDateTime": "", + "Keys": { + "my_partition_key": { + "S": "hello world" + } + }, + "NewImage": { + "binary_key": { + "B": "Zm9vYmFy" + }, + "my_partition_key": { + "S": "hello world" + } + }, + "SequenceNumber": "", + "SizeBytes": 70, + "StreamViewType": "NEW_IMAGE" + }, + "eventSourceARN": "arn::dynamodb::111111111111:table//stream/" + } + ] + }, + { + "Records": [ + { + "eventID": "", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "", + "dynamodb": { + "ApproximateCreationDateTime": "", + "Keys": { + "my_partition_key": { + "S": "hello world" + } + }, + "NewImage": { + "binary_key": { + "B": "Zm9vYmFy" + }, + "my_partition_key": { + "S": "hello world" + } + }, + "SequenceNumber": "", + "SizeBytes": 70, + "StreamViewType": "NEW_IMAGE" + }, + "eventSourceARN": "arn::dynamodb::111111111111:table//stream/" + } + ] + }, + { + "Records": [ + { + "eventID": "", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "", + "dynamodb": { + "ApproximateCreationDateTime": "", + "Keys": { + "my_partition_key": { + "S": "hello world" + } + }, + "NewImage": { + "binary_key": { + "B": "Zm9vYmFy" + }, + "my_partition_key": { + "S": "hello world" + } + }, + "SequenceNumber": "", + "SizeBytes": 70, + "StreamViewType": "NEW_IMAGE" + }, + "eventSourceARN": "arn::dynamodb::111111111111:table//stream/" + } + ] + }, + { + "Records": [ + { + "eventID": "", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "", + "dynamodb": { + "ApproximateCreationDateTime": "", + "Keys": { + "my_partition_key": { + "S": "hello world" + } + }, + "NewImage": { + "binary_key": { + "B": "Zm9vYmFy" + }, + "my_partition_key": { + "S": "hello world" + } + }, + "SequenceNumber": "", + "SizeBytes": 70, + "StreamViewType": "NEW_IMAGE" + }, + "eventSourceARN": "arn::dynamodb::111111111111:table//stream/" + } + ] + } + ] + } + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_dynamodbstreams.py::TestDynamoDBEventSourceMapping::test_dynamodb_report_batch_item_success_scenarios[empty_list_success]": { + "recorded-date": "11-09-2024, 19:13:50", + "recorded-content": { + "create-table-result": { + "TableDescription": { + "AttributeDefinitions": [ + { + "AttributeName": "my_partition_key", + "AttributeType": "S" + } + ], + "BillingModeSummary": { + "BillingMode": "PAY_PER_REQUEST" + }, + "CreationDateTime": "", + "DeletionProtectionEnabled": false, + "ItemCount": 0, + "KeySchema": [ + { + "AttributeName": "my_partition_key", + "KeyType": "HASH" + } + ], + "ProvisionedThroughput": { + "NumberOfDecreasesToday": 0, + "ReadCapacityUnits": 0, + "WriteCapacityUnits": 0 + }, + "TableArn": "arn::dynamodb::111111111111:table/", + "TableId": "", + "TableName": "", + "TableSizeBytes": 0, + "TableStatus": "CREATING" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "dynamodb_events": [ + { + "Records": [ + { + "eventID": "", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "", + "dynamodb": { + "ApproximateCreationDateTime": "", + "Keys": { + "my_partition_key": { + "S": "hello world" + } + }, + "NewImage": { + "binary_key": { + "B": "Zm9vYmFy" + }, + "my_partition_key": { + "S": "hello world" + } + }, + "SequenceNumber": "", + "SizeBytes": 70, + "StreamViewType": "NEW_IMAGE" + }, + "eventSourceARN": "arn::dynamodb::111111111111:table//stream/" + } + ] + } + ] + } + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_dynamodbstreams.py::TestDynamoDBEventSourceMapping::test_dynamodb_report_batch_item_success_scenarios[null_success]": { + "recorded-date": "11-09-2024, 19:15:36", + "recorded-content": { + "create-table-result": { + "TableDescription": { + "AttributeDefinitions": [ + { + "AttributeName": "my_partition_key", + "AttributeType": "S" + } + ], + "BillingModeSummary": { + "BillingMode": "PAY_PER_REQUEST" + }, + "CreationDateTime": "", + "DeletionProtectionEnabled": false, + "ItemCount": 0, + "KeySchema": [ + { + "AttributeName": "my_partition_key", + "KeyType": "HASH" + } + ], + "ProvisionedThroughput": { + "NumberOfDecreasesToday": 0, + "ReadCapacityUnits": 0, + "WriteCapacityUnits": 0 + }, + "TableArn": "arn::dynamodb::111111111111:table/", + "TableId": "", + "TableName": "", + "TableSizeBytes": 0, + "TableStatus": "CREATING" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "dynamodb_events": [ + { + "Records": [ + { + "eventID": "", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "", + "dynamodb": { + "ApproximateCreationDateTime": "", + "Keys": { + "my_partition_key": { + "S": "hello world" + } + }, + "NewImage": { + "binary_key": { + "B": "Zm9vYmFy" + }, + "my_partition_key": { + "S": "hello world" + } + }, + "SequenceNumber": "", + "SizeBytes": 70, + "StreamViewType": "NEW_IMAGE" + }, + "eventSourceARN": "arn::dynamodb::111111111111:table//stream/" + } + ] + } + ] + } + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_dynamodbstreams.py::TestDynamoDBEventSourceMapping::test_dynamodb_report_batch_item_success_scenarios[empty_dict_success]": { + "recorded-date": "11-09-2024, 19:17:40", + "recorded-content": { + "create-table-result": { + "TableDescription": { + "AttributeDefinitions": [ + { + "AttributeName": "my_partition_key", + "AttributeType": "S" + } + ], + "BillingModeSummary": { + "BillingMode": "PAY_PER_REQUEST" + }, + "CreationDateTime": "", + "DeletionProtectionEnabled": false, + "ItemCount": 0, + "KeySchema": [ + { + "AttributeName": "my_partition_key", + "KeyType": "HASH" + } + ], + "ProvisionedThroughput": { + "NumberOfDecreasesToday": 0, + "ReadCapacityUnits": 0, + "WriteCapacityUnits": 0 + }, + "TableArn": "arn::dynamodb::111111111111:table/", + "TableId": "", + "TableName": "", + "TableSizeBytes": 0, + "TableStatus": "CREATING" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "dynamodb_events": [ + { + "Records": [ + { + "eventID": "", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "", + "dynamodb": { + "ApproximateCreationDateTime": "", + "Keys": { + "my_partition_key": { + "S": "hello world" + } + }, + "NewImage": { + "binary_key": { + "B": "Zm9vYmFy" + }, + "my_partition_key": { + "S": "hello world" + } + }, + "SequenceNumber": "", + "SizeBytes": 70, + "StreamViewType": "NEW_IMAGE" + }, + "eventSourceARN": "arn::dynamodb::111111111111:table//stream/" + } + ] + } + ] + } + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_dynamodbstreams.py::TestDynamoDBEventSourceMapping::test_dynamodb_report_batch_item_success_scenarios[empty_batch_item_failure_success]": { + "recorded-date": "11-09-2024, 19:19:26", + "recorded-content": { + "create-table-result": { + "TableDescription": { + "AttributeDefinitions": [ + { + "AttributeName": "my_partition_key", + "AttributeType": "S" + } + ], + "BillingModeSummary": { + "BillingMode": "PAY_PER_REQUEST" + }, + "CreationDateTime": "", + "DeletionProtectionEnabled": false, + "ItemCount": 0, + "KeySchema": [ + { + "AttributeName": "my_partition_key", + "KeyType": "HASH" + } + ], + "ProvisionedThroughput": { + "NumberOfDecreasesToday": 0, + "ReadCapacityUnits": 0, + "WriteCapacityUnits": 0 + }, + "TableArn": "arn::dynamodb::111111111111:table/", + "TableId": "", + "TableName": "", + "TableSizeBytes": 0, + "TableStatus": "CREATING" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "dynamodb_events": [ + { + "Records": [ + { + "eventID": "", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "", + "dynamodb": { + "ApproximateCreationDateTime": "", + "Keys": { + "my_partition_key": { + "S": "hello world" + } + }, + "NewImage": { + "binary_key": { + "B": "Zm9vYmFy" + }, + "my_partition_key": { + "S": "hello world" + } + }, + "SequenceNumber": "", + "SizeBytes": 70, + "StreamViewType": "NEW_IMAGE" + }, + "eventSourceARN": "arn::dynamodb::111111111111:table//stream/" + } + ] + } + ] + } + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_dynamodbstreams.py::TestDynamoDBEventSourceMapping::test_dynamodb_report_batch_item_success_scenarios[null_batch_item_failure_success]": { + "recorded-date": "11-09-2024, 19:20:35", + "recorded-content": { + "create-table-result": { + "TableDescription": { + "AttributeDefinitions": [ + { + "AttributeName": "my_partition_key", + "AttributeType": "S" + } + ], + "BillingModeSummary": { + "BillingMode": "PAY_PER_REQUEST" + }, + "CreationDateTime": "", + "DeletionProtectionEnabled": false, + "ItemCount": 0, + "KeySchema": [ + { + "AttributeName": "my_partition_key", + "KeyType": "HASH" + } + ], + "ProvisionedThroughput": { + "NumberOfDecreasesToday": 0, + "ReadCapacityUnits": 0, + "WriteCapacityUnits": 0 + }, + "TableArn": "arn::dynamodb::111111111111:table/", + "TableId": "", + "TableName": "", + "TableSizeBytes": 0, + "TableStatus": "CREATING" + }, + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "dynamodb_events": [ + { + "Records": [ + { + "eventID": "", + "eventName": "INSERT", + "eventVersion": "1.1", + "eventSource": "aws:dynamodb", + "awsRegion": "", + "dynamodb": { + "ApproximateCreationDateTime": "", + "Keys": { + "my_partition_key": { + "S": "hello world" + } + }, + "NewImage": { + "binary_key": { + "B": "Zm9vYmFy" + }, + "my_partition_key": { + "S": "hello world" + } + }, + "SequenceNumber": "", + "SizeBytes": 70, + "StreamViewType": "NEW_IMAGE" + }, + "eventSourceARN": "arn::dynamodb::111111111111:table//stream/" + } + ] + } + ] + } } } diff --git a/tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_dynamodbstreams.validation.json b/tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_dynamodbstreams.validation.json index 31e62ab7e64d4..289f6233b4590 100644 --- a/tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_dynamodbstreams.validation.json +++ b/tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_dynamodbstreams.validation.json @@ -45,6 +45,39 @@ "last_validated_date": "2024-09-03T15:10:35+00:00" }, "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_dynamodbstreams.py::TestDynamoDBEventSourceMapping::test_dynamodb_invalid_event_filter[single-string]": { - "last_validated_date": "2024-09-03T15:10:17+00:00" + "last_validated_date": "2023-02-27T17:44:12+00:00" + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_dynamodbstreams.py::TestDynamoDBEventSourceMapping::test_dynamodb_report_batch_item_failure_scenarios[empty_string_item_identifier_failure]": { + "last_validated_date": "2024-09-12T21:07:39+00:00" + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_dynamodbstreams.py::TestDynamoDBEventSourceMapping::test_dynamodb_report_batch_item_failure_scenarios[invalid_key_foo_failure]": { + "last_validated_date": "2024-09-12T21:12:30+00:00" + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_dynamodbstreams.py::TestDynamoDBEventSourceMapping::test_dynamodb_report_batch_item_failure_scenarios[invalid_key_foo_null_value_failure]": { + "last_validated_date": "2024-09-12T21:14:16+00:00" + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_dynamodbstreams.py::TestDynamoDBEventSourceMapping::test_dynamodb_report_batch_item_failure_scenarios[null_item_identifier_failure]": { + "last_validated_date": "2024-09-12T21:09:41+00:00" + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_dynamodbstreams.py::TestDynamoDBEventSourceMapping::test_dynamodb_report_batch_item_failure_scenarios[unhandled_exception_in_function]": { + "last_validated_date": "2024-09-12T21:18:06+00:00" + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_dynamodbstreams.py::TestDynamoDBEventSourceMapping::test_dynamodb_report_batch_item_failures": { + "last_validated_date": "2024-09-11T18:47:28+00:00" + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_dynamodbstreams.py::TestDynamoDBEventSourceMapping::test_dynamodb_report_batch_item_success_scenarios[empty_batch_item_failure_success]": { + "last_validated_date": "2024-09-11T19:19:23+00:00" + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_dynamodbstreams.py::TestDynamoDBEventSourceMapping::test_dynamodb_report_batch_item_success_scenarios[empty_dict_success]": { + "last_validated_date": "2024-09-11T19:17:38+00:00" + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_dynamodbstreams.py::TestDynamoDBEventSourceMapping::test_dynamodb_report_batch_item_success_scenarios[empty_list_success]": { + "last_validated_date": "2024-09-11T19:13:49+00:00" + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_dynamodbstreams.py::TestDynamoDBEventSourceMapping::test_dynamodb_report_batch_item_success_scenarios[null_batch_item_failure_success]": { + "last_validated_date": "2024-09-11T19:20:33+00:00" + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_dynamodbstreams.py::TestDynamoDBEventSourceMapping::test_dynamodb_report_batch_item_success_scenarios[null_success]": { + "last_validated_date": "2024-09-11T19:15:34+00:00" } } diff --git a/tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_kinesis.py b/tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_kinesis.py index 3a8b66dead314..b20952aa4ccdc 100644 --- a/tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_kinesis.py +++ b/tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_kinesis.py @@ -11,6 +11,7 @@ _await_event_source_mapping_enabled, _await_event_source_mapping_state, _get_lambda_invocation_events, + get_lambda_log_events, lambda_role, s3_lambda_permission, ) @@ -18,7 +19,12 @@ from localstack.testing.pytest import markers from localstack.utils.strings import short_uid, to_bytes from localstack.utils.sync import ShortCircuitWaitException, retry, wait_until -from tests.aws.services.lambda_.event_source_mapping.utils import is_old_esm, is_v2_esm +from tests.aws.services.lambda_.event_source_mapping.utils import ( + LAMBDA_KINESIS_BATCH_ITEM_FAILURE, + create_lambda_with_response, + is_old_esm, + is_v2_esm, +) from tests.aws.services.lambda_.functions import FUNCTIONS_PATH, lambda_integration from tests.aws.services.lambda_.test_lambda import ( TEST_LAMBDA_PYTHON, @@ -55,7 +61,6 @@ def _snapshot_transformers(snapshot): "$..Records..eventID", "$..BisectBatchOnFunctionError", "$..DestinationConfig", - "$..FunctionResponseTypes", "$..LastProcessingResult", "$..MaximumBatchingWindowInSeconds", "$..MaximumRecordAgeInSeconds", @@ -537,12 +542,310 @@ def test_kinesis_event_source_mapping_with_on_failure_destination_config( def verify_failure_received(): result = aws_client.sqs.receive_message(QueueUrl=queue_event_source_mapping) - assert result["Messages"] + assert result.get("Messages") + return result + + sleep = 15 if is_aws_cloud() else 5 + sqs_payload = retry(verify_failure_received, retries=15, sleep=sleep, sleep_before=5) + snapshot.match("sqs_payload", sqs_payload) + + @pytest.mark.skipif( + is_old_esm(), + reason="ReportBatchItemFailures: Partial batch failure handling not implemented in ESM v1", + ) + @markers.snapshot.skip_snapshot_verify( + paths=[ + "$..Messages..Body.KinesisBatchInfo.shardId", + "$..Messages..Body.KinesisBatchInfo.streamArn", + "$..Records", # FIXME Figure out why there is an extra log record + ], + ) + @markers.aws.needs_fixing + def test_kinesis_report_batch_item_failures( + self, + create_lambda_function, + sqs_get_queue_arn, + sqs_create_queue, + create_iam_role_with_policy, + wait_for_stream_ready, + cleanups, + snapshot, + aws_client, + ): + # snapshot setup + snapshot.add_transformer(snapshot.transform.key_value("MD5OfBody")) + snapshot.add_transformer(snapshot.transform.key_value("ReceiptHandle")) + snapshot.add_transformer(snapshot.transform.key_value("startSequenceNumber")) + + function_name = f"lambda_func-{short_uid()}" + role = f"test-lambda-role-{short_uid()}" + policy_name = f"test-lambda-policy-{short_uid()}" + kinesis_name = f"test-kinesis-{short_uid()}" + role_arn = create_iam_role_with_policy( + RoleName=role, + PolicyName=policy_name, + RoleDefinition=lambda_role, + PolicyDefinition=s3_lambda_permission, + ) + + create_lambda_function( + handler_file=LAMBDA_KINESIS_BATCH_ITEM_FAILURE, + func_name=function_name, + runtime=Runtime.python3_12, + role=role_arn, + ) + aws_client.kinesis.create_stream(StreamName=kinesis_name, ShardCount=1) + cleanups.append( + lambda: aws_client.kinesis.delete_stream( + StreamName=kinesis_name, EnforceConsumerDeletion=True + ) + ) + result = aws_client.kinesis.describe_stream(StreamName=kinesis_name)["StreamDescription"] + kinesis_arn = result["StreamARN"] + wait_for_stream_ready(stream_name=kinesis_name) + + # Use OnFailure config with a DLQ to minimise flakiness instead of relying on Cloudwatch logs + queue_event_source_mapping = sqs_create_queue() + destination_queue = sqs_get_queue_arn(queue_event_source_mapping) + destination_config = {"OnFailure": {"Destination": destination_queue}} + + create_event_source_mapping_response = aws_client.lambda_.create_event_source_mapping( + FunctionName=function_name, + BatchSize=3, + StartingPosition="TRIM_HORIZON", + EventSourceArn=kinesis_arn, + MaximumBatchingWindowInSeconds=1, + MaximumRetryAttempts=3, + DestinationConfig=destination_config, + FunctionResponseTypes=["ReportBatchItemFailures"], + ) + cleanups.append( + lambda: aws_client.lambda_.delete_event_source_mapping(UUID=event_source_mapping_uuid) + ) + snapshot.match("create_event_source_mapping_response", create_event_source_mapping_response) + event_source_mapping_uuid = create_event_source_mapping_response["UUID"] + _await_event_source_mapping_enabled(aws_client.lambda_, event_source_mapping_uuid) + + kinesis_records = [ + {"Data": json.dumps({"should_fail": i == 5}), "PartitionKey": f"test_{i}"} + for i in range(6) + ] + + aws_client.kinesis.put_records( + Records=kinesis_records, + StreamName=kinesis_name, + ) + + def verify_failure_received(): + result = aws_client.sqs.receive_message(QueueUrl=queue_event_source_mapping) + assert result.get("Messages") return result - sqs_payload = retry(verify_failure_received, retries=50, sleep=5, sleep_before=5) + sleep = 15 if is_aws_cloud() else 5 + sqs_payload = retry(verify_failure_received, retries=15, sleep=sleep, sleep_before=5) snapshot.match("sqs_payload", sqs_payload) + batched_records = get_lambda_log_events(function_name, logs_client=aws_client.logs) + flattened_records = [ + record for batch in batched_records for record in batch.get("Records", []) + ] + sorted_records = sorted(flattened_records, key=lambda item: item["kinesis"]["partitionKey"]) + + snapshot.match("kinesis_records", {"Records": sorted_records}) + + @markers.aws.validated + @pytest.mark.skipif( + is_old_esm(), reason="ReportBatchItemFailures: Total batch fails not implemented in ESM v1" + ) + @markers.snapshot.skip_snapshot_verify( + paths=[ + "$..Messages..Body.KinesisBatchInfo.shardId", + ], + ) + @pytest.mark.parametrize( + "set_lambda_response", + [ + # Failures + {"batchItemFailures": [{"itemIdentifier": ""}]}, + {"batchItemFailures": [{"itemIdentifier": None}]}, + {"batchItemFailures": [{"foo": 123}]}, + {"batchItemFailures": [{"foo": None}]}, + # Unhandled Exceptions + "(lambda: 1 / 0)()", # This will (lazily) evaluate, raise an exception, and re-trigger the whole batch + ], + ids=[ + # Failures + "empty_string_item_identifier_failure", + "null_item_identifier_failure", + "invalid_key_foo_failure", + "invalid_key_foo_null_value_failure", + # Unhandled Exceptions + "unhandled_exception_in_function", + ], + ) + def test_kinesis_report_batch_item_failure_scenarios( + self, + create_lambda_function, + kinesis_create_stream, + lambda_su_role, + wait_for_stream_ready, + cleanups, + snapshot, + aws_client, + set_lambda_response, + sqs_get_queue_arn, + sqs_create_queue, + ): + snapshot.add_transformer(snapshot.transform.key_value("MD5OfBody")) + snapshot.add_transformer(snapshot.transform.key_value("ReceiptHandle")) + snapshot.add_transformer(snapshot.transform.key_value("startSequenceNumber")) + + function_name = f"lambda_func-{short_uid()}" + stream_name = f"test-foobar-{short_uid()}" + record_data = "hello" + + create_lambda_function( + handler_file=create_lambda_with_response(set_lambda_response), + func_name=function_name, + runtime=Runtime.python3_12, + role=lambda_su_role, + ) + + kinesis_create_stream(StreamName=stream_name, ShardCount=1) + wait_for_stream_ready(stream_name=stream_name) + stream_summary = aws_client.kinesis.describe_stream_summary(StreamName=stream_name) + assert stream_summary["StreamDescriptionSummary"]["OpenShardCount"] == 1 + stream_arn = aws_client.kinesis.describe_stream(StreamName=stream_name)[ + "StreamDescription" + ]["StreamARN"] + + queue_event_source_mapping = sqs_create_queue() + destination_queue = sqs_get_queue_arn(queue_event_source_mapping) + destination_config = {"OnFailure": {"Destination": destination_queue}} + + create_event_source_mapping_response = aws_client.lambda_.create_event_source_mapping( + EventSourceArn=stream_arn, + FunctionName=function_name, + StartingPosition="TRIM_HORIZON", + BatchSize=1, + MaximumBatchingWindowInSeconds=1, + FunctionResponseTypes=["ReportBatchItemFailures"], + MaximumRetryAttempts=2, + DestinationConfig=destination_config, + ) + snapshot.match("create_event_source_mapping_response", create_event_source_mapping_response) + uuid = create_event_source_mapping_response["UUID"] + cleanups.append(lambda: aws_client.lambda_.delete_event_source_mapping(UUID=uuid)) + _await_event_source_mapping_enabled(aws_client.lambda_, uuid) + + aws_client.kinesis.put_record( + Data=record_data, + PartitionKey="test", + StreamName=stream_name, + ) + + def verify_failure_received(): + result = aws_client.sqs.receive_message(QueueUrl=queue_event_source_mapping) + assert result.get("Messages") + return result + + sleep = 15 if is_aws_cloud() else 5 + sqs_payload = retry(verify_failure_received, retries=15, sleep=sleep, sleep_before=5) + snapshot.match("sqs_payload", sqs_payload) + + events = get_lambda_log_events(function_name, logs_client=aws_client.logs) + + # This will filter out exception messages being added to the log stream + invocation_events = [event for event in events if "Records" in event] + snapshot.match("kinesis_events", invocation_events) + + @markers.aws.validated + @markers.snapshot.skip_snapshot_verify( + paths=[ + "$..Messages..Body.KinesisBatchInfo.shardId", + ], + ) + @pytest.mark.parametrize( + "set_lambda_response", + [ + # Successes + [], + None, + {}, + {"batchItemFailures": []}, + {"batchItemFailures": None}, + ], + ids=[ + # Successes + "empty_list_success", + "null_success", + "empty_dict_success", + "empty_batch_item_failure_success", + "null_batch_item_failure_success", + ], + ) + def test_kinesis_report_batch_item_success_scenarios( + self, + create_lambda_function, + kinesis_create_stream, + lambda_su_role, + wait_for_stream_ready, + cleanups, + snapshot, + aws_client, + set_lambda_response, + ): + function_name = f"lambda_func-{short_uid()}" + stream_name = f"test-foobar-{short_uid()}" + record_data = "hello" + + create_lambda_function( + handler_file=create_lambda_with_response(set_lambda_response), + func_name=function_name, + runtime=Runtime.python3_12, + role=lambda_su_role, + ) + + kinesis_create_stream(StreamName=stream_name, ShardCount=1) + wait_for_stream_ready(stream_name=stream_name) + stream_summary = aws_client.kinesis.describe_stream_summary(StreamName=stream_name) + assert stream_summary["StreamDescriptionSummary"]["OpenShardCount"] == 1 + stream_arn = aws_client.kinesis.describe_stream(StreamName=stream_name)[ + "StreamDescription" + ]["StreamARN"] + + create_event_source_mapping_response = aws_client.lambda_.create_event_source_mapping( + EventSourceArn=stream_arn, + FunctionName=function_name, + StartingPosition="TRIM_HORIZON", + BatchSize=1, + MaximumBatchingWindowInSeconds=1, + FunctionResponseTypes=["ReportBatchItemFailures"], + MaximumRetryAttempts=2, + ) + snapshot.match("create_event_source_mapping_response", create_event_source_mapping_response) + uuid = create_event_source_mapping_response["UUID"] + cleanups.append(lambda: aws_client.lambda_.delete_event_source_mapping(UUID=uuid)) + _await_event_source_mapping_enabled(aws_client.lambda_, uuid) + + aws_client.kinesis.put_record( + Data=record_data, + PartitionKey="test", + StreamName=stream_name, + ) + + def _verify_messages_received(): + events = get_lambda_log_events(function_name, logs_client=aws_client.logs) + + # This will filter out exception messages being added to the log stream + record_events = [event for event in events if "Records" in event] + + assert len(record_events) >= 1 + return record_events + + invocation_events = retry(_verify_messages_received, retries=30, sleep=5) + snapshot.match("kinesis_events", invocation_events) + # TODO: add tests for different edge cases in filtering (e.g. message isn't json => needs to be dropped) # https://docs.aws.amazon.com/lambda/latest/dg/invocation-eventfiltering.html#filtering-kinesis diff --git a/tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_kinesis.snapshot.json b/tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_kinesis.snapshot.json index 49f68077edef3..ee716a8c7a1d7 100644 --- a/tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_kinesis.snapshot.json +++ b/tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_kinesis.snapshot.json @@ -1575,5 +1575,1175 @@ ] } } + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_kinesis.py::TestKinesisSource::test_kinesis_report_batch_item_failures": { + "recorded-date": "11-09-2024, 18:00:26", + "recorded-content": { + "create_event_source_mapping_response": { + "BatchSize": 3, + "BisectBatchOnFunctionError": false, + "DestinationConfig": { + "OnFailure": { + "Destination": "arn::sqs::111111111111:" + } + }, + "EventSourceArn": "arn::kinesis::111111111111:stream/", + "FunctionArn": "arn::lambda::111111111111:function:", + "FunctionResponseTypes": [ + "ReportBatchItemFailures" + ], + "LastModified": "", + "LastProcessingResult": "No records processed", + "MaximumBatchingWindowInSeconds": 1, + "MaximumRecordAgeInSeconds": -1, + "MaximumRetryAttempts": 3, + "ParallelizationFactor": 1, + "StartingPosition": "TRIM_HORIZON", + "State": "Creating", + "StateTransitionReason": "User action", + "TumblingWindowInSeconds": 0, + "UUID": "", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 202 + } + }, + "sqs_payload": { + "Messages": [ + { + "Body": { + "requestContext": { + "requestId": "", + "functionArn": "arn::lambda::111111111111:function:", + "condition": "RetryAttemptsExhausted", + "approximateInvokeCount": 4 + }, + "responseContext": { + "statusCode": 200, + "executedVersion": "$LATEST", + "functionError": null + }, + "version": "1.0", + "timestamp": "", + "KinesisBatchInfo": { + "shardId": "shardId-000000000000", + "startSequenceNumber": "", + "endSequenceNumber": "", + "approximateArrivalOfFirstRecord": "", + "approximateArrivalOfLastRecord": "", + "batchSize": 1, + "streamArn": "arn::kinesis::111111111111:stream/" + } + }, + "MD5OfBody": "", + "MessageId": "", + "ReceiptHandle": "" + } + ], + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "kinesis_records": { + "Records": [ + { + "awsRegion": "", + "eventID": "shardId-000000000000:", + "eventName": "aws:kinesis:record", + "eventSource": "aws:kinesis", + "eventSourceARN": "arn::kinesis::111111111111:stream/", + "eventVersion": "1.0", + "invokeIdentityArn": "arn::iam::111111111111:role/", + "kinesis": { + "approximateArrivalTimestamp": "", + "data": "eyJzaG91bGRfZmFpbCI6IGZhbHNlfQ==", + "kinesisSchemaVersion": "1.0", + "partitionKey": "test_0", + "sequenceNumber": "" + } + }, + { + "awsRegion": "", + "eventID": "shardId-000000000000:", + "eventName": "aws:kinesis:record", + "eventSource": "aws:kinesis", + "eventSourceARN": "arn::kinesis::111111111111:stream/", + "eventVersion": "1.0", + "invokeIdentityArn": "arn::iam::111111111111:role/", + "kinesis": { + "approximateArrivalTimestamp": "", + "data": "eyJzaG91bGRfZmFpbCI6IGZhbHNlfQ==", + "kinesisSchemaVersion": "1.0", + "partitionKey": "test_1", + "sequenceNumber": "" + } + }, + { + "awsRegion": "", + "eventID": "shardId-000000000000:", + "eventName": "aws:kinesis:record", + "eventSource": "aws:kinesis", + "eventSourceARN": "arn::kinesis::111111111111:stream/", + "eventVersion": "1.0", + "invokeIdentityArn": "arn::iam::111111111111:role/", + "kinesis": { + "approximateArrivalTimestamp": "", + "data": "eyJzaG91bGRfZmFpbCI6IGZhbHNlfQ==", + "kinesisSchemaVersion": "1.0", + "partitionKey": "test_2", + "sequenceNumber": "" + } + }, + { + "awsRegion": "", + "eventID": "shardId-000000000000:", + "eventName": "aws:kinesis:record", + "eventSource": "aws:kinesis", + "eventSourceARN": "arn::kinesis::111111111111:stream/", + "eventVersion": "1.0", + "invokeIdentityArn": "arn::iam::111111111111:role/", + "kinesis": { + "approximateArrivalTimestamp": "", + "data": "eyJzaG91bGRfZmFpbCI6IGZhbHNlfQ==", + "kinesisSchemaVersion": "1.0", + "partitionKey": "test_3", + "sequenceNumber": "" + } + }, + { + "awsRegion": "", + "eventID": "shardId-000000000000:", + "eventName": "aws:kinesis:record", + "eventSource": "aws:kinesis", + "eventSourceARN": "arn::kinesis::111111111111:stream/", + "eventVersion": "1.0", + "invokeIdentityArn": "arn::iam::111111111111:role/", + "kinesis": { + "approximateArrivalTimestamp": "", + "data": "eyJzaG91bGRfZmFpbCI6IGZhbHNlfQ==", + "kinesisSchemaVersion": "1.0", + "partitionKey": "test_4", + "sequenceNumber": "" + } + }, + { + "awsRegion": "", + "eventID": "shardId-000000000000:", + "eventName": "aws:kinesis:record", + "eventSource": "aws:kinesis", + "eventSourceARN": "arn::kinesis::111111111111:stream/", + "eventVersion": "1.0", + "invokeIdentityArn": "arn::iam::111111111111:role/", + "kinesis": { + "approximateArrivalTimestamp": "", + "data": "eyJzaG91bGRfZmFpbCI6IHRydWV9", + "kinesisSchemaVersion": "1.0", + "partitionKey": "test_5", + "sequenceNumber": "" + } + }, + { + "awsRegion": "", + "eventID": "shardId-000000000000:", + "eventName": "aws:kinesis:record", + "eventSource": "aws:kinesis", + "eventSourceARN": "arn::kinesis::111111111111:stream/", + "eventVersion": "1.0", + "invokeIdentityArn": "arn::iam::111111111111:role/", + "kinesis": { + "approximateArrivalTimestamp": "", + "data": "eyJzaG91bGRfZmFpbCI6IHRydWV9", + "kinesisSchemaVersion": "1.0", + "partitionKey": "test_5", + "sequenceNumber": "" + } + }, + { + "awsRegion": "", + "eventID": "shardId-000000000000:", + "eventName": "aws:kinesis:record", + "eventSource": "aws:kinesis", + "eventSourceARN": "arn::kinesis::111111111111:stream/", + "eventVersion": "1.0", + "invokeIdentityArn": "arn::iam::111111111111:role/", + "kinesis": { + "approximateArrivalTimestamp": "", + "data": "eyJzaG91bGRfZmFpbCI6IHRydWV9", + "kinesisSchemaVersion": "1.0", + "partitionKey": "test_5", + "sequenceNumber": "" + } + }, + { + "awsRegion": "", + "eventID": "shardId-000000000000:", + "eventName": "aws:kinesis:record", + "eventSource": "aws:kinesis", + "eventSourceARN": "arn::kinesis::111111111111:stream/", + "eventVersion": "1.0", + "invokeIdentityArn": "arn::iam::111111111111:role/", + "kinesis": { + "approximateArrivalTimestamp": "", + "data": "eyJzaG91bGRfZmFpbCI6IHRydWV9", + "kinesisSchemaVersion": "1.0", + "partitionKey": "test_5", + "sequenceNumber": "" + } + }, + { + "awsRegion": "", + "eventID": "shardId-000000000000:", + "eventName": "aws:kinesis:record", + "eventSource": "aws:kinesis", + "eventSourceARN": "arn::kinesis::111111111111:stream/", + "eventVersion": "1.0", + "invokeIdentityArn": "arn::iam::111111111111:role/", + "kinesis": { + "approximateArrivalTimestamp": "", + "data": "eyJzaG91bGRfZmFpbCI6IHRydWV9", + "kinesisSchemaVersion": "1.0", + "partitionKey": "test_5", + "sequenceNumber": "" + } + } + ] + } + } + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_kinesis.py::TestKinesisSource::test_kinesis_report_batch_item_success_scenarios[empty_list_success]": { + "recorded-date": "11-09-2024, 17:42:41", + "recorded-content": { + "create_event_source_mapping_response": { + "BatchSize": 1, + "BisectBatchOnFunctionError": false, + "DestinationConfig": { + "OnFailure": {} + }, + "EventSourceArn": "arn::kinesis::111111111111:stream/", + "FunctionArn": "arn::lambda::111111111111:function:", + "FunctionResponseTypes": [ + "ReportBatchItemFailures" + ], + "LastModified": "", + "LastProcessingResult": "No records processed", + "MaximumBatchingWindowInSeconds": 1, + "MaximumRecordAgeInSeconds": -1, + "MaximumRetryAttempts": 2, + "ParallelizationFactor": 1, + "StartingPosition": "TRIM_HORIZON", + "State": "Creating", + "StateTransitionReason": "User action", + "TumblingWindowInSeconds": 0, + "UUID": "", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 202 + } + }, + "kinesis_events": [ + { + "Records": [ + { + "kinesis": { + "kinesisSchemaVersion": "1.0", + "partitionKey": "test", + "sequenceNumber": "", + "data": "aGVsbG8=", + "approximateArrivalTimestamp": "" + }, + "eventSource": "aws:kinesis", + "eventVersion": "1.0", + "eventID": "shardId-000000000000:", + "eventName": "aws:kinesis:record", + "invokeIdentityArn": "arn::iam::111111111111:role/", + "awsRegion": "", + "eventSourceARN": "arn::kinesis::111111111111:stream/" + } + ] + } + ] + } + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_kinesis.py::TestKinesisSource::test_kinesis_report_batch_item_success_scenarios[null_success]": { + "recorded-date": "11-09-2024, 17:44:31", + "recorded-content": { + "create_event_source_mapping_response": { + "BatchSize": 1, + "BisectBatchOnFunctionError": false, + "DestinationConfig": { + "OnFailure": {} + }, + "EventSourceArn": "arn::kinesis::111111111111:stream/", + "FunctionArn": "arn::lambda::111111111111:function:", + "FunctionResponseTypes": [ + "ReportBatchItemFailures" + ], + "LastModified": "", + "LastProcessingResult": "No records processed", + "MaximumBatchingWindowInSeconds": 1, + "MaximumRecordAgeInSeconds": -1, + "MaximumRetryAttempts": 2, + "ParallelizationFactor": 1, + "StartingPosition": "TRIM_HORIZON", + "State": "Creating", + "StateTransitionReason": "User action", + "TumblingWindowInSeconds": 0, + "UUID": "", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 202 + } + }, + "kinesis_events": [ + { + "Records": [ + { + "kinesis": { + "kinesisSchemaVersion": "1.0", + "partitionKey": "test", + "sequenceNumber": "", + "data": "aGVsbG8=", + "approximateArrivalTimestamp": "" + }, + "eventSource": "aws:kinesis", + "eventVersion": "1.0", + "eventID": "shardId-000000000000:", + "eventName": "aws:kinesis:record", + "invokeIdentityArn": "arn::iam::111111111111:role/", + "awsRegion": "", + "eventSourceARN": "arn::kinesis::111111111111:stream/" + } + ] + } + ] + } + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_kinesis.py::TestKinesisSource::test_kinesis_report_batch_item_success_scenarios[empty_dict_success]": { + "recorded-date": "11-09-2024, 17:45:43", + "recorded-content": { + "create_event_source_mapping_response": { + "BatchSize": 1, + "BisectBatchOnFunctionError": false, + "DestinationConfig": { + "OnFailure": {} + }, + "EventSourceArn": "arn::kinesis::111111111111:stream/", + "FunctionArn": "arn::lambda::111111111111:function:", + "FunctionResponseTypes": [ + "ReportBatchItemFailures" + ], + "LastModified": "", + "LastProcessingResult": "No records processed", + "MaximumBatchingWindowInSeconds": 1, + "MaximumRecordAgeInSeconds": -1, + "MaximumRetryAttempts": 2, + "ParallelizationFactor": 1, + "StartingPosition": "TRIM_HORIZON", + "State": "Creating", + "StateTransitionReason": "User action", + "TumblingWindowInSeconds": 0, + "UUID": "", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 202 + } + }, + "kinesis_events": [ + { + "Records": [ + { + "kinesis": { + "kinesisSchemaVersion": "1.0", + "partitionKey": "test", + "sequenceNumber": "", + "data": "aGVsbG8=", + "approximateArrivalTimestamp": "" + }, + "eventSource": "aws:kinesis", + "eventVersion": "1.0", + "eventID": "shardId-000000000000:", + "eventName": "aws:kinesis:record", + "invokeIdentityArn": "arn::iam::111111111111:role/", + "awsRegion": "", + "eventSourceARN": "arn::kinesis::111111111111:stream/" + } + ] + } + ] + } + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_kinesis.py::TestKinesisSource::test_kinesis_report_batch_item_success_scenarios[empty_batch_item_failure_success]": { + "recorded-date": "11-09-2024, 17:47:33", + "recorded-content": { + "create_event_source_mapping_response": { + "BatchSize": 1, + "BisectBatchOnFunctionError": false, + "DestinationConfig": { + "OnFailure": {} + }, + "EventSourceArn": "arn::kinesis::111111111111:stream/", + "FunctionArn": "arn::lambda::111111111111:function:", + "FunctionResponseTypes": [ + "ReportBatchItemFailures" + ], + "LastModified": "", + "LastProcessingResult": "No records processed", + "MaximumBatchingWindowInSeconds": 1, + "MaximumRecordAgeInSeconds": -1, + "MaximumRetryAttempts": 2, + "ParallelizationFactor": 1, + "StartingPosition": "TRIM_HORIZON", + "State": "Creating", + "StateTransitionReason": "User action", + "TumblingWindowInSeconds": 0, + "UUID": "", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 202 + } + }, + "kinesis_events": [ + { + "Records": [ + { + "kinesis": { + "kinesisSchemaVersion": "1.0", + "partitionKey": "test", + "sequenceNumber": "", + "data": "aGVsbG8=", + "approximateArrivalTimestamp": "" + }, + "eventSource": "aws:kinesis", + "eventVersion": "1.0", + "eventID": "shardId-000000000000:", + "eventName": "aws:kinesis:record", + "invokeIdentityArn": "arn::iam::111111111111:role/", + "awsRegion": "", + "eventSourceARN": "arn::kinesis::111111111111:stream/" + } + ] + } + ] + } + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_kinesis.py::TestKinesisSource::test_kinesis_report_batch_item_success_scenarios[null_batch_item_failure_success]": { + "recorded-date": "11-09-2024, 17:48:37", + "recorded-content": { + "create_event_source_mapping_response": { + "BatchSize": 1, + "BisectBatchOnFunctionError": false, + "DestinationConfig": { + "OnFailure": {} + }, + "EventSourceArn": "arn::kinesis::111111111111:stream/", + "FunctionArn": "arn::lambda::111111111111:function:", + "FunctionResponseTypes": [ + "ReportBatchItemFailures" + ], + "LastModified": "", + "LastProcessingResult": "No records processed", + "MaximumBatchingWindowInSeconds": 1, + "MaximumRecordAgeInSeconds": -1, + "MaximumRetryAttempts": 2, + "ParallelizationFactor": 1, + "StartingPosition": "TRIM_HORIZON", + "State": "Creating", + "StateTransitionReason": "User action", + "TumblingWindowInSeconds": 0, + "UUID": "", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 202 + } + }, + "kinesis_events": [ + { + "Records": [ + { + "kinesis": { + "kinesisSchemaVersion": "1.0", + "partitionKey": "test", + "sequenceNumber": "", + "data": "aGVsbG8=", + "approximateArrivalTimestamp": "" + }, + "eventSource": "aws:kinesis", + "eventVersion": "1.0", + "eventID": "shardId-000000000000:", + "eventName": "aws:kinesis:record", + "invokeIdentityArn": "arn::iam::111111111111:role/", + "awsRegion": "", + "eventSourceARN": "arn::kinesis::111111111111:stream/" + } + ] + } + ] + } + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_kinesis.py::TestKinesisSource::test_kinesis_report_batch_item_failure_scenarios[empty_string_item_identifier_failure]": { + "recorded-date": "11-09-2024, 19:12:34", + "recorded-content": { + "create_event_source_mapping_response": { + "BatchSize": 1, + "BisectBatchOnFunctionError": false, + "DestinationConfig": { + "OnFailure": { + "Destination": "arn::sqs::111111111111:" + } + }, + "EventSourceArn": "arn::kinesis::111111111111:stream/", + "FunctionArn": "arn::lambda::111111111111:function:", + "FunctionResponseTypes": [ + "ReportBatchItemFailures" + ], + "LastModified": "", + "LastProcessingResult": "No records processed", + "MaximumBatchingWindowInSeconds": 1, + "MaximumRecordAgeInSeconds": -1, + "MaximumRetryAttempts": 2, + "ParallelizationFactor": 1, + "StartingPosition": "TRIM_HORIZON", + "State": "Creating", + "StateTransitionReason": "User action", + "TumblingWindowInSeconds": 0, + "UUID": "", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 202 + } + }, + "sqs_payload": { + "Messages": [ + { + "Body": { + "requestContext": { + "requestId": "", + "functionArn": "arn::lambda::111111111111:function:", + "condition": "RetryAttemptsExhausted", + "approximateInvokeCount": 3 + }, + "responseContext": { + "statusCode": 200, + "executedVersion": "$LATEST", + "functionError": null + }, + "version": "1.0", + "timestamp": "", + "KinesisBatchInfo": { + "shardId": "shardId-000000000000", + "startSequenceNumber": "", + "endSequenceNumber": "", + "approximateArrivalOfFirstRecord": "", + "approximateArrivalOfLastRecord": "", + "batchSize": 1, + "streamArn": "arn::kinesis::111111111111:stream/" + } + }, + "MD5OfBody": "", + "MessageId": "", + "ReceiptHandle": "" + } + ], + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "kinesis_events": [ + { + "Records": [ + { + "kinesis": { + "kinesisSchemaVersion": "1.0", + "partitionKey": "test", + "sequenceNumber": "", + "data": "aGVsbG8=", + "approximateArrivalTimestamp": "" + }, + "eventSource": "aws:kinesis", + "eventVersion": "1.0", + "eventID": "shardId-000000000000:", + "eventName": "aws:kinesis:record", + "invokeIdentityArn": "arn::iam::111111111111:role/", + "awsRegion": "", + "eventSourceARN": "arn::kinesis::111111111111:stream/" + } + ] + }, + { + "Records": [ + { + "kinesis": { + "kinesisSchemaVersion": "1.0", + "partitionKey": "test", + "sequenceNumber": "", + "data": "aGVsbG8=", + "approximateArrivalTimestamp": "" + }, + "eventSource": "aws:kinesis", + "eventVersion": "1.0", + "eventID": "shardId-000000000000:", + "eventName": "aws:kinesis:record", + "invokeIdentityArn": "arn::iam::111111111111:role/", + "awsRegion": "", + "eventSourceARN": "arn::kinesis::111111111111:stream/" + } + ] + }, + { + "Records": [ + { + "kinesis": { + "kinesisSchemaVersion": "1.0", + "partitionKey": "test", + "sequenceNumber": "", + "data": "aGVsbG8=", + "approximateArrivalTimestamp": "" + }, + "eventSource": "aws:kinesis", + "eventVersion": "1.0", + "eventID": "shardId-000000000000:", + "eventName": "aws:kinesis:record", + "invokeIdentityArn": "arn::iam::111111111111:role/", + "awsRegion": "", + "eventSourceARN": "arn::kinesis::111111111111:stream/" + } + ] + } + ] + } + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_kinesis.py::TestKinesisSource::test_kinesis_report_batch_item_failure_scenarios[null_item_identifier_failure]": { + "recorded-date": "11-09-2024, 19:15:41", + "recorded-content": { + "create_event_source_mapping_response": { + "BatchSize": 1, + "BisectBatchOnFunctionError": false, + "DestinationConfig": { + "OnFailure": { + "Destination": "arn::sqs::111111111111:" + } + }, + "EventSourceArn": "arn::kinesis::111111111111:stream/", + "FunctionArn": "arn::lambda::111111111111:function:", + "FunctionResponseTypes": [ + "ReportBatchItemFailures" + ], + "LastModified": "", + "LastProcessingResult": "No records processed", + "MaximumBatchingWindowInSeconds": 1, + "MaximumRecordAgeInSeconds": -1, + "MaximumRetryAttempts": 2, + "ParallelizationFactor": 1, + "StartingPosition": "TRIM_HORIZON", + "State": "Creating", + "StateTransitionReason": "User action", + "TumblingWindowInSeconds": 0, + "UUID": "", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 202 + } + }, + "sqs_payload": { + "Messages": [ + { + "Body": { + "requestContext": { + "requestId": "", + "functionArn": "arn::lambda::111111111111:function:", + "condition": "RetryAttemptsExhausted", + "approximateInvokeCount": 3 + }, + "responseContext": { + "statusCode": 200, + "executedVersion": "$LATEST", + "functionError": null + }, + "version": "1.0", + "timestamp": "", + "KinesisBatchInfo": { + "shardId": "shardId-000000000000", + "startSequenceNumber": "", + "endSequenceNumber": "", + "approximateArrivalOfFirstRecord": "", + "approximateArrivalOfLastRecord": "", + "batchSize": 1, + "streamArn": "arn::kinesis::111111111111:stream/" + } + }, + "MD5OfBody": "", + "MessageId": "", + "ReceiptHandle": "" + } + ], + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "kinesis_events": [ + { + "Records": [ + { + "kinesis": { + "kinesisSchemaVersion": "1.0", + "partitionKey": "test", + "sequenceNumber": "", + "data": "aGVsbG8=", + "approximateArrivalTimestamp": "" + }, + "eventSource": "aws:kinesis", + "eventVersion": "1.0", + "eventID": "shardId-000000000000:", + "eventName": "aws:kinesis:record", + "invokeIdentityArn": "arn::iam::111111111111:role/", + "awsRegion": "", + "eventSourceARN": "arn::kinesis::111111111111:stream/" + } + ] + }, + { + "Records": [ + { + "kinesis": { + "kinesisSchemaVersion": "1.0", + "partitionKey": "test", + "sequenceNumber": "", + "data": "aGVsbG8=", + "approximateArrivalTimestamp": "" + }, + "eventSource": "aws:kinesis", + "eventVersion": "1.0", + "eventID": "shardId-000000000000:", + "eventName": "aws:kinesis:record", + "invokeIdentityArn": "arn::iam::111111111111:role/", + "awsRegion": "", + "eventSourceARN": "arn::kinesis::111111111111:stream/" + } + ] + }, + { + "Records": [ + { + "kinesis": { + "kinesisSchemaVersion": "1.0", + "partitionKey": "test", + "sequenceNumber": "", + "data": "aGVsbG8=", + "approximateArrivalTimestamp": "" + }, + "eventSource": "aws:kinesis", + "eventVersion": "1.0", + "eventID": "shardId-000000000000:", + "eventName": "aws:kinesis:record", + "invokeIdentityArn": "arn::iam::111111111111:role/", + "awsRegion": "", + "eventSourceARN": "arn::kinesis::111111111111:stream/" + } + ] + } + ] + } + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_kinesis.py::TestKinesisSource::test_kinesis_report_batch_item_failure_scenarios[invalid_key_foo_failure]": { + "recorded-date": "11-09-2024, 19:19:39", + "recorded-content": { + "create_event_source_mapping_response": { + "BatchSize": 1, + "BisectBatchOnFunctionError": false, + "DestinationConfig": { + "OnFailure": { + "Destination": "arn::sqs::111111111111:" + } + }, + "EventSourceArn": "arn::kinesis::111111111111:stream/", + "FunctionArn": "arn::lambda::111111111111:function:", + "FunctionResponseTypes": [ + "ReportBatchItemFailures" + ], + "LastModified": "", + "LastProcessingResult": "No records processed", + "MaximumBatchingWindowInSeconds": 1, + "MaximumRecordAgeInSeconds": -1, + "MaximumRetryAttempts": 2, + "ParallelizationFactor": 1, + "StartingPosition": "TRIM_HORIZON", + "State": "Creating", + "StateTransitionReason": "User action", + "TumblingWindowInSeconds": 0, + "UUID": "", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 202 + } + }, + "sqs_payload": { + "Messages": [ + { + "Body": { + "requestContext": { + "requestId": "", + "functionArn": "arn::lambda::111111111111:function:", + "condition": "RetryAttemptsExhausted", + "approximateInvokeCount": 3 + }, + "responseContext": { + "statusCode": 200, + "executedVersion": "$LATEST", + "functionError": null + }, + "version": "1.0", + "timestamp": "", + "KinesisBatchInfo": { + "shardId": "shardId-000000000000", + "startSequenceNumber": "", + "endSequenceNumber": "", + "approximateArrivalOfFirstRecord": "", + "approximateArrivalOfLastRecord": "", + "batchSize": 1, + "streamArn": "arn::kinesis::111111111111:stream/" + } + }, + "MD5OfBody": "", + "MessageId": "", + "ReceiptHandle": "" + } + ], + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "kinesis_events": [ + { + "Records": [ + { + "kinesis": { + "kinesisSchemaVersion": "1.0", + "partitionKey": "test", + "sequenceNumber": "", + "data": "aGVsbG8=", + "approximateArrivalTimestamp": "" + }, + "eventSource": "aws:kinesis", + "eventVersion": "1.0", + "eventID": "shardId-000000000000:", + "eventName": "aws:kinesis:record", + "invokeIdentityArn": "arn::iam::111111111111:role/", + "awsRegion": "", + "eventSourceARN": "arn::kinesis::111111111111:stream/" + } + ] + }, + { + "Records": [ + { + "kinesis": { + "kinesisSchemaVersion": "1.0", + "partitionKey": "test", + "sequenceNumber": "", + "data": "aGVsbG8=", + "approximateArrivalTimestamp": "" + }, + "eventSource": "aws:kinesis", + "eventVersion": "1.0", + "eventID": "shardId-000000000000:", + "eventName": "aws:kinesis:record", + "invokeIdentityArn": "arn::iam::111111111111:role/", + "awsRegion": "", + "eventSourceARN": "arn::kinesis::111111111111:stream/" + } + ] + }, + { + "Records": [ + { + "kinesis": { + "kinesisSchemaVersion": "1.0", + "partitionKey": "test", + "sequenceNumber": "", + "data": "aGVsbG8=", + "approximateArrivalTimestamp": "" + }, + "eventSource": "aws:kinesis", + "eventVersion": "1.0", + "eventID": "shardId-000000000000:", + "eventName": "aws:kinesis:record", + "invokeIdentityArn": "arn::iam::111111111111:role/", + "awsRegion": "", + "eventSourceARN": "arn::kinesis::111111111111:stream/" + } + ] + } + ] + } + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_kinesis.py::TestKinesisSource::test_kinesis_report_batch_item_failure_scenarios[invalid_key_foo_null_value_failure]": { + "recorded-date": "11-09-2024, 19:22:51", + "recorded-content": { + "create_event_source_mapping_response": { + "BatchSize": 1, + "BisectBatchOnFunctionError": false, + "DestinationConfig": { + "OnFailure": { + "Destination": "arn::sqs::111111111111:" + } + }, + "EventSourceArn": "arn::kinesis::111111111111:stream/", + "FunctionArn": "arn::lambda::111111111111:function:", + "FunctionResponseTypes": [ + "ReportBatchItemFailures" + ], + "LastModified": "", + "LastProcessingResult": "No records processed", + "MaximumBatchingWindowInSeconds": 1, + "MaximumRecordAgeInSeconds": -1, + "MaximumRetryAttempts": 2, + "ParallelizationFactor": 1, + "StartingPosition": "TRIM_HORIZON", + "State": "Creating", + "StateTransitionReason": "User action", + "TumblingWindowInSeconds": 0, + "UUID": "", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 202 + } + }, + "sqs_payload": { + "Messages": [ + { + "Body": { + "requestContext": { + "requestId": "", + "functionArn": "arn::lambda::111111111111:function:", + "condition": "RetryAttemptsExhausted", + "approximateInvokeCount": 3 + }, + "responseContext": { + "statusCode": 200, + "executedVersion": "$LATEST", + "functionError": null + }, + "version": "1.0", + "timestamp": "", + "KinesisBatchInfo": { + "shardId": "shardId-000000000000", + "startSequenceNumber": "", + "endSequenceNumber": "", + "approximateArrivalOfFirstRecord": "", + "approximateArrivalOfLastRecord": "", + "batchSize": 1, + "streamArn": "arn::kinesis::111111111111:stream/" + } + }, + "MD5OfBody": "", + "MessageId": "", + "ReceiptHandle": "" + } + ], + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "kinesis_events": [ + { + "Records": [ + { + "kinesis": { + "kinesisSchemaVersion": "1.0", + "partitionKey": "test", + "sequenceNumber": "", + "data": "aGVsbG8=", + "approximateArrivalTimestamp": "" + }, + "eventSource": "aws:kinesis", + "eventVersion": "1.0", + "eventID": "shardId-000000000000:", + "eventName": "aws:kinesis:record", + "invokeIdentityArn": "arn::iam::111111111111:role/", + "awsRegion": "", + "eventSourceARN": "arn::kinesis::111111111111:stream/" + } + ] + }, + { + "Records": [ + { + "kinesis": { + "kinesisSchemaVersion": "1.0", + "partitionKey": "test", + "sequenceNumber": "", + "data": "aGVsbG8=", + "approximateArrivalTimestamp": "" + }, + "eventSource": "aws:kinesis", + "eventVersion": "1.0", + "eventID": "shardId-000000000000:", + "eventName": "aws:kinesis:record", + "invokeIdentityArn": "arn::iam::111111111111:role/", + "awsRegion": "", + "eventSourceARN": "arn::kinesis::111111111111:stream/" + } + ] + }, + { + "Records": [ + { + "kinesis": { + "kinesisSchemaVersion": "1.0", + "partitionKey": "test", + "sequenceNumber": "", + "data": "aGVsbG8=", + "approximateArrivalTimestamp": "" + }, + "eventSource": "aws:kinesis", + "eventVersion": "1.0", + "eventID": "shardId-000000000000:", + "eventName": "aws:kinesis:record", + "invokeIdentityArn": "arn::iam::111111111111:role/", + "awsRegion": "", + "eventSourceARN": "arn::kinesis::111111111111:stream/" + } + ] + } + ] + } + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_kinesis.py::TestKinesisSource::test_kinesis_report_batch_item_failure_scenarios[unhandled_exception_in_function]": { + "recorded-date": "11-09-2024, 19:25:09", + "recorded-content": { + "create_event_source_mapping_response": { + "BatchSize": 1, + "BisectBatchOnFunctionError": false, + "DestinationConfig": { + "OnFailure": { + "Destination": "arn::sqs::111111111111:" + } + }, + "EventSourceArn": "arn::kinesis::111111111111:stream/", + "FunctionArn": "arn::lambda::111111111111:function:", + "FunctionResponseTypes": [ + "ReportBatchItemFailures" + ], + "LastModified": "", + "LastProcessingResult": "No records processed", + "MaximumBatchingWindowInSeconds": 1, + "MaximumRecordAgeInSeconds": -1, + "MaximumRetryAttempts": 2, + "ParallelizationFactor": 1, + "StartingPosition": "TRIM_HORIZON", + "State": "Creating", + "StateTransitionReason": "User action", + "TumblingWindowInSeconds": 0, + "UUID": "", + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 202 + } + }, + "sqs_payload": { + "Messages": [ + { + "Body": { + "requestContext": { + "requestId": "", + "functionArn": "arn::lambda::111111111111:function:", + "condition": "RetryAttemptsExhausted", + "approximateInvokeCount": 3 + }, + "responseContext": { + "statusCode": 200, + "executedVersion": "$LATEST", + "functionError": "Unhandled" + }, + "version": "1.0", + "timestamp": "", + "KinesisBatchInfo": { + "shardId": "shardId-000000000000", + "startSequenceNumber": "", + "endSequenceNumber": "", + "approximateArrivalOfFirstRecord": "", + "approximateArrivalOfLastRecord": "", + "batchSize": 1, + "streamArn": "arn::kinesis::111111111111:stream/" + } + }, + "MD5OfBody": "", + "MessageId": "", + "ReceiptHandle": "" + } + ], + "ResponseMetadata": { + "HTTPHeaders": {}, + "HTTPStatusCode": 200 + } + }, + "kinesis_events": [ + { + "Records": [ + { + "kinesis": { + "kinesisSchemaVersion": "1.0", + "partitionKey": "test", + "sequenceNumber": "", + "data": "aGVsbG8=", + "approximateArrivalTimestamp": "" + }, + "eventSource": "aws:kinesis", + "eventVersion": "1.0", + "eventID": "shardId-000000000000:", + "eventName": "aws:kinesis:record", + "invokeIdentityArn": "arn::iam::111111111111:role/", + "awsRegion": "", + "eventSourceARN": "arn::kinesis::111111111111:stream/" + } + ] + }, + { + "Records": [ + { + "kinesis": { + "kinesisSchemaVersion": "1.0", + "partitionKey": "test", + "sequenceNumber": "", + "data": "aGVsbG8=", + "approximateArrivalTimestamp": "" + }, + "eventSource": "aws:kinesis", + "eventVersion": "1.0", + "eventID": "shardId-000000000000:", + "eventName": "aws:kinesis:record", + "invokeIdentityArn": "arn::iam::111111111111:role/", + "awsRegion": "", + "eventSourceARN": "arn::kinesis::111111111111:stream/" + } + ] + }, + { + "Records": [ + { + "kinesis": { + "kinesisSchemaVersion": "1.0", + "partitionKey": "test", + "sequenceNumber": "", + "data": "aGVsbG8=", + "approximateArrivalTimestamp": "" + }, + "eventSource": "aws:kinesis", + "eventVersion": "1.0", + "eventID": "shardId-000000000000:", + "eventName": "aws:kinesis:record", + "invokeIdentityArn": "arn::iam::111111111111:role/", + "awsRegion": "", + "eventSourceARN": "arn::kinesis::111111111111:stream/" + } + ] + } + ] + } } } diff --git a/tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_kinesis.validation.json b/tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_kinesis.validation.json index b7ab3f456797e..09f17a704ab56 100644 --- a/tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_kinesis.validation.json +++ b/tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_kinesis.validation.json @@ -25,5 +25,38 @@ }, "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_kinesis.py::TestKinesisSource::test_kinesis_event_source_trim_horizon": { "last_validated_date": "2023-02-27T15:56:17+00:00" + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_kinesis.py::TestKinesisSource::test_kinesis_report_batch_item_failure_scenarios[empty_string_item_identifier_failure]": { + "last_validated_date": "2024-09-11T19:12:32+00:00" + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_kinesis.py::TestKinesisSource::test_kinesis_report_batch_item_failure_scenarios[invalid_key_foo_failure]": { + "last_validated_date": "2024-09-11T19:19:37+00:00" + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_kinesis.py::TestKinesisSource::test_kinesis_report_batch_item_failure_scenarios[invalid_key_foo_null_value_failure]": { + "last_validated_date": "2024-09-11T19:22:48+00:00" + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_kinesis.py::TestKinesisSource::test_kinesis_report_batch_item_failure_scenarios[null_item_identifier_failure]": { + "last_validated_date": "2024-09-11T19:15:38+00:00" + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_kinesis.py::TestKinesisSource::test_kinesis_report_batch_item_failure_scenarios[unhandled_exception_in_function]": { + "last_validated_date": "2024-09-11T19:25:06+00:00" + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_kinesis.py::TestKinesisSource::test_kinesis_report_batch_item_failures": { + "last_validated_date": "2024-09-11T18:00:23+00:00" + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_kinesis.py::TestKinesisSource::test_kinesis_report_batch_item_success_scenarios[empty_batch_item_failure_success]": { + "last_validated_date": "2024-09-11T17:47:31+00:00" + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_kinesis.py::TestKinesisSource::test_kinesis_report_batch_item_success_scenarios[empty_dict_success]": { + "last_validated_date": "2024-09-11T17:45:41+00:00" + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_kinesis.py::TestKinesisSource::test_kinesis_report_batch_item_success_scenarios[empty_list_success]": { + "last_validated_date": "2024-09-11T17:42:39+00:00" + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_kinesis.py::TestKinesisSource::test_kinesis_report_batch_item_success_scenarios[null_batch_item_failure_success]": { + "last_validated_date": "2024-09-11T17:48:35+00:00" + }, + "tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_kinesis.py::TestKinesisSource::test_kinesis_report_batch_item_success_scenarios[null_success]": { + "last_validated_date": "2024-09-11T17:44:29+00:00" } } diff --git a/tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_sqs.py b/tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_sqs.py index 6478d196ed140..8df5ef1183e0b 100644 --- a/tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_sqs.py +++ b/tests/aws/services/lambda_/event_source_mapping/test_lambda_integration_sqs.py @@ -13,7 +13,10 @@ from localstack.utils.strings import short_uid from localstack.utils.sync import retry from localstack.utils.testutil import check_expected_lambda_log_events_length, get_lambda_log_events -from tests.aws.services.lambda_.event_source_mapping.utils import is_old_esm, is_v2_esm +from tests.aws.services.lambda_.event_source_mapping.utils import ( + is_old_esm, + is_v2_esm, +) from tests.aws.services.lambda_.functions import FUNCTIONS_PATH, lambda_integration from tests.aws.services.lambda_.test_lambda import ( TEST_LAMBDA_PYTHON, @@ -30,20 +33,6 @@ MAX_SQS_BATCH_SIZE_FIFO = 10 -def _await_queue_size(sqs_client, queue_url: str, qsize: int, retries=10, sleep=1): - # wait for all items to appear in the queue - def _verify_event_queue_size(): - attr = "ApproximateNumberOfMessages" - _approx = int( - sqs_client.get_queue_attributes(QueueUrl=queue_url, AttributeNames=[attr])[ - "Attributes" - ][attr] - ) - assert _approx >= qsize - - retry(_verify_event_queue_size, retries=retries, sleep=sleep) - - @pytest.fixture(autouse=True) def _snapshot_transformers(snapshot): # manual transformers since we are passing SQS attributes through lambdas and back again @@ -78,7 +67,6 @@ def _snapshot_transformers(snapshot): "$..Topics", "$..MaximumRetryAttempts", "$..MaximumBatchingWindowInSeconds", - "$..FunctionResponseTypes", "$..StartingPosition", "$..StateTransitionReason", ] @@ -265,7 +253,6 @@ def test_message_body_and_attributes_passed_correctly( "$..Topics", "$..MaximumRetryAttempts", "$..MaximumBatchingWindowInSeconds", - "$..FunctionResponseTypes", "$..StartingPosition", "$..StateTransitionReason", ] @@ -429,7 +416,6 @@ def receive_dlq(): # TODO: flaky against AWS -@pytest.mark.skipif(is_v2_esm(), reason="FunctionResponseTypes not yet implemented in ESM v2") @markers.snapshot.skip_snapshot_verify( paths=[ # FIXME: we don't seem to be returning SQS FIFO sequence numbers correctly @@ -445,7 +431,6 @@ def receive_dlq(): "$..create_event_source_mapping.Topics", "$..create_event_source_mapping.MaximumRetryAttempts", "$..create_event_source_mapping.MaximumBatchingWindowInSeconds", - "$..create_event_source_mapping.FunctionResponseTypes", "$..create_event_source_mapping.StartingPosition", "$..create_event_source_mapping.StateTransitionReason", "$..create_event_source_mapping.State", @@ -578,7 +563,7 @@ def test_report_batch_item_failures( ) snapshot.match("first_invocation", first_invocation) - # check that the DQL is empty + # check that the DLQ is empty dlq_messages = aws_client.sqs.receive_message(QueueUrl=event_dlq_url) assert "Messages" not in dlq_messages or dlq_messages["Messages"] == [] @@ -693,7 +678,6 @@ def _collect_message(): snapshot.match("dlq_messages", messages) -@pytest.mark.skipif(is_v2_esm(), reason="FunctionResponseTypes not yet implemented in ESM v2") @markers.aws.validated def test_report_batch_item_failures_invalid_result_json_batch_fails( create_lambda_function, @@ -1432,3 +1416,17 @@ def test_duplicate_event_source_mappings( FunctionName=function_name_2, EventSourceArn=event_source_arn, ) + + +def _await_queue_size(sqs_client, queue_url: str, qsize: int, retries=10, sleep=1): + # wait for all items to appear in the queue + def _verify_event_queue_size(): + attr = "ApproximateNumberOfMessages" + _approx = int( + sqs_client.get_queue_attributes(QueueUrl=queue_url, AttributeNames=[attr])[ + "Attributes" + ][attr] + ) + assert _approx >= qsize + + retry(_verify_event_queue_size, retries=retries, sleep=sleep) diff --git a/tests/aws/services/lambda_/event_source_mapping/utils.py b/tests/aws/services/lambda_/event_source_mapping/utils.py index 0270a6a4ff199..d2322d29f733b 100644 --- a/tests/aws/services/lambda_/event_source_mapping/utils.py +++ b/tests/aws/services/lambda_/event_source_mapping/utils.py @@ -1,6 +1,57 @@ from localstack.config import LAMBDA_EVENT_SOURCE_MAPPING from localstack.testing.aws.util import is_aws_cloud +# For DynamoDB Streams and Kinesis: +# If the batchItemFailures array contains multiple items, Lambda uses the record with the lowest sequence number as the checkpoint. +# Lambda then retries all records starting from that checkpoint. +LAMBDA_DYNAMODB_BATCH_ITEM_FAILURE = """ +import json + +def handler(event, context): + batch_item_failures = [] + print(json.dumps(event)) + + for record in event.get("Records", []): + new_image = record["dynamodb"].get("NewImage", {}) + + # Only 1 record allowed + if new_image.get("should_fail", {}).get("BOOL", False): + batch_item_failures.append({"itemIdentifier": record["dynamodb"]["SequenceNumber"]}) + + return {"batchItemFailures": batch_item_failures} +""" + + +LAMBDA_KINESIS_BATCH_ITEM_FAILURE = """ +import json +import base64 + +def handler(event, context): + batch_item_failures = [] + print(json.dumps(event)) + + for record in event.get("Records", []): + payload = json.loads(base64.b64decode(record["kinesis"]["data"])) + + if payload.get("should_fail", False): + batch_item_failures.append({"itemIdentifier": record["kinesis"]["sequenceNumber"]}) + + return {"batchItemFailures" : batch_item_failures} +""" + +_LAMBDA_WITH_RESPONSE = """ +import json + +def handler(event, context): + print(json.dumps(event)) + return {response} +""" + + +def create_lambda_with_response(response: str) -> str: + """Creates a lambda with pre-defined response""" + return _LAMBDA_WITH_RESPONSE.format(response=response) + def is_v2_esm(): return LAMBDA_EVENT_SOURCE_MAPPING == "v2" and not is_aws_cloud()