From 1f409b238f92c4d4d06b459f4ec40a129ae4e7a3 Mon Sep 17 00:00:00 2001 From: madhavi Date: Wed, 24 Apr 2024 22:48:18 +0530 Subject: [PATCH 1/7] 'branch-and-folder-created' --- lambda-sftp-connector-stepfunction/README.md | 60 +++++++++++++++++++ .../example-pattern.json | 59 ++++++++++++++++++ lambda-sftp-connector-stepfunction/src/app.js | 10 ++++ .../template.yaml | 16 +++++ sftp-madhavi-PEM-key.pub | 0 5 files changed, 145 insertions(+) create mode 100644 lambda-sftp-connector-stepfunction/README.md create mode 100644 lambda-sftp-connector-stepfunction/example-pattern.json create mode 100644 lambda-sftp-connector-stepfunction/src/app.js create mode 100644 lambda-sftp-connector-stepfunction/template.yaml create mode 100644 sftp-madhavi-PEM-key.pub diff --git a/lambda-sftp-connector-stepfunction/README.md b/lambda-sftp-connector-stepfunction/README.md new file mode 100644 index 000000000..e67e86d46 --- /dev/null +++ b/lambda-sftp-connector-stepfunction/README.md @@ -0,0 +1,60 @@ +# AWS Service 1 to AWS Service 2 + +This pattern << explain usage >> + +Learn more about this pattern at Serverless Land Patterns: << Add the live URL here >> + +Important: this application uses various AWS services and there are costs associated with these services after the Free Tier usage - please see the [AWS Pricing page](https://aws.amazon.com/pricing/) for details. You are responsible for any AWS costs incurred. No warranty is implied in this example. + +## Requirements + +* [Create an AWS account](https://portal.aws.amazon.com/gp/aws/developer/registration/index.html) if you do not already have one and log in. The IAM user that you use must have sufficient permissions to make necessary AWS service calls and manage AWS resources. +* [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html) installed and configured +* [Git Installed](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) +* [AWS Serverless Application Model](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/serverless-sam-cli-install.html) (AWS SAM) installed + +## Deployment Instructions + +1. Create a new directory, navigate to that directory in a terminal and clone the GitHub repository: + ``` + git clone https://github.com/aws-samples/serverless-patterns + ``` +1. Change directory to the pattern directory: + ``` + cd _patterns-model + ``` +1. From the command line, use AWS SAM to deploy the AWS resources for the pattern as specified in the template.yml file: + ``` + sam deploy --guided + ``` +1. During the prompts: + * Enter a stack name + * Enter the desired AWS Region + * Allow SAM CLI to create IAM roles with the required permissions. + + Once you have run `sam deploy --guided` mode once and saved arguments to a configuration file (samconfig.toml), you can use `sam deploy` in future to use these defaults. + +1. Note the outputs from the SAM deployment process. These contain the resource names and/or ARNs which are used for testing. + +## How it works + +Explain how the service interaction works. + +## Testing + +Provide steps to trigger the integration and show what should be observed if successful. + +## Cleanup + +1. Delete the stack + ```bash + aws cloudformation delete-stack --stack-name STACK_NAME + ``` +1. Confirm the stack has been deleted + ```bash + aws cloudformation list-stacks --query "StackSummaries[?contains(StackName,'STACK_NAME')].StackStatus" + ``` +---- +Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. + +SPDX-License-Identifier: MIT-0 \ No newline at end of file diff --git a/lambda-sftp-connector-stepfunction/example-pattern.json b/lambda-sftp-connector-stepfunction/example-pattern.json new file mode 100644 index 000000000..b7012bdfa --- /dev/null +++ b/lambda-sftp-connector-stepfunction/example-pattern.json @@ -0,0 +1,59 @@ +{ + "title": "Step Functions to Athena", + "description": "Create a Step Functions workflow to query Amazon Athena.", + "language": "Python", + "level": "200", + "framework": "CDK", + "introBox": { + "headline": "How it works", + "text": [ + "This sample project demonstrates how to use an AWS Step Functions state machine to query Athena and get the results. This pattern is leveraging the native integration between these 2 services which means only JSON-based, structured language is used to define the implementation.", + "With Amazon Athena you can get up to 1000 results per invocation of the GetQueryResults method and this is the reason why the Step Function has a loop to get more results. The results are sent to a Map which can be configured to handle (the DoSomething state) the items in parallel or one by one by modifying the max_concurrency parameter.", + "This pattern deploys one Step Functions, two S3 Buckets, one Glue table and one Glue database." + ] + }, + "gitHub": { + "template": { + "repoURL": "https://github.com/aws-samples/serverless-patterns/tree/main/sfn-athena-cdk-python", + "templateURL": "serverless-patterns/sfn-athena-cdk-python", + "projectFolder": "sfn-athena-cdk-python", + "templateFile": "sfn_athena_cdk_python_stack.py" + } + }, + "resources": { + "bullets": [ + { + "text": "Call Athena with Step Functions", + "link": "https://docs.aws.amazon.com/step-functions/latest/dg/connect-athena.html" + }, + { + "text": "Amazon Athena - Serverless Interactive Query Service", + "link": "https://aws.amazon.com/athena/" + } + ] + }, + "deploy": { + "text": [ + "sam deploy" + ] + }, + "testing": { + "text": [ + "See the GitHub repo for detailed testing instructions." + ] + }, + "cleanup": { + "text": [ + "Delete the stack: cdk delete." + ] + }, + "authors": [ + { + "name": "Your name", + "image": "link-to-your-photo.jpg", + "bio": "Your bio.", + "linkedin": "linked-in-ID", + "twitter": "twitter-handle" + } + ] +} diff --git a/lambda-sftp-connector-stepfunction/src/app.js b/lambda-sftp-connector-stepfunction/src/app.js new file mode 100644 index 000000000..cb3c4d9c1 --- /dev/null +++ b/lambda-sftp-connector-stepfunction/src/app.js @@ -0,0 +1,10 @@ +/*! Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: MIT-0 + */ + +'use strict' + +exports.handler = async (event) => { + // Lambda handler code + console.log(JSON.stringify(event, 0, null)) +} \ No newline at end of file diff --git a/lambda-sftp-connector-stepfunction/template.yaml b/lambda-sftp-connector-stepfunction/template.yaml new file mode 100644 index 000000000..269f82e41 --- /dev/null +++ b/lambda-sftp-connector-stepfunction/template.yaml @@ -0,0 +1,16 @@ +AWSTemplateFormatVersion: '2010-09-09' +Transform: AWS::Serverless-2016-10-31 +Description: Serverless patterns - Service to Service description + +# Comment on each global +Globals: + + +# Comment each resource section to explain usage +Resources: + + +# List all common outputs for usage +Outputs: + + diff --git a/sftp-madhavi-PEM-key.pub b/sftp-madhavi-PEM-key.pub new file mode 100644 index 000000000..e69de29bb From 504d45dfcbc55cc97a76cbdb824485df8ff3857a Mon Sep 17 00:00:00 2001 From: madhavi Date: Thu, 25 Apr 2024 16:22:16 +0530 Subject: [PATCH 2/7] 'updated-readme-and-examplepattern' --- .../example-pattern.json | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lambda-sftp-connector-stepfunction/example-pattern.json b/lambda-sftp-connector-stepfunction/example-pattern.json index b7012bdfa..41d53116e 100644 --- a/lambda-sftp-connector-stepfunction/example-pattern.json +++ b/lambda-sftp-connector-stepfunction/example-pattern.json @@ -1,23 +1,23 @@ { - "title": "Step Functions to Athena", - "description": "Create a Step Functions workflow to query Amazon Athena.", + "title": "SFTP Connector, Lambda, Step Functions to periodically transfer files from SFTP server to S3", + "description": "Create a Step Functions workflow to run list and transfer files from SFTP server using SFTP Connector, Lambda.", "language": "Python", "level": "200", "framework": "CDK", "introBox": { "headline": "How it works", "text": [ - "This sample project demonstrates how to use an AWS Step Functions state machine to query Athena and get the results. This pattern is leveraging the native integration between these 2 services which means only JSON-based, structured language is used to define the implementation.", - "With Amazon Athena you can get up to 1000 results per invocation of the GetQueryResults method and this is the reason why the Step Function has a loop to get more results. The results are sent to a Map which can be configured to handle (the DoSomething state) the items in parallel or one by one by modifying the max_concurrency parameter.", - "This pattern deploys one Step Functions, two S3 Buckets, one Glue table and one Glue database." + "This sample project demonstrates how to use an AWS Step Functions state machine to sequentially execute Lambda functions to create SFTP connector jobs to transfer files. This pattern is leveraging the native integration between these 2 services which means only JSON-based, structured language is used to define the implementation.", + "With AWS Transfer SFTP Connector you can retrieve list of directories, subdirectories and files from your remote SFTP server. It can also transfer these files to S3. The Step Function is used to retrieve file list in first step and transfer those files in following step. The failure events are sent to EventBridge and SQS. A Lambda function retrieve these events and retries file transfer by creating new transfer job with SFTP connector.", + "This pattern deploys one Step Functions, one SFTP Connector, three Lambda functions, one S3 bucket and three EventBridge events." ] }, "gitHub": { "template": { - "repoURL": "https://github.com/aws-samples/serverless-patterns/tree/main/sfn-athena-cdk-python", - "templateURL": "serverless-patterns/sfn-athena-cdk-python", - "projectFolder": "sfn-athena-cdk-python", - "templateFile": "sfn_athena_cdk_python_stack.py" + "repoURL": "https://github.com/aws-samples/serverless-patterns/tree/main/lambda-sftp-connector-stepfunction", + "templateURL": "serverless-patterns/lambda-sftp-connector-stepfunction", + "projectFolder": "slambda-sftp-connector-stepfunction", + "templateFile": "lambda-sftp-connector-stepfunction_stack.py" } }, "resources": { From af354b293ad43a54c0c37eaf61347bf46f86e307 Mon Sep 17 00:00:00 2001 From: madhavi Date: Thu, 25 Apr 2024 16:23:48 +0530 Subject: [PATCH 3/7] 'updated-readme-and-examplepattern' --- lambda-sftp-connector-stepfunction/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lambda-sftp-connector-stepfunction/README.md b/lambda-sftp-connector-stepfunction/README.md index e67e86d46..07f6391f7 100644 --- a/lambda-sftp-connector-stepfunction/README.md +++ b/lambda-sftp-connector-stepfunction/README.md @@ -1,6 +1,6 @@ -# AWS Service 1 to AWS Service 2 +# SFTP Server to Amazon S3 -This pattern << explain usage >> +This pattern shows how to use AWS Transfer Family SFTP connector in Lambda function to transfer files from SFTP server which can be running anywhere. Step functions is used to orchastrate the execution flow of 2 Lambda functions. EventBridge is used to schedule periodic execution of the Step function. The Lambda functions retrieves the list of files created or updated after the last execution of transfer jobs. These files are transferred from source SFTP server to S3. It uses SFTP connector to retrieve the file list and to transfer the files. Retrying file transfer is implemented in Lambda function. EventBridge and SQS are used to log failure events from the file transfer jobs. These events are processed by Lambda function to retry file transfer. Learn more about this pattern at Serverless Land Patterns: << Add the live URL here >> @@ -21,7 +21,7 @@ Important: this application uses various AWS services and there are costs associ ``` 1. Change directory to the pattern directory: ``` - cd _patterns-model + cd lambda-sftp-connector-stepfunction ``` 1. From the command line, use AWS SAM to deploy the AWS resources for the pattern as specified in the template.yml file: ``` @@ -42,7 +42,7 @@ Explain how the service interaction works. ## Testing -Provide steps to trigger the integration and show what should be observed if successful. +Go to EventBridge schedule and check the configured time of Step functions execution. You may manually execute the Step functions. Once the execution is complete, check S3 buckets. All files present in the source SFTP server are transferred to given S3 bucket. ## Cleanup From a390a68f66d416c5f9589b2d8abad9d115561f6c Mon Sep 17 00:00:00 2001 From: MadhaviWatve Date: Fri, 26 Apr 2024 13:54:26 +0530 Subject: [PATCH 4/7] Update README.md --- lambda-sftp-connector-stepfunction/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lambda-sftp-connector-stepfunction/README.md b/lambda-sftp-connector-stepfunction/README.md index 07f6391f7..8aa38fe98 100644 --- a/lambda-sftp-connector-stepfunction/README.md +++ b/lambda-sftp-connector-stepfunction/README.md @@ -1,6 +1,6 @@ # SFTP Server to Amazon S3 -This pattern shows how to use AWS Transfer Family SFTP connector in Lambda function to transfer files from SFTP server which can be running anywhere. Step functions is used to orchastrate the execution flow of 2 Lambda functions. EventBridge is used to schedule periodic execution of the Step function. The Lambda functions retrieves the list of files created or updated after the last execution of transfer jobs. These files are transferred from source SFTP server to S3. It uses SFTP connector to retrieve the file list and to transfer the files. Retrying file transfer is implemented in Lambda function. EventBridge and SQS are used to log failure events from the file transfer jobs. These events are processed by Lambda function to retry file transfer. +This pattern shows how to use AWS Transfer Family SFTP connector to transfer files from SFTP server which can be running anywhere to Amazon S3. EventBridge is used to schedule periodic execution of the Step function. The Step functions orchastrates the workflow. It invokes a Lambda function to retrieve the list of files created or updated after the last execution, followed by another Lambda function which creates file transfer jobs in SFTP connector. SFTP Connector transfers these files source SFTP server to S3. The status of each transfer job is pushed to DynamoDB using EventBridge. Retry mechanism for failed files is implemented using Lambda. Learn more about this pattern at Serverless Land Patterns: << Add the live URL here >> @@ -57,4 +57,4 @@ Go to EventBridge schedule and check the configured time of Step functions execu ---- Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. -SPDX-License-Identifier: MIT-0 \ No newline at end of file +SPDX-License-Identifier: MIT-0 From ca2ca371246a4de1b964b3ebce49ce4148370f07 Mon Sep 17 00:00:00 2001 From: MadhaviWatve Date: Fri, 26 Apr 2024 13:56:30 +0530 Subject: [PATCH 5/7] Update README.md --- lambda-sftp-connector-stepfunction/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambda-sftp-connector-stepfunction/README.md b/lambda-sftp-connector-stepfunction/README.md index 8aa38fe98..6ead9f877 100644 --- a/lambda-sftp-connector-stepfunction/README.md +++ b/lambda-sftp-connector-stepfunction/README.md @@ -2,7 +2,7 @@ This pattern shows how to use AWS Transfer Family SFTP connector to transfer files from SFTP server which can be running anywhere to Amazon S3. EventBridge is used to schedule periodic execution of the Step function. The Step functions orchastrates the workflow. It invokes a Lambda function to retrieve the list of files created or updated after the last execution, followed by another Lambda function which creates file transfer jobs in SFTP connector. SFTP Connector transfers these files source SFTP server to S3. The status of each transfer job is pushed to DynamoDB using EventBridge. Retry mechanism for failed files is implemented using Lambda. -Learn more about this pattern at Serverless Land Patterns: << Add the live URL here >> +Learn more about this pattern at Serverless Land Patterns: https://serverlessland.com/patterns/lambda-sftp-connector-stepfunction Important: this application uses various AWS services and there are costs associated with these services after the Free Tier usage - please see the [AWS Pricing page](https://aws.amazon.com/pricing/) for details. You are responsible for any AWS costs incurred. No warranty is implied in this example. From 3e7cc7fdc1610a2abed3eb2baedca1f507dfe89b Mon Sep 17 00:00:00 2001 From: MadhaviWatve Date: Fri, 26 Apr 2024 14:03:18 +0530 Subject: [PATCH 6/7] Update example-pattern.json --- lambda-sftp-connector-stepfunction/example-pattern.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lambda-sftp-connector-stepfunction/example-pattern.json b/lambda-sftp-connector-stepfunction/example-pattern.json index 41d53116e..1d55f1ff7 100644 --- a/lambda-sftp-connector-stepfunction/example-pattern.json +++ b/lambda-sftp-connector-stepfunction/example-pattern.json @@ -1,13 +1,13 @@ { "title": "SFTP Connector, Lambda, Step Functions to periodically transfer files from SFTP server to S3", - "description": "Create a Step Functions workflow to run list and transfer files from SFTP server using SFTP Connector, Lambda.", + "description": "Create a Step Functions for orchastration and Lambda functions to execute the business logic. Use EventBridge for scheduling and other events mnagament.", "language": "Python", "level": "200", "framework": "CDK", "introBox": { "headline": "How it works", "text": [ - "This sample project demonstrates how to use an AWS Step Functions state machine to sequentially execute Lambda functions to create SFTP connector jobs to transfer files. This pattern is leveraging the native integration between these 2 services which means only JSON-based, structured language is used to define the implementation.", + "This sample project demonstrates how to use an AWS Step Functions state machine to sequentially execute Lambda functions to create SFTP connector jobs to transfer files. This pattern is leveraging .", "With AWS Transfer SFTP Connector you can retrieve list of directories, subdirectories and files from your remote SFTP server. It can also transfer these files to S3. The Step Function is used to retrieve file list in first step and transfer those files in following step. The failure events are sent to EventBridge and SQS. A Lambda function retrieve these events and retries file transfer by creating new transfer job with SFTP connector.", "This pattern deploys one Step Functions, one SFTP Connector, three Lambda functions, one S3 bucket and three EventBridge events." ] From 54e5618fb24d7f7ed023f0c7b469a75b8ef320d1 Mon Sep 17 00:00:00 2001 From: MadhaviWatve Date: Mon, 29 Apr 2024 18:01:01 +0530 Subject: [PATCH 7/7] Update README.md --- lambda-sftp-connector-stepfunction/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambda-sftp-connector-stepfunction/README.md b/lambda-sftp-connector-stepfunction/README.md index 6ead9f877..b335045f7 100644 --- a/lambda-sftp-connector-stepfunction/README.md +++ b/lambda-sftp-connector-stepfunction/README.md @@ -1,6 +1,6 @@ # SFTP Server to Amazon S3 -This pattern shows how to use AWS Transfer Family SFTP connector to transfer files from SFTP server which can be running anywhere to Amazon S3. EventBridge is used to schedule periodic execution of the Step function. The Step functions orchastrates the workflow. It invokes a Lambda function to retrieve the list of files created or updated after the last execution, followed by another Lambda function which creates file transfer jobs in SFTP connector. SFTP Connector transfers these files source SFTP server to S3. The status of each transfer job is pushed to DynamoDB using EventBridge. Retry mechanism for failed files is implemented using Lambda. +This pattern shows how to use AWS Transfer Family SFTP connector to transfer files from SFTP server to Amazon S3. EventBridge is used to schedule periodic execution of the Step functions. The Step functions orchastrates the workflow. It invokes a Lambda function to retrieve the list of files created or updated after the previous execution of the workflow, followed by another Lambda function which creates file transfer jobs in SFTP connector. SFTP Connector transfers these files from source SFTP server to S3. The status of each transfer job is pushed to DynamoDB using EventBridge and Lambda function. Retry mechanism for failed files is implemented using Lambda. Learn more about this pattern at Serverless Land Patterns: https://serverlessland.com/patterns/lambda-sftp-connector-stepfunction