Update: [AEA-5928] - add log forwarding (#314)

## Summary

- Routine Change

### Details

- send api gateway logs to csoc

Author: bencegadanyi1-nhs

.gitallowed

@@ -8,6 +8,7 @@ self.token = token
 --token="\$GITHUB-TOKEN"
 "accountId": "123456789012"
 accountId: "123456789012"
+"AWS_ACCOUNT_ID": "123456789012"
 console\.log\(`access token : \${access_token}`\)
 .*CidrBlock.*
 .*Gemfile\.lock.*

.github/scripts/fix_cdk_json.sh

@@ -61,5 +61,7 @@ fix_string_key enableBedrockLogging "${ENABLE_BEDROCK_LOGGING:-false}"
 fix_string_key slackBotToken "${SLACK_BOT_TOKEN}"
 fix_string_key slackSigningSecret "${SLACK_SIGNING_SECRET}"
 fix_string_key cfnDriftDetectionGroup "${CFN_DRIFT_DETECTION_GROUP}"
+fix_string_key csocApiGatewayDestination "arn:aws:logs:eu-west-2:693466633220:destination:api_gateway_log_destination" # CSOC API GW log destination - do not change
+fix_boolean_number_key forwardCsocLogs "${FORWARD_CSOC_LOGS}"
 fix_boolean_number_key isPullRequest "${IS_PULL_REQUEST}"
 fix_boolean_number_key runRegressionTests "${RUN_REGRESSION_TESTS}"

.github/workflows/ci.yml

@@ -70,6 +70,7 @@ jobs:
       MARK_JIRA_RELEASED: false
       CREATE_INT_RC_RELEASE_NOTES: false
       IS_PULL_REQUEST: false
+      FORWARD_CSOC_LOGS: false
       RUN_REGRESSION_TESTS: true
     secrets:
       CLOUD_FORMATION_DEPLOY_ROLE: ${{ secrets.DEV_CLOUD_FORMATION_DEPLOY_ROLE }}
@@ -99,6 +100,7 @@ jobs:
       MARK_JIRA_RELEASED: false
       CREATE_INT_RC_RELEASE_NOTES: false
       IS_PULL_REQUEST: false
+      FORWARD_CSOC_LOGS: false
       RUN_REGRESSION_TESTS: true
     secrets:
       CLOUD_FORMATION_DEPLOY_ROLE: ${{ secrets.QA_CLOUD_FORMATION_DEPLOY_ROLE }}

.github/workflows/pull_request.yml

@@ -95,6 +95,7 @@ jobs:
       MARK_JIRA_RELEASED: false
       CREATE_INT_RC_RELEASE_NOTES: false
       IS_PULL_REQUEST: true
+      FORWARD_CSOC_LOGS: false
       RUN_REGRESSION_TESTS: true
     secrets:
       CLOUD_FORMATION_DEPLOY_ROLE: ${{ secrets.DEV_CLOUD_FORMATION_DEPLOY_ROLE }}

.github/workflows/release.yml

@@ -69,6 +69,7 @@ jobs:
       MARK_JIRA_RELEASED: false
       CREATE_INT_RC_RELEASE_NOTES: false
       IS_PULL_REQUEST: false
+      FORWARD_CSOC_LOGS: false
       RUN_REGRESSION_TESTS: true
     secrets:
       CLOUD_FORMATION_DEPLOY_ROLE: ${{ secrets.DEV_CLOUD_FORMATION_DEPLOY_ROLE }}
@@ -98,6 +99,7 @@ jobs:
       MARK_JIRA_RELEASED: false
       CREATE_INT_RC_RELEASE_NOTES: false
       IS_PULL_REQUEST: false
+      FORWARD_CSOC_LOGS: false
       RUN_REGRESSION_TESTS: true
     secrets:
       CLOUD_FORMATION_DEPLOY_ROLE: ${{ secrets.QA_CLOUD_FORMATION_DEPLOY_ROLE }}
@@ -127,6 +129,7 @@ jobs:
       MARK_JIRA_RELEASED: false
       CREATE_INT_RC_RELEASE_NOTES: true
       IS_PULL_REQUEST: false
+      FORWARD_CSOC_LOGS: false
       RUN_REGRESSION_TESTS: false
     secrets:
       CLOUD_FORMATION_DEPLOY_ROLE: ${{ secrets.INT_CLOUD_FORMATION_DEPLOY_ROLE }}
@@ -156,6 +159,7 @@ jobs:
       MARK_JIRA_RELEASED: true
       CREATE_INT_RC_RELEASE_NOTES: false
       IS_PULL_REQUEST: false
+      FORWARD_CSOC_LOGS: true
       RUN_REGRESSION_TESTS: false
     secrets:
       CLOUD_FORMATION_DEPLOY_ROLE: ${{ secrets.PROD_CLOUD_FORMATION_DEPLOY_ROLE }}
@@ -167,4 +171,3 @@ jobs:
       REGRESSION_TESTS_PEM: ${{ secrets.REGRESSION_TESTS_PEM }}
       SLACK_BOT_TOKEN: ${{ secrets.PROD_SLACK_BOT_TOKEN }}
       SLACK_SIGNING_SECRET: ${{ secrets.PROD_SLACK_SIGNING_SECRET }}
-  

.github/workflows/release_all_stacks.yml

@@ -45,6 +45,9 @@ on:
       IS_PULL_REQUEST:
         type: boolean
         default: false
+      FORWARD_CSOC_LOGS:
+        type: boolean
+        required: true
     secrets:
       CLOUD_FORMATION_DEPLOY_ROLE:
         required: true
@@ -148,6 +151,7 @@ jobs:
           SLACK_BOT_TOKEN: "${{ secrets.SLACK_BOT_TOKEN }}"
           SLACK_SIGNING_SECRET: "${{ secrets.SLACK_SIGNING_SECRET }}"
           CDK_APP_NAME: ${{ inputs.CDK_APP_NAME }}
+          FORWARD_CSOC_LOGS: ${{ inputs.FORWARD_CSOC_LOGS }}
           IS_PULL_REQUEST: ${{ inputs.IS_PULL_REQUEST }}
           RUN_REGRESSION_TESTS: ${{ inputs.RUN_REGRESSION_TESTS }}
 

Makefile

@@ -117,6 +117,7 @@ cdk-synth-non-pr:
 	SLACK_SIGNING_SECRET=dummy_secret \
 	LOG_RETENTION_IN_DAYS=30 \
 	LOG_LEVEL=debug \
+	FORWARD_CSOC_LOGS=false \
 	RUN_REGRESSION_TESTS=true \
 		 ./.github/scripts/fix_cdk_json.sh .local_config/epsam.config.json
 	CONFIG_FILE_NAME=.local_config/epsam.config.json npx cdk synth \
@@ -135,6 +136,7 @@ cdk-synth-pr:
 	SLACK_SIGNING_SECRET=dummy_secret \
 	LOG_RETENTION_IN_DAYS=30 \
 	LOG_LEVEL=debug \
+	FORWARD_CSOC_LOGS=false \
 	RUN_REGRESSION_TESTS=true \
 		 ./.github/scripts/fix_cdk_json.sh .local_config/epsam.config.json
 	CONFIG_FILE_NAME=.local_config/epsam.config.json npx cdk synth \

packages/cdk/constructs/RestApiGateway.ts

@@ -27,6 +27,8 @@ export interface RestApiGatewayProps {
   readonly logRetentionInDays: number
   readonly trustStoreKey: string
   readonly truststoreVersion: string
+  readonly forwardCsocLogs: boolean
+  readonly csocApiGatewayDestination: string
 }
 
 export class RestApiGateway extends Construct {
@@ -69,6 +71,15 @@ export class RestApiGateway extends Construct {
 
     })
 
+    if (props.forwardCsocLogs) {
+      new CfnSubscriptionFilter(this, "ApiGatewayAccessLogsCSOCSubscriptionFilter", {
+        destinationArn: props.csocApiGatewayDestination,
+        filterPattern: "",
+        logGroupName: logGroup.logGroupName,
+        roleArn: splunkSubscriptionFilterRole.roleArn
+      })
+    }
+
     const certificate = new Certificate(this, "Certificate", {
       domainName: serviceDomainName,
       validation: CertificateValidation.fromDns(hostedZone)

packages/cdk/constructs/RestApiGateway/accessLogFormat.ts

@@ -16,6 +16,7 @@ export const accessLogFormat = () => {
     responseLatency: "$context.responseLatency",
     responseLength: "$context.responseLength",
     domainName: "$context.domainName",
+    api_key: "$context.identity.apiKey",
     identity: {
       sourceIp: "$context.identity.sourceIp",
       userAgent: "$context.identity.userAgent",

packages/cdk/resources/Apis.ts

@@ -8,6 +8,8 @@ export interface ApisProps {
   readonly stackName: string
   readonly logRetentionInDays: number
   functions: {[key: string]: LambdaFunction}
+  readonly forwardCsocLogs: boolean
+  readonly csocApiGatewayDestination: string
 }
 
 export class Apis extends Construct {
@@ -21,7 +23,9 @@ export class Apis extends Construct {
       stackName: props.stackName,
       logRetentionInDays: props.logRetentionInDays,
       trustStoreKey: "unused",
-      truststoreVersion: "unused"
+      truststoreVersion: "unused",
+      forwardCsocLogs: props.forwardCsocLogs,
+      csocApiGatewayDestination: props.csocApiGatewayDestination
     })
     // Create /slack resource path
     const slackResource = apiGateway.api.root.addResource("slack")