diff --git a/packages/cdk/apps/core/app.ts b/packages/cdk/apps/core/app.ts index 858e7683..ae88c49f 100644 --- a/packages/cdk/apps/core/app.ts +++ b/packages/cdk/apps/core/app.ts @@ -42,6 +42,16 @@ if (customTagsJsonString) { customTagsMap = JSON.parse(customTagsJsonString); } +// If user specified adapter custom envs, add them to the stack parameters, so they will be persisted in SSM Parameter Store. +const customWesEnvVarsJsonString = getContextOrDefault>(app.node, "CUSTOM_WES_ENV_VARS"); +if (customWesEnvVarsJsonString) { + stackParameters.push({ + name: "customWesEnvVars", + value: customWesEnvVarsJsonString, + description: "JSON string of custom env vars to be used to pass to WES adapter", + }); +} + new CoreStack(app, `${PRODUCT_NAME}-Core`, { vpcId, bucketName, diff --git a/packages/cdk/lib/env/context-app-parameters.ts b/packages/cdk/lib/env/context-app-parameters.ts index 4f550a81..0f93d8f3 100644 --- a/packages/cdk/lib/env/context-app-parameters.ts +++ b/packages/cdk/lib/env/context-app-parameters.ts @@ -105,6 +105,10 @@ export class ContextAppParameters { * Map of custom tags to be applied to all the infrastructure in the context. */ public readonly customTags: { [key: string]: string }; + /** + * Environment variables to be passed to + */ + public readonly customWesEnvVars?: { [key: string]: string }; constructor(node: Node) { const instanceTypeStrings = getEnvStringListOrDefault(node, "BATCH_COMPUTE_INSTANCE_TYPES"); @@ -142,6 +146,12 @@ export class ContextAppParameters { } else { this.customTags = {}; } + + const customWesEnvVars = getEnvStringOrDefault(node, "CUSTOM_WES_ENV_VARS"); + + if (customWesEnvVars) { + this.customWesEnvVars = JSON.parse(customWesEnvVars); + } } public getContextBucketPath(): string { diff --git a/packages/cdk/lib/stacks/engines/cromwell-engine-construct.ts b/packages/cdk/lib/stacks/engines/cromwell-engine-construct.ts index db11f555..e605636e 100644 --- a/packages/cdk/lib/stacks/engines/cromwell-engine-construct.ts +++ b/packages/cdk/lib/stacks/engines/cromwell-engine-construct.ts @@ -69,6 +69,7 @@ export class CromwellEngineConstruct extends EngineConstruct { contextName: params.contextName, userId: params.userId, engineEndpoint: this.engine.loadBalancer.loadBalancerDnsName, + customEnvs: params.customWesEnvVars, }); this.adapterLogGroup = lambda.logGroup; @@ -130,21 +131,18 @@ export class CromwellEngineConstruct extends EngineConstruct { return engine; } - private renderAdapterLambda({ role, jobQueueArn, engineLogGroupName, projectName, contextName, userId, engineEndpoint, vpc }) { - return super.renderPythonLambda( - this, - "CromwellWesAdapterLambda", - role, - { - ENGINE_NAME: "cromwell", - ENGINE_ENDPOINT: engineEndpoint, - ENGINE_LOG_GROUP: engineLogGroupName, - JOB_QUEUE: jobQueueArn, - PROJECT_NAME: projectName, - CONTEXT_NAME: contextName, - USER_ID: userId, - }, - vpc - ); + private renderAdapterLambda({ role, jobQueueArn, engineLogGroupName, projectName, contextName, userId, engineEndpoint, vpc, customEnvs }) { + const environment = { + ...customEnvs, + ENGINE_NAME: "cromwell", + ENGINE_ENDPOINT: engineEndpoint, + ENGINE_LOG_GROUP: engineLogGroupName, + JOB_QUEUE: jobQueueArn, + PROJECT_NAME: projectName, + CONTEXT_NAME: contextName, + USER_ID: userId, + }; + + return super.renderPythonLambda(this, "CromwellWesAdapterLambda", role, environment, vpc); } } diff --git a/packages/cdk/lib/stacks/engines/miniwdl-engine-construct.ts b/packages/cdk/lib/stacks/engines/miniwdl-engine-construct.ts index 935f5b71..8573ad51 100644 --- a/packages/cdk/lib/stacks/engines/miniwdl-engine-construct.ts +++ b/packages/cdk/lib/stacks/engines/miniwdl-engine-construct.ts @@ -93,6 +93,7 @@ export class MiniwdlEngineConstruct extends EngineConstruct { jobDefinitionArn: this.miniwdlEngine.headJobDefinition.jobDefinitionArn, rootDirS3Uri: rootDirS3Uri, vpc: props.contextParameters.usePublicSubnets ? undefined : props.vpc, + customEnvs: props.contextParameters.customWesEnvVars, }); this.adapterLogGroup = lambda.logGroup; @@ -144,18 +145,15 @@ export class MiniwdlEngineConstruct extends EngineConstruct { return [this.batchHead.role, this.batchWorkers.role]; } - private renderAdapterLambda({ role, jobQueueArn, jobDefinitionArn, rootDirS3Uri, vpc }) { - return super.renderPythonLambda( - this, - "MiniWDLWesAdapterLambda", - role, - { - ENGINE_NAME: ENGINE_MINIWDL, - JOB_QUEUE: jobQueueArn, - JOB_DEFINITION: jobDefinitionArn, - OUTPUT_DIR_S3_URI: rootDirS3Uri, - }, - vpc - ); + private renderAdapterLambda({ role, jobQueueArn, jobDefinitionArn, rootDirS3Uri, vpc, customEnvs }) { + const environment = { + ...customEnvs, + ENGINE_NAME: ENGINE_MINIWDL, + JOB_QUEUE: jobQueueArn, + JOB_DEFINITION: jobDefinitionArn, + OUTPUT_DIR_S3_URI: rootDirS3Uri, + }; + + return super.renderPythonLambda(this, "MiniWDLWesAdapterLambda", role, environment, vpc); } } diff --git a/packages/cdk/lib/stacks/engines/nextflow-engine-construct.ts b/packages/cdk/lib/stacks/engines/nextflow-engine-construct.ts index b4227d73..ae012255 100644 --- a/packages/cdk/lib/stacks/engines/nextflow-engine-construct.ts +++ b/packages/cdk/lib/stacks/engines/nextflow-engine-construct.ts @@ -63,6 +63,7 @@ export class NextflowEngineConstruct extends EngineConstruct { jobDefinitionArn: this.nextflowEngine.headJobDefinition.jobDefinitionArn, engineLogGroupName: engineLogGroup.logGroupName, vpc: props.contextParameters.usePublicSubnets ? undefined : props.vpc, + customEnvs: props.contextParameters.customWesEnvVars, }); this.adapterLogGroup = lambda.logGroup; @@ -82,18 +83,15 @@ export class NextflowEngineConstruct extends EngineConstruct { }; } - private renderAdapterLambda({ role, jobQueueArn, jobDefinitionArn, engineLogGroupName, vpc }) { - return super.renderPythonLambda( - this, - "NextflowWesAdapterLambda", - role, - { - ENGINE_NAME: "nextflow", - JOB_QUEUE: jobQueueArn, - JOB_DEFINITION: jobDefinitionArn, - ENGINE_LOG_GROUP: engineLogGroupName, - }, - vpc - ); + private renderAdapterLambda({ role, jobQueueArn, jobDefinitionArn, engineLogGroupName, vpc, customEnvs }) { + const environment = { + ...customEnvs, + ENGINE_NAME: "nextflow", + JOB_QUEUE: jobQueueArn, + JOB_DEFINITION: jobDefinitionArn, + ENGINE_LOG_GROUP: engineLogGroupName, + }; + + return super.renderPythonLambda(this, "NextflowWesAdapterLambda", role, environment, vpc); } } diff --git a/packages/cdk/lib/stacks/engines/snakemake-engine-construct.ts b/packages/cdk/lib/stacks/engines/snakemake-engine-construct.ts index b7a73b51..a8e458e9 100644 --- a/packages/cdk/lib/stacks/engines/snakemake-engine-construct.ts +++ b/packages/cdk/lib/stacks/engines/snakemake-engine-construct.ts @@ -59,6 +59,7 @@ export class SnakemakeEngineConstruct extends EngineConstruct { taskQueueArn: this.batchWorkers.jobQueue.jobQueueArn, fsapId: this.snakemakeEngine.fsap.accessPointId, outputBucket: params.getEngineBucketPath(), + customEnvs: contextParameters.customWesEnvVars, }); this.adapterLogGroup = lambda.logGroup; @@ -173,22 +174,19 @@ export class SnakemakeEngineConstruct extends EngineConstruct { }); } - private renderAdapterLambda({ vpc, role, jobQueueArn, jobDefinitionArn, taskQueueArn, workflowRoleArn, fsapId, outputBucket }) { - return super.renderPythonLambda( - this, - "SnakemakeWesAdapterLambda", - role, - { - ENGINE_NAME: "snakemake", - JOB_QUEUE: jobQueueArn, - JOB_DEFINITION: jobDefinitionArn, - TASK_QUEUE: taskQueueArn, - WORKFLOW_ROLE: workflowRoleArn, - FSAP_ID: fsapId, - OUTPUT_DIR_S3_URI: outputBucket, - TIME: Date.now().toString(), - }, - vpc - ); + private renderAdapterLambda({ vpc, role, jobQueueArn, jobDefinitionArn, taskQueueArn, workflowRoleArn, fsapId, outputBucket, customEnvs }) { + const environment = { + ...customEnvs, + ENGINE_NAME: "snakemake", + JOB_QUEUE: jobQueueArn, + JOB_DEFINITION: jobDefinitionArn, + TASK_QUEUE: taskQueueArn, + WORKFLOW_ROLE: workflowRoleArn, + FSAP_ID: fsapId, + OUTPUT_DIR_S3_URI: outputBucket, + TIME: Date.now().toString(), + }; + + return super.renderPythonLambda(this, "SnakemakeWesAdapterLambda", role, environment, vpc); } } diff --git a/packages/cli/internal/pkg/aws/cdk/bootstrap.go b/packages/cli/internal/pkg/aws/cdk/bootstrap.go index 5ed4e3c6..8487ad98 100644 --- a/packages/cli/internal/pkg/aws/cdk/bootstrap.go +++ b/packages/cli/internal/pkg/aws/cdk/bootstrap.go @@ -22,6 +22,7 @@ func (client Client) Bootstrap(appDir string, context []string, executionName st // Add AGC version and custom tags to the bootstrap stack. agcVersionKey := constants.AgcVersionEnvKey + "=" customTagsKey := constants.CustomTagEnvKey + "=" + customWesEnvVarsKey := constants.CustomWesEnvVarEnvKey + "=" for _, c := range context { if strings.HasPrefix(c, agcVersionKey) { version := strings.TrimPrefix(c, agcVersionKey) @@ -38,6 +39,17 @@ func (client Client) Bootstrap(appDir string, context []string, executionName st cmdArgs = append(cmdArgs, "--tags", fmt.Sprintf("%s=%s", k, v)) } } + if strings.HasPrefix(c, customWesEnvVarsKey) { + jsonStr := strings.TrimPrefix(c, customWesEnvVarsKey) + customWesEnvVarsMap := make(map[string]interface{}) + err := json.Unmarshal([]byte(jsonStr), &customWesEnvVarsMap) + if err != nil { + return nil, err + } + for k, v := range customWesEnvVarsMap { + cmdArgs = append(cmdArgs, "--customWesEnvVars", fmt.Sprintf("%s=%s", k, v)) + } + } } cmdArgs = appendContextArguments(cmdArgs, context) diff --git a/packages/cli/internal/pkg/aws/ssm/get_common_parameter.go b/packages/cli/internal/pkg/aws/ssm/get_common_parameter.go index 799b856c..f39b8c3c 100644 --- a/packages/cli/internal/pkg/aws/ssm/get_common_parameter.go +++ b/packages/cli/internal/pkg/aws/ssm/get_common_parameter.go @@ -10,9 +10,10 @@ import ( ) const ( - parameterPrefix = "/agc/_common" - outputBucketParameter = "bucket" - customTagsParameter = "customTags" + parameterPrefix = "/agc/_common" + outputBucketParameter = "bucket" + customTagsParameter = "customTags" + customWesEnvVarsParameter = "customWesEnvVars" ) func (c *Client) GetCommonParameter(parameterSuffix string) (string, error) { @@ -40,3 +41,10 @@ func (c *Client) GetCustomTags() string { tags, _ := c.GetCommonParameter(customTagsParameter) return tags } + +func (c *Client) GetCustomWesEnvVars() string { + // Adapter custom envs may not exist, so ignore the error + + customWesEnvVars, _ := c.GetCommonParameter(customWesEnvVarsParameter) + return customWesEnvVars +} diff --git a/packages/cli/internal/pkg/aws/ssm/get_common_parameter_test.go b/packages/cli/internal/pkg/aws/ssm/get_common_parameter_test.go index 4e161822..7d033f40 100644 --- a/packages/cli/internal/pkg/aws/ssm/get_common_parameter_test.go +++ b/packages/cli/internal/pkg/aws/ssm/get_common_parameter_test.go @@ -75,3 +75,15 @@ func TestClient_GetCustomTags_ExpectedValue(t *testing.T) { actual := client.GetCustomTags() assert.Equal(t, tags, actual) } + +func TestClient_GetCustomWesEnvVars_ExpectedValue(t *testing.T) { + customWesEnvVars := "customWesEnvVars" + mockSsm := new(ssmMockClient) + client := &Client{mockSsm} + ctx := context.Background() + mockSsm.On("GetParameter", ctx, &ssm.GetParameterInput{Name: aws.String("/agc/_common/customWesEnvVars")}). + Return(&ssm.GetParameterOutput{Parameter: &types.Parameter{Value: aws.String(customWesEnvVars)}}, nil) + + actual := client.GetCustomWesEnvVars() + assert.Equal(t, customWesEnvVars, actual) +} diff --git a/packages/cli/internal/pkg/aws/ssm/interface.go b/packages/cli/internal/pkg/aws/ssm/interface.go index c0844b38..68fbabb1 100644 --- a/packages/cli/internal/pkg/aws/ssm/interface.go +++ b/packages/cli/internal/pkg/aws/ssm/interface.go @@ -10,6 +10,7 @@ type Interface interface { GetOutputBucket() (string, error) GetCommonParameter(parameterSuffix string) (string, error) GetCustomTags() string + GetCustomWesEnvVars() string } type ssmInterface interface { diff --git a/packages/cli/internal/pkg/cli/account_activate.go b/packages/cli/internal/pkg/cli/account_activate.go index 77f3f4d8..761257ca 100644 --- a/packages/cli/internal/pkg/cli/account_activate.go +++ b/packages/cli/internal/pkg/cli/account_activate.go @@ -32,6 +32,7 @@ const ( accountVpcFlag = "vpc" publicSubnetsFlag = "usePublicSubnets" accountTagsFlag = "tags" + accountCustomWesEnvVarsFlag = "customWesEnvVars" accountBucketFlagDescription = `The name of an S3 bucket that AGC will use to store its data. An autogenerated name will be used if not specified. A new bucket will be created if the bucket does not exist.` accountVpcFlagDescription = `The ID of a VPC that AGC will run in. @@ -41,6 +42,9 @@ You must enable the usePublicSubnets option in your project context if you use t accountTagsDescription = `A list of comma separated tags to be applied to all AGC resources in this account (i.e. --tags "k1=v1","k2=v2"). Each key-value pair must be quoted as shown in the example, otherwise the parsing will fail.` + customWesEnvVarsDescription = `A list of comma separated WES adapter custom env vars to be passed to WES adapter +(i.e. --customWesEnvVars "k1=v1","k2=v2","k3=[{ \"k4\": { \"k5\": \"v5\" } }]"). +Each key-value pair must be quoted as shown in the example, otherwise the parsing will fail.` cdkCoreDir = ".agc/cdk/apps/core" bucketPrefix = "agc" activateKey = "activate" @@ -48,10 +52,11 @@ otherwise the parsing will fail.` ) type accountActivateVars struct { - bucketName string - vpcId string - publicSubnets bool - customTags map[string]string + bucketName string + vpcId string + publicSubnets bool + customTags map[string]string + customWesEnvVars map[string]string } type accountActivateOpts struct { @@ -135,6 +140,14 @@ func (o accountActivateOpts) generateEnvVars() ([]string, error) { environmentVars = append(environmentVars, fmt.Sprintf("%s=%s", constants.CustomTagEnvKey, string(jsonBytes))) } + if o.customWesEnvVars != nil { + jsonBytes, err := json.Marshal(o.customWesEnvVars) + if err != nil { + return nil, err + } + environmentVars = append(environmentVars, fmt.Sprintf("%s=%s", constants.CustomWesEnvVarEnvKey, string(jsonBytes))) + } + vpcId, err := o.getVpcId() if err != nil { return nil, err @@ -236,5 +249,6 @@ Activate AGC in your AWS account with a custom S3 bucket and VPC. cmd.Flags().StringVar(&vars.vpcId, accountVpcFlag, "", accountVpcFlagDescription) cmd.Flags().BoolVar(&vars.publicSubnets, publicSubnetsFlag, false, publicSubnetsFlagDescription) cmd.Flags().StringToStringVar(&vars.customTags, accountTagsFlag, nil, accountTagsDescription) + cmd.Flags().StringToStringVar(&vars.customWesEnvVars, accountCustomWesEnvVarsFlag, nil, customWesEnvVarsDescription) return cmd } diff --git a/packages/cli/internal/pkg/cli/context/common_test.go b/packages/cli/internal/pkg/cli/context/common_test.go index 2c23b47e..691e4516 100644 --- a/packages/cli/internal/pkg/cli/context/common_test.go +++ b/packages/cli/internal/pkg/cli/context/common_test.go @@ -26,6 +26,7 @@ const ( testUserEmail = "bender@amazon.com" testUserId = "bender123" testTags = "{\"k1\":\"v1\",\"k2\":\"v2\"}" + testCustomWesEnvVars = "{\"k1\":\"v1\",\"k2\":\"v2\"}" ) var ( diff --git a/packages/cli/internal/pkg/cli/context/context_environment.go b/packages/cli/internal/pkg/cli/context/context_environment.go index 59a23cf3..99dd5cdf 100644 --- a/packages/cli/internal/pkg/cli/context/context_environment.go +++ b/packages/cli/internal/pkg/cli/context/context_environment.go @@ -11,12 +11,13 @@ const ( ) type contextEnvironment struct { - ProjectName string - ContextName string - UserId string - UserEmail string - OutputBucketName string - CustomTagsJson string + ProjectName string + ContextName string + UserId string + UserEmail string + OutputBucketName string + CustomTagsJson string + CustomWesEnvVarsJson string EngineName string FilesystemType string @@ -42,13 +43,14 @@ type contextEnvironment struct { func (input contextEnvironment) ToEnvironmentList() []string { return environmentMapToList(map[string]string{ - "PROJECT": input.ProjectName, - "CONTEXT": input.ContextName, - "USER_ID": input.UserId, - "USER_EMAIL": input.UserEmail, - "OUTPUT_BUCKET": input.OutputBucketName, - "AGC_VERSION": version.Version, - "CUSTOM_TAGS": input.CustomTagsJson, + "PROJECT": input.ProjectName, + "CONTEXT": input.ContextName, + "USER_ID": input.UserId, + "USER_EMAIL": input.UserEmail, + "OUTPUT_BUCKET": input.OutputBucketName, + "AGC_VERSION": version.Version, + "CUSTOM_TAGS": input.CustomTagsJson, + "CUSTOM_WES_ENV_VARS": input.CustomWesEnvVarsJson, "ENGINE_NAME": input.EngineName, "FILESYSTEM_TYPE": input.FilesystemType, diff --git a/packages/cli/internal/pkg/cli/context/manager.go b/packages/cli/internal/pkg/cli/context/manager.go index 1d9b903c..30764475 100644 --- a/packages/cli/internal/pkg/cli/context/manager.go +++ b/packages/cli/internal/pkg/cli/context/manager.go @@ -38,13 +38,14 @@ type baseProps struct { //nolint:structcheck type contextProps struct { - readBuckets []string - readWriteBuckets []string - outputBucket string - artifactBucket string - artifactUrl string - customTagsJson string - contextEnv contextEnvironment + readBuckets []string + readWriteBuckets []string + outputBucket string + artifactBucket string + artifactUrl string + customTagsJson string + customWesEnvVarsJson string + contextEnv contextEnvironment } //nolint:structcheck @@ -203,6 +204,14 @@ func (m *Manager) setCustomTags() { m.customTagsJson = m.Ssm.GetCustomTags() } +func (m *Manager) setCustomWesEnvVars() { + if m.err != nil { + return + } + + m.customWesEnvVarsJson = m.Ssm.GetCustomWesEnvVars() +} + func (m *Manager) setContextEnv(contextName string) { if m.err != nil { return @@ -221,6 +230,7 @@ func (m *Manager) setContextEnv(contextName string) { UserEmail: m.userEmail, OutputBucketName: m.outputBucket, CustomTagsJson: m.customTagsJson, + CustomWesEnvVarsJson: m.customWesEnvVarsJson, ArtifactBucketName: m.artifactBucket, ReadBucketArns: strings.Join(m.readBuckets, listDelimiter), ReadWriteBucketArns: strings.Join(m.readWriteBuckets, listDelimiter), diff --git a/packages/cli/internal/pkg/cli/context/manager_deploy.go b/packages/cli/internal/pkg/cli/context/manager_deploy.go index dc8ea20d..f7eeed20 100644 --- a/packages/cli/internal/pkg/cli/context/manager_deploy.go +++ b/packages/cli/internal/pkg/cli/context/manager_deploy.go @@ -58,6 +58,7 @@ func (m *Manager) setCdkConfigurationForDeployment() { m.setArtifactUrl() m.setArtifactBucket() m.setCustomTags() + m.setCustomWesEnvVars() } func (m *Manager) clearCdkContext(appDir string) { diff --git a/packages/cli/internal/pkg/cli/context/manager_deploy_test.go b/packages/cli/internal/pkg/cli/context/manager_deploy_test.go index e9330092..f137beae 100644 --- a/packages/cli/internal/pkg/cli/context/manager_deploy_test.go +++ b/packages/cli/internal/pkg/cli/context/manager_deploy_test.go @@ -44,9 +44,10 @@ func TestManager_Deploy(t *testing.T) { mockClients.ssmMock.EXPECT().GetOutputBucket().Return(testOutputBucket, nil) mockClients.ssmMock.EXPECT().GetCommonParameter("installed-artifacts/s3-root-url").Return(testArtifactBucket, nil) mockClients.ssmMock.EXPECT().GetCustomTags().Return(testTags) + mockClients.ssmMock.EXPECT().GetCustomWesEnvVars().Return(testCustomWesEnvVars) mockClients.ecrClientMock.EXPECT().VerifyImageExists(environment.CommonImages["NEXTFLOW"]).Return(nil) clearContext := mockClients.cdkMock.EXPECT().ClearContext(filepath.Join(testHomeDir, ".agc/cdk/apps/context")).Return(nil) - mockClients.cdkMock.EXPECT().DeployApp(filepath.Join(testHomeDir, ".agc/cdk/apps/context"), gomock.Len(43), testContextName3).After(clearContext).Return(mockClients.progressStream1, nil) + mockClients.cdkMock.EXPECT().DeployApp(filepath.Join(testHomeDir, ".agc/cdk/apps/context"), gomock.Len(44), testContextName3).After(clearContext).Return(mockClients.progressStream1, nil) displayProgressBar = mockClients.cdkMock.DisplayProgressBar mockClients.cdkMock.EXPECT().DisplayProgressBar(fmt.Sprintf("Deploying resources for context(s) %s", []string{testContextName3}), []cdk.ProgressStream{mockClients.progressStream1}).Return([]cdk.Result{{Outputs: []string{"some message"}, ExecutionName: testContextName3}}) return mockClients @@ -67,9 +68,10 @@ func TestManager_Deploy(t *testing.T) { mockClients.ssmMock.EXPECT().GetOutputBucket().Return(testOutputBucket, nil) mockClients.ssmMock.EXPECT().GetCommonParameter("installed-artifacts/s3-root-url").Return(testArtifactBucket, nil) mockClients.ssmMock.EXPECT().GetCustomTags().Return("") + mockClients.ssmMock.EXPECT().GetCustomWesEnvVars().Return("") mockClients.ecrClientMock.EXPECT().VerifyImageExists(environment.CommonImages["NEXTFLOW"]).Return(nil) clearContext := mockClients.cdkMock.EXPECT().ClearContext(filepath.Join(testHomeDir, ".agc/cdk/apps/context")).Return(nil) - mockClients.cdkMock.EXPECT().DeployApp(filepath.Join(testHomeDir, ".agc/cdk/apps/context"), gomock.Len(43), testContextName3).After(clearContext).Return(mockClients.progressStream1, nil) + mockClients.cdkMock.EXPECT().DeployApp(filepath.Join(testHomeDir, ".agc/cdk/apps/context"), gomock.Len(44), testContextName3).After(clearContext).Return(mockClients.progressStream1, nil) displayProgressBar = mockClients.cdkMock.DisplayProgressBar mockClients.cdkMock.EXPECT().DisplayProgressBar(fmt.Sprintf("Deploying resources for context(s) %s", []string{testContextName3}), []cdk.ProgressStream{mockClients.progressStream1}).Return([]cdk.Result{{Outputs: []string{"some message"}, ExecutionName: testContextName3}}) return mockClients @@ -91,11 +93,12 @@ func TestManager_Deploy(t *testing.T) { mockClients.ssmMock.EXPECT().GetOutputBucket().Times(2).Return(testOutputBucket, nil) mockClients.ssmMock.EXPECT().GetCommonParameter("installed-artifacts/s3-root-url").Times(2).Return(testArtifactBucket, nil) mockClients.ssmMock.EXPECT().GetCustomTags().Times(2).Return(testTags) + mockClients.ssmMock.EXPECT().GetCustomWesEnvVars().Times(2).Return(testCustomWesEnvVars) mockClients.ecrClientMock.EXPECT().VerifyImageExists(environment.CommonImages["CROMWELL"]).Times(2).Return(nil) clearContext := mockClients.cdkMock.EXPECT().ClearContext(filepath.Join(testHomeDir, ".agc/cdk/apps/context")).Return(nil) clearContext2 := mockClients.cdkMock.EXPECT().ClearContext(filepath.Join(testHomeDir, ".agc/cdk/apps/context")).Return(nil) - mockClients.cdkMock.EXPECT().DeployApp(filepath.Join(testHomeDir, ".agc/cdk/apps/context"), gomock.Len(43), testContextName1).After(clearContext).Return(mockClients.progressStream1, nil) - mockClients.cdkMock.EXPECT().DeployApp(filepath.Join(testHomeDir, ".agc/cdk/apps/context"), gomock.Len(43), testContextName2).After(clearContext2).Return(mockClients.progressStream2, nil) + mockClients.cdkMock.EXPECT().DeployApp(filepath.Join(testHomeDir, ".agc/cdk/apps/context"), gomock.Len(44), testContextName1).After(clearContext).Return(mockClients.progressStream1, nil) + mockClients.cdkMock.EXPECT().DeployApp(filepath.Join(testHomeDir, ".agc/cdk/apps/context"), gomock.Len(44), testContextName2).After(clearContext2).Return(mockClients.progressStream2, nil) displayProgressBar = mockClients.cdkMock.DisplayProgressBar expectedCdkResult := []cdk.Result{{Outputs: []string{"some message"}, ExecutionName: testContextName1}, {Outputs: []string{"some other message"}, ExecutionName: testContextName2}} mockClients.cdkMock.EXPECT().DisplayProgressBar(fmt.Sprintf("Deploying resources for context(s) %s", []string{testContextName1, testContextName2}), []cdk.ProgressStream{mockClients.progressStream1, mockClients.progressStream2}).Return(expectedCdkResult) @@ -117,6 +120,7 @@ func TestManager_Deploy(t *testing.T) { mockClients.ssmMock.EXPECT().GetOutputBucket().Return(testOutputBucket, nil) mockClients.ssmMock.EXPECT().GetCommonParameter("installed-artifacts/s3-root-url").Return(testArtifactBucket, nil) mockClients.ssmMock.EXPECT().GetCustomTags().Return(testTags) + mockClients.ssmMock.EXPECT().GetCustomWesEnvVars().Return(testCustomWesEnvVars) mockClients.cdkMock.EXPECT().ClearContext(filepath.Join(testHomeDir, ".agc/cdk/apps/context")).Return(nil) mockClients.ecrClientMock.EXPECT().VerifyImageExists(environment.CommonImages["CROMWELL"]).Return(fmt.Errorf("some error occurred")) return mockClients @@ -162,6 +166,7 @@ func TestManager_Deploy(t *testing.T) { mockClients.ssmMock.EXPECT().GetOutputBucket().Return(testOutputBucket, nil) mockClients.ssmMock.EXPECT().GetCommonParameter("installed-artifacts/s3-root-url").Return(testArtifactBucket, nil) mockClients.ssmMock.EXPECT().GetCustomTags().Return(testTags) + mockClients.ssmMock.EXPECT().GetCustomWesEnvVars().Return(testCustomWesEnvVars) mockClients.cdkMock.EXPECT().ClearContext(filepath.Join(testHomeDir, ".agc/cdk/apps/context")).Return(nil) return mockClients }, @@ -234,8 +239,9 @@ func TestManager_Deploy(t *testing.T) { mockClients.ssmMock.EXPECT().GetOutputBucket().Return(testOutputBucket, nil) mockClients.ssmMock.EXPECT().GetCommonParameter("installed-artifacts/s3-root-url").Return(testArtifactBucket, nil) mockClients.ssmMock.EXPECT().GetCustomTags().Return(testTags) + mockClients.ssmMock.EXPECT().GetCustomWesEnvVars().Return(testCustomWesEnvVars) mockClients.cdkMock.EXPECT().ClearContext(filepath.Join(testHomeDir, ".agc/cdk/apps/context")).Return(nil) - mockClients.cdkMock.EXPECT().DeployApp(filepath.Join(testHomeDir, ".agc/cdk/apps/context"), gomock.Len(43), testContextName1).Return(nil, fmt.Errorf("some context error")) + mockClients.cdkMock.EXPECT().DeployApp(filepath.Join(testHomeDir, ".agc/cdk/apps/context"), gomock.Len(44), testContextName1).Return(nil, fmt.Errorf("some context error")) mockClients.ecrClientMock.EXPECT().VerifyImageExists(environment.CommonImages["CROMWELL"]).Return(nil) return mockClients }, @@ -253,6 +259,7 @@ func TestManager_Deploy(t *testing.T) { mockClients.ssmMock.EXPECT().GetOutputBucket().Return(testOutputBucket, nil) mockClients.ssmMock.EXPECT().GetCommonParameter("installed-artifacts/s3-root-url").Return(testArtifactBucket, nil) mockClients.ssmMock.EXPECT().GetCustomTags().Return(testTags) + mockClients.ssmMock.EXPECT().GetCustomWesEnvVars().Return(testCustomWesEnvVars) mockClients.cdkMock.EXPECT().ClearContext(filepath.Join(testHomeDir, ".agc/cdk/apps/context")).Return(fmt.Errorf("failed to clear context")) return mockClients }, diff --git a/packages/cli/internal/pkg/constants/product.go b/packages/cli/internal/pkg/constants/product.go index 9f03f62a..fc454b1c 100644 --- a/packages/cli/internal/pkg/constants/product.go +++ b/packages/cli/internal/pkg/constants/product.go @@ -6,10 +6,11 @@ const ( AppTagValue = "agc" AgcVersionKey = "agc-version" - CustomTagEnvKey = "CUSTOM_TAGS" - AgcBucketNameEnvKey = "AGC_BUCKET_NAME" - CreateBucketEnvKey = "CREATE_AGC_BUCKET" - AgcVersionEnvKey = "AGC_VERSION" - VpcIdEnvKey = "VPC_ID" - PublicSubnetsEnvKey = "AGC_USE_PUBLIC_SUBNETS" + CustomTagEnvKey = "CUSTOM_TAGS" + AgcBucketNameEnvKey = "AGC_BUCKET_NAME" + CreateBucketEnvKey = "CREATE_AGC_BUCKET" + AgcVersionEnvKey = "AGC_VERSION" + VpcIdEnvKey = "VPC_ID" + PublicSubnetsEnvKey = "AGC_USE_PUBLIC_SUBNETS" + CustomWesEnvVarEnvKey = "CUSTOM_WES_ENV_VARS" ) diff --git a/packages/cli/internal/pkg/mocks/aws/mock_interfaces.go b/packages/cli/internal/pkg/mocks/aws/mock_interfaces.go index cfa3b1c0..70cc3912 100644 --- a/packages/cli/internal/pkg/mocks/aws/mock_interfaces.go +++ b/packages/cli/internal/pkg/mocks/aws/mock_interfaces.go @@ -355,6 +355,20 @@ func (mr *MockSsmClientMockRecorder) GetCustomTags() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCustomTags", reflect.TypeOf((*MockSsmClient)(nil).GetCustomTags)) } +// GetCustomWesEnvVars mocks base method. +func (m *MockSsmClient) GetCustomWesEnvVars() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetCustomWesEnvVars") + ret0, _ := ret[0].(string) + return ret0 +} + +// GetCustomWesEnvVars indicates an expected call of GetCustomWesEnvVars. +func (mr *MockSsmClientMockRecorder) GetCustomWesEnvVars() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCustomWesEnvVars", reflect.TypeOf((*MockSsmClient)(nil).GetCustomWesEnvVars)) +} + // GetOutputBucket mocks base method. func (m *MockSsmClient) GetOutputBucket() (string, error) { m.ctrl.T.Helper() diff --git a/packages/wes_adapter/amazon_genomics/wes/adapters/BatchAdapter.py b/packages/wes_adapter/amazon_genomics/wes/adapters/BatchAdapter.py index d5ac17fe..4c85cdbf 100644 --- a/packages/wes_adapter/amazon_genomics/wes/adapters/BatchAdapter.py +++ b/packages/wes_adapter/amazon_genomics/wes/adapters/BatchAdapter.py @@ -2,6 +2,7 @@ import os import typing import time +import json from abc import abstractmethod from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime @@ -163,14 +164,38 @@ def run_workflow( workflow_attachment=workflow_attachment, ) - submit_job_response = self.aws_batch.submit_job( - jobName="agc-run-workflow", - jobQueue=self.job_queue, - jobDefinition=self.job_definition, - containerOverrides={ - "command": command, - }, - ) + """ + E.g. [{ "name": "k1", "value": { "k2": 15 } }, { "name": "k3", "value": 10 }] + """ + if os.environ.get("CUSTOM_WORKFLOW_JOB_ENVIRONMENTS") is not None: + custom_environments = os.environ.get("CUSTOM_WORKFLOW_JOB_ENVIRONMENTS") + + environment = list( + map( + lambda key_value: { + "name": key_value["name"], + # Convert to string if a value is JSON + "value": json.dumps(key_value["value"]), + }, + json.loads(custom_environments), + ) + ) + + submit_job_response = self.aws_batch.submit_job( + jobName="agc-run-workflow", + jobQueue=self.job_queue, + jobDefinition=self.job_definition, + containerOverrides={"command": command, "environment": environment}, + ) + else: + submit_job_response = self.aws_batch.submit_job( + jobName="agc-run-workflow", + jobQueue=self.job_queue, + jobDefinition=self.job_definition, + containerOverrides={ + "command": command, + }, + ) return RunId(submit_job_response["jobId"]) def describe_job(self, job_id: str) -> Optional[JobDetailTypeDef]: diff --git a/site/content/en/docs/Concepts/accounts.md b/site/content/en/docs/Concepts/accounts.md index 9a7d3016..6a59ac2e 100644 --- a/site/content/en/docs/Concepts/accounts.md +++ b/site/content/en/docs/Concepts/accounts.md @@ -64,6 +64,12 @@ If you initially activated the account with `agc account activate --bucket my-ex and later invoked `agc account activate` then Amazon Genomics CLI will stop using the previous specified bucket, however the VPC will be recalled and re-used. *ALL* of the pre-existing S3 and VPC infrastructure will be retained and a new bucket will be created for use by Amazon Genomics CLI. +To pass custom environment variables into WES Adapter you can use `agc account activate --customWesEnvVars "k1=v1","k2=v2","k3=[{ \"k4\": { \"k5\": \"v5\" } }]"` + +To pass custom environment variables into a Workflow Job you can pass `CUSTOM_WORKFLOW_JOB_ENVIRONMENTS` environment variable +into WES Adapter `agc account activate --customWesEnvVars "CUSTOM_WORKFLOW_JOB_ENVIRONMENTS=[{ \"name\": \"MINIWDL__AWS__DESCRIBE_PERIOD\", \"value\": 10 }, { \"name\": \"MINIWDL__AWS__SUBMIT_PERIOD\", \"value\": 10 }]"` +so `MINIWDL__AWS__DESCRIBE_PERIOD` and `MINIWDL__AWS__SUBMIT_PERIOD` environment vars will be passed into a Workflow Job + ### `deactivate` The `deactivate` command is used to remove the core infrastructure deployed by Amazon Genomics CLI in the current region when an