diff --git a/packages/cdk/lib/constructs/engines/miniwdl/miniwdl-engine.ts b/packages/cdk/lib/constructs/engines/miniwdl/miniwdl-engine.ts index 21b42c1a..316d97d6 100644 --- a/packages/cdk/lib/constructs/engines/miniwdl/miniwdl-engine.ts +++ b/packages/cdk/lib/constructs/engines/miniwdl/miniwdl-engine.ts @@ -12,13 +12,14 @@ export interface MiniWdlEngineProps extends EngineProps { readonly engineBatch: Batch; readonly workerBatch: Batch; readonly iops?: Size; + readonly vcpus?: number; + readonly engineMemoryMiB?: number; } const MINIWDL_IMAGE_DESIGNATION = "miniwdl"; export class MiniWdlEngine extends Engine { readonly headJobDefinition: JobDefinition; - private readonly engineMemoryMiB = 4096; private readonly volumeName = "efs"; readonly fileSystem: FileSystem; @@ -41,7 +42,8 @@ export class MiniWdlEngine extends Engine { logGroup: this.logGroup, platformCapabilities: [PlatformCapabilities.FARGATE], container: { - memoryLimitMiB: this.engineMemoryMiB, + vcpus: props.vcpus || 1, + memoryLimitMiB: props.engineMemoryMiB || 4096, jobRole: engineBatch.role, executionRole: engineBatch.role, image: createEcrImage(this, MINIWDL_IMAGE_DESIGNATION), diff --git a/packages/cdk/lib/constructs/engines/nextflow/nextflow-engine.ts b/packages/cdk/lib/constructs/engines/nextflow/nextflow-engine.ts index a565946d..9709c280 100644 --- a/packages/cdk/lib/constructs/engines/nextflow/nextflow-engine.ts +++ b/packages/cdk/lib/constructs/engines/nextflow/nextflow-engine.ts @@ -8,6 +8,8 @@ import { Engine, EngineProps } from "../engine"; export interface NextflowEngineProps extends EngineProps { readonly jobQueueArn: string; readonly taskRole: IRole; + readonly vcpus?: number; + readonly engineMemoryMiB?: number; } const NEXTFLOW_IMAGE_DESIGNATION = "nextflow"; @@ -21,6 +23,8 @@ export class NextflowEngine extends Engine { this.headJobDefinition = new EngineJobDefinition(this, "NexflowHeadJobDef", { logGroup: this.logGroup, container: { + vcpus: props.vcpus, + memoryLimitMiB: props.engineMemoryMiB, jobRole: props.taskRole, image: createEcrImage(this, NEXTFLOW_IMAGE_DESIGNATION), command: [], diff --git a/packages/cdk/lib/constructs/engines/snakemake/snakemake-engine.ts b/packages/cdk/lib/constructs/engines/snakemake/snakemake-engine.ts index cca91f53..75e6290b 100644 --- a/packages/cdk/lib/constructs/engines/snakemake/snakemake-engine.ts +++ b/packages/cdk/lib/constructs/engines/snakemake/snakemake-engine.ts @@ -12,6 +12,8 @@ export interface SnakemakeEngineProps extends EngineProps { readonly engineBatch: Batch; readonly workerBatch: Batch; readonly iops?: Size; + readonly vcpus?: number; + readonly engineMemoryMiB?: number; } const SNAKEMAKE_IMAGE_DESIGNATION = "snakemake"; @@ -19,7 +21,6 @@ const SNAKEMAKE_IMAGE_DESIGNATION = "snakemake"; export class SnakemakeEngine extends Engine { readonly headJobDefinition: JobDefinition; private readonly volumeName = "efs"; - private readonly engineMemoryMiB = 4096; public readonly fsap: AccessPoint; public readonly fileSystem: FileSystem; @@ -41,7 +42,8 @@ export class SnakemakeEngine extends Engine { logGroup: this.logGroup, platformCapabilities: [PlatformCapabilities.FARGATE], container: { - memoryLimitMiB: this.engineMemoryMiB, + vcpus: props.vcpus || 1, + memoryLimitMiB: props.engineMemoryMiB || 4096, jobRole: engineBatch.role, executionRole: engineBatch.role, image: createEcrImage(this, SNAKEMAKE_IMAGE_DESIGNATION), diff --git a/packages/cdk/lib/env/context-app-parameters.ts b/packages/cdk/lib/env/context-app-parameters.ts index 0ce1447d..d8801e24 100644 --- a/packages/cdk/lib/env/context-app-parameters.ts +++ b/packages/cdk/lib/env/context-app-parameters.ts @@ -87,6 +87,14 @@ export class ContextAppParameters { * The types of EC2 instances that may be launched in the compute environment. */ public readonly instanceTypes?: InstanceType[]; + /** + * The maximum number of Amazon EC2 vCPUs that an environment can reach for the nextflow engine. + */ + public readonly vCpus?: number; + /** + * The maximum memory limit that an environment can reach for the nextflow engine. + */ + public readonly memoryLimitMiB?: number; /** * If true, put EC2 instances into public subnets instead of private subnets. * This allows you to obtain significantly lower ongoing costs if used in conjunction with the usePublicSubnets option @@ -133,6 +141,9 @@ export class ContextAppParameters { this.requestSpotInstances = getEnvBoolOrDefault(node, "REQUEST_SPOT_INSTANCES", false)!; this.instanceTypes = instanceTypeStrings ? instanceTypeStrings.map((instanceType) => new InstanceType(instanceType.trim())) : undefined; + this.vCpus = getEnvNumber(node, "V_CPUS"); + this.memoryLimitMiB = getEnvNumber(node, "MEMORY_LIMIT_MIB"); + this.usePublicSubnets = getEnvBoolOrDefault(node, "PUBLIC_SUBNETS", false); this.agcVersion = getEnvString(node, "AGC_VERSION"); diff --git a/packages/cdk/lib/stacks/engines/cromwell-engine-construct.ts b/packages/cdk/lib/stacks/engines/cromwell-engine-construct.ts index eed30d35..7ec67a2b 100644 --- a/packages/cdk/lib/stacks/engines/cromwell-engine-construct.ts +++ b/packages/cdk/lib/stacks/engines/cromwell-engine-construct.ts @@ -14,6 +14,7 @@ import { CromwellEngineRole } from "../../roles/cromwell-engine-role"; import { CromwellAdapterRole } from "../../roles/cromwell-adapter-role"; import { IJobQueue } from "@aws-cdk/aws-batch-alpha"; import { Construct } from "constructs"; +import {ContextAppParameters} from "../../env"; export interface CromwellEngineConstructProps extends EngineOptions { /** @@ -57,7 +58,7 @@ export class CromwellEngineConstruct extends EngineConstruct { }); // TODO: Move log group creation into service construct and make it a property - this.engine = this.getEngineServiceDefinition(props.vpc, props.subnets, engineContainer, this.engineLogGroup); + this.engine = this.getEngineServiceDefinition(props.vpc, props.subnets, engineContainer, this.engineLogGroup, params); this.adapterLogGroup = new LogGroup(this, "AdapterLogGroup"); const lambda = this.renderAdapterLambda({ @@ -89,7 +90,13 @@ export class CromwellEngineConstruct extends EngineConstruct { }; } - private getEngineServiceDefinition(vpc: IVpc, subnets: SubnetSelection, serviceContainer: ServiceContainer, logGroup: ILogGroup) { + private getEngineServiceDefinition( + vpc: IVpc, + subnets: SubnetSelection, + serviceContainer: ServiceContainer, + logGroup: ILogGroup, + params: ContextAppParameters + ) { const id = "Engine"; const fileSystem = new FileSystem(this, "EngineFileSystem", { vpc, @@ -99,8 +106,8 @@ export class CromwellEngineConstruct extends EngineConstruct { }); const definition = new FargateTaskDefinition(this, "EngineTaskDef", { taskRole: this.engineRole, - cpu: serviceContainer.cpu, - memoryLimitMiB: serviceContainer.memoryLimitMiB, + cpu: (params.vCpus ? params.vCpus * 1024 : undefined) || serviceContainer.cpu, + memoryLimitMiB: params.memoryLimitMiB || serviceContainer.memoryLimitMiB, }); const volumeName = "cromwell-executions"; @@ -112,8 +119,8 @@ export class CromwellEngineConstruct extends EngineConstruct { }); const container = definition.addContainer(serviceContainer.serviceName, { - cpu: serviceContainer.cpu, - memoryLimitMiB: serviceContainer.memoryLimitMiB, + cpu: (params.vCpus ? params.vCpus * 1024 : undefined) || serviceContainer.cpu, + memoryLimitMiB: params.memoryLimitMiB || serviceContainer.memoryLimitMiB, environment: serviceContainer.environment, containerName: serviceContainer.serviceName, image: createEcrImage(this, serviceContainer.imageConfig.designation), diff --git a/packages/cdk/lib/stacks/engines/miniwdl-engine-construct.ts b/packages/cdk/lib/stacks/engines/miniwdl-engine-construct.ts index a3e38ce6..14d17afd 100644 --- a/packages/cdk/lib/stacks/engines/miniwdl-engine-construct.ts +++ b/packages/cdk/lib/stacks/engines/miniwdl-engine-construct.ts @@ -52,6 +52,8 @@ export class MiniwdlEngineConstruct extends EngineConstruct { ); this.miniwdlEngine = new MiniWdlEngine(this, "MiniWdlEngine", { + vcpus: params.vCpus, + engineMemoryMiB: params.memoryLimitMiB, vpc: props.vpc, subnets: props.subnets, iops: props.iops, diff --git a/packages/cdk/lib/stacks/engines/nextflow-engine-construct.ts b/packages/cdk/lib/stacks/engines/nextflow-engine-construct.ts index 8174472a..d58ebb3f 100644 --- a/packages/cdk/lib/stacks/engines/nextflow-engine-construct.ts +++ b/packages/cdk/lib/stacks/engines/nextflow-engine-construct.ts @@ -46,6 +46,8 @@ export class NextflowEngineConstruct extends EngineConstruct { }); this.nextflowEngine = new NextflowEngine(this, "NextflowEngine", { + vcpus: params.vCpus, + engineMemoryMiB: params.memoryLimitMiB, vpc: props.vpc, subnets: props.subnets, jobQueueArn: props.jobQueue.jobQueueArn, diff --git a/packages/cdk/lib/stacks/engines/snakemake-engine-construct.ts b/packages/cdk/lib/stacks/engines/snakemake-engine-construct.ts index 3064dacf..865c593f 100644 --- a/packages/cdk/lib/stacks/engines/snakemake-engine-construct.ts +++ b/packages/cdk/lib/stacks/engines/snakemake-engine-construct.ts @@ -93,6 +93,8 @@ export class SnakemakeEngineConstruct extends EngineConstruct { private createSnakemakeEngine(props: EngineOptions, batchHead: Batch, batchWorkers: Batch): SnakemakeEngine { return new SnakemakeEngine(this, "SnakemakeEngine", { + vcpus: props.contextParameters.vCpus, + engineMemoryMiB: props.contextParameters.memoryLimitMiB, vpc: props.vpc, subnets: props.subnets, iops: props.iops, diff --git a/packages/cdk/lib/stacks/engines/toil-engine-construct.ts b/packages/cdk/lib/stacks/engines/toil-engine-construct.ts index 0ae6b91d..2c790c35 100644 --- a/packages/cdk/lib/stacks/engines/toil-engine-construct.ts +++ b/packages/cdk/lib/stacks/engines/toil-engine-construct.ts @@ -12,6 +12,7 @@ import { ToilJobRole } from "../../roles/toil-job-role"; import { ToilEngineRole } from "../../roles/toil-engine-role"; import { IJobQueue } from "@aws-cdk/aws-batch-alpha"; import { Construct } from "constructs"; +import { ContextAppParameters } from "../../env"; export interface ToilEngineConstructProps extends EngineOptions { /** @@ -55,7 +56,7 @@ export class ToilEngineConstruct extends EngineConstruct { TOIL_AWS_BATCH_JOB_ROLE_ARN: this.jobRole.roleArn, }); - this.engine = this.getEngineServiceDefinition(props.vpc, props.subnets, engineContainer, this.engineLogGroup); + this.engine = this.getEngineServiceDefinition(props.vpc, props.subnets, engineContainer, this.engineLogGroup, params); // We don't use an adapter, so put the access-controlling proxy right in // front of the engine load balancer. @@ -74,17 +75,23 @@ export class ToilEngineConstruct extends EngineConstruct { }; } - private getEngineServiceDefinition(vpc: IVpc, subnets: SubnetSelection, serviceContainer: ServiceContainer, logGroup: ILogGroup) { + private getEngineServiceDefinition( + vpc: IVpc, + subnets: SubnetSelection, + serviceContainer: ServiceContainer, + logGroup: ILogGroup, + params: ContextAppParameters + ) { const id = "Engine"; const definition = new FargateTaskDefinition(this, "EngineTaskDef", { taskRole: this.engineRole, - cpu: serviceContainer.cpu, - memoryLimitMiB: serviceContainer.memoryLimitMiB, + cpu: (params.vCpus ? params.vCpus * 1024 : undefined) || serviceContainer.cpu, + memoryLimitMiB: params.memoryLimitMiB || serviceContainer.memoryLimitMiB, }); definition.addContainer(serviceContainer.serviceName, { - cpu: serviceContainer.cpu, - memoryLimitMiB: serviceContainer.memoryLimitMiB, + cpu: (params.vCpus ? params.vCpus * 1024 : undefined) || serviceContainer.cpu, + memoryLimitMiB: params.memoryLimitMiB || serviceContainer.memoryLimitMiB, environment: serviceContainer.environment, containerName: serviceContainer.serviceName, image: createEcrImage(this, serviceContainer.imageConfig.designation), diff --git a/packages/cli/internal/pkg/cli/context/context_environment.go b/packages/cli/internal/pkg/cli/context/context_environment.go index 59a23cf3..a4fdff8e 100644 --- a/packages/cli/internal/pkg/cli/context/context_environment.go +++ b/packages/cli/internal/pkg/cli/context/context_environment.go @@ -36,6 +36,8 @@ type contextEnvironment struct { ResourceType string MaxVCpus int + VCpus int + MemoryLimitMiB int RequestSpotInstances bool UsePublicSubnets bool } @@ -68,5 +70,7 @@ func (input contextEnvironment) ToEnvironmentList() []string { "MAX_V_CPUS": strconv.Itoa(input.MaxVCpus), "REQUEST_SPOT_INSTANCES": strconv.FormatBool(input.RequestSpotInstances), "PUBLIC_SUBNETS": strconv.FormatBool(input.UsePublicSubnets), + "V_CPUS": strconv.Itoa(input.VCpus), + "MEMORY_LIMIT_MIB": strconv.Itoa(input.MemoryLimitMiB), }) } diff --git a/packages/cli/internal/pkg/cli/context/manager.go b/packages/cli/internal/pkg/cli/context/manager.go index b99486ae..63a285d5 100644 --- a/packages/cli/internal/pkg/cli/context/manager.go +++ b/packages/cli/internal/pkg/cli/context/manager.go @@ -237,6 +237,8 @@ func (m *Manager) setContextEnv(contextName string) { ReadWriteBucketArns: strings.Join(m.readWriteBuckets, listDelimiter), InstanceTypes: strings.Join(m.contextSpec.InstanceTypes, listDelimiter), MaxVCpus: m.contextSpec.MaxVCpus, + VCpus: context.Engines[0].ResourceRequirements.VCpus, + MemoryLimitMiB: context.Engines[0].ResourceRequirements.MemoryLimitMiB, RequestSpotInstances: m.contextSpec.RequestSpotInstances, UsePublicSubnets: m.contextSpec.UsePublicSubnets, // TODO: we default to a single engine in a context for now diff --git a/packages/cli/internal/pkg/cli/spec/context.go b/packages/cli/internal/pkg/cli/spec/context.go index 38fcbb02..44158997 100644 --- a/packages/cli/internal/pkg/cli/spec/context.go +++ b/packages/cli/internal/pkg/cli/spec/context.go @@ -10,10 +10,17 @@ type Filesystem struct { Configuration FSConfig `yaml:"configuration,omitempty"` } type Engine struct { - Type string `yaml:"type"` - Engine string `yaml:"engine"` - Filesystem Filesystem `yaml:"filesystem,omitempty"` + Type string `yaml:"type"` + Engine string `yaml:"engine"` + ResourceRequirements ResourceRequirement `yaml:"resourceRequirements,omitempty"` + Filesystem Filesystem `yaml:"filesystem,omitempty"` } + +type ResourceRequirement struct { + VCpus int `yaml:"vcpus,omitempty"` + MemoryLimitMiB int `yaml:"memoryLimit,omitempty"` +} + type Context struct { InstanceTypes []string `yaml:"instanceTypes,omitempty"` RequestSpotInstances bool `yaml:"requestSpotInstances,omitempty"` diff --git a/packages/cli/internal/pkg/cli/spec/project_schema.json b/packages/cli/internal/pkg/cli/spec/project_schema.json index 995f8b1c..129ad485 100644 --- a/packages/cli/internal/pkg/cli/spec/project_schema.json +++ b/packages/cli/internal/pkg/cli/spec/project_schema.json @@ -106,6 +106,21 @@ "type": "string", "minLength": 1 }, + "resourceRequirements": { + "type": "object", + "items": { + "properties": { + "vcpus":{ + "type":"integer", + "minimum": 1 + }, + "memoryLimit":{ + "type":"integer", + "minimum": 1 + } + } + } + }, "filesystem": { "type": "object", "items": { diff --git a/packages/cli/internal/pkg/cli/spec/project_test.go b/packages/cli/internal/pkg/cli/spec/project_test.go index ada31392..484c15b1 100644 --- a/packages/cli/internal/pkg/cli/spec/project_test.go +++ b/packages/cli/internal/pkg/cli/spec/project_test.go @@ -110,6 +110,10 @@ schemaVersion: 0 { Type: "nextflow", Engine: "nextflow", + ResourceRequirements: ResourceRequirement{ + VCpus: 2, + MemoryLimitMiB: 4048, + }, Filesystem: Filesystem{ FSType: "S3", }, @@ -122,6 +126,10 @@ schemaVersion: 0 { Type: "nextflow", Engine: "nextflow", + ResourceRequirements: ResourceRequirement{ + VCpus: 2, + MemoryLimitMiB: 4048, + }, }, }, }, @@ -161,6 +169,9 @@ contexts: engines: - type: nextflow engine: nextflow + resourceRequirements: + vcpus: 2 + memoryLimit: 4048 filesystem: fsType: S3 ctx3: @@ -168,6 +179,9 @@ contexts: engines: - type: nextflow engine: nextflow + resourceRequirements: + vcpus: 2 + memoryLimit: 4048 `, }, } diff --git a/site/content/en/docs/Concepts/contexts.md b/site/content/en/docs/Concepts/contexts.md index 27aee2d0..d274edea 100644 --- a/site/content/en/docs/Concepts/contexts.md +++ b/site/content/en/docs/Concepts/contexts.md @@ -105,7 +105,31 @@ contexts: - type: nextflow engine: nextflow ``` +### Engine vCpus and Memory default minimums: +| Engine | vCpus | Memory value (MiB) | +|-----------|-------|--------------------| +| Cromwell | 2 | 16384 | +| Nextflow | 1 | 2048 | +| miniwdl | 2 | 4096 | +| Snakemake | 2 | 4096 | +| Toil | 2 | 16384 | + + +You may optionally specify the number of vCpus and memory used in a context definition, vCpu count for server engines such as Cromwell or Toil would be converted to cpu units, 1024 per each vCPU. + +*note:* if these are not provided the defaults on the table above would be used. +```yaml +contexts: + spotContext: + requestSpotInstances: true + engines: + - type: nextflow + engine: nextflow + resourceRequirements: + vcpus: 2 + memoryLimit: 6144 +``` ### Public Subnets In the interest of saving money, in particular if you intend to have the AGC stack deployed for a long period, you may choose to deploy in "public subnet" mode. diff --git a/site/content/en/docs/Concepts/engines.md b/site/content/en/docs/Concepts/engines.md index 388c154f..da7e93fc 100644 --- a/site/content/en/docs/Concepts/engines.md +++ b/site/content/en/docs/Concepts/engines.md @@ -52,6 +52,21 @@ contexts: engine: cromwell ``` +The following snippet +defines a Nextflow DSL engine of type `Nextflow` as part of the context named `spotContext` with a minimum requirement count of Cpus and Memory, when these are not provided the defaults would be used. + +```yaml +contexts: + spotContext: + requestSpotInstances: true + engines: + - type: nextflow + engine: nextflow + resourceRequirements: + vcpus: 2 + memoryLimit: 4096 +``` + ## Commands There are no commands specific to engines. Engines are [deployed]( {{< relref "contexts#deploy" >}} ) along with contexts by the [`context` commands]( {{< relref "contexts#context-commands" >}} ) and workflows