Feat: Adding the option to provide cpu count and memory size for the …

packages/cdk/lib/constructs/engines/miniwdl/miniwdl-engine.ts

@@ -12,13 +12,14 @@ export interface MiniWdlEngineProps extends EngineProps {
   readonly engineBatch: Batch;
   readonly workerBatch: Batch;
   readonly iops?: Size;
+  readonly vcpus?: number;
+  readonly engineMemoryMiB?: number;
 }
 
 const MINIWDL_IMAGE_DESIGNATION = "miniwdl";
 
 export class MiniWdlEngine extends Engine {
   readonly headJobDefinition: JobDefinition;
-  private readonly engineMemoryMiB = 4096;
   private readonly volumeName = "efs";
   readonly fileSystem: FileSystem;
 
@@ -41,7 +42,8 @@ export class MiniWdlEngine extends Engine {
       logGroup: this.logGroup,
       platformCapabilities: [PlatformCapabilities.FARGATE],
       container: {
-        memoryLimitMiB: this.engineMemoryMiB,
+        vcpus: props.vcpus || 1,
+        memoryLimitMiB: props.engineMemoryMiB || 4096,
         jobRole: engineBatch.role,
         executionRole: engineBatch.role,
         image: createEcrImage(this, MINIWDL_IMAGE_DESIGNATION),

packages/cdk/lib/constructs/engines/nextflow/nextflow-engine.ts

@@ -8,6 +8,8 @@ import { Engine, EngineProps } from "../engine";
 export interface NextflowEngineProps extends EngineProps {
   readonly jobQueueArn: string;
   readonly taskRole: IRole;
+  readonly vcpus?: number;
+  readonly engineMemoryMiB?: number;
 }
 
 const NEXTFLOW_IMAGE_DESIGNATION = "nextflow";
@@ -21,6 +23,8 @@ export class NextflowEngine extends Engine {
     this.headJobDefinition = new EngineJobDefinition(this, "NexflowHeadJobDef", {
       logGroup: this.logGroup,
       container: {
+        vcpus: props.vcpus,
+        memoryLimitMiB: props.engineMemoryMiB,
         jobRole: props.taskRole,
         image: createEcrImage(this, NEXTFLOW_IMAGE_DESIGNATION),
         command: [],

packages/cdk/lib/constructs/engines/snakemake/snakemake-engine.ts

@@ -12,14 +12,15 @@ export interface SnakemakeEngineProps extends EngineProps {
   readonly engineBatch: Batch;
   readonly workerBatch: Batch;
   readonly iops?: Size;
+  readonly vcpus?: number;
+  readonly engineMemoryMiB?: number;
 }
 
 const SNAKEMAKE_IMAGE_DESIGNATION = "snakemake";
 
 export class SnakemakeEngine extends Engine {
   readonly headJobDefinition: JobDefinition;
   private readonly volumeName = "efs";
-  private readonly engineMemoryMiB = 4096;
   public readonly fsap: AccessPoint;
   public readonly fileSystem: FileSystem;
 
@@ -41,7 +42,8 @@ export class SnakemakeEngine extends Engine {
       logGroup: this.logGroup,
       platformCapabilities: [PlatformCapabilities.FARGATE],
       container: {
-        memoryLimitMiB: this.engineMemoryMiB,
+        vcpus: props.vcpus || 1,
+        memoryLimitMiB: props.engineMemoryMiB || 4096,
         jobRole: engineBatch.role,
         executionRole: engineBatch.role,
         image: createEcrImage(this, SNAKEMAKE_IMAGE_DESIGNATION),

packages/cdk/lib/env/context-app-parameters.ts

@@ -87,6 +87,14 @@ export class ContextAppParameters {
    * The types of EC2 instances that may be launched in the compute environment.
    */
   public readonly instanceTypes?: InstanceType[];
+  /**
+   * The maximum number of Amazon EC2 vCPUs that an environment can reach for the nextflow engine.
+   */
+  public readonly vCpus?: number;
+  /**
+   * The maximum memory limit that an environment can reach for the nextflow engine.
+   */
+  public readonly memoryLimitMiB?: number;
   /**
    * If true, put EC2 instances into public subnets instead of private subnets.
    * This allows you to obtain significantly lower ongoing costs if used in conjunction with the usePublicSubnets option
@@ -133,6 +141,9 @@ export class ContextAppParameters {
     this.requestSpotInstances = getEnvBoolOrDefault(node, "REQUEST_SPOT_INSTANCES", false)!;
     this.instanceTypes = instanceTypeStrings ? instanceTypeStrings.map((instanceType) => new InstanceType(instanceType.trim())) : undefined;
 
+    this.vCpus = getEnvNumber(node, "V_CPUS");
+    this.memoryLimitMiB = getEnvNumber(node, "MEMORY_LIMIT_MIB");
+
     this.usePublicSubnets = getEnvBoolOrDefault(node, "PUBLIC_SUBNETS", false);
     this.agcVersion = getEnvString(node, "AGC_VERSION");
 

packages/cdk/lib/stacks/engines/cromwell-engine-construct.ts

@@ -14,6 +14,7 @@ import { CromwellEngineRole } from "../../roles/cromwell-engine-role";
 import { CromwellAdapterRole } from "../../roles/cromwell-adapter-role";
 import { IJobQueue } from "@aws-cdk/aws-batch-alpha";
 import { Construct } from "constructs";
+import {ContextAppParameters} from "../../env";
 
 export interface CromwellEngineConstructProps extends EngineOptions {
   /**
@@ -57,7 +58,7 @@ export class CromwellEngineConstruct extends EngineConstruct {
     });
 
     // TODO: Move log group creation into service construct and make it a property
-    this.engine = this.getEngineServiceDefinition(props.vpc, props.subnets, engineContainer, this.engineLogGroup);
+    this.engine = this.getEngineServiceDefinition(props.vpc, props.subnets, engineContainer, this.engineLogGroup, params);
     this.adapterLogGroup = new LogGroup(this, "AdapterLogGroup");
 
     const lambda = this.renderAdapterLambda({
@@ -89,7 +90,13 @@ export class CromwellEngineConstruct extends EngineConstruct {
     };
   }
 
-  private getEngineServiceDefinition(vpc: IVpc, subnets: SubnetSelection, serviceContainer: ServiceContainer, logGroup: ILogGroup) {
+  private getEngineServiceDefinition(
+    vpc: IVpc,
+    subnets: SubnetSelection,
+    serviceContainer: ServiceContainer,
+    logGroup: ILogGroup,
+    params: ContextAppParameters
+  ) {
     const id = "Engine";
     const fileSystem = new FileSystem(this, "EngineFileSystem", {
       vpc,
@@ -99,8 +106,8 @@ export class CromwellEngineConstruct extends EngineConstruct {
     });
     const definition = new FargateTaskDefinition(this, "EngineTaskDef", {
       taskRole: this.engineRole,
-      cpu: serviceContainer.cpu,
-      memoryLimitMiB: serviceContainer.memoryLimitMiB,
+      cpu: (params.vCpus ? params.vCpus * 1024 : undefined) || serviceContainer.cpu,
+      memoryLimitMiB: params.memoryLimitMiB || serviceContainer.memoryLimitMiB,
     });
 
     const volumeName = "cromwell-executions";
@@ -112,8 +119,8 @@ export class CromwellEngineConstruct extends EngineConstruct {
     });
 
     const container = definition.addContainer(serviceContainer.serviceName, {
-      cpu: serviceContainer.cpu,
-      memoryLimitMiB: serviceContainer.memoryLimitMiB,
+      cpu: (params.vCpus ? params.vCpus * 1024 : undefined) || serviceContainer.cpu,
+      memoryLimitMiB: params.memoryLimitMiB || serviceContainer.memoryLimitMiB,
       environment: serviceContainer.environment,
       containerName: serviceContainer.serviceName,
       image: createEcrImage(this, serviceContainer.imageConfig.designation),

packages/cdk/lib/stacks/engines/miniwdl-engine-construct.ts

@@ -52,6 +52,8 @@ export class MiniwdlEngineConstruct extends EngineConstruct {
     );
 
     this.miniwdlEngine = new MiniWdlEngine(this, "MiniWdlEngine", {
+      vcpus: params.vCpus,
+      engineMemoryMiB: params.memoryLimitMiB,
       vpc: props.vpc,
       subnets: props.subnets,
       iops: props.iops,

packages/cdk/lib/stacks/engines/nextflow-engine-construct.ts

@@ -46,6 +46,8 @@ export class NextflowEngineConstruct extends EngineConstruct {
     });
 
     this.nextflowEngine = new NextflowEngine(this, "NextflowEngine", {
+      vcpus: params.vCpus,
+      engineMemoryMiB: params.memoryLimitMiB,
       vpc: props.vpc,
       subnets: props.subnets,
       jobQueueArn: props.jobQueue.jobQueueArn,

packages/cdk/lib/stacks/engines/snakemake-engine-construct.ts

@@ -93,6 +93,8 @@ export class SnakemakeEngineConstruct extends EngineConstruct {
 
   private createSnakemakeEngine(props: EngineOptions, batchHead: Batch, batchWorkers: Batch): SnakemakeEngine {
     return new SnakemakeEngine(this, "SnakemakeEngine", {
+      vcpus: props.contextParameters.vCpus,
+      engineMemoryMiB: props.contextParameters.memoryLimitMiB,
       vpc: props.vpc,
       subnets: props.subnets,
       iops: props.iops,

packages/cdk/lib/stacks/engines/toil-engine-construct.ts

@@ -12,6 +12,7 @@ import { ToilJobRole } from "../../roles/toil-job-role";
 import { ToilEngineRole } from "../../roles/toil-engine-role";
 import { IJobQueue } from "@aws-cdk/aws-batch-alpha";
 import { Construct } from "constructs";
+import { ContextAppParameters } from "../../env";
 
 export interface ToilEngineConstructProps extends EngineOptions {
   /**
@@ -55,7 +56,7 @@ export class ToilEngineConstruct extends EngineConstruct {
       TOIL_AWS_BATCH_JOB_ROLE_ARN: this.jobRole.roleArn,
     });
 
-    this.engine = this.getEngineServiceDefinition(props.vpc, props.subnets, engineContainer, this.engineLogGroup);
+    this.engine = this.getEngineServiceDefinition(props.vpc, props.subnets, engineContainer, this.engineLogGroup, params);
 
     // We don't use an adapter, so put the access-controlling proxy right in
     // front of the engine load balancer.
@@ -74,17 +75,23 @@ export class ToilEngineConstruct extends EngineConstruct {
     };
   }
 
-  private getEngineServiceDefinition(vpc: IVpc, subnets: SubnetSelection, serviceContainer: ServiceContainer, logGroup: ILogGroup) {
+  private getEngineServiceDefinition(
+    vpc: IVpc,
+    subnets: SubnetSelection,
+    serviceContainer: ServiceContainer,
+    logGroup: ILogGroup,
+    params: ContextAppParameters
+  ) {
     const id = "Engine";
     const definition = new FargateTaskDefinition(this, "EngineTaskDef", {
       taskRole: this.engineRole,
-      cpu: serviceContainer.cpu,
-      memoryLimitMiB: serviceContainer.memoryLimitMiB,
+      cpu: (params.vCpus ? params.vCpus * 1024 : undefined) || serviceContainer.cpu,
+      memoryLimitMiB: params.memoryLimitMiB || serviceContainer.memoryLimitMiB,
     });
 
     definition.addContainer(serviceContainer.serviceName, {
-      cpu: serviceContainer.cpu,
-      memoryLimitMiB: serviceContainer.memoryLimitMiB,
+      cpu: (params.vCpus ? params.vCpus * 1024 : undefined) || serviceContainer.cpu,
+      memoryLimitMiB: params.memoryLimitMiB || serviceContainer.memoryLimitMiB,
       environment: serviceContainer.environment,
       containerName: serviceContainer.serviceName,
       image: createEcrImage(this, serviceContainer.imageConfig.designation),

packages/cli/internal/pkg/cli/context/context_environment.go

@@ -36,6 +36,8 @@ type contextEnvironment struct {
 	ResourceType        string
 	MaxVCpus            int
 
+	VCpus                int
+	MemoryLimitMiB       int
 	RequestSpotInstances bool
 	UsePublicSubnets     bool
 }
@@ -68,5 +70,7 @@ func (input contextEnvironment) ToEnvironmentList() []string {
 		"MAX_V_CPUS":                   strconv.Itoa(input.MaxVCpus),
 		"REQUEST_SPOT_INSTANCES":       strconv.FormatBool(input.RequestSpotInstances),
 		"PUBLIC_SUBNETS":               strconv.FormatBool(input.UsePublicSubnets),
+		"V_CPUS":                       strconv.Itoa(input.VCpus),
+		"MEMORY_LIMIT_MIB":             strconv.Itoa(input.MemoryLimitMiB),
 	})
 }

packages/cli/internal/pkg/cli/context/manager.go

@@ -237,6 +237,8 @@ func (m *Manager) setContextEnv(contextName string) {
 		ReadWriteBucketArns:  strings.Join(m.readWriteBuckets, listDelimiter),
 		InstanceTypes:        strings.Join(m.contextSpec.InstanceTypes, listDelimiter),
 		MaxVCpus:             m.contextSpec.MaxVCpus,
+		VCpus:                context.Engines[0].ResourceRequirements.VCpus,
+		MemoryLimitMiB:       context.Engines[0].ResourceRequirements.MemoryLimitMiB,
 		RequestSpotInstances: m.contextSpec.RequestSpotInstances,
 		UsePublicSubnets:     m.contextSpec.UsePublicSubnets,
 		// TODO: we default to a single engine in a context for now

packages/cli/internal/pkg/cli/spec/context.go

@@ -10,10 +10,17 @@ type Filesystem struct {
 	Configuration FSConfig `yaml:"configuration,omitempty"`
 }
 type Engine struct {
-	Type       string     `yaml:"type"`
-	Engine     string     `yaml:"engine"`
-	Filesystem Filesystem `yaml:"filesystem,omitempty"`
+	Type                 string              `yaml:"type"`
+	Engine               string              `yaml:"engine"`
+	ResourceRequirements ResourceRequirement `yaml:"resourceRequirements,omitempty"`
+	Filesystem           Filesystem          `yaml:"filesystem,omitempty"`
 }
+
+type ResourceRequirement struct {
+	VCpus          int `yaml:"vcpus,omitempty"`
+	MemoryLimitMiB int `yaml:"memoryLimit,omitempty"`
+}
+
 type Context struct {
 	InstanceTypes        []string `yaml:"instanceTypes,omitempty"`
 	RequestSpotInstances bool     `yaml:"requestSpotInstances,omitempty"`

packages/cli/internal/pkg/cli/spec/project_schema.json

@@ -106,6 +106,21 @@
                     "type": "string",
                     "minLength": 1
                   },
+                  "resourceRequirements": {
+                    "type": "object",
+                    "items": {
+                      "properties": {
+                        "vcpus":{
+                          "type":"integer",
+                          "minimum": 1
+                        },
+                        "memoryLimit":{
+                          "type":"integer",
+                          "minimum": 1
+                        }
+                      }
+                    }
+                  },
                   "filesystem": {
                     "type": "object",
                     "items": {

packages/cli/internal/pkg/cli/spec/project_test.go

@@ -110,6 +110,10 @@ schemaVersion: 0
 							{
 								Type:   "nextflow",
 								Engine: "nextflow",
+								ResourceRequirements: ResourceRequirement{
+									VCpus:          2,
+									MemoryLimitMiB: 4048,
+								},
 								Filesystem: Filesystem{
 									FSType: "S3",
 								},
@@ -122,6 +126,10 @@ schemaVersion: 0
 							{
 								Type:   "nextflow",
 								Engine: "nextflow",
+								ResourceRequirements: ResourceRequirement{
+									VCpus:          2,
+									MemoryLimitMiB: 4048,
+								},
 							},
 						},
 					},
@@ -161,13 +169,19 @@ contexts:
         engines:
             - type: nextflow
               engine: nextflow
+              resourceRequirements:
+                vcpus: 2
+                memoryLimit: 4048
               filesystem:
                 fsType: S3
     ctx3:
         maxVCpus: 256
         engines:
             - type: nextflow
               engine: nextflow
+              resourceRequirements:
+                vcpus: 2
+                memoryLimit: 4048
 `,
 		},
 	}

site/content/en/docs/Concepts/contexts.md

@@ -105,7 +105,31 @@ contexts:
       - type: nextflow
         engine: nextflow
 ```
+### Engine vCpus and Memory default minimums:
 
+| Engine    | vCpus | Memory value (MiB) |
+|-----------|-------|--------------------|
+| Cromwell  | 2     | 16384              |
+| Nextflow  | 1     | 2048               |
+| miniwdl   | 2     | 4096               |
+| Snakemake | 2     | 4096               |
+| Toil      | 2     | 16384              |
+
+
+You may optionally specify the number of vCpus and memory used in a context definition, vCpu count for server engines such as Cromwell or Toil would be converted to cpu units, 1024 per each vCPU.
+
+*note:* if these are not provided the defaults on the table above would be used.
+```yaml
+contexts:
+  spotContext:
+    requestSpotInstances: true
+    engines:
+      - type: nextflow
+        engine: nextflow
+        resourceRequirements:
+          vcpus: 2
+          memoryLimit: 6144
+```
 ### Public Subnets
 
 In the interest of saving money, in particular if you intend to have the AGC stack deployed for a long period, you may choose to deploy in "public subnet" mode.

site/content/en/docs/Concepts/engines.md

@@ -52,6 +52,21 @@ contexts:
         engine: cromwell
 ```
 
+The following snippet
+defines a Nextflow DSL engine of type `Nextflow` as part of the context named `spotContext` with a minimum requirement count of Cpus and Memory, when these are not provided the defaults would be used.
+
+```yaml
+contexts:
+  spotContext:
+    requestSpotInstances: true
+    engines:
+      - type: nextflow
+        engine: nextflow
+        resourceRequirements:
+          vcpus: 2
+          memoryLimit: 4096
+```
+
 ## Commands
 
 There are no commands specific to engines. Engines are [deployed]( {{< relref "contexts#deploy" >}} ) along with contexts by the [`context` commands]( {{< relref "contexts#context-commands" >}} ) and workflows