From 1a35d0d056a76e5045feb1f14f4a266fef2484e3 Mon Sep 17 00:00:00 2001 From: Amarnath Valluri Date: Tue, 8 Dec 2020 14:27:15 +0200 Subject: [PATCH] operator: changes to support per-container resources Made changes to the `v1beta1` deployment API to support configuring per-container resources. In Kubernetes resource requiremets are per-container. Existing `ControllerResources` and `NodeResources` API fields are not suffice to configure resources for sidecar containers. Hence added new fields to deployment specification for per-container resources. Also chose the default cpu,memory resource requests/limits for the containers as suggested by the VPA (Verticall Pod Autoscaler). TODOS: - To support deprecated `v1alpha1` API, needs a conversion webhook in the cluster to handled conversions between different API versions. FIXES: #616 --- .../crd/pmem-csi.intel.com_deployments.yaml | 287 ++++++++++++++++++ docs/DEVELOPMENT.md | 9 + docs/design.md | 2 +- docs/install.md | 26 +- hack/stress-driver.sh | 30 ++ pkg/apis/addtoscheme_pmemcsi_v1beta1.go | 16 + pkg/apis/pmemcsi/v1alpha1/deployment_types.go | 8 + pkg/apis/pmemcsi/v1beta1/deployment_types.go | 150 ++++++--- .../pmemcsi/v1beta1/deployment_types_test.go | 109 +++++-- .../pmemcsi/v1beta1/zz_generated.deepcopy.go | 18 +- pkg/deployments/load.go | 43 ++- .../deployment/controller_driver.go | 8 +- .../deployment/deployment_controller_test.go | 22 +- .../deployment/testcases/testcases.go | 73 ++++- test/e2e/operator/deployment_api.go | 32 +- 15 files changed, 710 insertions(+), 123 deletions(-) create mode 100755 hack/stress-driver.sh create mode 100644 pkg/apis/addtoscheme_pmemcsi_v1beta1.go diff --git a/deploy/crd/pmem-csi.intel.com_deployments.yaml b/deploy/crd/pmem-csi.intel.com_deployments.yaml index 3df1c3b11b..46f5e2453f 100644 --- a/deploy/crd/pmem-csi.intel.com_deployments.yaml +++ b/deploy/crd/pmem-csi.intel.com_deployments.yaml @@ -470,6 +470,293 @@ spec: type: object type: object served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - jsonPath: .spec.deviceMode + name: DeviceMode + type: string + - jsonPath: .spec.nodeSelector + name: NodeSelector + type: string + - jsonPath: .spec.image + name: Image + type: string + - jsonPath: .status.phase + name: Status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: Deployment is the Schema for the deployments API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: DeploymentSpec defines the desired state of Deployment + properties: + caCert: + description: CACert encoded root certificate of the CA by which the + registry and node controller certificates are signed If not provided + operator uses a self-signed CA certificate + format: byte + type: string + controllerDriverResources: + description: ControllerDriverResources Compute resources required + by driver container running on master node + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute resources + allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + type: object + deviceMode: + description: DeviceMode to use to manage PMEM devices. One of lvm, + direct + type: string + image: + description: PMEM-CSI driver container image + type: string + imagePullPolicy: + description: PullPolicy image pull policy one of Always, Never, IfNotPresent + type: string + kubeletDir: + description: KubeletDir kubelet's root directory path + type: string + labels: + additionalProperties: + type: string + description: Labels contains additional labels for all objects created + by the operator. + type: object + logLevel: + description: LogLevel number for the log verbosity kubebuilder:default=3 + type: integer + nodeControllerCert: + description: NodeControllerCert encoded certificate signed by a CA + for node controller server authentication If not provided, provisioned + one by the operator using self-signed CA + format: byte + type: string + nodeControllerKey: + description: NodeControllerPrivateKey encoded private key used for + node controller server certificate If not provided, provisioned + one by the operator + format: byte + type: string + nodeDriverResources: + description: NodeDriverResources Compute resources required by driver + container running on worker nodes + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute resources + allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + type: object + nodeRegistrarImage: + description: NodeRegistrarImage CSI node driver registrar sidecar + image + type: string + nodeRegistrarResources: + description: NodeRegistrarResources Compute resources required by + node registrar sidecar container + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute resources + allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + type: object + nodeSelector: + additionalProperties: + type: string + description: NodeSelector node labels to use for selection of driver + node + type: object + pmemPercentage: + description: PMEMPercentage represents the percentage of space to + be used by the driver in each PMEM region on every node. This is + only valid for driver in LVM mode. -kubebuilder:validation:Minimum=1 + -kubebuilder:validation:Maximum=100 -kubebuilder:default=100 + type: integer + provisionerImage: + description: ProvisionerImage CSI provisioner sidecar image + type: string + provisionerResources: + description: ProvisionerResources Compute resources required by provisioner + sidecar container + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute resources + allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + type: object + registryCert: + description: RegistryCert encoded certificate signed by a CA for registry + server authentication If not provided, provisioned one by the operator + using self-signed CA + format: byte + type: string + registryKey: + description: RegistryPrivateKey encoded private key used for registry + server certificate If not provided, provisioned one by the operator + format: byte + type: string + type: object + status: + description: DeploymentStatus defines the observed state of Deployment + properties: + conditions: + description: Conditions + items: + description: DeploymentCondition type definition for driver deployment + status conditions + properties: + lastUpdateTime: + description: Last time the condition was probed. + format: date-time + type: string + reason: + description: Message human readable text that explain why this + condition is in this state + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition. + type: string + required: + - status + - type + type: object + type: array + driverComponents: + items: + description: DriverStatus type definition for representing deployed + driver status + properties: + component: + description: 'DriverComponent represents type of the driver: + controller or node' + type: string + lastUpdated: + description: LastUpdated time of the driver status + format: date-time + type: string + reason: + description: Reason represents the human readable text that + explains why the driver is in this state. + type: string + status: + description: Status represents the state of the component; one + of `Ready` or `NotReady`. Component becomes `Ready` if all + the instances(Pods) of the driver component are in running + state. Otherwise, `NotReady`. + type: string + required: + - component + - reason + - status + type: object + type: array + lastUpdated: + description: LastUpdated time of the deployment status + format: date-time + type: string + phase: + description: Phase indicates the state of the deployment + type: string + reason: + type: string + type: object + type: object + served: true storage: true subresources: status: {} diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 4b0933341b..ae067fc550 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -287,6 +287,12 @@ Resource requirements depend on the workload. To generate some load, run make test_e2e TEST_E2E_FOCUS=lvm-production.*late.binding.*stress.test ``` +Alternatively, could run [`hack/stress-driver.sh`](hack/stress-driver.sh) +helper script to generate the load on the driver +```console +ROUNDS=500 VOL_COUNT=5 ./hack/stress-driver.sh +``` + Now resource recommendations can be retrieved with: ```console @@ -295,6 +301,9 @@ kubectl describe vpa kubectl get vpa pmem-csi-node -o jsonpath='{range .status.recommendation.containerRecommendations[*]}{.containerName}{":\n\tRequests: "}{.lowerBound}{"\n\tLimits: "}{.upperBound}{"\n"}{end}' ``` +The default resource requirements used for the driver deployments by the operator +are chosen from the VPA recommendations described in this section. + ## Switching device mode If device mode is switched between LVM and direct(aka ndctl), please keep diff --git a/docs/design.md b/docs/design.md index ed6830c5f3..362ccbda6b 100644 --- a/docs/design.md +++ b/docs/design.md @@ -421,7 +421,7 @@ tools and APIs. The driver deployment is controlled by a cluster-scoped [custom resource](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/) named [`Deployment`](./install.md#pmem-csi-deployment-crd) in the -`pmem-csi.intel.com/v1alpha1` API group. The operator runs inside the cluster +`pmem-csi.intel.com/v1beta1` API group. The operator runs inside the cluster and listens for deployment changes. It makes sure that the required Kubernetes objects are created for a driver deployment. Refer to [Deployment CRD](./install.md#deployment) for details. diff --git a/docs/install.md b/docs/install.md index 185ca15eeb..8e42b88917 100644 --- a/docs/install.md +++ b/docs/install.md @@ -275,7 +275,7 @@ Here is a minimal example driver deployment created with a custom resource: ``` ShellSession $ kubectl create -f - < Annotations: -API Version: pmem-csi.intel.com/v1alpha1 +API Version: pmem-csi.intel.com/v1beta1 Kind: Deployment Metadata: Creation Timestamp: 2020-10-07T07:31:58Z Generation: 1 Managed Fields: - API Version: pmem-csi.intel.com/v1alpha1 + API Version: pmem-csi.intel.com/v1beta1 Fields Type: FieldsV1 fieldsV1: f:spec: @@ -320,7 +320,7 @@ Metadata: Manager: kubectl-create Operation: Update Time: 2020-10-07T07:31:58Z - API Version: pmem-csi.intel.com/v1alpha1 + API Version: pmem-csi.intel.com/v1beta1 Fields Type: FieldsV1 fieldsV1: f:status: @@ -333,7 +333,7 @@ Metadata: Operation: Update Time: 2020-10-07T07:32:22Z Resource Version: 1235740 - Self Link: /apis/pmem-csi.intel.com/v1alpha1/deployments/pmem-csi.intel.com + Self Link: /apis/pmem-csi.intel.com/v1beta1/deployments/pmem-csi.intel.com UID: d8635490-53fa-4eec-970d-cd4c76f53b23 Spec: Device Mode: lvm @@ -1200,13 +1200,15 @@ The current API for PMEM-CSI `Deployment` resources is: |Field | Type | Description | |---|---|---| -| apiVersion | string | `pmem-csi.intel.com/v1alpha1`| +| apiVersion | string | `pmem-csi.intel.com/v1beta1` or `pmem-csi.intel.com/v1alpha1` | | kind | string | `Deployment`| | metadata | [ObjectMeta](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#metadata) | Object metadata, name used for CSI driver and as prefix for sub-objects | | spec | [DeploymentSpec](#deployment-spec) | Specification of the desired behavior of the deployment | #### DeploymentSpec +Below specification fields are valid in all API revisions unless noted otherwise in the description. + |Field | Type | Description | Default Value | |---|---|---|---| | image | string | PMEM-CSI docker image name used for the deployment | the same image as the operator1 | @@ -1215,8 +1217,12 @@ The current API for PMEM-CSI `Deployment` resources is: | pullPolicy | string | Docker image pull policy. either one of `Always`, `Never`, `IfNotPresent` | `IfNotPresent` | | logLevel | integer | PMEM-CSI driver logging level | 3 | | deviceMode | string | Device management mode to use. Supports one of `lvm` or `direct` | `lvm` -| controllerResources | [ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#resourcerequirements-v1-core) | Describes the compute resource requirements for controller pod | -| nodeResources | [ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#resourcerequirements-v1-core) | Describes the compute resource requirements for the pods running on node(s) | +| controllerResources | [ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#resourcerequirements-v1-core) | Describes the compute resource requirements for controller pod.
3_Deprecated and only available in `v1alpha1` API revision._ | +| nodeResources | [ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#resourcerequirements-v1-core) | Describes the compute resource requirements for the pods running on node(s).
_3Deprecated and only available in `v1alpha1` API revision._ | +| controllerDriverResources | [ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#resourcerequirements-v1-core) | Describes the compute resource requirements for controller driver container running on master node. Available in `v1beta1` onwords. | +| nodeDriverResources | [ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#resourcerequirements-v1-core) | Describes the compute resource requirements for the driver container running on worker node(s).
_Available in `v1beta1` onwards._ | +| provisionerResources | [ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#resourcerequirements-v1-core) | Describes the compute resource requirements for the [external provisioner](https://kubernetes-csi.github.io/docs/external-provisioner.html) sidecar container.
_Available in `v1beta1` onwards._ | +| nodeRegistrarResources | [ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#resourcerequirements-v1-core) | Describes the compute resource requirements for the [driver registrar](https://kubernetes-csi.github.io/docs/node-driver-registrar.html) sidecar container running on worker node(s).
_Available in `v1beta1` onwards._ | | registryCert | string | Encoded tls certificate signed by a certificate authority used for driver's controller registry server | generated by operator self-signed CA | | nodeControllerCert | string | Encoded tls certificate signed by a certificate authority used for driver's node controllers | generated by operator self-signed CA | | registryKey | string | Encoded RSA private key used for signing by `registryCert` | generated by the operator | @@ -1237,6 +1243,10 @@ appropriate values: operator dynamically chooses suitable image versions. Users have to take care of that themselves when overriding the values. +3 Pod level resource requests(`nodeResources` and `controllerResources`) +are deprecated infavor of per-container(`nodeDriverResources`, `nodeRegistrarResources`, +`controllerDriverResources` and `provisionerResources`) resource requests. + **WARNING**: although all fields can be modified and changes will be propagated to the deployed driver, not all changes are safe. In particular, changing the `deviceMode` will not work when there are diff --git a/hack/stress-driver.sh b/hack/stress-driver.sh new file mode 100755 index 0000000000..02196affbe --- /dev/null +++ b/hack/stress-driver.sh @@ -0,0 +1,30 @@ +# !/bin/bash + +# Utility script to stress the driver by creating and deleting the volumes. It +# runs by default 100 runs, each run it creates and deletes 10(5 xfs and 5 ext) +# volumes of each 4GB size. Hence, it requires approximately 40 GB of PMEM memory +# +# The main intention of this script is to determine the default resource requirements +# to be set for the driver deployed via the operator. +# This is expected to run by setting up the metrics-server and VirtualPodAutoscaler +# describe in the 'Performance and resource measurements' section in the +# developer documentation(DEPLOYMENT.md). +# +# NOTE: This script is *not* expected to run on real clusters where the user +# has exisitng PV/PVCs. + +ROUNDS=${ROUNDS:-100} +VOL_COUNT=${VOL_COUNT:-5} # 5 ext4 + 5 xfs = 10 * 4Gi ~= 40Gi + +for i in $(seq 1 1 $ROUNDS) ; do + echo "Round #$i:" + echo "Creating volumes..." + for j in $(seq 1 1 $VOL_COUNT) ; do + sed -e "s;\(.*name:\)\(.*\);\1\2-$j;g" < deploy/common/pmem-pvc.yaml | kubectl create -f - + done + echo "Deleting all pvc..." + while [ "$(kubectl get pv --no-headers | wc -l)" -ne "0" ]; do + kubectl delete pvc --all + kubectl delete pv --all + done +done diff --git a/pkg/apis/addtoscheme_pmemcsi_v1beta1.go b/pkg/apis/addtoscheme_pmemcsi_v1beta1.go new file mode 100644 index 0000000000..52404e0425 --- /dev/null +++ b/pkg/apis/addtoscheme_pmemcsi_v1beta1.go @@ -0,0 +1,16 @@ +/* +Copyright 2020 The Kubernetes Authors. + +SPDX-License-Identifier: Apache-2.0 +*/ + +package apis + +import ( + "github.com/intel/pmem-csi/pkg/apis/pmemcsi/v1beta1" +) + +func init() { + // Register the types with the Scheme so the components can map objects to GroupVersionKinds and back + AddToSchemes = append(AddToSchemes, v1beta1.SchemeBuilder.AddToScheme) +} diff --git a/pkg/apis/pmemcsi/v1alpha1/deployment_types.go b/pkg/apis/pmemcsi/v1alpha1/deployment_types.go index 58b5e830b0..4a0dce8c87 100644 --- a/pkg/apis/pmemcsi/v1alpha1/deployment_types.go +++ b/pkg/apis/pmemcsi/v1alpha1/deployment_types.go @@ -340,6 +340,10 @@ func (d *Deployment) EnsureDefaults(operatorImage string) error { if d.Spec.ControllerResources == nil { d.Spec.ControllerResources = &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(DefaultControllerResourceCPU), + corev1.ResourceMemory: resource.MustParse(DefaultControllerResourceMemory), + }, Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse(DefaultControllerResourceCPU), corev1.ResourceMemory: resource.MustParse(DefaultControllerResourceMemory), @@ -366,6 +370,10 @@ func (d *Deployment) EnsureDefaults(operatorImage string) error { if d.Spec.NodeResources == nil { d.Spec.NodeResources = &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(DefaultNodeResourceCPU), + corev1.ResourceMemory: resource.MustParse(DefaultNodeResourceMemory), + }, Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse(DefaultNodeResourceCPU), corev1.ResourceMemory: resource.MustParse(DefaultNodeResourceMemory), diff --git a/pkg/apis/pmemcsi/v1beta1/deployment_types.go b/pkg/apis/pmemcsi/v1beta1/deployment_types.go index 8b13a42b0f..51b3d461b9 100644 --- a/pkg/apis/pmemcsi/v1beta1/deployment_types.go +++ b/pkg/apis/pmemcsi/v1beta1/deployment_types.go @@ -70,10 +70,14 @@ type DeploymentSpec struct { ProvisionerImage string `json:"provisionerImage,omitempty"` // NodeRegistrarImage CSI node driver registrar sidecar image NodeRegistrarImage string `json:"nodeRegistrarImage,omitempty"` - // ControllerResources Compute resources required by Controller driver - ControllerResources *corev1.ResourceRequirements `json:"controllerResources,omitempty"` - // NodeResources Compute resources required by Node driver - NodeResources *corev1.ResourceRequirements `json:"nodeResources,omitempty"` + // ProvisionerResources Compute resources required by provisioner sidecar container + ProvisionerResources *corev1.ResourceRequirements `json:"provisionerResources,omitempty"` + // NodeRegistrarResources Compute resources required by node registrar sidecar container + NodeRegistrarResources *corev1.ResourceRequirements `json:"nodeRegistrarResources,omitempty"` + // NodeDriverResources Compute resources required by driver container running on worker nodes + NodeDriverResources *corev1.ResourceRequirements `json:"nodeDriverResources,omitempty"` + // ControllerDriverResources Compute resources required by driver container running on master node + ControllerDriverResources *corev1.ResourceRequirements `json:"controllerDriverResources,omitempty"` // DeviceMode to use to manage PMEM devices. One of lvm, direct // +kubebuilder:default:lvm DeviceMode DeviceMode `json:"deviceMode,omitempty"` @@ -254,14 +258,44 @@ const ( // DefaultRegistrarImage default node driver registrar image to use DefaultRegistrarImage = defaultRegistrarImageName + ":" + defaultRegistrarImageTag - // DefaultControllerResourceCPU default CPU resource limit used for controller pod - DefaultControllerResourceCPU = "100m" // MilliSeconds - // DefaultControllerResourceMemory default memory resource limit used for controller pod - DefaultControllerResourceMemory = "250Mi" // MB - // DefaultNodeResourceCPU default CPU resource limit used for node driver pod - DefaultNodeResourceCPU = "100m" // MilliSeconds - // DefaultNodeResourceMemory default memory resource limit used for node driver pod - DefaultNodeResourceMemory = "250Mi" // MB + // Below resource requests and limits are derived(with minor adjustments) from + // recommendations reported by VirtualPodAutoscaler(LowerBound -> Requests and UpperBound -> Limits) + + // DefaultControllerResourceRequestCPU default CPU resource request used for controller driver container + DefaultControllerResourceRequestCPU = "12m" // MilliSeconds + + // DefaultControllerResourceRequestMemory default memory resource request used for controller driver container + DefaultControllerResourceRequestMemory = "128Mi" // MB + // DefaultNodeResourceRequestCPU default CPU resource request used for node driver container + DefaultNodeResourceRequestCPU = "100m" // MilliSeconds + // DefaultNodeResourceRequestMemory default memory resource request used for node driver container + DefaultNodeResourceRequestMemory = "250Mi" // MB + // DefaultNodeRegistrarRequestCPU default CPU resource request used for node registrar container + DefaultNodeRegistrarRequestCPU = "12m" // MilliSeconds + // DefaultNodeRegistrarRequestMemory default memory resource request used for node registrar container + DefaultNodeRegistrarRequestMemory = "128Mi" // MB + // DefaultProvisionerRequestCPU default CPU resource request used for provisioner container + DefaultProvisionerRequestCPU = "12m" // MilliSeconds + // DefaultProvisionerRequestMemory default memory resource request used for node registrar container + DefaultProvisionerRequestMemory = "128Mi" // MB + + // DefaultControllerResourceLimitCPU default CPU resource limit used for controller driver container + DefaultControllerResourceLimitCPU = "500m" // MilliSeconds + // DefaultControllerResourceLimitMemory default memory resource limit used for controller driver container + DefaultControllerResourceLimitMemory = "250Mi" // MB + // DefaultNodeResourceLimitCPU default CPU resource limit used for node driver container + DefaultNodeResourceLimitCPU = "600m" // MilliSeconds + // DefaultNodeResourceLimitMemory default memory resource limit used for node driver container + DefaultNodeResourceLimitMemory = "500Mi" // MB + // DefaultNodeRegistrarLimitCPU default CPU resource limit used for node registrar container + DefaultNodeRegistrarLimitCPU = "100m" // MilliSeconds + // DefaultNodeRegistrarLimitMemory default memory resource limit used for node registrar container + DefaultNodeRegistrarLimitMemory = "128Mi" // MB + // DefaultProvisionerLimitCPU default CPU resource limit used for provisioner container + DefaultProvisionerLimitCPU = "250m" // MilliSeconds + // DefaultProvisionerLimitMemory default memory resource limit used for node registrar container + DefaultProvisionerLimitMemory = "250Mi" // MB + // DefaultDeviceMode default device manger used for deployment DefaultDeviceMode = DeviceModeLVM // DefaultPMEMPercentage PMEM space to reserve for the driver @@ -318,6 +352,17 @@ func (d *Deployment) SetDriverStatus(t DriverType, status, reason string) { // EnsureDefaults make sure that the deployment object has all defaults set properly func (d *Deployment) EnsureDefaults(operatorImage string) error { + // Validate the given driver mode. + // In a realistic case this check might not needed as it should be + // handled by JSON schema as we defined deviceMode as enumeration. + switch d.Spec.DeviceMode { + case "": + d.Spec.DeviceMode = DefaultDeviceMode + case DeviceModeDirect, DeviceModeLVM: + default: + return fmt.Errorf("invalid device mode %q", d.Spec.DeviceMode) + } + if d.Spec.Image == "" { // If provided use operatorImage if operatorImage != "" { @@ -333,57 +378,76 @@ func (d *Deployment) EnsureDefaults(operatorImage string) error { d.Spec.LogLevel = DefaultLogLevel } - /* Controller Defaults */ - if d.Spec.ProvisionerImage == "" { d.Spec.ProvisionerImage = DefaultProvisionerImage } - if d.Spec.ControllerResources == nil { - d.Spec.ControllerResources = &corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse(DefaultControllerResourceCPU), - corev1.ResourceMemory: resource.MustParse(DefaultControllerResourceMemory), - }, - } + if d.Spec.NodeRegistrarImage == "" { + d.Spec.NodeRegistrarImage = DefaultRegistrarImage } - /* Node Defaults */ + if d.Spec.NodeSelector == nil { + d.Spec.NodeSelector = DefaultNodeSelector + } - // Validate the given driver mode. - // In a realistic case this check might not needed as it should be - // handled by JSON schema as we defined deviceMode as enumeration. - switch d.Spec.DeviceMode { - case "": - d.Spec.DeviceMode = DefaultDeviceMode - case DeviceModeDirect, DeviceModeLVM: - default: - return fmt.Errorf("invalid device mode %q", d.Spec.DeviceMode) + if d.Spec.PMEMPercentage == 0 { + d.Spec.PMEMPercentage = DefaultPMEMPercentage } - if d.Spec.NodeRegistrarImage == "" { - d.Spec.NodeRegistrarImage = DefaultRegistrarImage + if d.Spec.KubeletDir == "" { + d.Spec.KubeletDir = DefaultKubeletDir } - if d.Spec.NodeResources == nil { - d.Spec.NodeResources = &corev1.ResourceRequirements{ + if d.Spec.ControllerDriverResources == nil { + d.Spec.ControllerDriverResources = &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(DefaultControllerResourceRequestCPU), + corev1.ResourceMemory: resource.MustParse(DefaultControllerResourceRequestMemory), + }, Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse(DefaultNodeResourceCPU), - corev1.ResourceMemory: resource.MustParse(DefaultNodeResourceMemory), + corev1.ResourceCPU: resource.MustParse(DefaultControllerResourceLimitCPU), + corev1.ResourceMemory: resource.MustParse(DefaultControllerResourceLimitMemory), }, } } - if d.Spec.NodeSelector == nil { - d.Spec.NodeSelector = DefaultNodeSelector + if d.Spec.ProvisionerResources == nil { + d.Spec.ProvisionerResources = &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(DefaultProvisionerRequestCPU), + corev1.ResourceMemory: resource.MustParse(DefaultProvisionerRequestMemory), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(DefaultProvisionerLimitCPU), + corev1.ResourceMemory: resource.MustParse(DefaultProvisionerLimitMemory), + }, + } } - if d.Spec.PMEMPercentage == 0 { - d.Spec.PMEMPercentage = DefaultPMEMPercentage + if d.Spec.NodeDriverResources == nil { + d.Spec.NodeDriverResources = &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(DefaultNodeResourceRequestCPU), + corev1.ResourceMemory: resource.MustParse(DefaultNodeResourceRequestMemory), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(DefaultNodeResourceLimitCPU), + corev1.ResourceMemory: resource.MustParse(DefaultNodeResourceLimitMemory), + }, + } } - if d.Spec.KubeletDir == "" { - d.Spec.KubeletDir = DefaultKubeletDir + if d.Spec.NodeRegistrarResources == nil { + d.Spec.NodeRegistrarResources = &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(DefaultNodeRegistrarRequestCPU), + corev1.ResourceMemory: resource.MustParse(DefaultNodeRegistrarRequestMemory), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(DefaultNodeRegistrarLimitCPU), + corev1.ResourceMemory: resource.MustParse(DefaultNodeRegistrarLimitMemory), + }, + } } return nil diff --git a/pkg/apis/pmemcsi/v1beta1/deployment_types_test.go b/pkg/apis/pmemcsi/v1beta1/deployment_types_test.go index fa5acb1ce3..5e839f4a38 100644 --- a/pkg/apis/pmemcsi/v1beta1/deployment_types_test.go +++ b/pkg/apis/pmemcsi/v1beta1/deployment_types_test.go @@ -46,15 +46,37 @@ var _ = Describe("Operator", func() { Expect(d.Spec.ProvisionerImage).Should(BeEquivalentTo(api.DefaultProvisionerImage), "default provisioner image mismatch") Expect(d.Spec.NodeRegistrarImage).Should(BeEquivalentTo(api.DefaultRegistrarImage), "default node driver registrar image mismatch") - Expect(d.Spec.ControllerResources).ShouldNot(BeNil(), "default controller resources not set") - rs := d.Spec.ControllerResources.Limits - Expect(rs.Cpu().String()).Should(BeEquivalentTo(api.DefaultControllerResourceCPU), "controller driver 'cpu' resource mismatch") - Expect(rs.Memory().String()).Should(BeEquivalentTo(api.DefaultControllerResourceMemory), "controller driver 'memory' resource mismatch") - - Expect(d.Spec.NodeResources).ShouldNot(BeNil(), "default node resources not set") - nrs := d.Spec.NodeResources.Limits - Expect(nrs.Cpu().String()).Should(BeEquivalentTo(api.DefaultNodeResourceCPU), "node driver 'cpu' resource mismatch") - Expect(nrs.Memory().String()).Should(BeEquivalentTo(api.DefaultNodeResourceMemory), "node driver 'cpu' resource mismatch") + Expect(d.Spec.ControllerDriverResources).ShouldNot(BeNil(), "default controller resources not set") + rs := d.Spec.ControllerDriverResources.Limits + Expect(rs.Cpu().String()).Should(BeEquivalentTo(api.DefaultControllerResourceLimitCPU), "controller driver 'cpu' resource mismatch") + Expect(rs.Memory().String()).Should(BeEquivalentTo(api.DefaultControllerResourceLimitMemory), "controller driver 'memory' resource mismatch") + + Expect(d.Spec.NodeDriverResources).ShouldNot(BeNil(), "default node driver resources not set") + rs = d.Spec.NodeDriverResources.Requests + Expect(rs.Cpu().String()).Should(BeEquivalentTo(api.DefaultNodeResourceRequestCPU), "node driver 'cpu' resource request mismatch") + Expect(rs.Memory().String()).Should(BeEquivalentTo(api.DefaultNodeResourceRequestMemory), "node driver 'cpu' resource request mismatch") + + rs = d.Spec.NodeDriverResources.Limits + Expect(rs.Cpu().String()).Should(BeEquivalentTo(api.DefaultNodeResourceLimitCPU), "node driver 'cpu' resource limit mismatch") + Expect(rs.Memory().String()).Should(BeEquivalentTo(api.DefaultNodeResourceLimitMemory), "node driver 'cpu' resource limit mismatch") + Expect(d.Spec.NodeRegistrarResources).ShouldNot(BeNil(), "default node registrar resources not set") + + rs = d.Spec.NodeRegistrarResources.Requests + Expect(rs.Cpu().String()).Should(BeEquivalentTo(api.DefaultNodeRegistrarRequestCPU), "node registrar 'cpu' resource request mismatch") + Expect(rs.Memory().String()).Should(BeEquivalentTo(api.DefaultNodeRegistrarRequestMemory), "node registrar 'cpu' resource request mismatch") + + rs = d.Spec.NodeRegistrarResources.Limits + Expect(rs.Cpu().String()).Should(BeEquivalentTo(api.DefaultNodeRegistrarLimitCPU), "node registrar 'cpu' resource limit mismatch") + Expect(rs.Memory().String()).Should(BeEquivalentTo(api.DefaultNodeRegistrarLimitMemory), "node registrar 'cpu' resource limit mismatch") + + Expect(d.Spec.ProvisionerResources).ShouldNot(BeNil(), "default provisioner resources not set") + rs = d.Spec.ProvisionerResources.Requests + Expect(rs.Cpu().String()).Should(BeEquivalentTo(api.DefaultProvisionerRequestCPU), "provisioner 'cpu' resource request mismatch") + Expect(rs.Memory().String()).Should(BeEquivalentTo(api.DefaultProvisionerRequestMemory), "provisioner 'cpu' resource request mismatch") + + rs = d.Spec.ProvisionerResources.Limits + Expect(rs.Cpu().String()).Should(BeEquivalentTo(api.DefaultProvisionerLimitCPU), "provisioner 'cpu' resource limit mismatch") + Expect(rs.Memory().String()).Should(BeEquivalentTo(api.DefaultProvisionerLimitMemory), "provisioner 'cpu' resource limit mismatch") }) It("shall be able to set values", func() { @@ -69,14 +91,22 @@ spec: imagePullPolicy: Never provisionerImage: test-provisioner:v0.0.0 nodeRegistrarImage: test-driver-registrar:v0.0.0 - controllerResources: + controllerDriverResources: requests: cpu: 1000m memory: 10Mi - nodeResources: + nodeDriverResources: requests: cpu: 2000m memory: 100Mi + nodeRegistrarResources: + requests: + cpu: 100m + memory: 250Mi + provisionerResources: + requests: + cpu: 50m + memory: 150Mi ` decode := scheme.Codecs.UniversalDeserializer().Decode @@ -95,15 +125,25 @@ spec: Expect(d.Spec.ProvisionerImage).Should(BeEquivalentTo("test-provisioner:v0.0.0"), "provisioner image mismatch") Expect(d.Spec.NodeRegistrarImage).Should(BeEquivalentTo("test-driver-registrar:v0.0.0"), "node driver registrar image mismatch") - Expect(d.Spec.ControllerResources).ShouldNot(BeNil(), "controller resources not set") - rs := d.Spec.ControllerResources.Requests - Expect(rs.Cpu().Cmp(resource.MustParse("1000m"))).Should(BeZero(), "controller driver 'cpu' resource mismatch") - Expect(rs.Memory().Cmp(resource.MustParse("10Mi"))).Should(BeZero(), "controller driver 'memory' resource mismatch") - - Expect(d.Spec.NodeResources).ShouldNot(BeNil(), "node resources not set") - nrs := d.Spec.NodeResources.Requests - Expect(nrs.Cpu().Cmp(resource.MustParse("2000m"))).Should(BeZero(), "node driver 'cpu' resource mismatch") - Expect(nrs.Memory().Cmp(resource.MustParse("100Mi"))).Should(BeZero(), "node driver 'cpu' resource mismatch") + Expect(d.Spec.ControllerDriverResources).ShouldNot(BeNil(), "controller driver resources not set") + rs := d.Spec.ControllerDriverResources.Requests + Expect(rs.Cpu().Cmp(resource.MustParse("1000m"))).Should(BeZero(), "controller driver 'cpu' resource requests mismatch") + Expect(rs.Memory().Cmp(resource.MustParse("10Mi"))).Should(BeZero(), "controller driver 'memory' resource requests mismatch") + + Expect(d.Spec.NodeDriverResources).ShouldNot(BeNil(), "node driver resources not set") + rs = d.Spec.NodeDriverResources.Requests + Expect(rs.Cpu().Cmp(resource.MustParse("2000m"))).Should(BeZero(), "node driver 'cpu' resource requests mismatch") + Expect(rs.Memory().Cmp(resource.MustParse("100Mi"))).Should(BeZero(), "node driver 'memory' resource requests mismatch") + + Expect(d.Spec.NodeRegistrarResources).ShouldNot(BeNil(), "node registrar resources not set") + rs = d.Spec.NodeRegistrarResources.Requests + Expect(rs.Cpu().Cmp(resource.MustParse("100m"))).Should(BeZero(), "node registrar 'cpu' resource requests mismatch") + Expect(rs.Memory().Cmp(resource.MustParse("250Mi"))).Should(BeZero(), "node registrar 'memory' resource request mismatch") + + Expect(d.Spec.ProvisionerResources).ShouldNot(BeNil(), "provisioner resources not set") + rs = d.Spec.ProvisionerResources.Requests + Expect(rs.Cpu().Cmp(resource.MustParse("50m"))).Should(BeZero(), "provisioner 'cpu' resource requests mismatch") + Expect(rs.Memory().Cmp(resource.MustParse("150Mi"))).Should(BeZero(), "provisioner 'memory' resource requests mismatch") }) It("should have valid json schema", func() { @@ -117,7 +157,12 @@ spec: _, _, err = deserializer.Decode(data, nil, crd) Expect(err).ShouldNot(HaveOccurred(), "decode crd file") - crdProp := crd.Spec.Versions[0].Schema.OpenAPIV3Schema + var crdProp *apiextensions.JSONSchemaProps + for _, v := range crd.Spec.Versions { + if v.Name == api.SchemeBuilder.GroupVersion.Version { + crdProp = v.Schema.OpenAPIV3Schema + } + } Expect(crdProp).ShouldNot(BeNil(), "Nil CRD schmea") Expect(crdProp.Type).Should(BeEquivalentTo("object"), "Deployment JSON schema type mismatch") spec, ok := crdProp.Properties["spec"] @@ -126,14 +171,22 @@ spec: Expect(ok).Should(BeTrue(), "Deployment JSON schema does not have 'status'") specProperties := map[string]string{ - "logLevel": "integer", - "image": "string", - "imagePullPolicy": "string", - "provisionerImage": "string", - "nodeRegistrarImage": "string", - "controllerResources": "object", - "nodeResources": "object", + "logLevel": "integer", + "image": "string", + "imagePullPolicy": "string", + "provisionerImage": "string", + "nodeRegistrarImage": "string", + "controllerDriverResources": "object", + "nodeDriverResources": "object", + "provisionerResources": "object", + "nodeRegistrarResources": "object", + "kubeletDir": "string", } + + for key := range spec.Properties { + By(key) + } + for prop, tipe := range specProperties { jsonProp, ok := spec.Properties[prop] Expect(ok).Should(BeTrue(), "Missing %q property in deployment spec", prop) diff --git a/pkg/apis/pmemcsi/v1beta1/zz_generated.deepcopy.go b/pkg/apis/pmemcsi/v1beta1/zz_generated.deepcopy.go index c0f0940259..b8bfa8155a 100644 --- a/pkg/apis/pmemcsi/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/pmemcsi/v1beta1/zz_generated.deepcopy.go @@ -87,13 +87,23 @@ func (in *DeploymentList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) { *out = *in - if in.ControllerResources != nil { - in, out := &in.ControllerResources, &out.ControllerResources + if in.ProvisionerResources != nil { + in, out := &in.ProvisionerResources, &out.ProvisionerResources *out = new(v1.ResourceRequirements) (*in).DeepCopyInto(*out) } - if in.NodeResources != nil { - in, out := &in.NodeResources, &out.NodeResources + if in.NodeRegistrarResources != nil { + in, out := &in.NodeRegistrarResources, &out.NodeRegistrarResources + *out = new(v1.ResourceRequirements) + (*in).DeepCopyInto(*out) + } + if in.NodeDriverResources != nil { + in, out := &in.NodeDriverResources, &out.NodeDriverResources + *out = new(v1.ResourceRequirements) + (*in).DeepCopyInto(*out) + } + if in.ControllerDriverResources != nil { + in, out := &in.ControllerDriverResources, &out.ControllerDriverResources *out = new(v1.ResourceRequirements) (*in).DeepCopyInto(*out) } diff --git a/pkg/deployments/load.go b/pkg/deployments/load.go index e8357c4919..421c5d40be 100644 --- a/pkg/deployments/load.go +++ b/pkg/deployments/load.go @@ -85,12 +85,20 @@ func LoadAndCustomizeObjects(kubernetes version.Version, deviceMode api.DeviceMo case "CSIDriver": obj.SetName(deployment.GetName()) case "StatefulSet": - if err := patchPodTemplate(obj, deployment, deployment.Spec.ControllerResources); err != nil { + resources := map[string]*corev1.ResourceRequirements{ + "pmem-driver": deployment.Spec.ControllerDriverResources, + "external-provisioner": deployment.Spec.ProvisionerResources, + } + if err := patchPodTemplate(obj, deployment, resources); err != nil { // TODO: avoid panic panic(fmt.Errorf("set controller resources: %v", err)) } case "DaemonSet": - if err := patchPodTemplate(obj, deployment, deployment.Spec.NodeResources); err != nil { + resources := map[string]*corev1.ResourceRequirements{ + "pmem-driver": deployment.Spec.NodeDriverResources, + "driver-registrar": deployment.Spec.NodeRegistrarResources, + } + if err := patchPodTemplate(obj, deployment, resources); err != nil { // TODO: avoid panic panic(fmt.Errorf("set node resources: %v", err)) } @@ -110,7 +118,7 @@ func LoadAndCustomizeObjects(kubernetes version.Version, deviceMode api.DeviceMo return loadObjects(kubernetes, deviceMode, patchYAML, patchUnstructured) } -func patchPodTemplate(obj *unstructured.Unstructured, deployment api.Deployment, resources *corev1.ResourceRequirements) error { +func patchPodTemplate(obj *unstructured.Unstructured, deployment api.Deployment, resources map[string]*corev1.ResourceRequirements) error { if resources == nil { return nil } @@ -135,22 +143,27 @@ func patchPodTemplate(obj *unstructured.Unstructured, deployment api.Deployment, } // Convert through JSON. - resourcesObj := map[string]interface{}{} - data, err := json.Marshal(resources) - if err != nil { - return err - } - if err := json.Unmarshal(data, &resourcesObj); err != nil { - return err + resourceObj := func(r *corev1.ResourceRequirements) (map[string]interface{}, error) { + obj := map[string]interface{}{} + data, err := json.Marshal(r) + if err != nil { + return nil, err + } + if err := json.Unmarshal(data, &obj); err != nil { + return nil, err + } + return obj, nil } containers := spec["containers"].([]interface{}) for _, container := range containers { - // Mimick the current operator behavior - // (https://github.com/intel/pmem-csi/issues/616) and apply - // the resource requirements to all containers. container := container.(map[string]interface{}) - container["resources"] = resourcesObj + containerName := container["name"].(string) + obj, err := resourceObj(resources[containerName]) + if err != nil { + return err + } + container["resources"] = obj // Override driver name in env var. env := container["env"] @@ -166,7 +179,7 @@ func patchPodTemplate(obj *unstructured.Unstructured, deployment api.Deployment, } var image string - switch container["name"].(string) { + switch containerName { case "external-provisioner": image = deployment.Spec.ProvisionerImage case "driver-registrar": diff --git a/pkg/pmem-csi-operator/controller/deployment/controller_driver.go b/pkg/pmem-csi-operator/controller/deployment/controller_driver.go index b7a45c687e..c282f9daba 100644 --- a/pkg/pmem-csi-operator/controller/deployment/controller_driver.go +++ b/pkg/pmem-csi-operator/controller/deployment/controller_driver.go @@ -1312,7 +1312,7 @@ func (d *PmemCSIDriver) getControllerContainer() corev1.Container { }, }, Ports: d.getMetricsPorts(controllerMetricsPort), - Resources: *d.Spec.ControllerResources, + Resources: *d.Spec.ControllerDriverResources, TerminationMessagePath: "/tmp/termination-log", SecurityContext: &corev1.SecurityContext{ ReadOnlyRootFilesystem: &true, @@ -1395,7 +1395,7 @@ func (d *PmemCSIDriver) getNodeDriverContainer() corev1.Container { }, }, Ports: d.getMetricsPorts(nodeMetricsPort), - Resources: *d.Spec.NodeResources, + Resources: *d.Spec.NodeDriverResources, SecurityContext: &corev1.SecurityContext{ Privileged: &true, // Node driver must run as root user @@ -1429,7 +1429,7 @@ func (d *PmemCSIDriver) getProvisionerContainer() corev1.Container { }, }, Ports: d.getMetricsPorts(provisionerMetricsPort), - Resources: *d.Spec.ControllerResources, + Resources: *d.Spec.ProvisionerResources, SecurityContext: &corev1.SecurityContext{ ReadOnlyRootFilesystem: &true, }, @@ -1466,7 +1466,7 @@ func (d *PmemCSIDriver) getNodeRegistrarContainer() corev1.Container { Value: d.GetName(), }, }, - Resources: *d.Spec.NodeResources, + Resources: *d.Spec.NodeRegistrarResources, } } diff --git a/pkg/pmem-csi-operator/controller/deployment/deployment_controller_test.go b/pkg/pmem-csi-operator/controller/deployment/deployment_controller_test.go index 4347593a9c..fdd80b1b43 100644 --- a/pkg/pmem-csi-operator/controller/deployment/deployment_controller_test.go +++ b/pkg/pmem-csi-operator/controller/deployment/deployment_controller_test.go @@ -48,6 +48,8 @@ type pmemDeployment struct { image, pullPolicy, provisionerImage, registrarImage string controllerCPU, controllerMemory string nodeCPU, nodeMemory string + provisionerCPU, provisionerMemory string + nodeRegistarCPU, nodeRegistrarMemory string caCert, regCert, regKey, ncCert, ncKey []byte kubeletDir string } @@ -76,7 +78,7 @@ func getDeployment(d *pmemDeployment) *api.Deployment { spec.ProvisionerImage = d.provisionerImage spec.NodeRegistrarImage = d.registrarImage if d.controllerCPU != "" || d.controllerMemory != "" { - spec.ControllerResources = &corev1.ResourceRequirements{ + spec.ControllerDriverResources = &corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse(d.controllerCPU), corev1.ResourceMemory: resource.MustParse(d.controllerMemory), @@ -84,13 +86,29 @@ func getDeployment(d *pmemDeployment) *api.Deployment { } } if d.nodeCPU != "" || d.nodeMemory != "" { - spec.NodeResources = &corev1.ResourceRequirements{ + spec.NodeDriverResources = &corev1.ResourceRequirements{ Requests: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse(d.nodeCPU), corev1.ResourceMemory: resource.MustParse(d.nodeMemory), }, } } + if d.provisionerCPU != "" || d.provisionerMemory != "" { + spec.ProvisionerResources = &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(d.provisionerCPU), + corev1.ResourceMemory: resource.MustParse(d.provisionerMemory), + }, + } + } + if d.nodeRegistarCPU != "" || d.nodeRegistrarMemory != "" { + spec.NodeRegistrarResources = &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(d.nodeRegistarCPU), + corev1.ResourceMemory: resource.MustParse(d.nodeRegistrarMemory), + }, + } + } spec.CACert = d.caCert spec.RegistryCert = d.regCert spec.RegistryPrivateKey = d.regKey diff --git a/pkg/pmem-csi-operator/controller/deployment/testcases/testcases.go b/pkg/pmem-csi-operator/controller/deployment/testcases/testcases.go index b99c16207d..977d7b454b 100644 --- a/pkg/pmem-csi-operator/controller/deployment/testcases/testcases.go +++ b/pkg/pmem-csi-operator/controller/deployment/testcases/testcases.go @@ -41,22 +41,38 @@ func UpdateTests() []UpdateTest { "nodeRegistrarImage": func(d *api.Deployment) { d.Spec.NodeRegistrarImage = "still-no-such-registrar-image" }, - "controllerResources": func(d *api.Deployment) { - d.Spec.ControllerResources = &corev1.ResourceRequirements{ + "controllerDriverResources": func(d *api.Deployment) { + d.Spec.ControllerDriverResources = &corev1.ResourceRequirements{ Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("201m"), corev1.ResourceMemory: resource.MustParse("101Mi"), }, } }, - "nodeResources": func(d *api.Deployment) { - d.Spec.NodeResources = &corev1.ResourceRequirements{ + "nodeDriverResources": func(d *api.Deployment) { + d.Spec.NodeDriverResources = &corev1.ResourceRequirements{ Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("501m"), corev1.ResourceMemory: resource.MustParse("501Mi"), }, } }, + "provisionerResources": func(d *api.Deployment) { + d.Spec.ProvisionerResources = &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("101m"), + corev1.ResourceMemory: resource.MustParse("101Mi"), + }, + } + }, + "nodeRegistrarResources": func(d *api.Deployment) { + d.Spec.NodeRegistrarResources = &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("301m"), + corev1.ResourceMemory: resource.MustParse("301Mi"), + }, + } + }, "TLS": func(d *api.Deployment) { SetTLSOrDie(&d.Spec) }, @@ -91,33 +107,62 @@ func UpdateTests() []UpdateTest { PullPolicy: corev1.PullIfNotPresent, ProvisionerImage: "no-such-provisioner-image", NodeRegistrarImage: "no-such-registrar-image", - ControllerResources: &corev1.ResourceRequirements{ + DeviceMode: api.DeviceModeDirect, + LogLevel: 4, + NodeSelector: map[string]string{ + "no-such-label": "no-such-value", + }, + PMEMPercentage: 50, + Labels: map[string]string{ + "a": "b", + }, + ControllerDriverResources: &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("20m"), + corev1.ResourceMemory: resource.MustParse("10Mi"), + }, Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("200m"), corev1.ResourceMemory: resource.MustParse("100Mi"), }, }, - NodeResources: &corev1.ResourceRequirements{ + NodeDriverResources: &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("50m"), + corev1.ResourceMemory: resource.MustParse("50Mi"), + }, Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("500m"), corev1.ResourceMemory: resource.MustParse("500Mi"), }, }, - DeviceMode: api.DeviceModeDirect, - LogLevel: 4, - NodeSelector: map[string]string{ - "no-such-label": "no-such-value", + + ProvisionerResources: &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("10m"), + corev1.ResourceMemory: resource.MustParse("10Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("200Mi"), + }, }, - PMEMPercentage: 50, - Labels: map[string]string{ - "a": "b", + NodeRegistrarResources: &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("30m"), + corev1.ResourceMemory: resource.MustParse("30Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("300m"), + corev1.ResourceMemory: resource.MustParse("300Mi"), + }, }, }, } SetTLSOrDie(&full.Spec) baseDeployments := map[string]api.Deployment{ - "default deployment": api.Deployment{ + "default deployment": { ObjectMeta: metav1.ObjectMeta{ Name: "pmem-csi-with-defaults", }, diff --git a/test/e2e/operator/deployment_api.go b/test/e2e/operator/deployment_api.go index d034399d34..e86b404e2a 100644 --- a/test/e2e/operator/deployment_api.go +++ b/test/e2e/operator/deployment_api.go @@ -148,18 +148,30 @@ var _ = deploy.DescribeForSome("API", func(d *deploy.Deployment) bool { DeviceMode: api.DeviceModeDirect, PullPolicy: corev1.PullNever, Image: dummyImage, - ControllerResources: &corev1.ResourceRequirements{ + ControllerDriverResources: &corev1.ResourceRequirements{ Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("200m"), corev1.ResourceMemory: resource.MustParse("100Mi"), }, }, - NodeResources: &corev1.ResourceRequirements{ + NodeDriverResources: &corev1.ResourceRequirements{ Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("500m"), corev1.ResourceMemory: resource.MustParse("500Mi"), }, }, + ProvisionerResources: &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("210m"), + corev1.ResourceMemory: resource.MustParse("110Mi"), + }, + }, + NodeRegistrarResources: &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("300m"), + corev1.ResourceMemory: resource.MustParse("100Mi"), + }, + }, }, }, } @@ -237,18 +249,30 @@ var _ = deploy.DescribeForSome("API", func(d *deploy.Deployment) bool { spec.Image = "test-driver-image" spec.PullPolicy = corev1.PullNever spec.ProvisionerImage = "test-provisioner" - spec.ControllerResources = &corev1.ResourceRequirements{ + spec.ControllerDriverResources = &corev1.ResourceRequirements{ Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("200m"), corev1.ResourceMemory: resource.MustParse("100Mi"), }, } - spec.NodeResources = &corev1.ResourceRequirements{ + spec.NodeDriverResources = &corev1.ResourceRequirements{ Limits: corev1.ResourceList{ corev1.ResourceCPU: resource.MustParse("500m"), corev1.ResourceMemory: resource.MustParse("500Mi"), }, } + spec.ProvisionerResources = &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("300m"), + corev1.ResourceMemory: resource.MustParse("300Mi"), + }, + } + spec.NodeRegistrarResources = &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("200Mi"), + }, + } testcases.SetTLSOrDie(spec) deployment = deploy.UpdateDeploymentCR(f, deployment)