Add CPU/Memory request/limits and PriorityClasses for system components

Signed-off-by: tylerslaton <[email protected]>

Author: tylerslaton

pkg/imagesystem/buildertemplate.go

@@ -42,6 +42,7 @@ func BuilderObjects(name, namespace, forNamespace, buildKitImage, pub, privKey,
 					Labels: labels.ManagedByApp(namespace, name, "app", name),
 				},
 				Spec: corev1.PodSpec{
+					PriorityClassName:  system.AcornPriorityClass,
 					ServiceAccountName: "acorn-builder",
 					EnableServiceLinks: new(bool),
 					Containers: []corev1.Container{
@@ -54,6 +55,7 @@ func BuilderObjects(name, namespace, forNamespace, buildKitImage, pub, privKey,
 								"--addr",
 								"unix:///run/buildkit/buildkitd.sock",
 							},
+							Resources: system.BuildkitdResources(),
 							LivenessProbe: &corev1.Probe{
 								ProbeHandler: corev1.ProbeHandler{
 									Exec: &corev1.ExecAction{
@@ -138,6 +140,7 @@ func BuilderObjects(name, namespace, forNamespace, buildKitImage, pub, privKey,
 							Args: []string{
 								"build-server",
 							},
+							Resources: system.BuildkitdServiceResources(),
 							ReadinessProbe: &corev1.Probe{
 								ProbeHandler: corev1.ProbeHandler{
 									HTTPGet: &corev1.HTTPGetAction{

pkg/imagesystem/registrytemplate.go

@@ -58,6 +58,7 @@ func registryDeployment(namespace, registryImage string) []client.Object {
 					},
 				},
 				Spec: corev1.PodSpec{
+					PriorityClassName:  system.AcornPriorityClass,
 					EnableServiceLinks: new(bool),
 					Containers: []corev1.Container{
 						{
@@ -68,8 +69,9 @@ func registryDeployment(namespace, registryImage string) []client.Object {
 									Value: "true",
 								},
 							},
-							Image:   registryImage,
-							Command: []string{"/usr/local/bin/registry", "serve", "/etc/docker/registry/config.yml"},
+							Resources: system.RegistryResources(),
+							Image:     registryImage,
+							Command:   []string{"/usr/local/bin/registry", "serve", "/etc/docker/registry/config.yml"},
 							LivenessProbe: &corev1.Probe{
 								ProbeHandler: corev1.ProbeHandler{
 									TCPSocket: &corev1.TCPSocketAction{

pkg/install/apiserver.yaml

@@ -66,6 +66,11 @@ spec:
             - containerPort: 7443
           securityContext:
             runAsUser: 1000
+          resources:
+            requests:
+              cpu: 50m
+              memory: 100Mi
+      priorityClassName: system-cluster-critical
       serviceAccountName: acorn-system
       tolerations:
         - key: node-role.kubernetes.io/control-plane

pkg/install/controller.yaml

@@ -31,6 +31,11 @@ spec:
             httpGet:
               path: /healthz
               port: 8888
+          resources:
+            requests:
+              cpu: 150m
+              memory: 200Mi
+      priorityClassName: system-cluster-critical
       serviceAccountName: acorn-system
       tolerations:
         - key: node-role.kubernetes.io/control-plane

pkg/system/constants.go

@@ -13,6 +13,8 @@ const (
 	CustomCABundleSecretVolumeName
 	CustomCABundleDir      = "/etc/ssl/certs"
 	CustomCABundleCertName = "ca-certificates.crt"
+
+	AcornPriorityClass = "system-cluster-critical"
 )
 
 var (

pkg/system/resources.go

@@ -0,0 +1,75 @@
+package system
+
+import (
+	"os"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+)
+
+// Values will likely need to be tweaked as we get more usage data. They are currently based
+// on metrics we have collected from internal use. You can override these values by setting
+// the corresponding environment variable.
+var (
+	mi = int64(1_048_576) // 1 MiB in bytes
+
+	registryMemoryRequest = *resource.NewQuantity(40*mi, resource.BinarySI)    // REGISTRY_MEMORY_REQUEST
+	registryMemoryLimit   = *resource.NewQuantity(80*mi, resource.BinarySI)    // REGISTRY_MEMORY_LIMIT
+	registryCPURequest    = *resource.NewMilliQuantity(50, resource.DecimalSI) // REGISTRY_CPU_REQUEST
+
+	buildkitdMemoryRequest = *resource.NewQuantity(100*mi, resource.BinarySI)   // BUILDKITD_MEMORY_REQUEST
+	buildkitdMemoryLimit   = *resource.NewQuantity(200*mi, resource.BinarySI)   // BUILDKITD_MEMORY_LIMIT
+	buildkitdCPURequest    = *resource.NewMilliQuantity(50, resource.DecimalSI) // BUILDKITD_CPU_REQUEST
+
+	buildkitdServiceMemoryRequest = *resource.NewQuantity(70*mi, resource.BinarySI)    // BUILDKITD_SERVICE_MEMORY_REQUEST
+	buildkitdServiceMemoryLimit   = *resource.NewQuantity(140*mi, resource.BinarySI)   // BUILDKITD_SERVICE_MEMORY_LIMIT
+	buildkitdServiceCPURequest    = *resource.NewMilliQuantity(50, resource.DecimalSI) // BUILDKITD_SERVICE_CPU_REQUEST
+)
+
+func RegistryResources() corev1.ResourceRequirements {
+	return corev1.ResourceRequirements{
+		Requests: corev1.ResourceList{
+			corev1.ResourceMemory: envOrDefault("REGISTRY_MEMORY_REQUEST", registryMemoryRequest),
+			corev1.ResourceCPU:    envOrDefault("REGISTRY_CPU_REQUEST", registryCPURequest),
+		},
+		Limits: corev1.ResourceList{
+			corev1.ResourceMemory: envOrDefault("REGISTRY_MEMORY_LIMIT", registryMemoryLimit),
+		},
+	}
+}
+
+func BuildkitdResources() corev1.ResourceRequirements {
+	return corev1.ResourceRequirements{
+		Requests: corev1.ResourceList{
+			corev1.ResourceMemory: envOrDefault("BUILDKITD_MEMORY_REQUEST", buildkitdMemoryRequest),
+			corev1.ResourceCPU:    envOrDefault("BUILDKITD_CPU_REQUEST", buildkitdCPURequest),
+		},
+		Limits: corev1.ResourceList{
+			corev1.ResourceMemory: envOrDefault("BUILDKITD_MEMORY_LIMIT", buildkitdMemoryLimit),
+		},
+	}
+}
+
+func BuildkitdServiceResources() corev1.ResourceRequirements {
+	return corev1.ResourceRequirements{
+		Requests: corev1.ResourceList{
+			corev1.ResourceMemory: envOrDefault("BUILDKITD_SERVICE_MEMORY_REQUEST", buildkitdServiceMemoryRequest),
+			corev1.ResourceCPU:    envOrDefault("BUILDKITD_SERVICE_CPU_REQUEST", buildkitdServiceCPURequest),
+		},
+		Limits: corev1.ResourceList{
+			corev1.ResourceMemory: envOrDefault("BUILDKITD_SERVICE_MEMORY_LIMIT", buildkitdServiceMemoryLimit),
+		},
+	}
+}
+
+func envOrDefault(env string, def resource.Quantity) resource.Quantity {
+	if env = os.Getenv(env); env == "" {
+		return def
+	}
+
+	quantity, err := resource.ParseQuantity(env)
+	if err == nil {
+		return quantity
+	}
+	return def
+}