From 08440fea45df08d15a1c417fc09e608e727416b6 Mon Sep 17 00:00:00 2001 From: Zadkiel Aharonian Date: Fri, 24 Nov 2023 10:34:33 +0100 Subject: [PATCH] feat: add topologySpreadConstraints pod configuration (#337) * feat: add topologySpreadConstraints pod configuration Signed-off-by: GitHub * Add sample --------- Signed-off-by: GitHub --- api/v1alpha1/k6_types.go | 37 +++-- api/v1alpha1/zz_generated.deepcopy.go | 7 + config/crd/bases/k6.io_k6s.yaml | 156 ++++++++++++++++++ config/crd/bases/k6.io_testruns.yaml | 156 ++++++++++++++++++ .../k6-operator.clusterserviceversion.yaml | 8 +- ...ha1_k6_with_topologyspreadconstraints.yaml | 24 +++ pkg/resources/jobs/initializer.go | 1 + pkg/resources/jobs/initializer_test.go | 2 + pkg/resources/jobs/runner.go | 1 + pkg/resources/jobs/runner_test.go | 10 ++ pkg/resources/jobs/starter.go | 1 + pkg/resources/jobs/starter_test.go | 2 + pkg/resources/jobs/stopper_test.go | 2 + 13 files changed, 387 insertions(+), 20 deletions(-) create mode 100644 config/samples/k6_v1alpha1_k6_with_topologyspreadconstraints.yaml diff --git a/api/v1alpha1/k6_types.go b/api/v1alpha1/k6_types.go index 727c8b64..a1e0fbf7 100644 --- a/api/v1alpha1/k6_types.go +++ b/api/v1alpha1/k6_types.go @@ -26,24 +26,25 @@ type PodMetadata struct { } type Pod struct { - Affinity *corev1.Affinity `json:"affinity,omitempty"` - AutomountServiceAccountToken string `json:"automountServiceAccountToken,omitempty"` - Env []corev1.EnvVar `json:"env,omitempty"` - Image string `json:"image,omitempty"` - ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"` - ImagePullPolicy corev1.PullPolicy `json:"imagePullPolicy,omitempty"` - Metadata PodMetadata `json:"metadata,omitempty"` - NodeSelector map[string]string `json:"nodeSelector,omitempty"` - Tolerations []corev1.Toleration `json:"tolerations,omitempty"` - Resources corev1.ResourceRequirements `json:"resources,omitempty"` - ServiceAccountName string `json:"serviceAccountName,omitempty"` - SecurityContext corev1.PodSecurityContext `json:"securityContext,omitempty"` - EnvFrom []corev1.EnvFromSource `json:"envFrom,omitempty"` - ReadinessProbe *corev1.Probe `json:"readinessProbe,omitempty"` - LivenessProbe *corev1.Probe `json:"livenessProbe,omitempty"` - InitContainers []InitContainer `json:"initContainers,omitempty"` - Volumes []corev1.Volume `json:"volumes,omitempty"` - VolumeMounts []corev1.VolumeMount `json:"volumeMounts,omitempty"` + Affinity *corev1.Affinity `json:"affinity,omitempty"` + AutomountServiceAccountToken string `json:"automountServiceAccountToken,omitempty"` + Env []corev1.EnvVar `json:"env,omitempty"` + Image string `json:"image,omitempty"` + ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"` + ImagePullPolicy corev1.PullPolicy `json:"imagePullPolicy,omitempty"` + Metadata PodMetadata `json:"metadata,omitempty"` + NodeSelector map[string]string `json:"nodeSelector,omitempty"` + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + TopologySpreadConstraints []corev1.TopologySpreadConstraint `json:"topologySpreadConstraints,omitempty"` + Resources corev1.ResourceRequirements `json:"resources,omitempty"` + ServiceAccountName string `json:"serviceAccountName,omitempty"` + SecurityContext corev1.PodSecurityContext `json:"securityContext,omitempty"` + EnvFrom []corev1.EnvFromSource `json:"envFrom,omitempty"` + ReadinessProbe *corev1.Probe `json:"readinessProbe,omitempty"` + LivenessProbe *corev1.Probe `json:"livenessProbe,omitempty"` + InitContainers []InitContainer `json:"initContainers,omitempty"` + Volumes []corev1.Volume `json:"volumes,omitempty"` + VolumeMounts []corev1.VolumeMount `json:"volumeMounts,omitempty"` } type InitContainer struct { diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 871a0892..8046580b 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -229,6 +229,13 @@ func (in *Pod) DeepCopyInto(out *Pod) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.TopologySpreadConstraints != nil { + in, out := &in.TopologySpreadConstraints, &out.TopologySpreadConstraints + *out = make([]v1.TopologySpreadConstraint, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } in.Resources.DeepCopyInto(&out.Resources) in.SecurityContext.DeepCopyInto(&out.SecurityContext) if in.EnvFrom != nil { diff --git a/config/crd/bases/k6.io_k6s.yaml b/config/crd/bases/k6.io_k6s.yaml index 9079621e..7946a209 100644 --- a/config/crd/bases/k6.io_k6s.yaml +++ b/config/crd/bases/k6.io_k6s.yaml @@ -923,6 +923,58 @@ spec: type: string type: object type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + format: int32 + type: integer + minDomains: + format: int32 + type: integer + nodeAffinityPolicy: + type: string + nodeTaintsPolicy: + type: string + topologyKey: + type: string + whenUnsatisfiable: + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array volumeMounts: items: properties: @@ -2560,6 +2612,58 @@ spec: type: string type: object type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + format: int32 + type: integer + minDomains: + format: int32 + type: integer + nodeAffinityPolicy: + type: string + nodeTaintsPolicy: + type: string + topologyKey: + type: string + whenUnsatisfiable: + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array volumeMounts: items: properties: @@ -4218,6 +4322,58 @@ spec: type: string type: object type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + format: int32 + type: integer + minDomains: + format: int32 + type: integer + nodeAffinityPolicy: + type: string + nodeTaintsPolicy: + type: string + topologyKey: + type: string + whenUnsatisfiable: + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array volumeMounts: items: properties: diff --git a/config/crd/bases/k6.io_testruns.yaml b/config/crd/bases/k6.io_testruns.yaml index 3e2087aa..7b0ce871 100644 --- a/config/crd/bases/k6.io_testruns.yaml +++ b/config/crd/bases/k6.io_testruns.yaml @@ -921,6 +921,58 @@ spec: type: string type: object type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + format: int32 + type: integer + minDomains: + format: int32 + type: integer + nodeAffinityPolicy: + type: string + nodeTaintsPolicy: + type: string + topologyKey: + type: string + whenUnsatisfiable: + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array volumeMounts: items: properties: @@ -2558,6 +2610,58 @@ spec: type: string type: object type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + format: int32 + type: integer + minDomains: + format: int32 + type: integer + nodeAffinityPolicy: + type: string + nodeTaintsPolicy: + type: string + topologyKey: + type: string + whenUnsatisfiable: + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array volumeMounts: items: properties: @@ -4216,6 +4320,58 @@ spec: type: string type: object type: array + topologySpreadConstraints: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + matchLabelKeys: + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxSkew: + format: int32 + type: integer + minDomains: + format: int32 + type: integer + nodeAffinityPolicy: + type: string + nodeTaintsPolicy: + type: string + topologyKey: + type: string + whenUnsatisfiable: + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array volumeMounts: items: properties: diff --git a/config/manifests/bases/k6-operator.clusterserviceversion.yaml b/config/manifests/bases/k6-operator.clusterserviceversion.yaml index 0bf72eb3..b5585896 100644 --- a/config/manifests/bases/k6-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/k6-operator.clusterserviceversion.yaml @@ -13,8 +13,7 @@ spec: apiservicedefinitions: {} customresourcedefinitions: owned: - - description: K6 is the Schema for the k6s API - displayName: K6 + - displayName: K6 kind: K6 name: k6s.k6.io version: v1alpha1 @@ -23,6 +22,11 @@ spec: kind: PrivateLoadZone name: privateloadzones.k6.io version: v1alpha1 + - description: TestRun is the Schema for the testruns API + displayName: Test Run + kind: TestRun + name: testruns.k6.io + version: v1alpha1 description: k6-operator displayName: k6-operator icon: diff --git a/config/samples/k6_v1alpha1_k6_with_topologyspreadconstraints.yaml b/config/samples/k6_v1alpha1_k6_with_topologyspreadconstraints.yaml new file mode 100644 index 00000000..194ff7da --- /dev/null +++ b/config/samples/k6_v1alpha1_k6_with_topologyspreadconstraints.yaml @@ -0,0 +1,24 @@ +apiVersion: k6.io/v1alpha1 +kind: TestRun +metadata: + name: testrun-sample +spec: + parallelism: 4 + script: + configMap: + name: k6-test + file: test.js + runner: + metadata: + labels: + testrun: sample + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchExpressions: + - key: testrun + operator: "In" + values: + - sample diff --git a/pkg/resources/jobs/initializer.go b/pkg/resources/jobs/initializer.go index 3cf6a73b..db62dbdb 100644 --- a/pkg/resources/jobs/initializer.go +++ b/pkg/resources/jobs/initializer.go @@ -109,6 +109,7 @@ func NewInitializerJob(k6 v1alpha1.TestRunI, argLine string) (*batchv1.Job, erro Affinity: k6.GetSpec().Initializer.Affinity, NodeSelector: k6.GetSpec().Initializer.NodeSelector, Tolerations: k6.GetSpec().Initializer.Tolerations, + TopologySpreadConstraints: k6.GetSpec().Initializer.TopologySpreadConstraints, SecurityContext: &k6.GetSpec().Initializer.SecurityContext, RestartPolicy: corev1.RestartPolicyNever, ImagePullSecrets: k6.GetSpec().Initializer.ImagePullSecrets, diff --git a/pkg/resources/jobs/initializer_test.go b/pkg/resources/jobs/initializer_test.go index cb37a972..7f63a948 100644 --- a/pkg/resources/jobs/initializer_test.go +++ b/pkg/resources/jobs/initializer_test.go @@ -52,6 +52,8 @@ func TestNewInitializerJob(t *testing.T) { ServiceAccountName: "default", Affinity: nil, NodeSelector: nil, + Tolerations: nil, + TopologySpreadConstraints: nil, RestartPolicy: corev1.RestartPolicyNever, SecurityContext: &corev1.PodSecurityContext{}, Containers: []corev1.Container{ diff --git a/pkg/resources/jobs/runner.go b/pkg/resources/jobs/runner.go index d7452773..6f8bb49f 100644 --- a/pkg/resources/jobs/runner.go +++ b/pkg/resources/jobs/runner.go @@ -167,6 +167,7 @@ func NewRunnerJob(k6 v1alpha1.TestRunI, index int, token string) (*batchv1.Job, Affinity: k6.GetSpec().Runner.Affinity, NodeSelector: k6.GetSpec().Runner.NodeSelector, Tolerations: k6.GetSpec().Runner.Tolerations, + TopologySpreadConstraints: k6.GetSpec().Runner.TopologySpreadConstraints, SecurityContext: &k6.GetSpec().Runner.SecurityContext, ImagePullSecrets: k6.GetSpec().Runner.ImagePullSecrets, InitContainers: getInitContainers(k6.GetSpec(), script), diff --git a/pkg/resources/jobs/runner_test.go b/pkg/resources/jobs/runner_test.go index b8fcb904..9aa1e882 100644 --- a/pkg/resources/jobs/runner_test.go +++ b/pkg/resources/jobs/runner_test.go @@ -317,6 +317,7 @@ func TestNewRunnerJob(t *testing.T) { Affinity: nil, NodeSelector: nil, Tolerations: nil, + TopologySpreadConstraints: nil, ServiceAccountName: "default", AutomountServiceAccountToken: &automountServiceAccountToken, Containers: []corev1.Container{{ @@ -447,6 +448,7 @@ func TestNewRunnerJobNoisy(t *testing.T) { Affinity: nil, NodeSelector: nil, Tolerations: nil, + TopologySpreadConstraints: nil, ServiceAccountName: "default", AutomountServiceAccountToken: &automountServiceAccountToken, SecurityContext: &corev1.PodSecurityContext{}, @@ -560,6 +562,7 @@ func TestNewRunnerJobUnpaused(t *testing.T) { Affinity: nil, NodeSelector: nil, Tolerations: nil, + TopologySpreadConstraints: nil, ServiceAccountName: "default", AutomountServiceAccountToken: &automountServiceAccountToken, SecurityContext: &corev1.PodSecurityContext{}, @@ -673,6 +676,7 @@ func TestNewRunnerJobArguments(t *testing.T) { Affinity: nil, NodeSelector: nil, Tolerations: nil, + TopologySpreadConstraints: nil, ServiceAccountName: "default", AutomountServiceAccountToken: &automountServiceAccountToken, SecurityContext: &corev1.PodSecurityContext{}, @@ -787,6 +791,7 @@ func TestNewRunnerJobServiceAccount(t *testing.T) { Affinity: nil, NodeSelector: nil, Tolerations: nil, + TopologySpreadConstraints: nil, ServiceAccountName: "test", AutomountServiceAccountToken: &automountServiceAccountToken, SecurityContext: &corev1.PodSecurityContext{}, @@ -902,6 +907,7 @@ func TestNewRunnerJobIstio(t *testing.T) { Affinity: nil, NodeSelector: nil, Tolerations: nil, + TopologySpreadConstraints: nil, ServiceAccountName: "default", AutomountServiceAccountToken: &automountServiceAccountToken, SecurityContext: &corev1.PodSecurityContext{}, @@ -1029,6 +1035,7 @@ func TestNewRunnerJobCloud(t *testing.T) { Affinity: nil, NodeSelector: nil, Tolerations: nil, + TopologySpreadConstraints: nil, ServiceAccountName: "default", SecurityContext: &corev1.PodSecurityContext{}, AutomountServiceAccountToken: &automountServiceAccountToken, @@ -1153,6 +1160,7 @@ func TestNewRunnerJobLocalFile(t *testing.T) { Affinity: nil, NodeSelector: nil, Tolerations: nil, + TopologySpreadConstraints: nil, ServiceAccountName: "default", AutomountServiceAccountToken: &automountServiceAccountToken, SecurityContext: &corev1.PodSecurityContext{}, @@ -1266,6 +1274,7 @@ func TestNewRunnerJobWithInitContainer(t *testing.T) { Affinity: nil, NodeSelector: nil, Tolerations: nil, + TopologySpreadConstraints: nil, ServiceAccountName: "default", AutomountServiceAccountToken: &automountServiceAccountToken, InitContainers: []corev1.Container{ @@ -1442,6 +1451,7 @@ func TestNewRunnerJobWithVolume(t *testing.T) { Affinity: nil, NodeSelector: nil, Tolerations: nil, + TopologySpreadConstraints: nil, ServiceAccountName: "default", AutomountServiceAccountToken: &automountServiceAccountToken, InitContainers: []corev1.Container{ diff --git a/pkg/resources/jobs/starter.go b/pkg/resources/jobs/starter.go index b5b043cd..ca8adffd 100644 --- a/pkg/resources/jobs/starter.go +++ b/pkg/resources/jobs/starter.go @@ -62,6 +62,7 @@ func NewStarterJob(k6 v1alpha1.TestRunI, hostname []string) *batchv1.Job { Affinity: k6.GetSpec().Starter.Affinity, NodeSelector: k6.GetSpec().Starter.NodeSelector, Tolerations: k6.GetSpec().Starter.Tolerations, + TopologySpreadConstraints: k6.GetSpec().Starter.TopologySpreadConstraints, RestartPolicy: corev1.RestartPolicyNever, SecurityContext: &k6.GetSpec().Starter.SecurityContext, ImagePullSecrets: k6.GetSpec().Starter.ImagePullSecrets, diff --git a/pkg/resources/jobs/starter_test.go b/pkg/resources/jobs/starter_test.go index d727e532..bb8281ae 100644 --- a/pkg/resources/jobs/starter_test.go +++ b/pkg/resources/jobs/starter_test.go @@ -46,6 +46,7 @@ func TestNewStarterJob(t *testing.T) { Affinity: nil, NodeSelector: nil, Tolerations: nil, + TopologySpreadConstraints: nil, RestartPolicy: corev1.RestartPolicyNever, SecurityContext: &corev1.PodSecurityContext{}, Containers: []corev1.Container{ @@ -126,6 +127,7 @@ func TestNewStarterJobIstio(t *testing.T) { Affinity: nil, NodeSelector: nil, Tolerations: nil, + TopologySpreadConstraints: nil, RestartPolicy: corev1.RestartPolicyNever, SecurityContext: &corev1.PodSecurityContext{}, Containers: []corev1.Container{ diff --git a/pkg/resources/jobs/stopper_test.go b/pkg/resources/jobs/stopper_test.go index 6016a890..6c4166e1 100644 --- a/pkg/resources/jobs/stopper_test.go +++ b/pkg/resources/jobs/stopper_test.go @@ -45,6 +45,7 @@ func TestNewStopperJob(t *testing.T) { Affinity: nil, NodeSelector: nil, Tolerations: nil, + TopologySpreadConstraints: nil, RestartPolicy: corev1.RestartPolicyNever, SecurityContext: &corev1.PodSecurityContext{}, Containers: []corev1.Container{ @@ -124,6 +125,7 @@ func TestNewStopJobIstio(t *testing.T) { Affinity: nil, NodeSelector: nil, Tolerations: nil, + TopologySpreadConstraints: nil, RestartPolicy: corev1.RestartPolicyNever, SecurityContext: &corev1.PodSecurityContext{}, Containers: []corev1.Container{