✨ Support EKS upgrade policy (#5471)

* Support EKS upgrade policy

* Apply suggestions from code review

Co-authored-by: Damiano Donati <[email protected]>

* regenerate

* Apply suggestions from code review

Co-authored-by: Damiano Donati <[email protected]>

* Update pkg/cloud/services/eks/cluster.go

Co-authored-by: Damiano Donati <[email protected]>

* remove log

* Update config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml

Co-authored-by: Faiq <[email protected]>

* docstring typo

* set NotReady if cluster was automatically upgraded

* Update pkg/cloud/services/eks/cluster.go

Co-authored-by: Faiq <[email protected]>

* fix version compare logic

* e2e

* Apply suggestions from code review

Co-authored-by: Damiano Donati <[email protected]>

* syntax

* WaitForEKSClusterUpgradePolicy fail early on NotFound

---------

Co-authored-by: Damiano Donati <[email protected]>
Co-authored-by: Faiq <[email protected]>

Author: 3 people

config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml

@@ -3243,6 +3243,17 @@ spec:
                 - iam-authenticator
                 - aws-cli
                 type: string
+              upgradePolicy:
+                description: |-
+                  The cluster upgrade policy to use for the cluster.
+                  (Official AWS docs for this policy: https://docs.aws.amazon.com/eks/latest/userguide/view-upgrade-policy.html)
+                  `extended` upgrade policy indicates that the cluster will enter into extended support once the Kubernetes version reaches end of standard support. You will incur extended support charges with this setting. You can upgrade your cluster to a standard supported Kubernetes version to stop incurring extended support charges.
+                  `standard` upgrade policy indicates that the cluster is eligible for automatic upgrade at the end of standard support. You will not incur extended support charges with this setting but your EKS cluster will automatically upgrade to the next supported Kubernetes version in standard support.
+                  If omitted, new clusters will use the AWS default upgrade policy (which at the time of writing is "extended") and existing clusters will have their upgrade policy unchanged.
+                enum:
+                - extended
+                - standard
+                type: string
               version:
                 description: |-
                   Version defines the desired Kubernetes version. If no version number

config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanetemplates.yaml

@@ -1044,6 +1044,17 @@ spec:
                         - iam-authenticator
                         - aws-cli
                         type: string
+                      upgradePolicy:
+                        description: |-
+                          The cluster upgrade policy to use for the cluster.
+                          (Official AWS docs for this policy: https://docs.aws.amazon.com/eks/latest/userguide/view-upgrade-policy.html)
+                          `extended` upgrade policy indicates that the cluster will enter into extended support once the Kubernetes version reaches end of standard support. You will incur extended support charges with this setting. You can upgrade your cluster to a standard supported Kubernetes version to stop incurring extended support charges.
+                          `standard` upgrade policy indicates that the cluster is eligible for automatic upgrade at the end of standard support. You will not incur extended support charges with this setting but your EKS cluster will automatically upgrade to the next supported Kubernetes version in standard support.
+                          If omitted, new clusters will use the AWS default upgrade policy (which at the time of writing is "extended") and existing clusters will have their upgrade policy unchanged.
+                        enum:
+                        - extended
+                        - standard
+                        type: string
                       version:
                         description: |-
                           Version defines the desired Kubernetes version. If no version number

controlplane/eks/api/v1beta1/conversion.go

@@ -122,6 +122,7 @@ func (r *AWSManagedControlPlane) ConvertTo(dstRaw conversion.Hub) error {
 	dst.Spec.RolePermissionsBoundary = restored.Spec.RolePermissionsBoundary
 	dst.Status.Version = restored.Status.Version
 	dst.Spec.BootstrapSelfManagedAddons = restored.Spec.BootstrapSelfManagedAddons
+	dst.Spec.UpgradePolicy = restored.Spec.UpgradePolicy
 	return nil
 }
 

controlplane/eks/api/v1beta1/zz_generated.conversion.go

@@ -212,6 +212,15 @@ type AWSManagedControlPlaneSpec struct { //nolint: maligned
 
 	// KubeProxy defines managed attributes of the kube-proxy daemonset
 	KubeProxy KubeProxy `json:"kubeProxy,omitempty"`
+
+	// The cluster upgrade policy to use for the cluster.
+	// (Official AWS docs for this policy: https://docs.aws.amazon.com/eks/latest/userguide/view-upgrade-policy.html)
+	// `extended` upgrade policy indicates that the cluster will enter into extended support once the Kubernetes version reaches end of standard support. You will incur extended support charges with this setting. You can upgrade your cluster to a standard supported Kubernetes version to stop incurring extended support charges.
+	// `standard` upgrade policy indicates that the cluster is eligible for automatic upgrade at the end of standard support. You will not incur extended support charges with this setting but your EKS cluster will automatically upgrade to the next supported Kubernetes version in standard support.
+	// If omitted, new clusters will use the AWS default upgrade policy (which at the time of writing is "extended") and existing clusters will have their upgrade policy unchanged.
+	// +kubebuilder:validation:Enum=extended;standard
+	// +optional
+	UpgradePolicy UpgradePolicy `json:"upgradePolicy,omitempty"`
 }
 
 // KubeProxy specifies how the kube-proxy daemonset is managed.

controlplane/eks/api/v1beta2/awsmanagedcontrolplane_types.go

@@ -241,6 +241,24 @@ type AddonIssue struct {
 	ResourceIDs []string `json:"resourceIds,omitempty"`
 }
 
+// UpgradePolicy defines the support policy to use for the cluster.
+type UpgradePolicy string
+
+var (
+	// UpgradePolicyExtended indicates that the cluster will enter into extended support once the Kubernetes version reaches end of standard support.
+	// You will incur extended support charges with this setting.
+	// You can upgrade your cluster to a standard supported Kubernetes version to stop incurring extended support charges.
+	UpgradePolicyExtended = UpgradePolicy("extended")
+
+	// UpgradePolicyStandard indicates that the cluster is eligible for automatic upgrade at the end of standard support.
+	// You will not incur extended support charges with this setting but your EKS cluster will automatically upgrade to the next supported Kubernetes version in standard support.
+	UpgradePolicyStandard = UpgradePolicy("standard")
+)
+
+func (e UpgradePolicy) String() string {
+	return string(e)
+}
+
 const (
 	// SecurityGroupCluster is the security group for communication between EKS
 	// control plane and managed node groups.

controlplane/eks/api/v1beta2/types.go

@@ -14,6 +14,9 @@ clusterctl generate cluster capi-eks-quickstart --flavor eks-managedmachinepool
 
 NOTE: When creating an EKS cluster only the **MAJOR.MINOR** of the `-kubernetes-version` is taken into consideration.
 
+By default CAPA relies on the default EKS cluster upgrade policy, which at the moment of writing is EXTENDED support.
+See more info about [cluster upgrade policy](https://docs.aws.amazon.com/eks/latest/userguide/view-upgrade-policy.html)
+
 ## Kubeconfig
 
 When creating an EKS cluster 2 kubeconfigs are generated and stored as secrets in the management cluster. This is different to when you create a non-managed cluster using the AWS provider.

docs/book/src/topics/eks/creating-a-cluster.md

@@ -278,3 +278,11 @@ func AddonConflictResolutionFromSDK(conflict ekstypes.ResolveConflicts) *string
 	}
 	return aws.String(string(ekscontrolplanev1.AddonResolutionOverwrite))
 }
+
+// SupportTypeToSDK converts CAPA upgrade support policy types to SDK types.
+func SupportTypeToSDK(input ekscontrolplanev1.UpgradePolicy) ekstypes.SupportType {
+	if input == ekscontrolplanev1.UpgradePolicyStandard {
+		return ekstypes.SupportTypeStandard
+	}
+	return ekstypes.SupportTypeExtended
+}

pkg/cloud/converters/eks.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	"net"
+	"strings"
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
@@ -35,6 +36,7 @@ import (
 	infrav1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2"
 	ekscontrolplanev1 "sigs.k8s.io/cluster-api-provider-aws/v2/controlplane/eks/api/v1beta2"
 	"sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/awserrors"
+	"sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/converters"
 	"sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/wait"
 	"sigs.k8s.io/cluster-api-provider-aws/v2/pkg/internal/cidr"
 	"sigs.k8s.io/cluster-api-provider-aws/v2/pkg/internal/cmp"
@@ -147,23 +149,7 @@ func (s *Service) reconcileCluster(ctx context.Context) error {
 // computeCurrentStatusVersion returns the computed current EKS cluster kubernetes version.
 // The computation has awareness of the fact that EKS clusters only return a major.minor kubernetes version,
 // and returns a compatible version for te status according to the one the user specified in the spec.
-func computeCurrentStatusVersion(specV *string, clusterV *string) *string {
-	specVersion := ""
-	if specV != nil {
-		specVersion = *specV
-	}
-
-	clusterVersion := ""
-	if clusterV != nil {
-		clusterVersion = *clusterV
-	}
-
-	// Ignore parsing errors as these are already validated by the kubebuilder validation and the AWS API.
-	// Also specVersion might not be specified in the spec.Version for AWSManagedControlPlane, this results in a "0.0.0" version.
-	// Also clusterVersion might not yet be returned by the AWS EKS API, as the cluster might still be initializing, this results in a "0.0.0" version.
-	specSemverVersion, _ := semver.ParseTolerant(specVersion)
-	currentSemverVersion, _ := semver.ParseTolerant(clusterVersion)
-
+func computeCurrentStatusVersion(clusterV *string, specSemverVersion semver.Version, currentSemverVersion semver.Version) *string {
 	// If AWS EKS API is not returning a version, set the status.Version to empty string.
 	if currentSemverVersion.String() == "0.0.0" {
 		return ptr.To("")
@@ -187,9 +173,27 @@ func computeCurrentStatusVersion(specV *string, clusterV *string) *string {
 	return clusterV
 }
 
+// parseClusterVersionString parse a version string to semver version.
+// If the string cannot be parsed to semver, returning 0.0.0.
+func parseClusterVersionString(str *string) semver.Version {
+	version := ""
+	if str != nil {
+		version = *str
+	}
+
+	// Ignore parsing errors as these are already validated by the kubebuilder validation and the AWS API.
+	semverVersion, _ := semver.ParseTolerant(version)
+	return semverVersion
+}
+
 func (s *Service) setStatus(cluster *ekstypes.Cluster) error {
+	// specSemver might not be specified in the spec.Version for AWSManagedControlPlane, this results in a "0.0.0" version.
+	specSemver := parseClusterVersionString(s.scope.ControlPlane.Spec.Version)
+	// clusterSemver might not yet be returned by the AWS EKS API, as the cluster might still be initializing, this results in a "0.0.0" version.
+	clusterSemver := parseClusterVersionString(cluster.Version)
+
 	// Set the current Kubernetes control plane version in the status.
-	s.scope.ControlPlane.Status.Version = computeCurrentStatusVersion(s.scope.ControlPlane.Spec.Version, cluster.Version)
+	s.scope.ControlPlane.Status.Version = computeCurrentStatusVersion(cluster.Version, specSemver, clusterSemver)
 
 	// Set the current cluster status in the control plane status.
 	switch cluster.Status {
@@ -211,6 +215,19 @@ func (s *Service) setStatus(cluster *ekstypes.Cluster) error {
 			conditions.MarkFalse(s.scope.ControlPlane, ekscontrolplanev1.EKSControlPlaneUpdatingCondition, "updated", clusterv1.ConditionSeverityInfo, "")
 			record.Eventf(s.scope.ControlPlane, "SuccessfulUpdateEKSControlPlane", "Updated EKS control plane %s", s.scope.KubernetesClusterName())
 		}
+		if s.scope.ControlPlane.Spec.UpgradePolicy == ekscontrolplanev1.UpgradePolicyStandard &&
+			(specSemver.Major < clusterSemver.Major ||
+				(specSemver.Major == clusterSemver.Major && specSemver.Minor < clusterSemver.Minor)) {
+			s.scope.ControlPlane.Status.Ready = false
+			failureMsg := fmt.Sprintf(
+				"EKS control plane %s was automatically upgraded to version %s because %s is out of standard support. "+
+					"This can be fixed by changing to the version of the AWSManagedControlPlane to the one reported in the status",
+				s.scope.KubernetesClusterName(),
+				clusterSemver.String(),
+				specSemver.String(),
+			)
+			s.scope.ControlPlane.Status.FailureMessage = &failureMsg
+		}
 		// TODO FailureReason
 	case ekstypes.ClusterStatusCreating:
 		s.scope.ControlPlane.Status.Ready = false
@@ -478,6 +495,14 @@ func (s *Service) createCluster(ctx context.Context, eksClusterName string) (*ek
 		eksVersion = &v
 	}
 
+	var upgradePolicy *ekstypes.UpgradePolicyRequest
+
+	if s.scope.ControlPlane.Spec.UpgradePolicy != "" {
+		upgradePolicy = &ekstypes.UpgradePolicyRequest{
+			SupportType: converters.SupportTypeToSDK(s.scope.ControlPlane.Spec.UpgradePolicy),
+		}
+	}
+
 	bootstrapAddon := s.scope.BootstrapSelfManagedAddons()
 	input := &eks.CreateClusterInput{
 		Name:                       aws.String(eksClusterName),
@@ -490,6 +515,7 @@ func (s *Service) createCluster(ctx context.Context, eksClusterName string) (*ek
 		Tags:                       tags,
 		KubernetesNetworkConfig:    netConfig,
 		BootstrapSelfManagedAddons: bootstrapAddon,
+		UpgradePolicy:              upgradePolicy,
 	}
 
 	var out *eks.CreateClusterOutput
@@ -545,6 +571,11 @@ func (s *Service) reconcileClusterConfig(ctx context.Context, cluster *ekstypes.
 		input.ResourcesVpcConfig = updateVpcConfig
 	}
 
+	if updateUpgradePolicy := s.reconcileUpgradePolicy(cluster.UpgradePolicy); updateUpgradePolicy != nil {
+		needsUpdate = true
+		input.UpgradePolicy = updateUpgradePolicy
+	}
+
 	if needsUpdate {
 		if err := wait.WaitForWithRetryable(wait.NewBackoff(), func() (bool, error) {
 			if _, err := s.EKSClient.UpdateClusterConfig(ctx, input); err != nil {
@@ -782,6 +813,26 @@ func (s *Service) reconcileClusterVersion(ctx context.Context, cluster *ekstypes
 	return nil
 }
 
+func (s *Service) reconcileUpgradePolicy(upgradePolicy *ekstypes.UpgradePolicyResponse) *ekstypes.UpgradePolicyRequest {
+	// Should not update when cluster upgrade policy is unknown
+	if upgradePolicy == nil {
+		return nil
+	}
+
+	// Cluster stay unchanged when upgrade policy omitted
+	if s.scope.ControlPlane.Spec.UpgradePolicy == "" {
+		return nil
+	}
+
+	if strings.ToLower(string(upgradePolicy.SupportType)) == s.scope.ControlPlane.Spec.UpgradePolicy.String() {
+		return nil
+	}
+
+	return &ekstypes.UpgradePolicyRequest{
+		SupportType: converters.SupportTypeToSDK(s.scope.ControlPlane.Spec.UpgradePolicy),
+	}
+}
+
 func (s *Service) describeEKSCluster(ctx context.Context, eksClusterName string) (*ekstypes.Cluster, error) {
 	input := &eks.DescribeClusterInput{
 		Name: aws.String(eksClusterName),

pkg/cloud/services/eks/cluster.go

@@ -652,6 +652,7 @@ func TestCreateCluster(t *testing.T) {
 						RoleName:                   tc.role,
 						NetworkSpec:                infrav1.NetworkSpec{Subnets: tc.subnets},
 						BootstrapSelfManagedAddons: false,
+						UpgradePolicy:              ekscontrolplanev1.UpgradePolicyStandard,
 					},
 				},
 			})
@@ -674,6 +675,9 @@ func TestCreateCluster(t *testing.T) {
 					Tags:                       tc.tags,
 					Version:                    version,
 					BootstrapSelfManagedAddons: aws.Bool(false),
+					UpgradePolicy: &ekstypes.UpgradePolicyRequest{
+						SupportType: ekstypes.SupportTypeStandard,
+					},
 				}).Return(&eks.CreateClusterOutput{}, nil)
 			}
 			s := NewService(scope)
@@ -805,6 +809,91 @@ func TestReconcileEKSEncryptionConfig(t *testing.T) {
 	}
 }
 
+func TestReconcileUpgradePolicy(t *testing.T) {
+	clusterName := "default.cluster"
+	tests := []struct {
+		name             string
+		oldUpgradePolicy *ekstypes.UpgradePolicyResponse
+		newUpgradePolicy ekscontrolplanev1.UpgradePolicy
+		expect           *ekstypes.UpgradePolicyRequest
+		expectError      bool
+	}{
+		{
+			name: "no update necessary - upgrade policy omitted",
+			oldUpgradePolicy: &ekstypes.UpgradePolicyResponse{
+				SupportType: ekstypes.SupportTypeStandard,
+			},
+			expect:      nil,
+			expectError: false,
+		},
+		{
+			name:             "no update necessary - cannot get cluster upgrade policy",
+			newUpgradePolicy: ekscontrolplanev1.UpgradePolicyStandard,
+			expect:           nil,
+			expectError:      false,
+		},
+		{
+			name: "no update necessary - upgrade policy unchanged",
+			oldUpgradePolicy: &ekstypes.UpgradePolicyResponse{
+				SupportType: ekstypes.SupportTypeStandard,
+			},
+			newUpgradePolicy: ekscontrolplanev1.UpgradePolicyStandard,
+			expect:           nil,
+			expectError:      false,
+		},
+		{
+			name: "needs update",
+			oldUpgradePolicy: &ekstypes.UpgradePolicyResponse{
+				SupportType: ekstypes.SupportTypeStandard,
+			},
+			newUpgradePolicy: ekscontrolplanev1.UpgradePolicyExtended,
+			expect: &ekstypes.UpgradePolicyRequest{
+				SupportType: ekstypes.SupportTypeExtended,
+			},
+			expectError: false,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			g := NewWithT(t)
+
+			mockControl := gomock.NewController(t)
+			defer mockControl.Finish()
+
+			scheme := runtime.NewScheme()
+			_ = infrav1.AddToScheme(scheme)
+			_ = ekscontrolplanev1.AddToScheme(scheme)
+			client := fake.NewClientBuilder().WithScheme(scheme).Build()
+			scope, err := scope.NewManagedControlPlaneScope(scope.ManagedControlPlaneScopeParams{
+				Client: client,
+				Cluster: &clusterv1.Cluster{
+					ObjectMeta: metav1.ObjectMeta{
+						Namespace: "ns",
+						Name:      clusterName,
+					},
+				},
+				ControlPlane: &ekscontrolplanev1.AWSManagedControlPlane{
+					Spec: ekscontrolplanev1.AWSManagedControlPlaneSpec{
+						Version:       aws.String("1.16"),
+						UpgradePolicy: tc.newUpgradePolicy,
+					},
+				},
+			})
+			g.Expect(err).To(BeNil())
+
+			s := NewService(scope)
+
+			upgradePolicyRequest := s.reconcileUpgradePolicy(tc.oldUpgradePolicy)
+			if tc.expectError {
+				g.Expect(err).To(HaveOccurred())
+				return
+			}
+			g.Expect(upgradePolicyRequest).To(Equal(tc.expect))
+		})
+	}
+}
+
 func TestCreateIPv6Cluster(t *testing.T) {
 	g := NewWithT(t)