Skip to content

✨ Add ability to control "EKS Auto Mode" for EKS clusters #5642

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
+1,443 −81
Open

✨ Add ability to control "EKS Auto Mode" for EKS clusters #5642

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanes.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2345,6 +2345,33 @@ spec:
AssociateOIDCProvider can be enabled to automatically create an identity
provider for the controller for use with IAM roles for service accounts
type: boolean
autoMode:
description: |-
AutoMode is the EKS Auto Mode.
allows to create cluster with aws compute, ebs, elb capabilities.
properties:
compute:
description: Compute capability configuration for EKS Auto Mode.
properties:
nodePools:
description: NodePools that defines the compute resources
for your EKS Auto Mode cluster.
items:
type: string
type: array
nodeRoleArn:
description: |-
NodeRoleArn the ARN of the IAM Role EKS will assign to EC2 Managed Instances in your EKS
Auto Mode cluster. This value cannot be changed after the compute capability of
EKS Auto Mode is enabled. For more information, see the IAM Reference in the
Amazon EKS User Guide.
type: string
type: object
enabled:
default: false
description: Enabled will enable EKS Auto Mode.
type: boolean
type: object
bastion:
description: Bastion contains options to configure the bastion host.
properties:
Expand Down
28 changes: 28 additions & 0 deletions config/crd/bases/controlplane.cluster.x-k8s.io_awsmanagedcontrolplanetemplates.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,34 @@ spec:
AssociateOIDCProvider can be enabled to automatically create an identity
provider for the controller for use with IAM roles for service accounts
type: boolean
autoMode:
description: |-
AutoMode is the EKS Auto Mode.
allows to create cluster with aws compute, ebs, elb capabilities.
properties:
compute:
description: Compute capability configuration for EKS
Auto Mode.
properties:
nodePools:
description: NodePools that defines the compute resources
for your EKS Auto Mode cluster.
items:
type: string
type: array
nodeRoleArn:
description: |-
NodeRoleArn the ARN of the IAM Role EKS will assign to EC2 Managed Instances in your EKS
Auto Mode cluster. This value cannot be changed after the compute capability of
EKS Auto Mode is enabled. For more information, see the IAM Reference in the
Amazon EKS User Guide.
type: string
type: object
enabled:
default: false
description: Enabled will enable EKS Auto Mode.
type: boolean
type: object
bastion:
description: Bastion contains options to configure the bastion
host.
Expand Down
1 change: 1 addition & 0 deletions controlplane/eks/api/v1beta1/conversion.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,7 @@ func (r *AWSManagedControlPlane) ConvertTo(dstRaw conversion.Hub) error {
dst.Spec.RolePermissionsBoundary = restored.Spec.RolePermissionsBoundary
dst.Status.Version = restored.Status.Version
dst.Spec.BootstrapSelfManagedAddons = restored.Spec.BootstrapSelfManagedAddons
dst.Spec.AutoMode = restored.Spec.AutoMode
return nil
}

Expand Down
1 change: 1 addition & 0 deletions controlplane/eks/api/v1beta1/zz_generated.conversion.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

26 changes: 26 additions & 0 deletions controlplane/eks/api/v1beta2/awsmanagedcontrolplane_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,9 @@ type AWSManagedControlPlaneSpec struct { //nolint: maligned
// +kubebuilder:default=true
BootstrapSelfManagedAddons bool `json:"bootstrapSelfManagedAddons,omitempty"`

// +optional
AutoMode *AutoMode `json:"autoMode,omitempty"`

// RestrictPrivateSubnets indicates that the EKS control plane should only use private subnets.
// +kubebuilder:default=false
RestrictPrivateSubnets bool `json:"restrictPrivateSubnets,omitempty"`
Expand All @@ -225,6 +228,29 @@ type KubeProxy struct {
Disable bool `json:"disable,omitempty"`
}

// AutoMode is the EKS Auto Mode.
// allows to create cluster with aws compute, ebs, elb capabilities.
type AutoMode struct {
// Enabled will enable EKS Auto Mode.
// +kubebuilder:default=false
Enabled bool `json:"enabled,omitempty"`
// Compute capability configuration for EKS Auto Mode.
// +optional
Compute Compute `json:"compute,omitempty"`
}

// Compute allows to run compute capability with EKS AutoMode.
type Compute struct {
// NodePools that defines the compute resources for your EKS Auto Mode cluster.
NodePools []string `json:"nodePools,omitempty"`
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

needs a comment describing the field.

// NodeRoleArn the ARN of the IAM Role EKS will assign to EC2 Managed Instances in your EKS
// Auto Mode cluster. This value cannot be changed after the compute capability of
// EKS Auto Mode is enabled. For more information, see the IAM Reference in the
// Amazon EKS User Guide.
// +optional
NodeRoleArn *string `json:"nodeRoleArn,omitempty"`
}

// VpcCni specifies configuration related to the VPC CNI.
type VpcCni struct {
// Disable indicates that the Amazon VPC CNI should be disabled. With EKS clusters the
Expand Down
54 changes: 52 additions & 2 deletions controlplane/eks/api/v1beta2/awsmanagedcontrolplane_webhook.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"github.com/pkg/errors"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/validation/field"
"k8s.io/apimachinery/pkg/util/version"
"k8s.io/klog/v2"
Expand Down Expand Up @@ -52,6 +53,9 @@ const (
cidrSizeMin = 16
vpcCniAddon = "vpc-cni"
kubeProxyAddon = "kube-proxy"

autoModeComputeNodePoolSystem = "system"
autoModeComputeNodePoolGeneral = "general-purpose"
)

// SetupWebhookWithManager will setup the webhooks for the AWSManagedControlPlane.
Expand Down Expand Up @@ -102,6 +106,7 @@ func (*awsManagedControlPlaneWebhook) ValidateCreate(_ context.Context, obj runt
allErrs = append(allErrs, r.validateSecondaryCIDR()...)
allErrs = append(allErrs, r.validateEKSAddons()...)
allErrs = append(allErrs, r.validateDisableVPCCNI()...)
allErrs = append(allErrs, r.validateAutoMode(nil)...)
allErrs = append(allErrs, r.validateRestrictPrivateSubnets()...)
allErrs = append(allErrs, r.validateKubeProxy()...)
allErrs = append(allErrs, r.Spec.AdditionalTags.Validate()...)
Expand Down Expand Up @@ -144,6 +149,7 @@ func (*awsManagedControlPlaneWebhook) ValidateUpdate(ctx context.Context, oldObj
allErrs = append(allErrs, r.validateAccessConfigUpdate(oldAWSManagedControlplane)...)
allErrs = append(allErrs, r.validateIAMAuthConfig()...)
allErrs = append(allErrs, r.validateSecondaryCIDR()...)
allErrs = append(allErrs, r.validateAutoMode(oldAWSManagedControlplane)...)
allErrs = append(allErrs, r.validateEKSAddons()...)
allErrs = append(allErrs, r.validateDisableVPCCNI()...)
allErrs = append(allErrs, r.validateRestrictPrivateSubnets()...)
Expand Down Expand Up @@ -472,6 +478,52 @@ func validateDisableVPCCNI(vpcCni VpcCni, addons *[]Addon, path *field.Path) fie
return allErrs
}

func (r *AWSManagedControlPlane) validateAutoMode(old *AWSManagedControlPlane) field.ErrorList {
return validateAutoMode(r.Spec, old, field.NewPath("spec"))
}

func validateAutoMode(spec AWSManagedControlPlaneSpec, old *AWSManagedControlPlane, path *field.Path) field.ErrorList {
var allErrs field.ErrorList

if spec.AutoMode == nil {
return nil
}

if spec.AutoMode.Enabled {
// EKS Auto mode is not compatible with configmap AuthenticationMode.
if spec.AccessConfig.AuthenticationMode == EKSAuthenticationModeConfigMap {
authConfigField := path.Child("accessConfig", "authenticationMode")
allErrs = append(allErrs, field.Invalid(authConfigField, spec.AccessConfig.AuthenticationMode, "authenticationMode CONFIG_MAP couldn't be used with autoMode"))
}

if old != nil {
// nodeRoleArn cannot be changed after the compute capability of EKS Auto Mode is enabled.
if old.Spec.AutoMode.Compute.NodeRoleArn != spec.AutoMode.Compute.NodeRoleArn {
nodeRoleArnField := path.Child("autoMode", "compute", "nodeRoleArn")
allErrs = append(allErrs, field.Invalid(nodeRoleArnField, spec.AutoMode.Compute.NodeRoleArn, "nodeRoleArn could not be changed"))
}
}

if len(spec.AutoMode.Compute.NodePools) > 0 {
// nodeRoleArn should be always defined with node pools.
if spec.AutoMode.Compute.NodeRoleArn == nil {
nodeRoleArnField := path.Child("autoMode", "compute", "nodeRoleArn")
allErrs = append(allErrs, field.Invalid(nodeRoleArnField, spec.AutoMode.Compute.NodeRoleArn, "nodeRoleArn is required when nodePools specified"))
}

allowedPoolNames := sets.New[string](autoModeComputeNodePoolSystem, autoModeComputeNodePoolGeneral)
for _, poolName := range spec.AutoMode.Compute.NodePools {
nodePoolsField := path.Child("autoMode", "compute", "nodePools")
if !allowedPoolNames.Has(poolName) {
allErrs = append(allErrs, field.Invalid(nodePoolsField, poolName, "nodePools contains an invalid pool"))
}
}
}
}

return allErrs
}

func (r *AWSManagedControlPlane) validateRestrictPrivateSubnets() field.ErrorList {
return validateRestrictPrivateSubnets(r.Spec.RestrictPrivateSubnets, r.Spec.NetworkSpec, r.Spec.EKSClusterName, field.NewPath("spec"))
}
Expand Down Expand Up @@ -620,7 +672,5 @@ func (*awsManagedControlPlaneWebhook) Default(_ context.Context, obj runtime.Obj
infrav1.SetDefaults_Bastion(&r.Spec.Bastion)
infrav1.SetDefaults_NetworkSpec(&r.Spec.NetworkSpec)

// Set default value for BootstrapSelfManagedAddons
r.Spec.BootstrapSelfManagedAddons = true
return nil
}
60 changes: 60 additions & 0 deletions controlplane/eks/api/v1beta2/awsmanagedcontrolplane_webhook_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,7 @@ func TestWebhookCreate(t *testing.T) {
secondaryCidr *string
secondaryCidrBlocks []infrav1.VpcCidrBlock
kubeProxy KubeProxy
AutoMode *AutoMode
accessConfig *AccessConfig
}{
{
Expand Down Expand Up @@ -364,6 +365,30 @@ func TestWebhookCreate(t *testing.T) {
BootstrapClusterCreatorAdminPermissions: ptr.To(false),
},
},
{
name: "autoMode compute not allowed with authenticationMode CONFIG_MAP",
eksClusterName: "default_cluster1",
eksVersion: "v1.19",
expectError: true,
vpcCNI: VpcCni{Disable: false},
AutoMode: &AutoMode{Enabled: true},
},
{
name: "autoMode compute nodeRoleArn should be defined with nodePools",
eksClusterName: "default_cluster1",
eksVersion: "v1.19",
expectError: true,
vpcCNI: VpcCni{Disable: false},
AutoMode: &AutoMode{Enabled: true, Compute: Compute{NodePools: []string{"system", "general-purpose"}}},
},
{
name: "autoMode compute nodeRoleArn defined with nodePools",
eksClusterName: "default_cluster1",
eksVersion: "v1.19",
expectError: false,
vpcCNI: VpcCni{Disable: false},
AutoMode: &AutoMode{Enabled: true, Compute: Compute{NodePools: []string{"system", "general-purpose"}, NodeRoleArn: aws.String("foo")}},
},
}

for _, tc := range tests {
Expand Down Expand Up @@ -411,6 +436,10 @@ func TestWebhookCreate(t *testing.T) {
mcp.Spec.AccessConfig = tc.accessConfig
}

if tc.AutoMode != nil {
mcp.Spec.AutoMode = tc.AutoMode
}

err := testEnv.Create(ctx, mcp)

if tc.expectError {
Expand Down Expand Up @@ -877,6 +906,37 @@ func TestWebhookUpdate(t *testing.T) {
},
expectError: true,
},
{
name: "changing noderolearn is not allowed after it has been set",
oldClusterSpec: AWSManagedControlPlaneSpec{
EKSClusterName: "default_cluster1",
NetworkSpec: infrav1.NetworkSpec{
VPC: infrav1.VPCSpec{},
},
Version: ptr.To[string]("1.22"),
AutoMode: &AutoMode{
Compute: Compute{
NodeRoleArn: aws.String("fooarn"),
NodePools: []string{"pool1", "pool2"},
},
},
},
newClusterSpec: AWSManagedControlPlaneSpec{
EKSClusterName: "default_cluster1",
NetworkSpec: infrav1.NetworkSpec{
VPC: infrav1.VPCSpec{
IPv6: &infrav1.IPv6{},
},
},
AutoMode: &AutoMode{
Compute: Compute{
NodeRoleArn: aws.String("bararn"),
NodePools: []string{"pool1", "pool2"},
},
},
},
expectError: true,
},
}

for _, tc := range tests {
Expand Down
46 changes: 46 additions & 0 deletions controlplane/eks/api/v1beta2/zz_generated.deepcopy.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading