Starting aws registration by spoke by assuming IAM role on startup and adding annotations to ManagedCluster CR

Signed-off-by: Erica Jin <[email protected]>

Author: jaswalkiranavtar

manifests/klusterlet/managed/klusterlet-registration-serviceaccount.yaml

@@ -9,5 +9,9 @@ metadata:
     "{{ $key }}": "{{ $value }}"
     {{ end }}
     {{ end }}
+    {{ if and .ManagedClusterRoleArn (eq .RegistrationDriver.AuthType "awsirsa") }}
+  annotations:
+    eks.amazonaws.com/role-arn: {{ .ManagedClusterRoleArn }}
+    {{ end }}
 imagePullSecrets:
 - name: open-cluster-management-image-pull-credentials

manifests/klusterlet/managed/klusterlet-work-serviceaccount.yaml

@@ -9,5 +9,9 @@ metadata:
     "{{ $key }}": "{{ $value }}"
     {{ end }}
     {{ end }}
+    {{ if and .ManagedClusterRoleArn (eq .RegistrationDriver.AuthType "awsirsa") }}
+  annotations:
+    eks.amazonaws.com/role-arn: {{ .ManagedClusterRoleArn }}
+    {{ end }}
 imagePullSecrets:
 - name: open-cluster-management-image-pull-credentials

manifests/klusterlet/management/klusterlet-agent-deployment.yaml

@@ -109,11 +109,15 @@ spec:
           {{if .AppliedManifestWorkEvictionGracePeriod}}
           - "--appliedmanifestwork-eviction-grace-period={{ .AppliedManifestWorkEvictionGracePeriod }}"
           {{end}}
-          {{if .RegistrationDriver.AuthType}}
+          {{if and .RegistrationDriver .RegistrationDriver.AuthType}}
           - "--registration-auth={{ .RegistrationDriver.AuthType }}"
-          {{end}}
           {{if eq .RegistrationDriver.AuthType "awsirsa"}}
           - "--hub-cluster-arn={{ .RegistrationDriver.AwsIrsa.HubClusterArn }}"
+          - "--managed-cluster-arn={{ .RegistrationDriver.AwsIrsa.ManagedClusterArn }}"
+          {{if .ManagedClusterRoleSuffix}}
+          - "--managed-cluster-role-suffix={{ .ManagedClusterRoleSuffix }}"
+          {{end}}
+          {{end}}
           {{end}}
         env:
           - name: POD_NAME
@@ -144,6 +148,10 @@ spec:
           mountPath: "/spoke/hub-kubeconfig"
         - name: tmpdir
           mountPath: /tmp
+        {{if and .RegistrationDriver .RegistrationDriver.AuthType (eq .RegistrationDriver.AuthType "awsirsa")}}
+        - name: dot-aws
+          mountPath: /.aws
+        {{end}}
         {{if eq .InstallMode "SingletonHosted"}}
         - name: spoke-kubeconfig-secret
           mountPath: "/spoke/config"
@@ -195,6 +203,10 @@ spec:
           medium: Memory
       - name: tmpdir
         emptyDir: { }
+      {{if and .RegistrationDriver .RegistrationDriver.AuthType (eq .RegistrationDriver.AuthType "awsirsa")}}
+      - name: dot-aws
+        emptyDir: { }
+      {{end}}
       {{if eq .InstallMode "SingletonHosted"}}
       - name: spoke-kubeconfig-secret
         secret:

manifests/klusterlet/management/klusterlet-registration-deployment.yaml

@@ -97,11 +97,15 @@ spec:
           {{if gt .RegistrationKubeAPIBurst 0}}
           - "--kube-api-burst={{ .RegistrationKubeAPIBurst }}"
           {{end}}
-          {{if .RegistrationDriver.AuthType}}
+          {{if and .RegistrationDriver .RegistrationDriver.AuthType}}
           - "--registration-auth={{ .RegistrationDriver.AuthType }}"
-          {{end}}
           {{if eq .RegistrationDriver.AuthType "awsirsa"}}
           - "--hub-cluster-arn={{ .RegistrationDriver.AwsIrsa.HubClusterArn }}"
+          - "--managed-cluster-arn={{ .RegistrationDriver.AwsIrsa.ManagedClusterArn }}"
+          {{if .ManagedClusterRoleSuffix}}
+          - "--managed-cluster-role-suffix={{ .ManagedClusterRoleSuffix }}"
+          {{end}}
+          {{end}}
           {{end}}
         env:
           - name: POD_NAME
@@ -132,6 +136,10 @@ spec:
           mountPath: "/spoke/hub-kubeconfig"
         - name: tmpdir
           mountPath: /tmp
+        {{if and .RegistrationDriver .RegistrationDriver.AuthType (eq .RegistrationDriver.AuthType "awsirsa")}}
+        - name: dot-aws
+          mountPath: /.aws
+        {{end}}
         {{if eq .InstallMode "Hosted"}}
         - name: spoke-kubeconfig-secret
           mountPath: "/spoke/config"
@@ -183,6 +191,10 @@ spec:
           medium: Memory
       - name: tmpdir
         emptyDir: { }
+      {{if and .RegistrationDriver .RegistrationDriver.AuthType (eq .RegistrationDriver.AuthType "awsirsa")}}
+      - name: dot-aws
+        emptyDir: { }
+      {{end}}
       {{if eq .InstallMode "Hosted"}}
       - name: spoke-kubeconfig-secret
         secret:

manifests/klusterlet/management/klusterlet-work-deployment.yaml

@@ -107,6 +107,10 @@ spec:
           readOnly: true
         - name: tmpdir
           mountPath: /tmp
+        {{if and .RegistrationDriver .RegistrationDriver.AuthType (eq .RegistrationDriver.AuthType "awsirsa")}}
+        - name: dot-aws
+          mountPath: /.aws
+        {{end}}
         {{if eq .InstallMode "Hosted"}}
         - name: spoke-kubeconfig-secret
           mountPath: "/spoke/config"
@@ -147,6 +151,10 @@ spec:
           secretName: {{ .HubKubeConfigSecret }}
       - name: tmpdir
         emptyDir: { }
+      {{if and .RegistrationDriver .RegistrationDriver.AuthType (eq .RegistrationDriver.AuthType "awsirsa")}}
+      - name: dot-aws
+        emptyDir: { }
+      {{end}}
       {{if eq .InstallMode "Hosted"}}
       - name: spoke-kubeconfig-secret
         secret:

pkg/common/helpers/aws.go

@@ -0,0 +1,14 @@
+package helpers
+
+import (
+	"regexp"
+)
+
+func IsEksArnWellFormed(eksArn string) bool {
+	pattern := "^arn:aws:eks:([a-zA-Z0-9-]+):(\\d{12}):cluster/([a-zA-Z0-9-]+)$"
+	matched, err := regexp.MatchString(pattern, eksArn)
+	if err != nil {
+		return false
+	}
+	return matched
+}

pkg/operator/operators/klusterlet/controllers/klusterletcontroller/klusterlet_controller.go

@@ -2,6 +2,9 @@ package klusterletcontroller
 
 import (
 	"context"
+	"crypto/md5"
+	"encoding/hex"
+	er "errors"
 	"fmt"
 	"strings"
 	"time"
@@ -114,7 +117,8 @@ func NewKlusterletController(
 }
 
 type AwsIrsa struct {
-	HubClusterArn string
+	HubClusterArn     string
+	ManagedClusterArn string
 }
 
 type RegistrationDriver struct {
@@ -187,6 +191,10 @@ type klusterletConfig struct {
 	// Labels of the agents are synced from klusterlet CR.
 	Labels             map[string]string
 	RegistrationDriver RegistrationDriver
+
+	ManagedClusterArn        string
+	ManagedClusterRoleArn    string
+	ManagedClusterRoleSuffix string
 }
 
 // If multiplehubs feature gate is enabled, using the bootstrapkubeconfigs from klusterlet CR.
@@ -329,12 +337,32 @@ func (n *klusterletController) sync(ctx context.Context, controllerContext facto
 		//Configuring Registration driver depending on registration auth
 		if &klusterlet.Spec.RegistrationConfiguration.RegistrationDriver != nil &&
 			klusterlet.Spec.RegistrationConfiguration.RegistrationDriver.AuthType == AwsIrsaAuthType {
+
+			hubClusterArn := klusterlet.Spec.RegistrationConfiguration.RegistrationDriver.AwsIrsa.HubClusterArn
+			managedClusterArn := klusterlet.Spec.RegistrationConfiguration.RegistrationDriver.AwsIrsa.ManagedClusterArn
+
+			if !commonhelpers.IsEksArnWellFormed(hubClusterArn) {
+				errorMsg := fmt.Sprintf("HubClusterArn %s is not well formed", hubClusterArn)
+				klog.Errorf(errorMsg)
+				return er.New(errorMsg)
+			}
+
+			if !commonhelpers.IsEksArnWellFormed(managedClusterArn) {
+				errorMsg := fmt.Sprintf("ManagedClusterArn %s is not well formed", managedClusterArn)
+				klog.Errorf(errorMsg)
+				return er.New(errorMsg)
+			}
+
 			config.RegistrationDriver = RegistrationDriver{
 				AuthType: klusterlet.Spec.RegistrationConfiguration.RegistrationDriver.AuthType,
 				AwsIrsa: &AwsIrsa{
-					HubClusterArn: klusterlet.Spec.RegistrationConfiguration.RegistrationDriver.AwsIrsa.HubClusterArn,
+					HubClusterArn:     hubClusterArn,
+					ManagedClusterArn: managedClusterArn,
 				},
 			}
+			managedClusterRoleArn, managedClusterRoleSuffix := n.generateManagedRoleArnAndSuffix(klusterlet)
+			config.ManagedClusterRoleArn = managedClusterRoleArn
+			config.ManagedClusterRoleSuffix = managedClusterRoleSuffix
 		} else {
 			config.RegistrationDriver = RegistrationDriver{
 				AuthType: klusterlet.Spec.RegistrationConfiguration.RegistrationDriver.AuthType,
@@ -433,6 +461,23 @@ func (n *klusterletController) sync(ctx context.Context, controllerContext facto
 	return utilerrors.NewAggregate(errs)
 }
 
+func (n *klusterletController) generateManagedRoleArnAndSuffix(klusterlet *operatorapiv1.Klusterlet) (string, string) {
+	hubClusterArn := klusterlet.Spec.RegistrationConfiguration.RegistrationDriver.AwsIrsa.HubClusterArn
+	managedClusterArn := klusterlet.Spec.RegistrationConfiguration.RegistrationDriver.AwsIrsa.ManagedClusterArn
+
+	hubClusterStringParts := strings.Split(hubClusterArn, ":")
+
+	managedClusterStringParts := strings.Split(managedClusterArn, ":")
+	hubClusterName := strings.Split(hubClusterStringParts[5], "/")[1]
+	hubClusterAccountId := hubClusterStringParts[4]
+	managedClusterName := strings.Split(managedClusterStringParts[5], "/")[1]
+	managedClusterAccountId := managedClusterStringParts[4]
+	md5HashUniqueIdentifier := generateMd5HashUniqueIdentifier(hubClusterAccountId, hubClusterName, managedClusterAccountId, managedClusterName)
+	//arn:aws:iam::<managed-cluster-account-id>:role/ocm-managed-cluster-<md5-hash-unique-identifier>
+	managedClusterRoleArn := "arn:aws:iam::" + managedClusterAccountId + ":role/ocm-managed-cluster-" + md5HashUniqueIdentifier
+	return managedClusterRoleArn, md5HashUniqueIdentifier
+}
+
 // TODO also read CABundle from ExternalServerURLs and set into registration deployment
 func getServersFromKlusterlet(klusterlet *operatorapiv1.Klusterlet) string {
 	if klusterlet.Spec.ExternalServerURLs == nil {
@@ -536,3 +581,8 @@ func serviceAccountName(suffix string, klusterlet *operatorapiv1.Klusterlet) str
 	}
 	return fmt.Sprintf("%s-%s", klusterlet.Name, suffix)
 }
+
+func generateMd5HashUniqueIdentifier(hubClusterAccountId string, hubClusterName string, managedClusterAccountId string, managedClusterName string) string {
+	hash := md5.Sum([]byte(hubClusterAccountId + "#" + hubClusterName + "#" + managedClusterAccountId + "#" + managedClusterName))
+	return hex.EncodeToString(hash[:])
+}

pkg/operator/operators/klusterlet/controllers/klusterletcontroller/klusterlet_controller_test.go

@@ -388,6 +388,9 @@ func assertKlusterletDeployment(t *testing.T, actions []clienttesting.Action, ve
 	}
 
 	args := deployment.Spec.Template.Spec.Containers[0].Args
+	volumeMounts := deployment.Spec.Template.Spec.Containers[0].VolumeMounts
+	volumes := deployment.Spec.Template.Spec.Volumes
+
 	expectedArgs := []string{
 		"/registration-operator",
 		"agent",
@@ -405,14 +408,39 @@ func assertKlusterletDeployment(t *testing.T, actions []clienttesting.Action, ve
 		expectedArgs = append(expectedArgs, "--disable-leader-election")
 	}
 
-	expectedArgs = append(expectedArgs, "--status-sync-interval=60s", "--kube-api-qps=20", "--kube-api-burst=60",
-		"--registration-auth=awsirsa", "--hub-cluster-arn=arneks:us-west-2:123456789012:cluster/hub-cluster1")
+	expectedArgs = append(expectedArgs, "--status-sync-interval=60s", "--kube-api-qps=20", "--kube-api-burst=60")
+
+	expectedArgs = append(expectedArgs, "--registration-auth=awsirsa",
+		"--hub-cluster-arn=arn:aws:eks:us-west-2:123456789012:cluster/hub-cluster1",
+		"--managed-cluster-arn=arn:aws:eks:us-west-2:123456789012:cluster/managed-cluster1",
+		"--managed-cluster-role-suffix=7f8141296c75f2871e3d030f85c35692")
 
 	if !equality.Semantic.DeepEqual(args, expectedArgs) {
 		t.Errorf("Expect args %v, but got %v", expectedArgs, args)
 		return
 	}
 
+	assert.True(t, isDotAwsMounted(volumeMounts))
+	assert.True(t, isDotAwsVolumePresent(volumes))
+
+}
+
+func isDotAwsVolumePresent(volumes []corev1.Volume) bool {
+	for _, volume := range volumes {
+		if volume.Name == "dot-aws" {
+			return true
+		}
+	}
+	return false
+}
+
+func isDotAwsMounted(mounts []corev1.VolumeMount) bool {
+	for _, mount := range mounts {
+		if mount.Name == "dot-aws" && mount.MountPath == "/.aws" {
+			return true
+		}
+	}
+	return false
 }
 
 func assertRegistrationDeployment(t *testing.T, actions []clienttesting.Action, verb, serverURL, clusterName string, replica int32, awsAuth bool) {
@@ -444,7 +472,9 @@ func assertRegistrationDeployment(t *testing.T, actions []clienttesting.Action,
 
 	expectedArgs = append(expectedArgs, "--kube-api-qps=10", "--kube-api-burst=60")
 	if awsAuth {
-		expectedArgs = append(expectedArgs, "--registration-auth=awsirsa", "--hub-cluster-arn=arneks:us-west-2:123456789012:cluster/hub-cluster1")
+		expectedArgs = append(expectedArgs, "--registration-auth=awsirsa", "--hub-cluster-arn=arn:aws:eks:us-west-2:123456789012:cluster/hub-cluster1")
+		expectedArgs = append(expectedArgs, "--managed-cluster-arn=arn:aws:eks:us-west-2:123456789012:cluster/managed-cluster1",
+			"--managed-cluster-role-suffix=7f8141296c75f2871e3d030f85c35692")
 	}
 	if !equality.Semantic.DeepEqual(args, expectedArgs) {
 		t.Errorf("Expect args %v, but got %v", expectedArgs, args)
@@ -988,18 +1018,50 @@ func TestGetServersFromKlusterlet(t *testing.T) {
 	}
 }
 
+func TestAWSIrsaAuthInSingletonModeWithInvalidClusterArns(t *testing.T) {
+	klusterlet := newKlusterlet("klusterlet", "testns", "cluster1")
+	awsIrsaRegistrationDriver := operatorapiv1.RegistrationDriver{
+		AuthType: AwsIrsaAuthType,
+		AwsIrsa: &operatorapiv1.AwsIrsa{
+			HubClusterArn:     "arn:aws:bks:us-west-2:123456789012:cluster/hub-cluster1",
+			ManagedClusterArn: "arn:aws:eks:us-west-2:123456789012:cluster/managed-cluster1",
+		},
+	}
+	klusterlet.Spec.RegistrationConfiguration.RegistrationDriver = awsIrsaRegistrationDriver
+	klusterlet.Spec.DeployOption.Mode = operatorapiv1.InstallModeSingleton
+	hubSecret := newSecret(helpers.HubKubeConfig, "testns")
+	hubSecret.Data["kubeconfig"] = []byte("dummykubeconfig")
+	hubSecret.Data["cluster-name"] = []byte("cluster1")
+	objects := []runtime.Object{
+		newNamespace("testns"),
+		newSecret(helpers.BootstrapHubKubeConfig, "testns"),
+		hubSecret,
+	}
+
+	syncContext := testingcommon.NewFakeSyncContext(t, "klusterlet")
+	controller := newTestController(t, klusterlet, syncContext.Recorder(), nil, false,
+		objects...)
+
+	err := controller.controller.sync(context.TODO(), syncContext)
+	if err != nil {
+		assert.Equal(t, err.Error(), "HubClusterArn arn:aws:bks:us-west-2:123456789012:cluster/hub-cluster1 is not well formed")
+	}
+
+}
+
 func TestAWSIrsaAuthInSingletonMode(t *testing.T) {
 	klusterlet := newKlusterlet("klusterlet", "testns", "cluster1")
 	awsIrsaRegistrationDriver := operatorapiv1.RegistrationDriver{
 		AuthType: AwsIrsaAuthType,
 		AwsIrsa: &operatorapiv1.AwsIrsa{
-			HubClusterArn: "arneks:us-west-2:123456789012:cluster/hub-cluster1",
+			HubClusterArn:     "arn:aws:eks:us-west-2:123456789012:cluster/hub-cluster1",
+			ManagedClusterArn: "arn:aws:eks:us-west-2:123456789012:cluster/managed-cluster1",
 		},
 	}
 	klusterlet.Spec.RegistrationConfiguration.RegistrationDriver = awsIrsaRegistrationDriver
 	klusterlet.Spec.DeployOption.Mode = operatorapiv1.InstallModeSingleton
 	hubSecret := newSecret(helpers.HubKubeConfig, "testns")
-	hubSecret.Data["kubeconfig"] = []byte("dummuykubeconnfig")
+	hubSecret.Data["kubeconfig"] = []byte("dummykubeconfig")
 	hubSecret.Data["cluster-name"] = []byte("cluster1")
 	objects := []runtime.Object{
 		newNamespace("testns"),
@@ -1024,7 +1086,8 @@ func TestAWSIrsaAuthInNonSingletonMode(t *testing.T) {
 	awsIrsaRegistrationDriver := operatorapiv1.RegistrationDriver{
 		AuthType: AwsIrsaAuthType,
 		AwsIrsa: &operatorapiv1.AwsIrsa{
-			HubClusterArn: "arneks:us-west-2:123456789012:cluster/hub-cluster1",
+			HubClusterArn:     "arn:aws:eks:us-west-2:123456789012:cluster/hub-cluster1",
+			ManagedClusterArn: "arn:aws:eks:us-west-2:123456789012:cluster/managed-cluster1",
 		},
 	}
 	klusterlet.Spec.RegistrationConfiguration.RegistrationDriver = awsIrsaRegistrationDriver

pkg/registration/helpers/helpers.go

@@ -2,9 +2,6 @@ package helpers
 
 import (
 	"embed"
-	"net/url"
-	"regexp"
-
 	"github.com/openshift/library-go/pkg/assets"
 	"github.com/openshift/library-go/pkg/operator/resource/resourceapply"
 	certificatesv1 "k8s.io/api/certificates/v1"
@@ -13,6 +10,7 @@ import (
 	"k8s.io/client-go/discovery/cached/memory"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/restmapper"
+	"net/url"
 
 	clusterv1 "open-cluster-management.io/api/cluster/v1"
 )
@@ -177,14 +175,3 @@ func IsCSRSupported(nativeClient kubernetes.Interface) (bool, bool, error) {
 	}
 	return v1CSRSupported, v1beta1CSRSupported, nil
 }
-
-// IsEksArnWellFormed checks if the EKS cluster ARN is well-formed
-// Example of a well-formed ARN: arn:aws:eks:us-west-2:123456789012:cluster/my-cluster
-func IsEksArnWellFormed(eksArn string) bool {
-	pattern := "^arn:aws:eks:([a-zA-Z0-9-]+):(\\d{12}):cluster/([a-zA-Z0-9-]+)$"
-	matched, err := regexp.MatchString(pattern, eksArn)
-	if err != nil {
-		return false
-	}
-	return matched
-}