super wip br

br/Dockerfile

@@ -0,0 +1,15 @@
+FROM public.ecr.aws/bottlerocket/bottlerocket-admin:v0.11.16 as br-admin
+
+FROM public.ecr.aws/amazonlinux/amazonlinux:2023
+
+RUN yum install -y awscli jq wget
+
+RUN wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_$([ "x86_64" = "$(uname -m)" ] && echo amd64 || echo arm64) && chmod +x /usr/local/bin/yq
+
+# iam-auth exec's aws_signing_helper via the aws sdk which execs it thru sh
+# https://github.com/aws/aws-sdk-go-v2/blob/061fd1b6d9940c6b37974e6aa64a5f9dc4ee1dc4/credentials/processcreds/provider.go#L93
+COPY --from=br-admin /bin/bash /static-sh
+
+COPY start-br-setup.sh /
+
+ENTRYPOINT [ "/start-br-setup.sh" ]

br/start-br-setup.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+set -e
+set -x
+
+declare -r PERSISTENT_STORAGE_BASE_DIR="/.bottlerocket/host-containers/current"
+declare -r USER_DATA="${PERSISTENT_STORAGE_BASE_DIR}/user-data"
+
+if [[ "$(yq '.iamra'  ${USER_DATA})" == "true" ]]; then
+    echo "iam-ra"
+
+    readarray files < <(yq e -o=j -I=0 '.write_files[]' ${USER_DATA} )
+
+    for file in "${files[@]}"; do
+        content=$(echo "$file" | yq e '.content' -)
+        path=$(echo "$file" | yq e '.path' -)
+
+        mkdir -p $(dirname $path)
+        echo "$content" > $path    
+    done
+
+    # copy created pki files via cloud-init to var/lib
+    mkdir -p /.bottlerocket/rootfs/var/lib/eks-hybrid/roles-anywhere/pki/
+    cp /etc/roles-anywhere/pki/node.* /.bottlerocket/rootfs/var/lib/eks-hybrid/roles-anywhere/pki/
+
+    mkdir -p /.bottlerocket/rootfs/var/lib/eks-hybrid/bin/.overlay/{upper,work}
+    mv /static-sh /.bottlerocket/rootfs/var/lib/eks-hybrid/bin/.overlay/upper/sh
+    chmod -R 775 /.bottlerocket/rootfs/var/lib/eks-hybrid/bin
+
+    # add static build of sh to bin on host for iam-auth to be able to exec aws-singing-helper
+    chroot /.bottlerocket/rootfs/ mount -t overlay overlay -o rw,nosuid,nodev,noatime,context=system_u:object_r:os_t:s0,lowerdir=/bin,upperdir=/var/lib/eks-hybrid/bin/.overlay/upper,workdir=/var/lib/eks-hybrid/bin/.overlay/work /x86_64-bottlerocket-linux-gnu/sys-root/usr/bin
+else
+    # wait for ssm to register in the control container and copy the aws config to the host and set hostname
+    echo "ssm"
+    while [ ! -f /.bottlerocket/rootfs/run/host-containerd/io.containerd.runtime.v2.task/default/control/rootfs/root/.aws/credentials ]; do sleep 1; done
+    while ! AWS_SHARED_CREDENTIALS_FILE=/.bottlerocket/rootfs/run/host-containerd/io.containerd.runtime.v2.task/default/control/rootfs/root/.aws/credentials aws sts get-caller-identity; do sleep 1; done
+    while [ ! -f /.bottlerocket/rootfs//local/host-containers/control/ssm/registration ]; do sleep 1; done
+    hostname="$(jq -r ".ManagedInstanceID" /.bottlerocket/rootfs//local/host-containers/control/ssm/registration)"
+
+    apiclient set kubernetes.hostname-override=$hostname
+    apiclient set network.hostname=$hostname
+
+    cluster_name="$(apiclient get settings.kubernetes.cluster-name | jq -r ".settings.kubernetes.\"cluster-name\"")"
+    region="$(apiclient get settings.aws.region | jq -r ".settings.aws.region")"
+    apiclient set kubernetes.provider-id="eks-hybrid:///$region/$cluster_name/$hostname"
+
+    # copy aws cred file from ssm control container to host for kubelet
+    apiclient set aws.credentials="$(cat /.bottlerocket/rootfs/run/host-containerd/io.containerd.runtime.v2.task/default/control/rootfs/root/.aws/credentials | base64 -w0)"
+fi
+

cmd/e2e-test/ssh/ssh.go

@@ -84,7 +84,7 @@ func (s *command) Run(log *zap.Logger, opts *cli.GlobalOptions) error {
 		"--document",
 		"AWS-StartInteractiveCommand",
 		"--parameters",
-		fmt.Sprintf("{\"command\":[\"sudo ssh %s\"]}", *targetInstance.PrivateIpAddress),
+		fmt.Sprintf("{\"command\":[\"sudo ssh ec2-user@%s\"]}", *targetInstance.PrivateIpAddress),
 		"--target",
 		*jumpbox.InstanceId,
 	)

test/e2e/credentials/rolesanywhere.go

@@ -39,7 +39,7 @@ func (i *IamRolesAnywhereProvider) NodeadmConfig(ctx context.Context, spec e2e.N
 			},
 			Hybrid: &api.HybridOptions{
 				IAMRolesAnywhere: &api.IAMRolesAnywhere{
-					NodeName:        i.nodeName(spec),
+					NodeName:        i.NodeName(spec),
 					RoleARN:         i.RoleARN,
 					TrustAnchorARN:  i.TrustAnchorARN,
 					ProfileARN:      i.ProfileARN,
@@ -52,7 +52,7 @@ func (i *IamRolesAnywhereProvider) NodeadmConfig(ctx context.Context, spec e2e.N
 	}, nil
 }
 
-func (i *IamRolesAnywhereProvider) nodeName(node e2e.NodeSpec) string {
+func (i *IamRolesAnywhereProvider) NodeName(node e2e.NodeSpec) string {
 	return node.NamePrefix + "-node-" + string(i.Name()) + "-" + node.OS.Name()
 }
 
@@ -61,7 +61,7 @@ func (i *IamRolesAnywhereProvider) VerifyUninstall(ctx context.Context, instance
 }
 
 func (i *IamRolesAnywhereProvider) FilesForNode(spec e2e.NodeSpec) ([]e2e.File, error) {
-	nodeCertificate, err := CreateCertificateForNode(i.CA.Cert, i.CA.Key, i.nodeName(spec))
+	nodeCertificate, err := CreateCertificateForNode(i.CA.Cert, i.CA.Key, i.NodeName(spec))
 	if err != nil {
 		return nil, err
 	}

test/e2e/credentials/ssm.go

@@ -27,6 +27,11 @@ func (s *SsmProvider) Name() creds.CredentialProvider {
 	return creds.SsmCredentialProvider
 }
 
+func (s *SsmProvider) NodeName(e2e.NodeSpec) string {
+	// seems to need to be something otherwise BR wont boot
+	return "foo"
+}
+
 func (s *SsmProvider) NodeadmConfig(ctx context.Context, node e2e.NodeSpec) (*api.NodeConfig, error) {
 	ssmActivationDetails, err := createSSMActivation(ctx, s.SSM, s.Role, ssmActivationName, node.Cluster.Name)
 	if err != nil {

test/e2e/nodeadm.go

@@ -23,11 +23,18 @@ type UserDataInput struct {
 	CredsProviderName string
 	KubernetesVersion string
 	NodeadmUrls       NodeadmURLs
+	NodeadmConfig     *api.NodeConfig
 	NodeadmConfigYaml string
 	Provider          string
 	PublicKey         string
 	RootPasswordHash  string
 	Files             []File
+
+	KubernetesAPIServer string
+	HostName            string
+	Region              string
+	ClusterName         string
+	ClusterCert         []byte
 }
 
 type NodeadmURLs struct {
@@ -49,6 +56,7 @@ type File struct {
 
 type NodeadmCredentialsProvider interface {
 	Name() creds.CredentialProvider
+	NodeName(NodeSpec) string
 	NodeadmConfig(ctx context.Context, node NodeSpec) (*api.NodeConfig, error)
 	VerifyUninstall(ctx context.Context, instanceId string) error
 	FilesForNode(spec NodeSpec) ([]File, error)

test/e2e/os/bottlerocket.go

@@ -0,0 +1,132 @@
+package os
+
+import (
+	"context"
+	_ "embed"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/ssm"
+
+	"github.com/aws/eks-hybrid/test/e2e"
+)
+
+//go:embed testdata/bottlerocket/settings.toml
+var brSettingsToml []byte
+
+//go:embed testdata/bottlerocket/files.txt
+var filesData []byte
+
+type brSettingsTomlInitData struct {
+	e2e.UserDataInput
+	NodeadmUrl               string
+	AdminContainerUserData   string
+	AWSConfig                string
+	ClusterCertificate       string
+	HybridContainerUserData  string
+	ControlContainerUserData string
+	IamRA                    bool
+}
+
+type BottleRocket struct {
+	amiArchitecture string
+	architecture    architecture
+}
+
+func NewBottleRocket() *BottleRocket {
+	br := new(BottleRocket)
+	br.amiArchitecture = x8664Arch
+	br.architecture = amd64
+	return br
+}
+
+func NewBottleRocketARM() *BottleRocket {
+	br := new(BottleRocket)
+	br.amiArchitecture = arm64Arch
+	br.architecture = arm64
+	return br
+}
+
+func (a BottleRocket) Name() string {
+	return "br-" + a.architecture.String()
+}
+
+func (a BottleRocket) InstanceType(region string) string {
+	return getInstanceTypeFromRegionAndArch(region, a.architecture)
+}
+
+func (a BottleRocket) AMIName(ctx context.Context, awsConfig aws.Config) (string, error) {
+	amiId, err := getAmiIDFromSSM(ctx, ssm.NewFromConfig(awsConfig), "/aws/service/bottlerocket/aws-k8s-1.31/"+a.amiArchitecture+"/latest/image_id")
+	return *amiId, err
+}
+
+func (a BottleRocket) BuildUserData(userDataInput e2e.UserDataInput) ([]byte, error) {
+	if err := populateBaseScripts(&userDataInput); err != nil {
+		return nil, err
+	}
+	sshData := map[string]interface{}{
+		"user":          "ec2-user",
+		"password-hash": userDataInput.RootPasswordHash,
+		"ssh": map[string][]string{
+			"authorized-keys": {
+				strings.TrimSuffix(userDataInput.PublicKey, "\n"),
+			},
+		},
+	}
+
+	jsonData, err := json.Marshal(sshData)
+	if err != nil {
+		return nil, err
+	}
+	sshKey := base64.StdEncoding.EncodeToString([]byte(jsonData))
+
+	awsConfig := ""
+	if userDataInput.NodeadmConfig.Spec.Hybrid.IAMRolesAnywhere != nil {
+		awsConfig = fmt.Sprintf(`
+		[default]
+			region = us-west-2
+			credential_process = aws_signing_helper credential-process --certificate /var/lib/eks-hybrid/roles-anywhere/pki/node.crt --private-key /var/lib/eks-hybrid/roles-anywhere/pki/node.key --profile-arn %s --role-arn %s --trust-anchor-arn %s --role-session-name %s
+	`, userDataInput.NodeadmConfig.Spec.Hybrid.IAMRolesAnywhere.ProfileARN, userDataInput.NodeadmConfig.Spec.Hybrid.IAMRolesAnywhere.RoleARN, userDataInput.NodeadmConfig.Spec.Hybrid.IAMRolesAnywhere.TrustAnchorARN, userDataInput.HostName)
+	}
+	ssmData := ""
+	if userDataInput.NodeadmConfig.Spec.Hybrid.SSM != nil {
+
+		ssmConfigData := map[string]interface{}{
+			"ssm": map[string]string{
+				"activation-id":   userDataInput.NodeadmConfig.Spec.Hybrid.SSM.ActivationID,
+				"activation-code": userDataInput.NodeadmConfig.Spec.Hybrid.SSM.ActivationCode,
+				"region":          userDataInput.Region,
+			},
+		}
+
+		jsonData, err = json.Marshal(ssmConfigData)
+		if err != nil {
+			return nil, err
+		}
+		ssmData = base64.StdEncoding.EncodeToString([]byte(jsonData))
+	}
+	data := brSettingsTomlInitData{
+		UserDataInput:            userDataInput,
+		NodeadmUrl:               userDataInput.NodeadmUrls.AMD,
+		AdminContainerUserData:   sshKey,
+		AWSConfig:                base64.StdEncoding.EncodeToString([]byte(awsConfig)),
+		ClusterCertificate:       base64.StdEncoding.EncodeToString(userDataInput.ClusterCert),
+		ControlContainerUserData: ssmData,
+		IamRA:                    userDataInput.NodeadmConfig.Spec.Hybrid.SSM == nil,
+	}
+
+	if a.architecture.arm() {
+		data.NodeadmUrl = userDataInput.NodeadmUrls.ARM
+	}
+
+	cloudInitData, err := executeTemplate(filesData, data)
+	if err != nil {
+		return nil, err
+	}
+	data.HybridContainerUserData = base64.StdEncoding.EncodeToString([]byte(cloudInitData))
+
+	return executeTemplate(brSettingsToml, data)
+}

test/e2e/os/testdata/bottlerocket/files.txt

@@ -0,0 +1,10 @@
+iamra: {{ .IamRA }}
+write_files:
+{{ range $file := .Files }}
+  - content: |
+{{ $file.Content | indent 6 }}
+    path: {{ $file.Path }}
+{{if $file.Permissions}}
+    permissions: '{{ $file.Permissions }}'
+{{- end }}
+{{- end }}

test/e2e/os/testdata/bottlerocket/settings.toml

@@ -0,0 +1,43 @@
+[settings.kubernetes]
+cluster-name = "{{ .ClusterName }}"
+api-server = "{{ .KubernetesAPIServer }}"
+cluster-certificate = "{{ .ClusterCertificate }}"
+
+authentication-mode = "aws"
+cloud-provider = ""
+
+hostname-override = "{{ .HostName}}"
+provider-id = "eks-hybrid:///{{ .Region }}/{{ .ClusterName }}/{{ .HostName}}"
+
+[settings.network]
+hostname = "{{ .HostName }}"
+
+[settings.aws]
+region = "{{ .Region }}"
+config = "{{ .AWSConfig }}"
+
+[settings.kubernetes.node-labels]
+"eks.amazonaws.com/compute-type" = "hybrid"
+{{- if .IamRA }}
+"eks.amazonaws.com/hybrid-credential-provider" = "iam-ra"
+{{- else }}
+"eks.amazonaws.com/hybrid-credential-provider" = "ssm"
+{{- end}}
+
+[settings.host-containers.hybrid-setup]
+superpowered = true
+enabled = true
+source = "public.ecr.aws/k1e6s8o8/br-test:1"
+user-data = "{{ .HybridContainerUserData }}"
+
+[settings.host-containers.admin]
+enabled = true
+source = "public.ecr.aws/bottlerocket/bottlerocket-admin:v0.11.16"
+user-data = "{{ .AdminContainerUserData }}"
+
+{{- if not .IamRA }}
+[settings.host-containers.control]
+enabled = true
+source = "public.ecr.aws/bottlerocket/bottlerocket-control:v0.7.20"
+user-data = "{{ .ControlContainerUserData }}"
+{{- end }}

test/e2e/peered/node.go

@@ -17,6 +17,7 @@ import (
 	"github.com/go-logr/logr"
 	gssh "golang.org/x/crypto/ssh"
 	clientgo "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
 	"sigs.k8s.io/yaml"
 
 	"github.com/aws/eks-hybrid/test/e2e"
@@ -41,6 +42,7 @@ type Node struct {
 	SSM                 *ssm.Client
 	S3                  *s3sdk.Client
 	K8s                 *clientgo.Clientset
+	K8sClientConfig     *rest.Config
 	RemoteCommandRunner commands.RemoteCommandRunner
 	Logger              logr.Logger
 	SkipDelete          bool
@@ -105,11 +107,18 @@ func (c Node) Create(ctx context.Context, spec *NodeSpec) (ec2.Instance, error)
 	userdata, err := spec.OS.BuildUserData(e2e.UserDataInput{
 		KubernetesVersion: spec.NodeK8sVersion,
 		NodeadmUrls:       c.NodeadmURLs,
+		NodeadmConfig:     nodeadmConfig,
 		NodeadmConfigYaml: string(nodeadmConfigYaml),
 		Provider:          string(spec.Provider.Name()),
 		RootPasswordHash:  rootPasswordHash,
 		Files:             files,
 		PublicKey:         c.PublicKey,
+
+		KubernetesAPIServer: c.K8sClientConfig.Host,
+		HostName:            string(spec.Provider.NodeName(nodeSpec)),
+		Region:              c.Cluster.Region,
+		ClusterName:         c.Cluster.Name,
+		ClusterCert:         c.K8sClientConfig.CAData,
 	})
 	if err != nil {
 		return ec2.Instance{}, fmt.Errorf("expected to successfully build user data: %w", err)
@@ -137,7 +146,7 @@ func (c Node) Create(ctx context.Context, spec *NodeSpec) (ec2.Instance, error)
 	if err != nil {
 		return ec2.Instance{}, fmt.Errorf("EC2 Instance should have been created successfully: %w", err)
 	}
-
+	c.Logger.Info("EC2 Instance created successfully.", "instanceId", instance.ID)
 	return instance, nil
 }
 

test/e2e/suite/nodeadm_test.go

@@ -187,6 +187,7 @@ var _ = Describe("Hybrid Nodes", func() {
 		osystem.NewRedHat8ARM(os.Getenv("RHEL_USERNAME"), os.Getenv("RHEL_PASSWORD")),
 		osystem.NewRedHat9AMD(os.Getenv("RHEL_USERNAME"), os.Getenv("RHEL_PASSWORD")),
 		osystem.NewRedHat9ARM(os.Getenv("RHEL_USERNAME"), os.Getenv("RHEL_PASSWORD")),
+		osystem.NewBottleRocket(),
 	}
 	credentialProviders := []e2e.NodeadmCredentialsProvider{
 		&credentials.SsmProvider{},
@@ -490,6 +491,7 @@ func (t *peeredVPCTest) newPeeredNode() *peered.Node {
 		Cluster:             t.cluster,
 		EC2:                 t.ec2Client,
 		K8s:                 t.k8sClient,
+		K8sClientConfig:     t.k8sClientConfig,
 		Logger:              t.logger,
 		LogsBucket:          t.logsBucket,
 		NodeadmURLs:         t.nodeadmURLs,