Migrate to Managed Nodegroups (#191)

* add and clean up byo infra options

* work in progress to move to managed node group

* wip

* working version

* PIn packages, remove unneeded bits and add upgrade instructions

* review comments

eks-hosted/01-iam/Pulumi.README.yaml

@@ -11,13 +11,11 @@ config:
   # Currently this is just passed through and consumed by later stacks to enable the k8s provider to assume the role and deploy 
   # k8s infra to the eks cluster.
   # A future iteration may create this sso role as part of the stack.
-  ssoRoleArn: arn:aws:iam::123456789012:role/SSO-Role-Name
 
   #### BRINGING YOUR OWN IAM INFRASTRUCTURE ###
   # If you are not using the `01-iam` stack, then set the following values that would have otherwise been provided by the iam stack.
   # The stack will then "pretend" it created the resources and output the values for the other stacks to use.
   databaseMonitoringRoleArn: 
   eksServiceRoleName:
   eksInstanceRoleName:      
-  instanceProfileName:
 

eks-hosted/01-iam/albControllerPolicy.ts

@@ -33,6 +33,7 @@ JSON.stringify({
               "elasticloadbalancing:DescribeLoadBalancers",
               "elasticloadbalancing:DescribeLoadBalancerAttributes",
               "elasticloadbalancing:DescribeListeners",
+              "elasticloadbalancing:DescribeListenerAttributes",
               "elasticloadbalancing:DescribeListenerCertificates",
               "elasticloadbalancing:DescribeSSLPolicies",
               "elasticloadbalancing:DescribeRules",
@@ -182,6 +183,7 @@ JSON.stringify({
       {
           "Effect": "Allow",
           "Action": [
+              "elasticloadbalancing:ModifyListenerAttributes",
               "elasticloadbalancing:ModifyLoadBalancerAttributes",
               "elasticloadbalancing:SetIpAddressType",
               "elasticloadbalancing:SetSecurityGroups",

eks-hosted/01-iam/config.ts

@@ -4,10 +4,8 @@ const pulumiConfig = new pulumi.Config();
 
 export const config = {
     baseName: pulumiConfig.require("baseName"),
-    ssoRoleArn: pulumiConfig.require("ssoRoleArn"),
     // Optional: If bringing your own IAM configs - see Pulumi.README.yaml
     eksServiceRoleName: pulumiConfig.get("eksServiceRoleName"),
     eksInstanceRoleName: pulumiConfig.get("eksInstanceRoleName"), 
-    instanceProfileName: pulumiConfig.get("instanceProfileName"),
     databaseMonitoringRoleArn: pulumiConfig.get("databaseMonitoringRoleArn"),
 };

eks-hosted/01-iam/index.ts

@@ -5,26 +5,27 @@ import { albControllerPolicyStatement } from "./albControllerPolicy";
 
 /// SSO Role ///
 // This is currently managed outside of the stack and passed through for later stacks to use.
-export const ssoRoleArn = config.ssoRoleArn;
 
 // These roles are either provided by the user or created in this stack.
 export let eksServiceRoleName: string | pulumi.Output<string>;
+export let eksServiceRole: aws.iam.Role | pulumi.Output<aws.iam.Role>;
 export let eksInstanceRoleName: string | pulumi.Output<string>; 
-export let instanceProfileName: string | pulumi.Output<string>;
+export let eksInstanceRole: aws.iam.Role | pulumi.Output<aws.iam.Role>;
 export let databaseMonitoringRoleArn: string | pulumi.Output<string>;
 
 
 // If the user provided the roles, use them instead of creating new ones.
 // It's an all-or-nothing situation, so if one is provided, they all must be.
-if (config.eksServiceRoleName && config.eksInstanceRoleName && config.instanceProfileName && config.databaseMonitoringRoleArn) {
+if (config.eksServiceRoleName && config.eksInstanceRoleName && config.databaseMonitoringRoleArn) {
     eksServiceRoleName = config.eksServiceRoleName;
     eksInstanceRoleName = config.eksInstanceRoleName;
-    instanceProfileName = config.instanceProfileName;
     databaseMonitoringRoleArn = config.databaseMonitoringRoleArn;
+    eksServiceRole = aws.iam.Role.get("eksServiceRole", eksServiceRoleName);
+    eksInstanceRole = aws.iam.Role.get("eksInstanceRole", eksInstanceRoleName);
 } else {
     // Create the roles.
     /// Cluster Role ///
-    const eksRole = new aws.iam.Role(`${config.baseName}-eksRole`, {
+    eksServiceRole = new aws.iam.Role(`${config.baseName}-eksRole`, {
         assumeRolePolicy: {
             Statement: [
                 {   Action:"sts:AssumeRole",
@@ -41,10 +42,10 @@ if (config.eksServiceRoleName && config.eksInstanceRoleName && config.instancePr
             "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy"
         ],
     });
-    eksServiceRoleName = eksRole.name;
+    eksServiceRoleName = eksServiceRole.name;
 
     /// Instance Role ///
-    const instanceRole = new aws.iam.Role(`${config.baseName}-instanceRole`, {
+    eksInstanceRole = new aws.iam.Role(`${config.baseName}-instanceRole`, {
         assumeRolePolicy: aws.iam.assumeRolePolicyForPrincipal(aws.iam.Principals.Ec2Principal),
         managedPolicyArns: [
             "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly",
@@ -56,7 +57,7 @@ if (config.eksServiceRoleName && config.eksInstanceRoleName && config.instancePr
     // S3 policy used by Pulumi services
     const instanceRoleS3Policy = new aws.iam.RolePolicyAttachment("instanceRoleS3Policy", {
         policyArn: "arn:aws:iam::aws:policy/AmazonS3FullAccess",
-        role: instanceRole 
+        role: eksInstanceRole 
     })
 
     // ALB management used by ingress controller
@@ -65,7 +66,7 @@ if (config.eksServiceRoleName && config.eksInstanceRoleName && config.instancePr
     });
     const rpaAlbPolicy = new aws.iam.RolePolicyAttachment("albPolicy", {
         policyArn: albControllerPolicy.arn,
-        role: instanceRole
+        role: eksInstanceRole
     })
 
     // Opensearch access
@@ -85,13 +86,10 @@ if (config.eksServiceRoleName && config.eksInstanceRoleName && config.instancePr
     });
     const openSearchPolicyAttachment = new aws.iam.RolePolicyAttachment("opensearchPolicy", {
         policyArn: opensearchPolicy.arn,
-        role: instanceRole
+        role: eksInstanceRole
     })
 
-    eksInstanceRoleName = instanceRole.name;
-
-    const instanceProfile =  new aws.iam.InstanceProfile("ng-standard", {role: eksInstanceRoleName})
-    instanceProfileName = instanceProfile.name;
+    eksInstanceRoleName = eksInstanceRole.name;
 
     // used by RDS to publish metrics to CloudWatch
     const databaseMonitoringRole = new aws.iam.Role("databaseMonitoringRole", {
@@ -115,3 +113,5 @@ if (config.eksServiceRoleName && config.eksInstanceRoleName && config.instancePr
     databaseMonitoringRoleArn = databaseMonitoringRole.arn;
 }
 
+
+

eks-hosted/01-iam/package.json

@@ -4,7 +4,7 @@
     "typescript": "^3.0.0"
   },
   "dependencies": {
-    "@pulumi/aws": "^6.54.0",
-    "@pulumi/pulumi": "^3.136.0"
+    "@pulumi/aws": "6.64.0",
+    "@pulumi/pulumi": "3.142.0"
   }
 }

eks-hosted/02-networking/package.json

@@ -4,8 +4,8 @@
     "typescript": "^3.0.0"
   },
   "dependencies": {
-    "@pulumi/aws": "^6.54.0",
-    "@pulumi/awsx": "^2.16.0",
-    "@pulumi/pulumi": "^3.136.0"
+    "@pulumi/aws": "6.64.0",
+    "@pulumi/awsx": "2.19.0",
+    "@pulumi/pulumi": "3.142.0"
   }
 }

eks-hosted/05-eks-cluster/Pulumi.README.yaml

@@ -20,3 +20,7 @@ config:
   pulumiNodeGroupDesiredCapacity: 3
   pulumiNodeGroupMinSize: 3
   pulumiNodeGroupMaxSize: 5
+
+  # Settings for instance metadata support (IMDSv2) for the EKS cluster nodegroups. If not set, a defaults (see config.ts) will be used.
+  httpTokens: 
+  httpPutResponseHopLimit:

eks-hosted/05-eks-cluster/config.ts

@@ -10,9 +10,9 @@ const stackName = pulumi.getStack();
 // IAM stack values
 const iamStackRef = new pulumi.StackReference(`${orgName}/selfhosted-01-iam/${stackName}`);
 const eksInstanceRoleName = iamStackRef.requireOutput("eksInstanceRoleName");
-const instanceProfileName = iamStackRef.requireOutput("instanceProfileName");
+const eksInstanceRole = iamStackRef.requireOutput("eksInstanceRole");
 const eksServiceRoleName = iamStackRef.requireOutput("eksServiceRoleName");
-const ssoRoleArn = iamStackRef.requireOutput("ssoRoleArn");
+const eksServiceRole = iamStackRef.requireOutput("eksServiceRole");
 
 // Networking Stack values
 // Get the needed values from the networking stack.
@@ -45,16 +45,18 @@ export const config = {
     pulumiNodeGroupMinSize: pulumiConfig.getNumber("pulumiNodeGroupMinSize") ?? 3,
     pulumiNodeGroupMaxSize: pulumiConfig.getNumber("pulumiNodeGroupMaxSize") ?? 5,
 
+    httpTokens: pulumiConfig.get("httpTokens") || "required",
+    httpPutResponseHopLimit: pulumiConfig.getNumber("httpPutResponseHopLimit") ?? 2,
+
     // IAM stack values
     eksInstanceRoleName: eksInstanceRoleName,
-    instanceProfileName: instanceProfileName,
+    eksInstanceRole: eksInstanceRole,
     eksServiceRoleName: eksServiceRoleName,
-    ssoRoleArn: ssoRoleArn,
+    eksServiceRole: eksServiceRole,
 
     // Networking stack values
     clusterName: clusterName,
     vpcId: vpcId,
     publicSubnetIds: publicSubnetIds,
     privateSubnetIds: privateSubnetIds,
-
 };

eks-hosted/05-eks-cluster/index.ts

@@ -8,102 +8,81 @@ const tags = { "Project": "pulumi-k8s-aws-cluster", "Owner": "pulumi"};
 
 /////////////////////
 // --- EKS Cluster ---
-const serviceRole = aws.iam.Role.get("eksServiceRole", config.eksServiceRoleName)
-const instanceRole = aws.iam.Role.get("instanceRole", config.eksInstanceRoleName)
-const instanceProfile = aws.iam.InstanceProfile.get("ng-standard", config.instanceProfileName)
-
 // Create an EKS cluster.
 const cluster = new eks.Cluster(`${baseName}`, {
     name: config.clusterName,
     authenticationMode: "API",
     // We keep these serviceRole and instanceRole properties to prevent the EKS provider from creating its own roles.
-    serviceRole: serviceRole,
-    instanceRole: instanceRole,
+    serviceRole: config.eksServiceRole,
+    instanceRole: config.eksInstanceRole,
     vpcId: config.vpcId,
     publicSubnetIds: config.publicSubnetIds,
     privateSubnetIds: config.privateSubnetIds,
     providerCredentialOpts: { profileName: process.env.AWS_PROFILE}, 
     nodeAssociatePublicIpAddress: false,
     skipDefaultNodeGroup: true,
-    deployDashboard: false,
     version: config.clusterVersion,
     createOidcProvider: false,
     tags: tags,
     enabledClusterLogTypes: ["api", "audit", "authenticator", "controllerManager", "scheduler"],
-}, {
-    transformations: [(args) => {
-        if (args.type === "aws:eks/cluster:Cluster") {
-            return {
-                props: args.props,
-                opts: pulumi.mergeOptions(args.opts, {
-                    protect: true,
-                })
-            }
-        }
-        return undefined;
-    }],
-});
+}, { protect: true });
 
 // Export the cluster details.
 export const kubeconfig = pulumi.secret(cluster.kubeconfig.apply(JSON.stringify));
 export const clusterName = cluster.core.cluster.name;
 export const region = aws.config.region;
-export const nodeSecurityGroupId = cluster.nodeSecurityGroup.id; // For RDS
+export const nodeSecurityGroupId = cluster.nodeSecurityGroupId;
 export const nodeGroupInstanceType = config.pulumiNodeGroupInstanceType;
 
 /////////////////////
-/// Build node groups
-const ssmParam = pulumi.output(aws.ssm.getParameter({
-    // https://docs.aws.amazon.com/eks/latest/userguide/retrieve-ami-id.html
-    name: `/aws/service/eks/optimized-ami/${config.clusterVersion}/amazon-linux-2/recommended`,
-}))
-const amiId = ssmParam.value.apply(s => JSON.parse(s).image_id)
+// Build managed nodegroup for the service to run on.
 
-// Create a standard node group.
-const ngStandard = new eks.NodeGroup(`${baseName}-ng-standard`, {
-    cluster: cluster,
-    instanceProfile: instanceProfile,
-    nodeAssociatePublicIpAddress: false,
-    nodeSecurityGroup: cluster.nodeSecurityGroup,
-    clusterIngressRule: cluster.eksClusterIngressRule,
-    amiId: amiId,
-
-    instanceType: <aws.ec2.InstanceType>config.standardNodeGroupInstanceType,
-    desiredCapacity: config.standardNodeGroupDesiredCapacity,
-    minSize: config.standardNodeGroupMinSize,
-    maxSize: config.standardNodeGroupMaxSize,
+const instanceRoleArn = config.eksInstanceRole.apply(role => role.arn); 
 
-    labels: {"amiId": `${amiId}`},
-    cloudFormationTags: clusterName.apply(clusterName => ({
-        "k8s.io/cluster-autoscaler/enabled": "true",
-        [`k8s.io/cluster-autoscaler/${clusterName}`]: "true",
-        ...tags,
-    })),
-}, {
-    providers: { kubernetes: cluster.provider},
-});
+// Launch template for the managed node group to manage settings.
+const ngManagedLaunchTemplate = new aws.ec2.LaunchTemplate(`${baseName}-ng-managed-launch-template`, {
+    vpcSecurityGroupIds: [cluster.nodeSecurityGroupId],
+    metadataOptions: {
+        httpTokens: config.httpTokens,
+        httpPutResponseHopLimit: config.httpPutResponseHopLimit,
+    },
+})
 
-// Create a standard node group tainted for use only by self-hosted pulumi.
-const ngStandardPulumi = new eks.NodeGroup(`${baseName}-ng-standard-pulumi`, {
+const ngManagedStandard = new eks.ManagedNodeGroup(`${baseName}-ng-managed-standard`, {
     cluster: cluster,
-    instanceProfile: instanceProfile,
-    nodeAssociatePublicIpAddress: false,
-    nodeSecurityGroup: cluster.nodeSecurityGroup,
-    clusterIngressRule: cluster.eksClusterIngressRule,
-    amiId: amiId,
-
-    instanceType: <aws.ec2.InstanceType>config.pulumiNodeGroupInstanceType,
-    desiredCapacity: config.pulumiNodeGroupDesiredCapacity,
-    minSize: config.pulumiNodeGroupMinSize,
-    maxSize: config.pulumiNodeGroupMaxSize,
+    instanceTypes: [<aws.ec2.InstanceType>config.standardNodeGroupInstanceType],
+    launchTemplate: {
+        id: ngManagedLaunchTemplate.id,
+        version: ngManagedLaunchTemplate.latestVersion.apply(v => v.toString()),
+    },
+    nodeRoleArn: instanceRoleArn,
+    scalingConfig: {
+        desiredSize: config.standardNodeGroupDesiredCapacity,
+        minSize: config.standardNodeGroupMinSize,
+        maxSize: config.standardNodeGroupMaxSize,
+    },
+    subnetIds: config.privateSubnetIds,
+    tags: tags,
+})
 
-    labels: {"amiId": `${amiId}`},
-    taints: { "self-hosted-pulumi": { value: "true", effect: "NoSchedule"}},
-    cloudFormationTags: clusterName.apply(clusterName => ({
-        "k8s.io/cluster-autoscaler/enabled": "true",
-        [`k8s.io/cluster-autoscaler/${clusterName}`]: "true",
-        ...tags,
-    })),
-}, {
-    providers: { kubernetes: cluster.provider},
+const ngManagedPulumi = new eks.ManagedNodeGroup(`${baseName}-ng-managed-pulumi`, {
+    cluster: cluster,
+    instanceTypes: [<aws.ec2.InstanceType>config.pulumiNodeGroupInstanceType],
+    launchTemplate: {
+        id: ngManagedLaunchTemplate.id,
+        version: ngManagedLaunchTemplate.latestVersion.apply(v => v.toString()),
+    },
+    nodeRoleArn: instanceRoleArn,
+    scalingConfig: {
+        desiredSize: config.pulumiNodeGroupDesiredCapacity,
+        minSize: config.pulumiNodeGroupMinSize,
+        maxSize: config.pulumiNodeGroupMaxSize,
+    },
+    subnetIds: config.privateSubnetIds,
+    taints: [{ 
+        key: "self-hosted-pulumi",
+        value: "true", 
+        effect: "NO_SCHEDULE"
+    }],
+    tags: tags,
 });

eks-hosted/05-eks-cluster/package.json

@@ -4,8 +4,8 @@
     "typescript": "^3.0.0"
   },
   "dependencies": {
-    "@pulumi/aws": "^6.54.0",
-    "@pulumi/pulumi": "^3.136.0",
-    "@pulumi/eks": "^2.8.1"
+    "@pulumi/aws": "6.64.0",
+    "@pulumi/pulumi": "3.142.0",
+    "@pulumi/eks": "3.4.0"
   }
 }

eks-hosted/10-cluster-svcs/index.ts

@@ -23,7 +23,6 @@ const coreDnsAddon = new aws.eks.Addon("coreDns", {
 const albSecurityGroup = createAlbSecurityGroup(config.baseName, {
     vpcId: config.vpcId,
     nodeSecurityGroupId: config.nodeSecurityGroupId,
-    // tags: tags,
     clusterName: config.clusterName,
 });
 

eks-hosted/10-cluster-svcs/ingress-controller.ts

@@ -21,8 +21,8 @@ export function createAlbIngressController(name: string, args: AlbIngressControl
   const albServiceAccount = new k8s.core.v1.ServiceAccount("albServiceAccount", {
     metadata: {
         name: "aws-load-balancer-controller",
-        namespace: "kube-system"
-    }
+        namespace: "kube-system",
+    },
   }, {provider: args.k8sprovider})
 
   // If you are not able to access the repository, you can reference it as a local chart by doing the following:
@@ -35,6 +35,7 @@ export function createAlbIngressController(name: string, args: AlbIngressControl
         repo: "https://aws.github.io/eks-charts"
     },
     chart: "aws-load-balancer-controller",
+    version: "1.10.1",
     // chart: "./alb-ingress-chart-local/aws-load-balancer-controller",
     namespace: "kube-system",
     values: {

eks-hosted/10-cluster-svcs/package.json

@@ -4,8 +4,8 @@
     "typescript": "^3.0.0"
   },
   "dependencies": {
-    "@pulumi/aws": "^6.21.0",
-    "@pulumi/kubernetes": "^3.0.0",
-    "@pulumi/pulumi": "^3.136.0"
+    "@pulumi/aws": "6.42.0",
+    "@pulumi/kubernetes": "4.18.4",
+    "@pulumi/pulumi": "3.142.0"
   }
 }