init

README.md

@@ -0,0 +1,33 @@
+# Preface
+
+This is a very simple, opinionated module to give you an eks cluster and associated efs store so your pvcs can cross AZ's. Note that efs is considerably slow, so if you're running a website or something of the sort, you're going to want either nginx micro caching or varnish in front of it. However you want to limit page rendering.  
+
+Its a very simple and powerful architechture.
+
+![arch](docs/arch.jpeg)
+
+I don't expose instance size and a handful of other things at this time. This will accompany a blog post, of which the goal is to provide one of the cheapest, cost stable, k8s implimentations out there. EFS is cheap, you get 12 spot instances totaling 24 cores and 24 gigs of ram for 60/mo. For just managed eks and nodes that's about 132/mo. You'll see a little creep on this with alb and efs costs, but it should be negliglble. I'd say I highly doubt this ecosystem will exceed 200/mo. Some of the fully abstracted k8's services are cheap at a first glance, but they scale rapidly with compute needs. This maintains a mostly fixed cost (I think everyone nails you on network costs, it's like pennies though for fairly massive amounts of data).
+
+I use the default public subnets, otherwise you need nat instances for private outbound and that costs money. 
+
+# K8 Components
+
+Out of the box, it will configure and install the EFS csi driver and generate the associated iam policy and service accounts against the OIDC endpoint. 
+
+It will also configure external DNS, again with associated iam policies and service accounts. 
+
+You also get nginx as a service which creates an assoicarted loadbalncer. External dns will key on ingress objects you create and create alias records to your load balancer. 
+
+
+<!-- BEGIN_TF_DOCS -->
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_name"></a> [name](#input\_name) | n/a | `string` | `"k8s"` | no |
+| <a name="input_namespace"></a> [namespace](#input\_namespace) | n/a | `string` | `"playground"` | no |
+| <a name="input_region"></a> [region](#input\_region) | n/a | `string` | n/a | yes |
+| <a name="input_stage"></a> [stage](#input\_stage) | n/a | `string` | `"dev"` | no |
+| <a name="input_subnet_ids"></a> [subnet\_ids](#input\_subnet\_ids) | n/a | `list(string)` | n/a | yes |
+| <a name="input_vpc_id"></a> [vpc\_id](#input\_vpc\_id) | n/a | `string` | n/a | yes |
+<!-- END_TF_DOCS -->

files/storageclass.yaml

@@ -0,0 +1,11 @@
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: efs-sc
+provisioner: efs.csi.aws.com
+parameters:
+  provisioningMode: efs-ap
+  fileSystemId: ##FS_ID##
+  directoryPerms: "755"
+  gidRangeStart: "1000" # optional
+  gidRangeEnd: "2000" # optional

iam.tf

@@ -0,0 +1,142 @@
+resource "aws_iam_policy" "external_dns_policy" {
+  name        = "${module.eks_cluster.eks_cluster_id}-external-dns"
+  description = "policy to allow k8s external dns for ${module.eks_cluster.eks_cluster_id}"
+
+  policy = jsonencode({
+    "Version" : "2012-10-17",
+    "Statement" : [
+      {
+        "Effect" : "Allow",
+        "Action" : [
+          "route53:ChangeResourceRecordSets"
+        ],
+        "Resource" : [
+          "arn:aws:route53:::hostedzone/*"
+        ]
+      },
+      {
+        "Effect" : "Allow",
+        "Action" : [
+          "route53:ListHostedZones",
+          "route53:ListResourceRecordSets"
+        ],
+        "Resource" : [
+          "*"
+        ]
+      }
+    ]
+  })
+}
+
+resource "aws_iam_policy" "efs_csi_policy" {
+  name        = "${module.eks_cluster.eks_cluster_id}-efs_csi_policy"
+  description = "${module.eks_cluster.eks_cluster_id} efs csi policy"
+
+  policy = jsonencode({
+    "Version" : "2012-10-17",
+    "Statement" : [
+      {
+        "Effect" : "Allow",
+        "Action" : [
+          "elasticfilesystem:DescribeAccessPoints",
+          "elasticfilesystem:DescribeFileSystems"
+        ],
+        "Resource" : "*"
+      },
+      {
+        "Effect" : "Allow",
+        "Action" : [
+          "elasticfilesystem:CreateAccessPoint"
+        ],
+        "Resource" : "*",
+        "Condition" : {
+          "StringLike" : {
+            "aws:RequestTag/efs.csi.aws.com/cluster" : "true"
+          }
+        }
+      },
+      {
+        "Effect" : "Allow",
+        "Action" : "elasticfilesystem:DeleteAccessPoint",
+        "Resource" : "*",
+        "Condition" : {
+          "StringEquals" : {
+            "aws:ResourceTag/efs.csi.aws.com/cluster" : "true"
+          }
+        }
+      }
+    ]
+  })
+}
+
+resource "aws_iam_role" "efs_csi_role" {
+  depends_on = [module.eks_cluster.eks_cluster_identity_oidc_issuer_arn]
+  name       = "${module.eks_cluster.eks_cluster_id}-efs-csi-role"
+
+  assume_role_policy = jsonencode(
+    {
+      "Version" : "2012-10-17",
+      "Statement" : [
+        {
+          "Effect" : "Allow",
+          "Principal" : {
+            "Federated" : "${module.eks_cluster.eks_cluster_identity_oidc_issuer_arn}"
+          },
+          "Action" : "sts:AssumeRoleWithWebIdentity",
+          "Condition" : {
+            "StringEquals" : {
+              "${replace(module.eks_cluster.eks_cluster_identity_oidc_issuer, "https://", "")}:sub" : "system:serviceaccount:kube-system:efs-csi-controller-sa",
+              "${replace(module.eks_cluster.eks_cluster_identity_oidc_issuer, "https://", "")}:aud" : "sts.amazonaws.com"
+            }
+          }
+        }
+      ]
+    }
+  )
+}
+
+resource "kubernetes_service_account" "efs-csi-controller-sa" {
+  metadata {
+    name        = "efs-csi-controller-sa"
+    namespace   = "kube-system"
+    annotations = { "eks.amazonaws.com/role-arn" : "${aws_iam_role.efs_csi_role.arn}" }
+  }
+}
+
+resource "aws_iam_role" "external_dns_controller_role" {
+  depends_on = [module.eks_cluster.eks_cluster_identity_oidc_issuer_arn]
+  name       = "${module.eks_cluster.eks_cluster_id}-external-dns"
+
+  assume_role_policy = jsonencode(
+    {
+      "Version" : "2012-10-17",
+      "Statement" : [
+        {
+          "Effect" : "Allow",
+          "Principal" : {
+            "Federated" : "${module.eks_cluster.eks_cluster_identity_oidc_issuer_arn}"
+          },
+          "Action" : "sts:AssumeRoleWithWebIdentity",
+          "Condition" : {
+            "StringEquals" : {
+              "${replace(module.eks_cluster.eks_cluster_identity_oidc_issuer, "https://", "")}:sub" : "system:serviceaccount:external-dns:external-dns-controller",
+              "${replace(module.eks_cluster.eks_cluster_identity_oidc_issuer, "https://", "")}:aud" : "sts.amazonaws.com"
+            }
+          }
+        }
+      ]
+    }
+  )
+}
+
+resource "aws_iam_role_policy_attachment" "external_dns_policy_external_dns_controller_role" {
+  policy_arn = aws_iam_policy.external_dns_policy.arn
+  role       = aws_iam_role.external_dns_controller_role.name
+  depends_on = [aws_iam_role.external_dns_controller_role]
+}
+
+resource "aws_iam_role_policy_attachment" "efs_controller" {
+  policy_arn = aws_iam_policy.efs_csi_policy.arn
+  role       = aws_iam_role.efs_csi_role.name
+  depends_on = [aws_iam_role.efs_csi_role]
+}

main.tf

@@ -0,0 +1,152 @@
+module "eks_cluster" {
+  source  = "cloudposse/eks-cluster/aws"
+  version = "0.45.0"
+
+  namespace = var.namespace
+  stage     = var.stage
+  name      = var.name
+
+  region                = var.region
+  create_security_group = true
+
+  vpc_id                = var.vpc_id
+  subnet_ids            = var.subnet_ids
+  kubernetes_version    = "1.22"
+  oidc_provider_enabled = true
+
+  enabled_cluster_log_types = []
+
+  wait_for_cluster_command = "curl --silent --fail --retry 90 --retry-delay 5 --retry-connrefused --insecure --output /dev/null $ENDPOINT/healthz"
+}
+
+
+module "eks_node_group" {
+  source  = "cloudposse/eks-node-group/aws"
+  version = "0.28.0"
+
+  namespace = var.namespace
+  stage     = var.stage
+  name      = var.name
+
+  instance_types     = ["t3a.small"]
+  subnet_ids         = var.subnet_ids
+  cluster_name       = module.eks_cluster.eks_cluster_id
+  desired_size       = 12
+  min_size           = 6
+  max_size           = 12
+  kubernetes_version = ["1.22"]
+  resources_to_tag   = ["instance", "volume", "network-interface"]
+  label_key_case     = "title"
+  capacity_type      = "SPOT"
+
+  associated_security_group_ids = [module.eks_cluster.security_group_id]
+
+  depends_on = [module.eks_cluster.kubernetes_config_map_id]
+
+  create_before_destroy = true
+
+  node_group_terraform_timeouts = [{
+    create = "40m"
+    update = null
+    delete = "20m"
+  }]
+}
+
+module "efs" {
+  source  = "cloudposse/efs/aws"
+  version = "0.32.6"
+
+
+  namespace = var.namespace
+  stage     = var.stage
+  name      = var.name
+  region    = var.region
+  vpc_id    = var.vpc_id
+  subnets   = var.subnet_ids
+
+  allowed_security_group_ids = [module.eks_cluster.security_group_id]
+}
+
+resource "helm_release" "cert_manager" {
+  depends_on = [module.eks_node_group]
+
+  name             = "cert-manager"
+  repository       = "https://charts.jetstack.io"
+  chart            = "cert-manager"
+  version          = "v1.7.1"
+  namespace        = "cert-manager"
+  create_namespace = true
+
+
+  set {
+    name  = "installCRDs"
+    value = "true"
+  }
+}
+
+resource "helm_release" "nginx_ingress" {
+  depends_on = [helm_release.cert_manager, module.eks_node_group]
+
+  name             = "nginx-ingress"
+  repository       = "https://charts.bitnami.com/bitnami"
+  chart            = "nginx-ingress-controller"
+  version          = "9.1.5"
+  namespace        = "nginx"
+  create_namespace = true
+
+  set {
+    name  = "ingressClassResource.default"
+    value = "true"
+  }
+
+  set {
+    name  = "publishService.enabled"
+    value = "true"
+  }
+
+  values = [
+    <<EOF
+config: 
+  hsts: "false"
+EOF
+  ]
+}
+
+resource "helm_release" "external_dns_helm" {
+  depends_on = [aws_iam_role.external_dns_controller_role, module.eks_node_group]
+
+  name             = "external-dns"
+  repository       = "https://charts.bitnami.com/bitnami"
+  chart            = "external-dns"
+  version          = "6.1.5"
+  namespace        = "external-dns"
+  create_namespace = true
+  force_update     = true
+
+  set {
+    name  = "serviceAccount.name"
+    value = "external-dns-controller"
+  }
+
+  set {
+    name  = "sources"
+    value = "{ingress}"
+  }
+
+  values = [
+    <<EOF
+serviceAccount: 
+  annotations: 
+    eks.amazonaws.com/role-arn: ${aws_iam_role.external_dns_controller_role.arn}
+EOF
+  ]
+}
+
+resource "aws_route53_zone" "hireryan" {
+  name = "hireryan.today"
+}
+
+resource "aws_route53_zone" "tailswiki" {
+  name = "tailswiki.com"
+}
+

manifests.tf

@@ -0,0 +1,40 @@
+resource "null_resource" "manifests" {
+  depends_on = [
+    module.efs,
+    module.eks_cluster.kubernetes_config_map_id,
+    helm_release.aws_efs
+  ]
+  provisioner "local-exec" {
+    command = <<-EOF
+    aws eks update-kubeconfig --name ${module.eks_cluster.eks_cluster_id}
+    cat <<EOD | sed 's/##FS_ID##/${module.efs.id}/g' | kubectl apply -f -
+    ${file("${path.module}/files/storageclass.yaml")}
+    EOD
+    EOF
+  }
+}
+
+resource "helm_release" "aws_efs" {
+  depends_on = [
+    module.efs,
+    module.eks_cluster.kubernetes_config_map_id,
+    kubernetes_service_account.efs-csi-controller-sa
+  ]
+
+  name         = "aws-efs-csi-driver"
+  repository   = "https://kubernetes-sigs.github.io/aws-efs-csi-driver/"
+  chart        = "aws-efs-csi-driver"
+  namespace    = "kube-system"
+  force_update = true
+
+  set {
+    name  = "controller.serviceAccount.create"
+    value = "false"
+  }
+
+  set {
+    name  = "controller.serviceAccount.name"
+    value = "efs-csi-controller-sa"
+  }
+
+}