Feat: Use homelab molecules to setup a Dev Kubernetes Environment

molecules/cluster-resources/config/externals/ingress-nginx/values.yaml

@@ -1,4 +1,4 @@
-tcp:
-  53: "homelab/pihole-dns-tcp:53"
-udp:
-  53: "homelab/pihole-dns-udp:53"
+# tcp:
+#   53: "homelab/pihole-dns-tcp:53"
+# udp:
+#   53: "homelab/pihole-dns-udp:53"

molecules/cluster-resources/config/externals/kafka/ingress.values.tftpl

@@ -0,0 +1,20 @@
+flags:
+  rewrite: false
+
+service:
+  name: kafka
+  port:
+    number: 9092
+
+annotations:
+  hajimari.io/enable: "true"
+  hajimari.io/appName: "kafka"
+
+ingress:
+  subdomain: kafka
+  routePath: /
+
+hosts:
+%{ for host in hosts ~}
+- name: ${host}
+%{ endfor ~}

molecules/cluster-resources/config/externals/keycloak/values.yaml

@@ -1,2 +1,20 @@
 postgresql:
   enabled: false
+
+extraEnv: |
+  - name: DB_VENDOR
+    value:
+  - name: DB_ADDR
+    value:
+  - name: DB_PORT
+    value:
+  - name: DB_DATABASE
+    value:
+  - name: DB_USER
+    value:
+  - name: DB_PASSWORD
+    value:
+  - name: KEYCLOAK_USER
+    value:
+  - name: KEYCLOAK_PASSWORD
+    value:

molecules/cluster-resources/config/externals/opentelemetry-collector/values.yaml

@@ -6,21 +6,25 @@ config:
       include: [/var/log/*.log]
 
   processors:
-    batch: {}
+    attributes:
+      actions:
+        - action: insert
+          key: log_file_name
+          from_attribute: log.file.name
+        - action: insert
+          key: loki.attribute.labels
+          value: log_file_name
 
   exporters:
     logging:
       loglevel: debug
 
     loki:
       endpoint: "http://loki.homelab.svc.cluster.local:3100/loki/api/v1/push"
-      labels:
-        attributes:
-          log.file.name: "filename"
 
   service:
     pipelines:
       logs:
         receivers: [filelog]
-        processors: [batch]
+        processors: [attributes]
         exporters: [loki]

molecules/cluster-resources/config/externals/otel-collector-mrss/values.yaml

@@ -0,0 +1,26 @@
+mode: "deployment"
+
+config:
+  receivers:
+    filelog:
+      include: [/var/log/*.log]
+
+  processors:
+    batch: {}
+
+  exporters:
+    logging:
+      loglevel: debug
+
+    loki:
+      endpoint: "http://loki:3100/loki/api/v1/push"
+      labels:
+        attributes:
+          log.file.name: "filename"
+
+  service:
+    pipelines:
+      logs:
+        receivers: [filelog]
+        processors: [batch]
+        exporters: [loki]

molecules/cluster-resources/config/externals/otel-collector-template-monorepo-dev/values.yaml

@@ -0,0 +1,26 @@
+mode: "deployment"
+
+config:
+  receivers:
+    filelog:
+      include: [/var/log/*.log]
+
+  processors:
+    batch: {}
+
+  exporters:
+    logging:
+      loglevel: debug
+
+    loki:
+      endpoint: "http://loki:3100/loki/api/v1/push"
+      labels:
+        attributes:
+          log.file.name: "filename"
+
+  service:
+    pipelines:
+      logs:
+        receivers: [filelog]
+        processors: [batch]
+        exporters: [loki]

molecules/cluster-resources/config/externals/prometheus/values.yaml

@@ -0,0 +1,5 @@
+nodeExporter:
+  # hostRootfs: false
+  hostRootFsMount:
+    enabled: false
+    mountPropagation: HostToContainer

molecules/cluster-resources/externals.tf

@@ -2,7 +2,7 @@ resource "helm_release" "external_apps" {
   for_each = { for external in var.externals : external.name => external }
   repository = each.value.repo
   name       = each.value.name
-  chart      = each.value.name
+  chart      = coalesce(each.value.chart_name, each.value.name)
   namespace  = each.value.namespace != null ? each.value.namespace : each.value.name
   create_namespace = true
   values = fileexists("./config/externals/${each.value.name}/values.yaml") == true ? ["${file("./config/externals/${each.value.name}/values.yaml")}"] : []

molecules/cluster-resources/homelab.inputs.tfvars

@@ -0,0 +1,101 @@
+cluster = {
+  name = "colima"
+  config_path = "~/.kube/config"
+  host = "https://127.0.0.1:6443"
+  hosts = [
+    "127.0.0.1.nip.io"
+  ]
+}
+
+externals = [
+    {
+      name = "ingress-nginx"
+      repo = "https://kubernetes.github.io/ingress-nginx"
+      version = "4.4.0"
+    },
+    #{
+    #  name = "hajimari"
+    #  repo = "https://hajimari.io"
+    #  version = "1.2.0"
+    #  namespace = "homelab"
+    #},
+    #{
+    #  name = "postgresql"
+    #  repo = "https://cetic.github.io/helm-charts"
+    #  version = "0.2.3"
+    #  namespace = "homelab"
+    #},
+    // TODO: nextcloud fails due to slow startup, need to figure this out
+    # {
+    #   name = "nextcloud"
+    #   repo = "https://nextcloud.github.io/helm"
+    #   version = "3.3.6"
+    #   namespace = "homelab"
+    # },
+    # {
+    #   name = "coredns",
+    #   repo = "https://coredns.github.io/helm",
+    #   version = "1.19.7",
+    #   namespace = "homelab"
+    # }
+    # {
+    #   name = "pihole",
+    #   repo = "https://mojo2600.github.io/pihole-kubernetes/",
+    #   version = "2.11.0",
+    #   namespace = "homelab"
+    # },
+    # {
+    #   name = "keycloak",
+    #   repo = "https://codecentric.github.io/helm-charts",
+    #   version = "20.0.3",
+    #   namespace = "homelab"
+    # },
+    {
+     name = "prometheus",
+     repo = "https://prometheus-community.github.io/helm-charts",
+     version = "19.3.1",
+     namespace = "homelab"
+    },
+    {
+      name = "grafana",
+      repo = "https://grafana.github.io/helm-charts",
+      version = "0.2.3",
+      namespace = "homelab"
+    },
+    {
+      name = "loki",
+      repo = "https://grafana.github.io/loki/charts",
+      version = "2.1.1",
+      namespace = "homelab"
+    },
+    # https://grafana.com/blog/2022/06/23/how-to-send-logs-to-grafana-loki-with-the-opentelemetry-collector-using-fluent-forward-and-filelog-receivers
+    {
+      name = "opentelemetry-collector",
+      repo = "https://open-telemetry.github.io/opentelemetry-helm-charts"
+      version = "v0.72.0",
+      namespace = "payobills-dev"
+    },
+   #{
+   #   name = "opentelemetry-collector",
+   #   repo = "https://open-telemetry.github.io/opentelemetry-helm-charts"
+   #   version = "v0.72.0",
+   #   namespace = "homelab"
+   # },
+  #{
+  #    name = "kafka",
+  #    repo = "https://charts.bitnami.com/bitnami",
+  #    version = "21.3.0"
+  #},
+  # {
+  #   name = "zeebe-full-helm",
+  #   repo = "https://helm.camunda.io"
+  #   version = "1.3.5",
+  #   namespace = "homelab"
+  # },
+  #{
+  #  name = "dapr",
+  #  repo = "https://dapr.github.io/helm-charts/",
+  #  version = "1.10.5",
+  #  namespace = "dapr-system"
+  #}
+  ]

molecules/cluster-resources/inputs.tf

@@ -22,6 +22,7 @@ variable "externals" {
   description = "Details about external apps to install"
   type = list(object({
     name = string
+    chart_name = optional(string)
     repo = string
     version = string
     namespace = optional(string)
@@ -38,71 +39,61 @@ variable "externals" {
       version = "1.2.0"
       namespace = "homelab"
     },
-    {
-      name = "postgresql"
-      repo = "https://cetic.github.io/helm-charts"
-      version = "0.2.3"
-      namespace = "homelab"
-    },
-
     {
       name = "pihole",
       repo = "https://mojo2600.github.io/pihole-kubernetes/",
       version = "2.11.0",
       namespace = "homelab"
     },
     {
-      name = "prometheus",
-      repo = "https://prometheus-community.github.io/helm-charts",
-      version = "19.3.1",
-      namespace = "homelab"
-    },
-    {
-      name = "grafana",
-      repo = "https://grafana.github.io/helm-charts",
-      version = "0.2.3",
-      namespace = "homelab"
-    },
-    {
-      name = "loki",
-      repo = "https://grafana.github.io/loki/charts",
-      version = "2.1.1",
-      namespace = "homelab"
-    },
-    # https://grafana.com/blog/2022/06/23/how-to-send-logs-to-grafana-loki-with-the-opentelemetry-collector-using-fluent-forward-and-filelog-receivers
-    {
-      name = "opentelemetry-collector",
-      repo = "https://open-telemetry.github.io/opentelemetry-helm-charts"
-      version = "v0.72.0",
+      name = "postgresql"
+      repo = "https://cetic.github.io/helm-charts"
+      version = "0.2.3"
       namespace = "homelab"
     },
-    # TODO: I don't think this is needed, need to check
-    # {
-    #   name = "grafana-agent-operator",
-    #   repo = "https://grafana.github.io/helm-charts",
-    #   version = "9.3.1",
-    #   namespace = "homelab"
-    # },
-    # TODO: nextcloud fails due to slow startup, need to figure this out
+    // TODO: nextcloud fails due to slow startup, need to figure this out
     # {
     #   name = "nextcloud"
     #   repo = "https://nextcloud.github.io/helm"
     #   version = "3.3.6"
     #   namespace = "homelab"
     # },
-    # TODO: Use coredns for DNS. Still not sure about this
     # {
     #   name = "coredns",
     #   repo = "https://coredns.github.io/helm",
     #   version = "1.19.7",
     #   namespace = "homelab"
     # },
-    # TODO: Add configuration for this
     # {
     #   name = "keycloak",
     #   repo = "https://codecentric.github.io/helm-charts",
     #   version = "20.0.3",
     #   namespace = "homelab"
     # },
+    {
+      name = "prometheus",
+      repo = "https://prometheus-community.github.io/helm-charts",
+      version = "19.3.1",
+      namespace = "homelab"
+    },
+    {
+      name = "grafana",
+      repo = "https://grafana.github.io/helm-charts",
+      version = "0.2.3",
+      namespace = "homelab"
+    },
+    {
+      name = "loki",
+      repo = "https://grafana.github.io/loki/charts",
+      version = "2.1.1",
+      namespace = "homelab"
+    },
+    # https://grafana.com/blog/2022/06/23/how-to-send-logs-to-grafana-loki-with-the-opentelemetry-collector-using-fluent-forward-and-filelog-receivers
+    #{
+    #  name = "opentelemetry-collector",
+    #  repo = "https://open-telemetry.github.io/opentelemetry-helm-charts"
+    #  version = "v0.72.0",
+    #  namespace = "homelab"
+    #},
   ]
 }

molecules/cluster-resources/providers.tf

@@ -13,7 +13,7 @@ terraform {
     }
   }
 
-  backend "http" { }
+  # backend "http" { }
 }
 
 provider "helm" {