Merge pull request #79 from torumakabe/feat

feat: add Prometheus/Grafana, refactor and bump
torumakabe · Jun 7, 2023 · b00b6e4 · b00b6e4
2 parents 7e6d9eb + 7e244e7
commit b00b6e4
Show file tree

Hide file tree

Showing 44 changed files with 538 additions and 68 deletions.
diff --git a/.github/workflows/ci-terraform-blue.yaml b/.github/workflows/ci-terraform-blue.yaml
@@ -67,7 +67,7 @@ jobs:
       - name: "Setup Terraform"
         uses: hashicorp/setup-terraform@v2
         with:
-          terraform_version: 1.4.5
+          terraform_version: 1.4.6
           terraform_wrapper: false
 
       - name: "Terraform Format"

diff --git a/.github/workflows/ci-terraform-green.yaml b/.github/workflows/ci-terraform-green.yaml
@@ -67,7 +67,7 @@ jobs:
       - name: "Setup Terraform"
         uses: hashicorp/setup-terraform@v2
         with:
-          terraform_version: 1.4.5
+          terraform_version: 1.4.6
           terraform_wrapper: false
 
       - name: "Terraform Format"

diff --git a/.github/workflows/ci-terraform-shared.yaml b/.github/workflows/ci-terraform-shared.yaml
@@ -31,7 +31,7 @@ jobs:
       - name: "Setup Terraform"
         uses: hashicorp/setup-terraform@v2
         with:
-          terraform_version: 1.4.5
+          terraform_version: 1.4.6
           terraform_wrapper: false
 
       - name: "Terraform Format"

diff --git a/README.md b/README.md
@@ -29,14 +29,14 @@ DRY is a great concept, and you should be aware that it will come true in the fu
 
 Prerequisites & tested
 
-- [Terraform](https://www.terraform.io/docs/index.html): 1.4.5
-  - hashicorp/azurerm: 3.53.0
-  - hashicorp/kubernetes: 2.20
+- [Terraform](https://www.terraform.io/docs/index.html): 1.4.6
+  - hashicorp/azurerm: 3.59.0
+  - hashicorp/kubernetes: 2.21
   - State store: Local
-- [TFLint](https://github.com/terraform-linters/tflint): 0.46.0
+- [TFLint](https://github.com/terraform-linters/tflint): 0.46.1
   - [azurerm plugin](https://github.com/terraform-linters/tflint-ruleset-azurerm): 0.23.0
-- [Flux(v2)](https://fluxcd.io/docs/): 2.0.0-rc.1
-- [Azure/kubelogin](https://github.com/Azure/kubelogin): 0.0.28
+- [Flux(v2)](https://fluxcd.io/docs/): 2.0.0-rc.5
+- [Azure/kubelogin](https://github.com/Azure/kubelogin): 0.0.29
 
 ### Privileges required for execution
 

diff --git a/flux/apps/base/demoapp/deployment.yaml b/flux/apps/base/demoapp/deployment.yaml
@@ -16,7 +16,7 @@ spec:
       serviceAccountName: demoapp-sa
       containers:
       - name: session-checker
-        image: ghcr.io/torumakabe/session-checker:v1.2.20 # {"$imagepolicy": "flux-system:session-checker"}
+        image: ghcr.io/torumakabe/session-checker:v1.2.21 # {"$imagepolicy": "flux-system:session-checker"}
         imagePullPolicy: IfNotPresent
         command: ["/session-checker"]
         volumeMounts:

diff --git a/flux/infrastructure/base/chaos-mesh/release.yaml b/flux/infrastructure/base/chaos-mesh/release.yaml
@@ -11,7 +11,7 @@ spec:
         kind: HelmRepository
         name: chaos-mesh
         namespace: flux-system
-      version: "2.5.2"
+      version: "2.6.0"
   interval: 5m
   install:
     remediation:

diff --git a/flux/infrastructure/base/nginx/release.yaml b/flux/infrastructure/base/nginx/release.yaml
@@ -11,7 +11,7 @@ spec:
         kind: HelmRepository
         name: ingress-nginx
         namespace: flux-system
-      version: "4.6.0"
+      version: "4.7.0"
   interval: 5m
   install:
     remediation:

diff --git a/terraform/blue/aks/locals.tf b/terraform/blue/aks/locals.tf
@@ -1,41 +1,54 @@
 locals {
   tenant_id      = data.azurerm_client_config.current.tenant_id
   subnet_id_base = "/subscriptions/${data.azurerm_client_config.current.subscription_id}/resourceGroups/${local.shared_rg.name}/providers/Microsoft.Network/virtualNetworks/vnet-default/subnets"
+
   shared_rg = {
     name = "rg-aks-anti-dry-iac-shared-${var.suffix}"
   }
+
   aks = {
     rg = {
       name     = "rg-aks-anti-dry-iac-${var.aks.switch}-${var.suffix}"
       location = var.aks.rg.location
     }
+
     cluster_name = "${var.prefix}-aks-anti-dry-iac-${var.aks.switch}-${var.suffix}"
+
     default = {
       orchestrator_version = "1.26.3"
       vm_size              = "Standard_D2ds_v5"
       os_disk_size_gb      = 75
       os_disk_type         = "Ephemeral"
       os_sku               = "Mariner"
     }
+
     network = {
       node_system_subnet_id         = "${local.subnet_id_base}/snet-aks-${var.aks.switch}-node-system"
       node_user_az_subnet_id_prefix = "${local.subnet_id_base}/snet-aks-${var.aks.switch}-node-user-az"
       svc_lb_subnet_id              = "${local.subnet_id_base}/snet-aks-${var.aks.switch}-svc-lb"
     }
   }
+
   log_analytics = {
     workspace_id = "/subscriptions/${data.azurerm_client_config.current.subscription_id}/resourceGroups/${var.log_analytics.workspace.rg_name}/providers/Microsoft.OperationalInsights/workspaces/${var.log_analytics.workspace.name}"
   }
+
   demoapp = {
     service_account = {
       name      = "demoapp-sa"
       namespace = "demoapp"
     }
+
     key_vault = {
       name = "${var.prefix}-${var.demoapp.key_vault.name_body}-${var.suffix}"
       id   = "/subscriptions/${data.azurerm_client_config.current.subscription_id}/resourceGroups/${local.shared_rg.name}/providers/Microsoft.KeyVault/vaults/${var.prefix}-${var.demoapp.key_vault.name_body}-${var.suffix}" # repeat key vault name interpolation to avoid self-referencing local value
     }
   }
+
+  prometheus = {
+    data_collection_endpoint_id = "/subscriptions/${data.azurerm_client_config.current.subscription_id}/resourceGroups/${local.shared_rg.name}/providers/Microsoft.Insights/dataCollectionEndpoints/${var.prometheus.data_collection_endpoint_name}"
+    data_collection_rule_id     = "/subscriptions/${data.azurerm_client_config.current.subscription_id}/resourceGroups/${local.shared_rg.name}/providers/Microsoft.Insights/dataCollectionRules/${var.prometheus.data_collection_rule_name}"
+  }
 }
 
 data "azurerm_client_config" "current" {}

diff --git a/terraform/blue/aks/main.tf b/terraform/blue/aks/main.tf
@@ -1,5 +1,5 @@
 terraform {
-  required_version = "~> 1.4.5"
+  required_version = "~> 1.4.6"
 
   required_providers {
     azurerm = {
@@ -184,8 +184,12 @@ resource "azurerm_kubernetes_cluster" "default" {
     log_analytics_workspace_id      = local.log_analytics.workspace_id
     msi_auth_for_monitoring_enabled = true
   }
-  // Just enabled. Waiting for this issue https://github.com/hashicorp/terraform-provider-azurerm/issues/20702
-  monitor_metrics {}
+
+  dynamic "monitor_metrics" {
+    for_each = var.prometheus.enabled ? toset(["1"]) : toset([])
+    content {}
+  }
+
   open_service_mesh_enabled = false
   key_vault_secrets_provider {
     secret_rotation_enabled = true
@@ -298,26 +302,19 @@ resource "azurerm_monitor_diagnostic_setting" "aks" {
   }
 }
 
-// Replace this after resolution this issue https://github.com/hashicorp/terraform-provider-azurerm/issues/18809
-resource "azapi_resource" "amw_prometheus" {
-  schema_validation_enabled = false
-  type                      = "microsoft.monitor/accounts@2023-04-03"
-  name                      = "amw-prom-${local.aks.cluster_name}"
-  parent_id                 = azurerm_resource_group.aks.id
-  location                  = azurerm_resource_group.aks.location
-
-  response_export_values = ["*"]
-}
-
 resource "azurerm_monitor_data_collection_rule_association" "dce_amw_prometheus" {
+  for_each = var.prometheus.enabled ? toset(["1"]) : toset([])
+
   target_resource_id          = azurerm_kubernetes_cluster.default.id
-  data_collection_endpoint_id = jsondecode(azapi_resource.amw_prometheus.output).properties.defaultIngestionSettings.dataCollectionEndpointResourceId
+  data_collection_endpoint_id = local.prometheus.data_collection_endpoint_id
 }
 
 resource "azurerm_monitor_data_collection_rule_association" "dcra_amw_prometheus" {
+  for_each = var.prometheus.enabled ? toset(["1"]) : toset([])
+
   name                    = "dcra-amw-prom-${local.aks.cluster_name}"
   target_resource_id      = azurerm_kubernetes_cluster.default.id
-  data_collection_rule_id = jsondecode(azapi_resource.amw_prometheus.output).properties.defaultIngestionSettings.dataCollectionRuleResourceId
+  data_collection_rule_id = local.prometheus.data_collection_rule_id
 }
 
 resource "azurerm_user_assigned_identity" "demoapp" {

diff --git a/terraform/blue/aks/variables.tf b/terraform/blue/aks/variables.tf
@@ -44,3 +44,11 @@ variable "demoapp" {
     })
   })
 }
+
+variable "prometheus" {
+  type = object({
+    enabled                       = bool
+    data_collection_endpoint_name = string
+    data_collection_rule_name     = string
+  })
+}
diff --git a/terraform/blue/dev.tfvars b/terraform/blue/dev.tfvars
@@ -41,3 +41,9 @@ demoapp = {
     name_body = "kv-demoapp"
   }
 }
+
+prometheus = {
+  enabled                       = false
+  data_collection_endpoint_name = "dce-amw-prom"
+  data_collection_rule_name     = "dcr-amw-prom"
+}
diff --git a/terraform/blue/kubernetes-config/main.tf b/terraform/blue/kubernetes-config/main.tf
@@ -1,5 +1,5 @@
 terraform {
-  required_version = "~> 1.4.5"
+  required_version = "~> 1.4.6"
 
   required_providers {
     kubernetes = {

diff --git a/terraform/blue/main.tf b/terraform/blue/main.tf
@@ -1,12 +1,12 @@
 terraform {
-  required_version = "~> 1.4.5"
+  required_version = "~> 1.4.6"
   # Choose the backend according to your requirements
   # backend "remote" {}
 
   required_providers {
     azurerm = {
       source  = "hashicorp/azurerm"
-      version = "~> 3.53.0"
+      version = "~> 3.59.0"
     }
 
     azapi = {
@@ -16,7 +16,7 @@ terraform {
 
     kubernetes = {
       source  = "hashicorp/kubernetes"
-      version = "~> 2.20"
+      version = "~> 2.21"
     }
   }
 }
@@ -64,6 +64,7 @@ module "aks" {
   aks           = var.aks
   log_analytics = var.log_analytics
   demoapp       = var.demoapp
+  prometheus    = var.prometheus
 }
 
 module "kubernetes-config" {

diff --git a/terraform/blue/prod.tfvars b/terraform/blue/prod.tfvars
@@ -41,3 +41,9 @@ demoapp = {
     name_body = "kv-demoapp"
   }
 }
+
+prometheus = {
+  enabled                       = true
+  data_collection_endpoint_name = "dce-amw-prom"
+  data_collection_rule_name     = "dcr-amw-prom"
+}
diff --git a/terraform/blue/variables.tf b/terraform/blue/variables.tf
@@ -47,3 +47,11 @@ variable "demoapp" {
     })
   })
 }
+
+variable "prometheus" {
+  type = object({
+    enabled                       = bool
+    data_collection_endpoint_name = string
+    data_collection_rule_name     = string
+  })
+}
diff --git a/terraform/green/aks/locals.tf b/terraform/green/aks/locals.tf
@@ -1,41 +1,54 @@
 locals {
   tenant_id      = data.azurerm_client_config.current.tenant_id
   subnet_id_base = "/subscriptions/${data.azurerm_client_config.current.subscription_id}/resourceGroups/${local.shared_rg.name}/providers/Microsoft.Network/virtualNetworks/vnet-default/subnets"
+
   shared_rg = {
     name = "rg-aks-anti-dry-iac-shared-${var.suffix}"
   }
+
   aks = {
     rg = {
       name     = "rg-aks-anti-dry-iac-${var.aks.switch}-${var.suffix}"
       location = var.aks.rg.location
     }
+
     cluster_name = "${var.prefix}-aks-anti-dry-iac-${var.aks.switch}-${var.suffix}"
+
     default = {
       orchestrator_version = "1.26.3"
       vm_size              = "Standard_D2ds_v5"
       os_disk_size_gb      = 75
       os_disk_type         = "Ephemeral"
       os_sku               = "Mariner"
     }
+
     network = {
       node_system_subnet_id         = "${local.subnet_id_base}/snet-aks-${var.aks.switch}-node-system"
       node_user_az_subnet_id_prefix = "${local.subnet_id_base}/snet-aks-${var.aks.switch}-node-user-az"
       svc_lb_subnet_id              = "${local.subnet_id_base}/snet-aks-${var.aks.switch}-svc-lb"
     }
   }
+
   log_analytics = {
     workspace_id = "/subscriptions/${data.azurerm_client_config.current.subscription_id}/resourceGroups/${var.log_analytics.workspace.rg_name}/providers/Microsoft.OperationalInsights/workspaces/${var.log_analytics.workspace.name}"
   }
+
   demoapp = {
     service_account = {
       name      = "demoapp-sa"
       namespace = "demoapp"
     }
+
     key_vault = {
       name = "${var.prefix}-${var.demoapp.key_vault.name_body}-${var.suffix}"
       id   = "/subscriptions/${data.azurerm_client_config.current.subscription_id}/resourceGroups/${local.shared_rg.name}/providers/Microsoft.KeyVault/vaults/${var.prefix}-${var.demoapp.key_vault.name_body}-${var.suffix}" # repeat key vault name interpolation to avoid self-referencing local value
     }
   }
+
+  prometheus = {
+    data_collection_endpoint_id = "/subscriptions/${data.azurerm_client_config.current.subscription_id}/resourceGroups/${local.shared_rg.name}/providers/Microsoft.Insights/dataCollectionEndpoints/${var.prometheus.data_collection_endpoint_name}"
+    data_collection_rule_id     = "/subscriptions/${data.azurerm_client_config.current.subscription_id}/resourceGroups/${local.shared_rg.name}/providers/Microsoft.Insights/dataCollectionRules/${var.prometheus.data_collection_rule_name}"
+  }
 }
 
 data "azurerm_client_config" "current" {}

diff --git a/terraform/green/aks/main.tf b/terraform/green/aks/main.tf
@@ -1,5 +1,5 @@
 terraform {
-  required_version = "~> 1.4.5"
+  required_version = "~> 1.4.6"
 
   required_providers {
     azurerm = {
@@ -184,8 +184,12 @@ resource "azurerm_kubernetes_cluster" "default" {
     log_analytics_workspace_id      = local.log_analytics.workspace_id
     msi_auth_for_monitoring_enabled = true
   }
-  // Just enabled. Waiting for this issue https://github.com/hashicorp/terraform-provider-azurerm/issues/20702
-  monitor_metrics {}
+
+  dynamic "monitor_metrics" {
+    for_each = var.prometheus.enabled ? toset(["1"]) : toset([])
+    content {}
+  }
+
   open_service_mesh_enabled = false
   key_vault_secrets_provider {
     secret_rotation_enabled = true
@@ -298,26 +302,19 @@ resource "azurerm_monitor_diagnostic_setting" "aks" {
   }
 }
 
-// Replace this after resolution this issue https://github.com/hashicorp/terraform-provider-azurerm/issues/18809
-resource "azapi_resource" "amw_prometheus" {
-  schema_validation_enabled = false
-  type                      = "microsoft.monitor/accounts@2023-04-03"
-  name                      = "amw-prom-${local.aks.cluster_name}"
-  parent_id                 = azurerm_resource_group.aks.id
-  location                  = azurerm_resource_group.aks.location
-
-  response_export_values = ["*"]
-}
-
 resource "azurerm_monitor_data_collection_rule_association" "dce_amw_prometheus" {
+  for_each = var.prometheus.enabled ? toset(["1"]) : toset([])
+
   target_resource_id          = azurerm_kubernetes_cluster.default.id
-  data_collection_endpoint_id = jsondecode(azapi_resource.amw_prometheus.output).properties.defaultIngestionSettings.dataCollectionEndpointResourceId
+  data_collection_endpoint_id = local.prometheus.data_collection_endpoint_id
 }
 
 resource "azurerm_monitor_data_collection_rule_association" "dcra_amw_prometheus" {
+  for_each = var.prometheus.enabled ? toset(["1"]) : toset([])
+
   name                    = "dcra-amw-prom-${local.aks.cluster_name}"
   target_resource_id      = azurerm_kubernetes_cluster.default.id
-  data_collection_rule_id = jsondecode(azapi_resource.amw_prometheus.output).properties.defaultIngestionSettings.dataCollectionRuleResourceId
+  data_collection_rule_id = local.prometheus.data_collection_rule_id
 }
 
 resource "azurerm_user_assigned_identity" "demoapp" {

diff --git a/terraform/green/aks/variables.tf b/terraform/green/aks/variables.tf
@@ -44,3 +44,11 @@ variable "demoapp" {
     })
   })
 }
+
+variable "prometheus" {
+  type = object({
+    enabled                       = bool
+    data_collection_endpoint_name = string
+    data_collection_rule_name     = string
+  })
+}
diff --git a/terraform/green/dev.tfvars b/terraform/green/dev.tfvars
@@ -41,3 +41,9 @@ demoapp = {
     name_body = "kv-demoapp"
   }
 }
+
+prometheus = {
+  enabled                       = false
+  data_collection_endpoint_name = "dce-amw-prom"
+  data_collection_rule_name     = "dcr-amw-prom"
+}