azure: use MSI for native fencing (not user/pw needed)

Author: yeoldegrove

azure/main.tf

@@ -171,11 +171,6 @@ module "drbd_node" {
   iscsi_srv_ip        = join("", module.iscsi_server.iscsisrv_ip)
   nfs_mounting_point  = var.drbd_nfs_mounting_point
   nfs_export_name     = var.netweaver_sid
-  # only used by azure fence agent (native fencing)
-  subscription_id           = data.azurerm_subscription.current.subscription_id
-  tenant_id                 = data.azurerm_subscription.current.tenant_id
-  fence_agent_app_id        = var.fence_agent_app_id
-  fence_agent_client_secret = var.fence_agent_client_secret
 }
 
 module "netweaver_node" {
@@ -215,11 +210,6 @@ module "netweaver_node" {
   anf_pool_name              = local.anf_pool_name
   anf_pool_service_level     = var.anf_pool_service_level
   netweaver_anf_quota_sapmnt = var.netweaver_anf_quota_sapmnt
-  # only used by azure fence agent (native fencing)
-  subscription_id           = data.azurerm_subscription.current.subscription_id
-  tenant_id                 = data.azurerm_subscription.current.tenant_id
-  fence_agent_app_id        = var.fence_agent_app_id
-  fence_agent_client_secret = var.fence_agent_client_secret
 }
 
 module "hana_node" {
@@ -254,11 +244,6 @@ module "hana_node" {
   hana_scale_out_anf_quota_log    = var.hana_scale_out_anf_quota_log
   hana_scale_out_anf_quota_backup = var.hana_scale_out_anf_quota_backup
   hana_scale_out_anf_quota_shared = var.hana_scale_out_anf_quota_shared
-  # only used by azure fence agent (native fencing)
-  subscription_id           = data.azurerm_subscription.current.subscription_id
-  tenant_id                 = data.azurerm_subscription.current.tenant_id
-  fence_agent_app_id        = var.fence_agent_app_id
-  fence_agent_client_secret = var.fence_agent_client_secret
   # passed to majority_maker module
   majority_maker_vm_size = var.hana_majority_maker_vm_size
   majority_maker_ip      = local.hana_majority_maker_ip

azure/modules/drbd_node/salt_provisioner.tf

@@ -37,11 +37,6 @@ partitions:
   1:
     start: 0%
     end: 100%
-subscription_id: ${var.subscription_id}
-tenant_id: ${var.tenant_id}
-resource_group_name: ${var.resource_group_name}
-fence_agent_app_id: ${var.fence_agent_app_id}
-fence_agent_client_secret: ${var.fence_agent_client_secret}
   EOF
     destination = "/tmp/grains"
   }

azure/modules/drbd_node/variables.tf

@@ -86,22 +86,3 @@ variable "nfs_export_name" {
   type        = string
 }
 
-variable "subscription_id" {
-  description = "ID of the azure subscription."
-  type        = string
-}
-
-variable "tenant_id" {
-  description = "ID of the azure tenant."
-  type        = string
-}
-
-variable "fence_agent_app_id" {
-  description = "ID of the azure service principal / application that is used for native fencing."
-  type        = string
-}
-
-variable "fence_agent_client_secret" {
-  description = "Secret for the azure service principal / application that is used for native fencing."
-  type        = string
-}

azure/modules/hana_node/main.tf

@@ -20,7 +20,26 @@ locals {
   ]) : toset([])
 
   hana_lb_rules_ports_secondary = local.create_active_active_infra == 1 ? local.hana_lb_rules_ports : toset([])
+
   hostname                      = var.common_variables["deployment_name_in_hostname"] ? format("%s-%s", var.common_variables["deployment_name"], var.name) : var.name
+  hostnames_hana = [ for h in range(var.hana_count) : format("%s%02d", local.hostname, h + 1) ]
+  hostname_mm = format("%s%s", local.hostname, "mm")
+  hostnames = local.create_scale_out == 1 ? concat(local.hostnames_hana, [local.hostname_mm]) : local.hostnames_hana
+
+  principal_ids = concat(azurerm_virtual_machine.hana.*.identity.0.principal_id, [module.hana_majority_maker.fence_principal_id])
+  fence_scopes = flatten([
+    for c in range(var.hana_count + local.create_scale_out) : [
+      for n in local.hostnames : {
+        count = c
+        node = element(local.hostnames, c)
+        principal = element(local.principal_ids, c)
+        scope = format("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/virtualMachines/%s", data.azurerm_subscription.current.subscription_id, var.resource_group_name, n)
+      }
+    ]
+  ])
+}
+
+data "azurerm_subscription" "current" {
 }
 
 resource "azurerm_availability_set" "hana-availability-set" {
@@ -401,7 +420,7 @@ resource "azurerm_virtual_machine" "hana" {
   }
 
   os_profile {
-    computer_name  = "${local.hostname}${format("%02d", count.index + 1)}"
+    computer_name  = element(local.hostnames_hana, count.index)
     admin_username = var.common_variables["authorized_user"]
   }
 
@@ -419,15 +438,48 @@ resource "azurerm_virtual_machine" "hana" {
     storage_uri = var.storage_account
   }
 
+  identity {
+    type = "SystemAssigned"
+  }
+
   tags = {
     workspace = var.common_variables["deployment_name"]
   }
 }
 
+resource "azurerm_role_definition" "fence" {
+  count                            = var.common_variables["hana"]["ha_enabled"] && var.common_variables["hana"]["fencing_mechanism"] == "native" ? 1 : 0
+  name                             = "role-fence-${local.hostname}"
+  # It is recommended to use the first entry of the assignable_scopes.
+  # https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/role_definition#scope
+  scope =  format("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/virtualMachines/%s%02d", data.azurerm_subscription.current.subscription_id, var.resource_group_name, local.hostname, count.index + 1)
+
+  permissions {
+    actions     = [
+      "Microsoft.Compute/*/read",
+      "Microsoft.Compute/virtualMachines/powerOff/action",
+      "Microsoft.Compute/virtualMachines/restart/action",
+      "Microsoft.Compute/virtualMachines/start/action"
+    ]
+    not_actions = []
+  }
+
+  assignable_scopes = [
+      for h in local.hostnames : format("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/virtualMachines/%s", data.azurerm_subscription.current.subscription_id, var.resource_group_name, h)
+  ]
+}
+
+resource "azurerm_role_assignment" "host" {
+  for_each = { for r in local.fence_scopes : "${r.node}_${r.scope}" => r if var.common_variables["hana"]["ha_enabled"] && var.common_variables["hana"]["fencing_mechanism"] == "native" }
+  scope =  each.value.scope
+  role_definition_id = element(azurerm_role_definition.fence.*.role_definition_resource_id, 0)
+  principal_id       = each.value.principal
+}
+
 module "hana_majority_maker" {
   source                        = "../majority_maker_node"
   node_count                    = local.create_scale_out
-  name                          = var.name
+  name                          = local.hostname
   common_variables              = var.common_variables
   bastion_host                  = var.bastion_host
   az_region                     = var.az_region
@@ -446,11 +498,6 @@ module "hana_majority_maker" {
   cluster_ssh_key               = var.cluster_ssh_key
   os_image                      = var.os_image
   iscsi_srv_ip                  = var.iscsi_srv_ip
-  # only used by azure fence agent (native fencing)
-  subscription_id           = var.subscription_id
-  tenant_id                 = var.tenant_id
-  fence_agent_app_id        = var.fence_agent_app_id
-  fence_agent_client_secret = var.fence_agent_client_secret
 }
 
 module "hana_on_destroy" {

azure/modules/hana_node/salt_provisioner.tf

@@ -32,11 +32,6 @@ sbd_lun_index: 0
 iscsi_srv_ip: ${var.iscsi_srv_ip}
 cluster_ssh_pub:  ${var.cluster_ssh_pub}
 cluster_ssh_key: ${var.cluster_ssh_key}
-subscription_id: ${var.subscription_id}
-tenant_id: ${var.tenant_id}
-resource_group_name: ${var.resource_group_name}
-fence_agent_app_id: ${var.fence_agent_app_id}
-fence_agent_client_secret: ${var.fence_agent_client_secret}
 anf_mount_ip:
   data: [ ${local.shared_storage_anf == 1 ? join(", ", azurerm_netapp_volume.hana-netapp-volume-data.*.mount_ip_addresses.0) : ""} ]
   log: [ ${local.shared_storage_anf == 1 ? join(", ", azurerm_netapp_volume.hana-netapp-volume-log.*.mount_ip_addresses.0) : ""} ]

azure/modules/hana_node/variables.tf

@@ -123,26 +123,6 @@ variable "hana_data_disks_configuration" {
   EOF
 }
 
-variable "subscription_id" {
-  description = "ID of the azure subscription."
-  type        = string
-}
-
-variable "tenant_id" {
-  description = "ID of the azure tenant."
-  type        = string
-}
-
-variable "fence_agent_app_id" {
-  description = "ID of the azure service principal / application that is used for native fencing."
-  type        = string
-}
-
-variable "fence_agent_client_secret" {
-  description = "Secret for the azure service principal / application that is used for native fencing."
-  type        = string
-}
-
 variable "anf_account_name" {
   description = "Name of ANF Accounts"
   type        = string

azure/modules/majority_maker_node/main.tf

@@ -3,7 +3,6 @@ locals {
   provisioning_address = local.bastion_enabled ? data.azurerm_network_interface.majority_maker.*.private_ip_address : data.azurerm_public_ip.majority_maker.*.ip_address
 }
 
-
 # majority maker network configuration
 
 resource "azurerm_network_interface" "majority_maker" {
@@ -66,7 +65,7 @@ module "os_image_reference" {
 
 resource "azurerm_virtual_machine" "majority_maker" {
   count                 = var.node_count
-  name                  = "vm${var.name}mm"
+  name                  = "${var.name}mm"
   location              = var.az_region
   resource_group_name   = var.resource_group_name
   network_interface_ids = [element(azurerm_network_interface.majority_maker.*.id, count.index)]
@@ -90,7 +89,7 @@ resource "azurerm_virtual_machine" "majority_maker" {
   }
 
   os_profile {
-    computer_name  = "vm${var.name}mm"
+    computer_name  = "${var.name}mm"
     admin_username = var.common_variables["authorized_user"]
   }
 
@@ -108,6 +107,10 @@ resource "azurerm_virtual_machine" "majority_maker" {
     storage_uri = var.storage_account
   }
 
+  identity {
+    type = "SystemAssigned"
+  }
+
   tags = {
     workspace = var.common_variables["deployment_name"]
   }

azure/modules/majority_maker_node/outputs.tf

@@ -29,3 +29,7 @@ output "cluster_nodes_name" {
 output "cluster_nodes_public_name" {
   value = [data.azurerm_public_ip.majority_maker.*.fqdn]
 }
+
+output "fence_principal_id" {
+  value = var.common_variables["hana"]["fencing_mechanism"] == "native" ? azurerm_virtual_machine.majority_maker.0.identity.0.principal_id : null
+}

azure/modules/majority_maker_node/salt_provisioner.tf

@@ -31,11 +31,6 @@ sbd_lun_index: 0
 iscsi_srv_ip: ${var.iscsi_srv_ip}
 cluster_ssh_pub:  ${var.cluster_ssh_pub}
 cluster_ssh_key: ${var.cluster_ssh_key}
-subscription_id: ${var.subscription_id}
-tenant_id: ${var.tenant_id}
-resource_group_name: ${var.resource_group_name}
-fence_agent_app_id: ${var.fence_agent_app_id}
-fence_agent_client_secret: ${var.fence_agent_client_secret}
 node_count: ${var.hana_count + var.node_count}
 majority_maker_node: vm${var.name}mm
 majority_maker_ip: ${var.majority_maker_ip}

azure/modules/majority_maker_node/variables.tf

@@ -90,23 +90,3 @@ variable "cluster_ssh_key" {
   description = "path for the private key needed by the cluster"
   type        = string
 }
-
-variable "subscription_id" {
-  description = "ID of the azure subscription."
-  type        = string
-}
-
-variable "tenant_id" {
-  description = "ID of the azure tenant."
-  type        = string
-}
-
-variable "fence_agent_app_id" {
-  description = "ID of the azure service principal / application that is used for native fencing."
-  type        = string
-}
-
-variable "fence_agent_client_secret" {
-  description = "Secret for the azure service principal / application that is used for native fencing."
-  type        = string
-}

azure/modules/netweaver_node/salt_provisioner.tf

@@ -36,11 +36,6 @@ iscsi_srv_ip: ${var.iscsi_srv_ip}
 storage_account_name: ${var.storage_account_name}
 storage_account_key: ${var.storage_account_key}
 storage_account_path: ${var.storage_account_path}
-subscription_id: ${var.subscription_id}
-tenant_id: ${var.tenant_id}
-resource_group_name: ${var.resource_group_name}
-fence_agent_app_id: ${var.fence_agent_app_id}
-fence_agent_client_secret: ${var.fence_agent_client_secret}
 anf_mount_ip:
   sapmnt: [ ${local.shared_storage_anf == 1 ? join(", ", azurerm_netapp_volume.netweaver-netapp-volume-sapmnt.*.mount_ip_addresses.0) : ""} ]
   EOF

azure/modules/netweaver_node/variables.tf

@@ -149,26 +149,6 @@ variable "cluster_ssh_key" {
   type        = string
 }
 
-variable "subscription_id" {
-  description = "ID of the azure subscription."
-  type        = string
-}
-
-variable "tenant_id" {
-  description = "ID of the azure tenant."
-  type        = string
-}
-
-variable "fence_agent_app_id" {
-  description = "ID of the azure service principal / application that is used for native fencing."
-  type        = string
-}
-
-variable "fence_agent_client_secret" {
-  description = "Secret for the azure service principal / application that is used for native fencing."
-  type        = string
-}
-
 variable "anf_account_name" {
   description = "Name of ANF Accounts"
   type        = string

azure/terraform.tfvars.example

@@ -132,13 +132,6 @@ cluster_ssh_key = "salt://sshkeys/cluster.id_rsa"
 # true or false (default)
 #hwcct = false
 
-# Variables used with native fencing (azure fence agent)
-# Make sure to check out the documentation:
-# https://docs.microsoft.com/en-us/azure/virtual-machines/workloads/sap/high-availability-guide-suse-pacemaker#create-azure-fence-agent-stonith-device
-# The fencing mechanism has to be defined on a per cluster basis.
-# fence_agent_app_id = "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX"       # login
-# fence_agent_client_secret = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"  # password
-
 ##########################
 # Bastion (jumpbox) machine variables
 ##########################

azure/variables.tf

@@ -1001,19 +1001,6 @@ variable "pre_deployment" {
   default     = false
 }
 
-# native fencing
-variable "fence_agent_app_id" {
-  description = "ID of the azure service principal / application that is used for native fencing."
-  type        = string
-  default     = ""
-}
-
-variable "fence_agent_client_secret" {
-  description = "Secret for the azure service principal / application that is used for native fencing."
-  type        = string
-  default     = ""
-}
-
 # ANF shared storage
 variable "anf_account_name" {
   description = "Name of ANF Accounts"

pillar_examples/automatic/drbd/cluster.sls

@@ -64,13 +64,3 @@ cluster:
         virtual_ip_mask: 24
         {% endif %}
         native_fencing: {{ grains['fencing_mechanism'] == 'native' }}
-        {% if grains['fencing_mechanism'] == 'native' %}
-        {% if grains['provider'] == 'azure' %}
-        # only used by azure fence agent (native fencing)
-        azure_subscription_id: {{ grains['subscription_id'] }}
-        azure_resource_group_name: {{ grains['resource_group_name'] }}
-        azure_tenant_id: {{ grains['tenant_id'] }}
-        azure_fence_agent_app_id: {{ grains['fence_agent_app_id'] }}
-        azure_fence_agent_client_secret: {{ grains['fence_agent_client_secret'] }}
-        {% endif %}
-        {% endif %}

pillar_examples/automatic/hana/cluster.sls

@@ -112,16 +112,6 @@ cluster:
         virtual_ip_secondary: {{ grains['hana_cluster_vip_secondary'] }}
         {% endif %}
         native_fencing: {{ grains['fencing_mechanism'] == 'native' }}
-        {% if grains['fencing_mechanism'] == 'native' %}
-        {% if grains['provider'] == 'azure' %}
-        # only used by azure fence agent (native fencing)
-        azure_subscription_id: {{ grains['subscription_id'] }}
-        azure_resource_group_name: {{ grains['resource_group_name'] }}
-        azure_tenant_id: {{ grains['tenant_id'] }}
-        azure_fence_agent_app_id: {{ grains['fence_agent_app_id'] }}
-        azure_fence_agent_client_secret: {{ grains['fence_agent_client_secret'] }}
-        {% endif %}
-        {% endif %}
         {% if grains['scenario_type'] == 'cost-optimized' %}
         prefer_takeover: false
         {% else %}