From d6d3c207da77fee2567e4d2b3dd06b908c7e621b Mon Sep 17 00:00:00 2001 From: Jiawei Tao Date: Mon, 2 Dec 2024 14:50:54 +0800 Subject: [PATCH 1/3] hdinsight_spark_cluster_resource: Add support for availability_zones --- .../services/hdinsight/common_hdinsight.go | 11 ++ .../hdinsight_spark_cluster_resource.go | 10 ++ .../hdinsight_spark_cluster_resource_test.go | 160 ++++++++++++++++++ .../r/hdinsight_spark_cluster.html.markdown | 2 + 4 files changed, 183 insertions(+) diff --git a/internal/services/hdinsight/common_hdinsight.go b/internal/services/hdinsight/common_hdinsight.go index c785b861ad9c..3820fe6e9d9e 100644 --- a/internal/services/hdinsight/common_hdinsight.go +++ b/internal/services/hdinsight/common_hdinsight.go @@ -6,6 +6,7 @@ package hdinsight import ( "context" "fmt" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" "log" "time" @@ -404,6 +405,16 @@ func flattenHDInsightAzureMonitor(input *extensions.AzureMonitorResponse) []inte return output } +func expandHDInsightAvailabilityZones(input interface{}) []string { + output := make([]string, 0) + + v := input.(*schema.Set) + for _, az := range v.List() { + output = append(output, az.(string)) + } + return output +} + func enableHDInsightMonitoring(ctx context.Context, client *extensions.ExtensionsClient, clusterId commonids.HDInsightClusterId, input []interface{}) error { payload := ExpandHDInsightsMonitor(input) diff --git a/internal/services/hdinsight/hdinsight_spark_cluster_resource.go b/internal/services/hdinsight/hdinsight_spark_cluster_resource.go index 59eb0e8c0b12..65ff5299662f 100644 --- a/internal/services/hdinsight/hdinsight_spark_cluster_resource.go +++ b/internal/services/hdinsight/hdinsight_spark_cluster_resource.go @@ -150,6 +150,8 @@ func resourceHDInsightSparkCluster() *pluginsdk.Resource { "monitor": SchemaHDInsightsMonitor(), "extension": SchemaHDInsightsExtension(), + + "availability_zones": commonschema.ZonesMultipleOptional(), }, } } @@ -272,6 +274,10 @@ func resourceHDInsightSparkClusterCreate(d *pluginsdk.ResourceData, meta interfa } } + if v, ok := d.GetOk("availability_zones"); ok { + payload.Zones = pointer.To(expandHDInsightAvailabilityZones(v)) + } + if err := client.CreateThenPoll(ctx, id, payload); err != nil { return fmt.Errorf("creating Spark %s: %+v", id, err) } @@ -384,6 +390,10 @@ func resourceHDInsightSparkClusterRead(d *pluginsdk.ResourceData, meta interface return fmt.Errorf("flattening setting `disk_encryption`: %+v", err) } + if model.Zones != nil { + d.Set("availability_zones", pointer.From(model.Zones)) + } + if err := d.Set("network", flattenHDInsightsNetwork(props.NetworkProperties)); err != nil { return fmt.Errorf("flattening `network`: %+v", err) } diff --git a/internal/services/hdinsight/hdinsight_spark_cluster_resource_test.go b/internal/services/hdinsight/hdinsight_spark_cluster_resource_test.go index 8e0df13bfd03..d08c25e40566 100644 --- a/internal/services/hdinsight/hdinsight_spark_cluster_resource_test.go +++ b/internal/services/hdinsight/hdinsight_spark_cluster_resource_test.go @@ -40,6 +40,26 @@ func TestAccHDInsightSparkCluster_basic(t *testing.T) { }) } +func TestAccHDInsightSparkCluster_availabilityZones(t *testing.T) { + data := acceptance.BuildTestData(t, "azurerm_hdinsight_spark_cluster", "test") + r := HDInsightSparkClusterResource{} + data.ResourceTest(t, r, []acceptance.TestStep{ + { + Config: r.availabilityZones(data), + Check: acceptance.ComposeTestCheckFunc( + check.That(data.ResourceName).ExistsInAzure(r), + ), + }, + data.ImportStep("roles.0.head_node.0.password", + "roles.0.head_node.0.vm_size", + "roles.0.worker_node.0.password", + "roles.0.worker_node.0.vm_size", + "roles.0.zookeeper_node.0.password", + "roles.0.zookeeper_node.0.vm_size", + "storage_account"), + }) +} + func TestAccHDInsightSparkCluster_roleScriptActions(t *testing.T) { data := acceptance.BuildTestData(t, "azurerm_hdinsight_spark_cluster", "test") r := HDInsightSparkClusterResource{} @@ -742,6 +762,146 @@ resource "azurerm_hdinsight_spark_cluster" "test" { `, r.template(data), data.RandomInteger) } +func (r HDInsightSparkClusterResource) availabilityZones(data acceptance.TestData) string { + return fmt.Sprintf(` +%s + +resource "azurerm_mssql_server" "test" { + name = "acctestsql-%d" + resource_group_name = azurerm_resource_group.test.name + location = azurerm_resource_group.test.location + administrator_login = "sql_admin" + administrator_login_password = "TerrAform123!" + version = "12.0" +} + +resource "azurerm_mssql_database" "hive" { + name = "hive" + server_id = azurerm_mssql_server.test.id + collation = "SQL_Latin1_General_CP1_CI_AS" + create_mode = "Default" +} + +resource "azurerm_mssql_database" "oozie" { + name = "oozie" + server_id = azurerm_mssql_server.test.id + collation = "SQL_Latin1_General_CP1_CI_AS" + create_mode = "Default" +} + +resource "azurerm_mssql_database" "ambari" { + name = "ambari" + server_id = azurerm_mssql_server.test.id + collation = "SQL_Latin1_General_CP1_CI_AS" + create_mode = "Default" +} + +resource "azurerm_mssql_firewall_rule" "AzureServices" { + name = "allow-azure-services" + server_id = azurerm_mssql_server.test.id + start_ip_address = "0.0.0.0" + end_ip_address = "0.0.0.0" +} + +resource "azurerm_virtual_network" "test" { + name = "acctestvirtnet%d" + address_space = ["172.16.0.0/16"] + location = azurerm_resource_group.test.location + resource_group_name = azurerm_resource_group.test.name +} + +resource "azurerm_subnet" "test" { + name = "acctestsubnet%d" + resource_group_name = azurerm_resource_group.test.name + virtual_network_name = azurerm_virtual_network.test.name + address_prefixes = ["172.16.11.0/26"] +} + +resource "azurerm_hdinsight_spark_cluster" "test" { + name = "acctesthdi-%d" + resource_group_name = azurerm_resource_group.test.name + location = azurerm_resource_group.test.location + cluster_version = "4.0" + tier = "Standard" + availability_zones = ["1"] + + component_version { + spark = "2.4" + } + + gateway { + username = "acctestusrgw" + password = "TerrAform123!" + } + + storage_account_gen2 { + storage_resource_id = azurerm_storage_account.gen2test.id + filesystem_id = azurerm_storage_data_lake_gen2_filesystem.gen2test.id + managed_identity_resource_id = azurerm_user_assigned_identity.test.id + is_default = true + } + + private_link_configuration { + name = "testconfig" + group_id = "headnode" + ip_configuration { + name = "testipconfig" + primary = false + private_ip_allocation_method = "dynamic" + subnet_id = azurerm_subnet.test.id + } + } + + roles { + head_node { + vm_size = "Standard_A4_V2" + username = "acctestusrvm" + password = "AccTestvdSC4daf986!" + subnet_id = azurerm_subnet.test.id + virtual_network_id = azurerm_virtual_network.test.id + } + + worker_node { + vm_size = "Standard_A4_V2" + username = "acctestusrvm" + password = "AccTestvdSC4daf986!" + target_instance_count = 3 + subnet_id = azurerm_subnet.test.id + virtual_network_id = azurerm_virtual_network.test.id + } + + zookeeper_node { + vm_size = "Medium" + username = "acctestusrvm" + password = "AccTestvdSC4daf986!" + subnet_id = azurerm_subnet.test.id + virtual_network_id = azurerm_virtual_network.test.id + } + } + metastores { + hive { + server = azurerm_mssql_server.test.fully_qualified_domain_name + database_name = azurerm_mssql_database.hive.name + username = azurerm_mssql_server.test.administrator_login + password = azurerm_mssql_server.test.administrator_login_password + } + oozie { + server = azurerm_mssql_server.test.fully_qualified_domain_name + database_name = azurerm_mssql_database.oozie.name + username = azurerm_mssql_server.test.administrator_login + password = azurerm_mssql_server.test.administrator_login_password + } + ambari { + server = azurerm_mssql_server.test.fully_qualified_domain_name + database_name = azurerm_mssql_database.ambari.name + username = azurerm_mssql_server.test.administrator_login + password = azurerm_mssql_server.test.administrator_login_password + } + } +} +`, r.gen2template(data), data.RandomInteger, data.RandomInteger, data.RandomInteger, data.RandomInteger) +} + func (r HDInsightSparkClusterResource) gen2basic(data acceptance.TestData) string { return fmt.Sprintf(` %s diff --git a/website/docs/r/hdinsight_spark_cluster.html.markdown b/website/docs/r/hdinsight_spark_cluster.html.markdown index b43ebe8ea21a..e5ee464b9f38 100644 --- a/website/docs/r/hdinsight_spark_cluster.html.markdown +++ b/website/docs/r/hdinsight_spark_cluster.html.markdown @@ -127,6 +127,8 @@ The following arguments are supported: * `security_profile` - (Optional) A `security_profile` block as defined below. Changing this forces a new resource to be created. +* `availability_zones` - (Optional) A list of Availability Zones which should be used for this HDInsight Spark Cluster. + --- A `component_version` block supports the following: From 87bab69bf4ff5f0ece6b55058024710efc02d63d Mon Sep 17 00:00:00 2001 From: Jiawei Tao Date: Mon, 2 Dec 2024 15:08:33 +0800 Subject: [PATCH 2/3] sort imports --- internal/services/hdinsight/common_hdinsight.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/services/hdinsight/common_hdinsight.go b/internal/services/hdinsight/common_hdinsight.go index 3820fe6e9d9e..53591c817d82 100644 --- a/internal/services/hdinsight/common_hdinsight.go +++ b/internal/services/hdinsight/common_hdinsight.go @@ -6,7 +6,6 @@ package hdinsight import ( "context" "fmt" - "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" "log" "time" @@ -17,6 +16,7 @@ import ( "github.com/hashicorp/go-azure-sdk/resource-manager/hdinsight/2021-06-01/clusters" "github.com/hashicorp/go-azure-sdk/resource-manager/hdinsight/2021-06-01/extensions" "github.com/hashicorp/go-azure-sdk/sdk/client/pollers" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" "github.com/hashicorp/terraform-provider-azurerm/internal/clients" "github.com/hashicorp/terraform-provider-azurerm/internal/services/hdinsight/custompollers" "github.com/hashicorp/terraform-provider-azurerm/internal/tf/pluginsdk" From 274d024b07807a116c62560af494506a590c8fe9 Mon Sep 17 00:00:00 2001 From: Jiawei Tao Date: Tue, 3 Dec 2024 17:29:36 +0800 Subject: [PATCH 3/3] fix acc test --- .../hdinsight_spark_cluster_resource_test.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/internal/services/hdinsight/hdinsight_spark_cluster_resource_test.go b/internal/services/hdinsight/hdinsight_spark_cluster_resource_test.go index d08c25e40566..456d3a792ca5 100644 --- a/internal/services/hdinsight/hdinsight_spark_cluster_resource_test.go +++ b/internal/services/hdinsight/hdinsight_spark_cluster_resource_test.go @@ -841,17 +841,6 @@ resource "azurerm_hdinsight_spark_cluster" "test" { is_default = true } - private_link_configuration { - name = "testconfig" - group_id = "headnode" - ip_configuration { - name = "testipconfig" - primary = false - private_ip_allocation_method = "dynamic" - subnet_id = azurerm_subnet.test.id - } - } - roles { head_node { vm_size = "Standard_A4_V2"