Deploy Databricks in a VNET

Author: dtzar

Python/azureml-pipeline-databricks/environment_setup/terraform/databricks/main.tf

@@ -1,6 +1,50 @@
-resource "azurerm_databricks_workspace" "aml" {
-  name                = "dbricks${var.appname}${var.environment}"
+data "azurerm_virtual_network" "main" {
+  name                = var.vnet_name
+  resource_group_name  = var.resource_group_name
+}
+
+data "azurerm_subnet" "databricks-private" {
+  name                 = var.private_subnet_name
+  virtual_network_name = var.vnet_name
+  resource_group_name  = var.resource_group_name
+}
+
+data "azurerm_subnet" "databricks-public" {
+  name                 = var.public_subnet_name
+  virtual_network_name = var.vnet_name
+  resource_group_name  = var.resource_group_name
+}
+
+resource "azurerm_network_security_group" "databricks" {
+  name     = "nsg-${var.appname}-${var.environment}-databricks"
+  location            = var.location
+  resource_group_name = var.resource_group_name
+}
+
+resource "azurerm_subnet_network_security_group_association" "private" {
+  subnet_id                 = data.azurerm_subnet.databricks-private.id
+  network_security_group_id = azurerm_network_security_group.databricks.id
+}
+
+resource "azurerm_subnet_network_security_group_association" "public" {
+  subnet_id                 = data.azurerm_subnet.databricks-public.id
+  network_security_group_id = azurerm_network_security_group.databricks.id
+}
+
+resource "azurerm_databricks_workspace" "main" {
+  name                = "dbricks-${var.appname}-${var.environment}"
   resource_group_name = var.resource_group_name
   location            = var.location
   sku                 = "standard"
+
+  custom_parameters {
+    public_subnet_name = data.azurerm_subnet.databricks-public.name
+    private_subnet_name = data.azurerm_subnet.databricks-private.name
+    virtual_network_id = data.azurerm_virtual_network.main.id
+  }
+
+  depends_on = [
+    azurerm_subnet_network_security_group_association.private,
+    azurerm_subnet_network_security_group_association.public,
+  ]
 }

Python/azureml-pipeline-databricks/environment_setup/terraform/databricks/outputs.tf

@@ -1,11 +1,11 @@
 output "id" {
-  value = azurerm_databricks_workspace.aml.id
+  value = azurerm_databricks_workspace.main.id
 }
 
 output "name" {
-  value = azurerm_databricks_workspace.aml.name
+  value = azurerm_databricks_workspace.main.name
 }
 
 output "location" {
-  value = azurerm_databricks_workspace.aml.location
+  value = azurerm_databricks_workspace.main.location
 }

Python/azureml-pipeline-databricks/environment_setup/terraform/databricks/variables.tf

@@ -13,3 +13,15 @@ variable "resource_group_name" {
 variable "location" {
   type    = string
 }
+
+variable "vnet_name" {
+  type = string
+}
+
+variable "private_subnet_name" {
+  type    = string
+}
+
+variable "public_subnet_name" {
+  type    = string
+}

Python/azureml-pipeline-databricks/environment_setup/terraform/main.tf

@@ -6,31 +6,42 @@
 # Resource Group
 
 resource "azurerm_resource_group" "main" {
-  name   = "rg-${var.appname}-${var.environment}-main"
+  name     = "rg-${var.appname}-${var.environment}-main"
   location = var.location
 }
 
+module "vnet" {
+  source                         = "./vnet"
+  appname                        = var.appname
+  environment                    = var.environment
+  location                       = var.location
+  resource_group_name            = azurerm_resource_group.main.name
+}
+
 module "azureml" {
-  source = "./azureml"
-  appname = var.appname
-  environment = var.environment
-  location = var.location
-  tenant_id = data.azurerm_client_config.current.tenant_id
-  resource_group_name = azurerm_resource_group.main.name
+  source                         = "./azureml"
+  appname                        = var.appname
+  environment                    = var.environment
+  location                       = var.location
+  tenant_id                      = data.azurerm_client_config.current.tenant_id
+  resource_group_name            = azurerm_resource_group.main.name
 }
 
 module "training-data" {
-  source = "./training-data"
-  appname = var.appname
-  environment = var.environment
-  resource_group_name = azurerm_resource_group.main.name
-  location = var.location
+  source                         = "./training-data"
+  appname                        = var.appname
+  environment                    = var.environment
+  resource_group_name            = azurerm_resource_group.main.name
+  location                       = var.location
 }
 
 module "databricks" {
-  source = "./databricks"
-  appname = var.appname
-  environment = var.environment
-  resource_group_name = azurerm_resource_group.main.name
-  location = var.location
+  source                         = "./databricks"
+  appname                        = var.appname
+  environment                    = var.environment
+  resource_group_name            = azurerm_resource_group.main.name
+  location                       = var.location
+  vnet_name                      = module.vnet.vnet_name
+  private_subnet_name            = module.vnet.databricks_private_subnet_name
+  public_subnet_name             = module.vnet.databricks_public_subnet_name
 }

Python/azureml-pipeline-databricks/environment_setup/terraform/provider.tf

@@ -6,7 +6,9 @@ terraform {
 # Configure the Azure Provider
 provider "azurerm" {
   # It is recommended to pin to a given version of the Provider
-  version = "=1.44.0"
+  version = "=2.6.0"
+  skip_provider_registration = true
+  features {}
 }
 
 # Data

Python/azureml-pipeline-databricks/environment_setup/terraform/variables.tf

@@ -10,6 +10,9 @@ variable "environment" {
   default = "dev"
 }
 
+# The total number of characters of the appname and environment variables additioned
+# Should not exceed 8 characters
+
 variable "location" {
   type    = string
   description = "Azure region where to create resources."

Python/azureml-pipeline-databricks/environment_setup/terraform/vnet/main.tf

@@ -0,0 +1,48 @@
+# Create virtual network
+
+resource "azurerm_virtual_network" "main" {
+  name                = "vnet-${var.appname}-${var.environment}"
+  address_space       = ["10.100.0.0/16"]
+  location            = var.location
+  resource_group_name = var.resource_group_name
+}
+
+# Create subnets
+
+resource "azurerm_subnet" "databricks-private" {
+  name                 = "databricks-private-subnet"
+  resource_group_name = var.resource_group_name
+  virtual_network_name = azurerm_virtual_network.main.name
+  address_prefix       = "10.100.1.0/24"
+
+  delegation {
+    name = "acctestdelegation"
+    service_delegation {
+      name = "Microsoft.Databricks/workspaces"
+      actions = [
+        "Microsoft.Network/virtualNetworks/subnets/action",
+        "Microsoft.Network/virtualNetworks/subnets/join/action",
+        "Microsoft.Network/virtualNetworks/subnets/prepareNetworkPolicies/action"
+      ]
+    }
+  }
+}
+
+resource "azurerm_subnet" "databricks-public" {
+  name                 = "databricks-public-subnet"
+  resource_group_name = var.resource_group_name
+  virtual_network_name = azurerm_virtual_network.main.name
+  address_prefix       = "10.100.2.0/24"
+
+  delegation {
+    name = "acctestdelegation"
+    service_delegation {
+      name = "Microsoft.Databricks/workspaces"
+      actions = [
+        "Microsoft.Network/virtualNetworks/subnets/action",
+        "Microsoft.Network/virtualNetworks/subnets/join/action",
+        "Microsoft.Network/virtualNetworks/subnets/prepareNetworkPolicies/action"
+      ]
+    }
+  }
+}

Python/azureml-pipeline-databricks/environment_setup/terraform/vnet/outputs.tf

@@ -0,0 +1,11 @@
+output "vnet_name" {
+  value = azurerm_virtual_network.main.name
+}
+
+output "databricks_private_subnet_name" {
+  value = azurerm_subnet.databricks-private.name
+}
+
+output "databricks_public_subnet_name" {
+  value = azurerm_subnet.databricks-public.name
+}

Python/azureml-pipeline-databricks/environment_setup/terraform/vnet/variables.tf

@@ -0,0 +1,15 @@
+variable "appname" {
+  type = string
+}
+
+variable "environment" {
+  type = string
+}
+
+variable "resource_group_name" {
+  type    = string
+}
+
+variable "location" {
+  type = string
+}