diff --git a/examples/terraform/azure/README.md b/examples/terraform/azure/README.md
index 97db3bd26..c476a2aca 100644
--- a/examples/terraform/azure/README.md
+++ b/examples/terraform/azure/README.md
@@ -47,6 +47,9 @@ No modules.
| [azurerm_virtual_network.vpc](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/virtual_network) | resource |
| [time_sleep.wait_30_seconds](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource |
| [azurerm_public_ip.control_plane](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/public_ip) | data source |
+| [azurerm_network_interface.lb_vm_network](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/network_interface) | resource |
+| [azurerm_virtual_machine.lb_vm](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/virtual_machine) | resource |
+| [null_resource.lb_config](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource |
## Inputs
@@ -65,8 +68,10 @@ No modules.
| [initial\_machinedeployment\_operating\_system\_profile](#input\_initial\_machinedeployment\_operating\_system\_profile) | Name of operating system profile for MachineDeployments, only applicable if operating-system-manager addon is enabled.
If not specified, the default value will be added by machine-controller addon. | `string` | `""` | no |
| [initial\_machinedeployment\_replicas](#input\_initial\_machinedeployment\_replicas) | Number of replicas per MachineDeployment | `number` | `2` | no |
| [ip\_sku](#input\_ip\_sku) | SKU to use for IP addresses | `string` | `"Basic"` | no |
+| [lb_vm_size](#input\_lb_vm_size) | VM Size for Load Balancer machines | `string` | `"Standard_B2s"` | no |
| [location](#input\_location) | Azure datacenter to use | `string` | `"westeurope"` | no |
| [os](#input\_os) | Operating System to use for finding image reference and in MachineDeployment | `string` | `"ubuntu"` | no |
+| [private_networking_only](#input\_private_networking_only) | Enable this to use private networking only | `bool` | `"false"` | no
| [rhsm\_offline\_token](#input\_rhsm\_offline\_token) | RHSM offline token | `string` | `""` | no |
| [rhsm\_password](#input\_rhsm\_password) | RHSM password | `string` | `""` | no |
| [rhsm\_username](#input\_rhsm\_username) | RHSM username | `string` | `""` | no |
diff --git a/examples/terraform/azure/etc_gobetween.tpl b/examples/terraform/azure/etc_gobetween.tpl
new file mode 100644
index 000000000..fdaae71a7
--- /dev/null
+++ b/examples/terraform/azure/etc_gobetween.tpl
@@ -0,0 +1,26 @@
+[api]
+enabled = false
+
+[servers.default]
+protocol = "tcp"
+bind = "0.0.0.0:6443"
+balance = "roundrobin"
+max_connections = 10000
+client_idle_timeout = "10m"
+backend_idle_timeout = "10m"
+backend_connection_timeout = "2s"
+
+[servers.default.discovery]
+kind = "static"
+static_list = [
+ %{ for target in lb_targets ~}
+ "${target}:6443",
+ %{ endfor ~}
+]
+
+[servers.default.healthcheck]
+kind = "ping"
+interval = "10s"
+timeout = "2s"
+fails = 2
+passes = 1
diff --git a/examples/terraform/azure/gobetween.sh b/examples/terraform/azure/gobetween.sh
new file mode 100755
index 000000000..4a8da2204
--- /dev/null
+++ b/examples/terraform/azure/gobetween.sh
@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+
+# Copyright 2019 The KubeOne Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script is mostly used in CI
+# It installs dependencies and starts the tests
+
+set -euf -o pipefail
+
+GOBETWEEN_VERSION=0.7.0
+
+noop() { : "didn't detected package manager, noop"; }
+
+PKG_MANAGER="noop"
+
+[ "$(command -v yum)" ] && PKG_MANAGER=yum
+[ "$(command -v apt-get)" ] && PKG_MANAGER=apt-get
+
+sudo ${PKG_MANAGER} install tar -y
+
+mkdir -p /tmp/gobetween
+cd /tmp/gobetween
+curl -L -o gobetween_${GOBETWEEN_VERSION}_linux_amd64.tar.gz \
+ https://github.com/yyyar/gobetween/releases/download/${GOBETWEEN_VERSION}/gobetween_${GOBETWEEN_VERSION}_linux_amd64.tar.gz
+tar xvf gobetween_${GOBETWEEN_VERSION}_linux_amd64.tar.gz
+sudo mkdir -p /opt/bin
+sudo mv gobetween /opt/bin/gobetween
+sudo chown root:root /opt/bin/gobetween
+
+cat < 0 ? var.control_plane_vm_count : 0
worker_os = var.worker_os == "" ? var.image_references[var.os].worker_os : var.worker_os
ssh_username = var.ssh_username == "" ? var.image_references[var.os].ssh_username : var.ssh_username
- cluster_autoscaler_min_replicas = var.cluster_autoscaler_min_replicas > 0 ? var.cluster_autoscaler_min_replicas : var.initial_machinedeployment_replicas
- cluster_autoscaler_max_replicas = var.cluster_autoscaler_max_replicas > 0 ? var.cluster_autoscaler_max_replicas : var.initial_machinedeployment_replicas
+ cluster_autoscaler_min_replicas = var.cluster_autoscaler_min_replicas > 0 ? var.cluster_autoscaler_min_replicas : var.initial_machinedeployment_replicas
+ cluster_autoscaler_max_replicas = var.cluster_autoscaler_max_replicas > 0 ? var.cluster_autoscaler_max_replicas : var.initial_machinedeployment_replicas
+
+ lb_vm_count = var.private_networking_only ? 2 : 0
}
provider "time" {
@@ -139,7 +144,7 @@ resource "azurerm_network_security_group" "sg" {
}
resource "azurerm_public_ip" "lbip" {
- count = local.loadbalancer_count
+ count = var.private_networking_only ? 0 : local.loadbalancer_count
name = "${var.cluster_name}-lbip"
location = var.location
@@ -154,7 +159,7 @@ resource "azurerm_public_ip" "lbip" {
}
resource "azurerm_public_ip" "control_plane" {
- count = var.control_plane_vm_count
+ count = var.private_networking_only ? 0 : var.control_plane_vm_count
name = "${var.cluster_name}-cp-${count.index}"
location = var.location
@@ -175,9 +180,21 @@ resource "azurerm_lb" "lb" {
name = "kubernetes"
location = var.location
- frontend_ip_configuration {
- name = "KubeApi"
- public_ip_address_id = azurerm_public_ip.lbip.0.id
+ dynamic "frontend_ip_configuration" {
+ for_each = var.private_networking_only ? [1] : []
+ content {
+ name = "KubeApi"
+ subnet_id = azurerm_subnet.subnet.id
+ private_ip_address_allocation = "Dynamic"
+ }
+ }
+
+ dynamic "frontend_ip_configuration" {
+ for_each = var.private_networking_only ? [] : [1]
+ content {
+ name = "KubeApi"
+ public_ip_address_id = azurerm_public_ip.lbip.0.id
+ }
}
tags = {
@@ -227,11 +244,23 @@ resource "azurerm_network_interface" "control_plane" {
location = var.location
resource_group_name = azurerm_resource_group.rg.name
- ip_configuration {
- name = "${var.cluster_name}-cp-${count.index}"
- subnet_id = azurerm_subnet.subnet.id
- private_ip_address_allocation = "Dynamic"
- public_ip_address_id = element(azurerm_public_ip.control_plane.*.id, count.index)
+ dynamic "ip_configuration" {
+ for_each = var.private_networking_only ? [] : [1]
+ content {
+ name = "${var.cluster_name}-cp-${count.index}"
+ subnet_id = azurerm_subnet.subnet.id
+ private_ip_address_allocation = "Dynamic"
+ public_ip_address_id = element(azurerm_public_ip.control_plane.*.id, count.index)
+ }
+ }
+
+ dynamic "ip_configuration" {
+ for_each = var.private_networking_only ? [1] : []
+ content {
+ name = "${var.cluster_name}-cp-${count.index}"
+ subnet_id = azurerm_subnet.subnet.id
+ private_ip_address_allocation = "Dynamic"
+ }
}
}
@@ -314,7 +343,116 @@ data "azurerm_public_ip" "control_plane" {
depends_on = [
time_sleep.wait_30_seconds
]
- count = var.control_plane_vm_count
+ count = var.private_networking_only ? 0 : var.control_plane_vm_count
name = "${var.cluster_name}-cp-${count.index}"
resource_group_name = azurerm_resource_group.rg.name
}
+
+resource "azurerm_network_interface" "lb_vm_network" {
+ depends_on = [
+ azurerm_lb.lb,
+ ]
+ count = var.private_networking_only ? local.lb_vm_count : 0
+
+ name = "${var.cluster_name}-lb-${count.index}"
+ location = var.location
+ resource_group_name = azurerm_resource_group.rg.name
+ ip_configuration {
+ name = "${var.cluster_name}-lb-${count.index}"
+ subnet_id = azurerm_subnet.subnet.id
+ private_ip_address_allocation = "Dynamic"
+ }
+
+}
+resource "azurerm_virtual_machine" "lb_vm" {
+ depends_on = [
+ azurerm_network_interface.lb_vm_network,
+ azurerm_virtual_machine.control_plane
+ ]
+ count = var.private_networking_only ? local.lb_vm_count : 0
+
+ name = "${var.cluster_name}-lb-${count.index}"
+ location = var.location
+ resource_group_name = azurerm_resource_group.rg.name
+ availability_set_id = azurerm_availability_set.avset.id
+ vm_size = var.lb_vm_size
+ network_interface_ids = [element(azurerm_network_interface.lb_vm_network.*.id, count.index)]
+ delete_os_disk_on_termination = true
+ delete_data_disks_on_termination = true
+
+ dynamic "plan" {
+ for_each = var.image_references[var.os].plan
+
+ content {
+ name = plan.value["name"]
+ publisher = plan.value["publisher"]
+ product = plan.value["product"]
+ }
+ }
+ storage_image_reference {
+ publisher = var.image_references[var.os].image.publisher
+ offer = var.image_references[var.os].image.offer
+ sku = var.image_references[var.os].image.sku
+ version = var.image_references[var.os].image.version
+ }
+
+ storage_os_disk {
+ name = "${var.cluster_name}-lb-${count.index}"
+ caching = "ReadWrite"
+ create_option = "FromImage"
+ managed_disk_type = "Standard_LRS"
+ }
+
+ os_profile {
+ computer_name = "${var.cluster_name}-lb-${count.index}"
+ admin_username = local.ssh_username
+ custom_data = file("gobetween.sh")
+ }
+
+ os_profile_linux_config {
+ disable_password_authentication = true
+
+ ssh_keys {
+ key_data = file(var.ssh_public_key_file)
+ path = "/home/${local.ssh_username}/.ssh/authorized_keys"
+ }
+ }
+
+ tags = {
+ environment = "kubeone"
+ cluster = var.cluster_name
+ }
+
+}
+
+
+resource "null_resource" "lb_config" {
+ depends_on = [
+ azurerm_virtual_machine.lb_vm
+ ]
+ triggers = {
+ lb_instance_ids = join(",", azurerm_virtual_machine.lb_vm.*.id)
+ cluster_instance_ids = join(",", azurerm_virtual_machine.control_plane.*.id)
+ config = local.rendered_lb_config
+ }
+
+ count = var.private_networking_only ? local.lb_vm_count : 0
+ connection {
+ type = "ssh"
+ user = var.ssh_username
+ private_key = file(var.ssh_private_key_file)
+ host = element(azurerm_network_interface.lb_vm_network.*.private_ip_address, count.index)
+ }
+
+ provisioner "file" {
+ content = local.rendered_lb_config
+ destination = "/tmp/gobetween.toml"
+ }
+
+ provisioner "remote-exec" {
+ inline = [
+ "sudo mv /tmp/gobetween.toml /etc/gobetween.toml",
+ "sudo systemctl restart gobetween",
+ ]
+ }
+}
diff --git a/examples/terraform/azure/output.tf b/examples/terraform/azure/output.tf
index 7920213e2..e801e9ef9 100644
--- a/examples/terraform/azure/output.tf
+++ b/examples/terraform/azure/output.tf
@@ -95,7 +95,7 @@ output "kubeone_workers" {
availabilitySet = azurerm_availability_set.avset_workers.name
# assignAvailabilitySet = true/false
securityGroupName = azurerm_network_security_group.sg.name
- assignPublicIP = true
+ assignPublicIP = var.private_networking_only ? false : true
imageReference = var.os != "rhel" ? var.image_references[var.os].image : null
imagePlan = length(var.image_references[var.os].plan) > 0 && var.os != "rhel" ? var.image_references[var.os].plan[0] : null
# Zones (optional)
diff --git a/examples/terraform/azure/variables.tf b/examples/terraform/azure/variables.tf
index f63e73d3f..1b8d7654d 100644
--- a/examples/terraform/azure/variables.tf
+++ b/examples/terraform/azure/variables.tf
@@ -24,6 +24,12 @@ variable "cluster_name" {
}
}
+variable "private_networking_only" {
+ description = "Enable this to use private networking only"
+ default = false
+ type = bool
+}
+
variable "apiserver_alternative_names" {
description = "subject alternative names for the API Server signing cert."
default = []
@@ -281,3 +287,9 @@ variable "disable_auto_update" {
type = bool
default = false
}
+
+variable "lb_vm_size" {
+ description = "VM Size for Load Balancer machines"
+ type = string
+ default = "Standard_B2s"
+}