diff --git a/examples/terraform/azure/README.md b/examples/terraform/azure/README.md index 97db3bd26..c476a2aca 100644 --- a/examples/terraform/azure/README.md +++ b/examples/terraform/azure/README.md @@ -47,6 +47,9 @@ No modules. | [azurerm_virtual_network.vpc](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/virtual_network) | resource | | [time_sleep.wait_30_seconds](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource | | [azurerm_public_ip.control_plane](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/data-sources/public_ip) | data source | +| [azurerm_network_interface.lb_vm_network](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/network_interface) | resource | +| [azurerm_virtual_machine.lb_vm](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/virtual_machine) | resource | +| [null_resource.lb_config](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | ## Inputs @@ -65,8 +68,10 @@ No modules. | [initial\_machinedeployment\_operating\_system\_profile](#input\_initial\_machinedeployment\_operating\_system\_profile) | Name of operating system profile for MachineDeployments, only applicable if operating-system-manager addon is enabled.
If not specified, the default value will be added by machine-controller addon. | `string` | `""` | no | | [initial\_machinedeployment\_replicas](#input\_initial\_machinedeployment\_replicas) | Number of replicas per MachineDeployment | `number` | `2` | no | | [ip\_sku](#input\_ip\_sku) | SKU to use for IP addresses | `string` | `"Basic"` | no | +| [lb_vm_size](#input\_lb_vm_size) | VM Size for Load Balancer machines | `string` | `"Standard_B2s"` | no | | [location](#input\_location) | Azure datacenter to use | `string` | `"westeurope"` | no | | [os](#input\_os) | Operating System to use for finding image reference and in MachineDeployment | `string` | `"ubuntu"` | no | +| [private_networking_only](#input\_private_networking_only) | Enable this to use private networking only | `bool` | `"false"` | no | [rhsm\_offline\_token](#input\_rhsm\_offline\_token) | RHSM offline token | `string` | `""` | no | | [rhsm\_password](#input\_rhsm\_password) | RHSM password | `string` | `""` | no | | [rhsm\_username](#input\_rhsm\_username) | RHSM username | `string` | `""` | no | diff --git a/examples/terraform/azure/etc_gobetween.tpl b/examples/terraform/azure/etc_gobetween.tpl new file mode 100644 index 000000000..fdaae71a7 --- /dev/null +++ b/examples/terraform/azure/etc_gobetween.tpl @@ -0,0 +1,26 @@ +[api] +enabled = false + +[servers.default] +protocol = "tcp" +bind = "0.0.0.0:6443" +balance = "roundrobin" +max_connections = 10000 +client_idle_timeout = "10m" +backend_idle_timeout = "10m" +backend_connection_timeout = "2s" + +[servers.default.discovery] +kind = "static" +static_list = [ + %{ for target in lb_targets ~} + "${target}:6443", + %{ endfor ~} +] + +[servers.default.healthcheck] +kind = "ping" +interval = "10s" +timeout = "2s" +fails = 2 +passes = 1 diff --git a/examples/terraform/azure/gobetween.sh b/examples/terraform/azure/gobetween.sh new file mode 100755 index 000000000..4a8da2204 --- /dev/null +++ b/examples/terraform/azure/gobetween.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +# Copyright 2019 The KubeOne Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script is mostly used in CI +# It installs dependencies and starts the tests + +set -euf -o pipefail + +GOBETWEEN_VERSION=0.7.0 + +noop() { : "didn't detected package manager, noop"; } + +PKG_MANAGER="noop" + +[ "$(command -v yum)" ] && PKG_MANAGER=yum +[ "$(command -v apt-get)" ] && PKG_MANAGER=apt-get + +sudo ${PKG_MANAGER} install tar -y + +mkdir -p /tmp/gobetween +cd /tmp/gobetween +curl -L -o gobetween_${GOBETWEEN_VERSION}_linux_amd64.tar.gz \ + https://github.com/yyyar/gobetween/releases/download/${GOBETWEEN_VERSION}/gobetween_${GOBETWEEN_VERSION}_linux_amd64.tar.gz +tar xvf gobetween_${GOBETWEEN_VERSION}_linux_amd64.tar.gz +sudo mkdir -p /opt/bin +sudo mv gobetween /opt/bin/gobetween +sudo chown root:root /opt/bin/gobetween + +cat < 0 ? var.control_plane_vm_count : 0 worker_os = var.worker_os == "" ? var.image_references[var.os].worker_os : var.worker_os ssh_username = var.ssh_username == "" ? var.image_references[var.os].ssh_username : var.ssh_username - cluster_autoscaler_min_replicas = var.cluster_autoscaler_min_replicas > 0 ? var.cluster_autoscaler_min_replicas : var.initial_machinedeployment_replicas - cluster_autoscaler_max_replicas = var.cluster_autoscaler_max_replicas > 0 ? var.cluster_autoscaler_max_replicas : var.initial_machinedeployment_replicas + cluster_autoscaler_min_replicas = var.cluster_autoscaler_min_replicas > 0 ? var.cluster_autoscaler_min_replicas : var.initial_machinedeployment_replicas + cluster_autoscaler_max_replicas = var.cluster_autoscaler_max_replicas > 0 ? var.cluster_autoscaler_max_replicas : var.initial_machinedeployment_replicas + + lb_vm_count = var.private_networking_only ? 2 : 0 } provider "time" { @@ -139,7 +144,7 @@ resource "azurerm_network_security_group" "sg" { } resource "azurerm_public_ip" "lbip" { - count = local.loadbalancer_count + count = var.private_networking_only ? 0 : local.loadbalancer_count name = "${var.cluster_name}-lbip" location = var.location @@ -154,7 +159,7 @@ resource "azurerm_public_ip" "lbip" { } resource "azurerm_public_ip" "control_plane" { - count = var.control_plane_vm_count + count = var.private_networking_only ? 0 : var.control_plane_vm_count name = "${var.cluster_name}-cp-${count.index}" location = var.location @@ -175,9 +180,21 @@ resource "azurerm_lb" "lb" { name = "kubernetes" location = var.location - frontend_ip_configuration { - name = "KubeApi" - public_ip_address_id = azurerm_public_ip.lbip.0.id + dynamic "frontend_ip_configuration" { + for_each = var.private_networking_only ? [1] : [] + content { + name = "KubeApi" + subnet_id = azurerm_subnet.subnet.id + private_ip_address_allocation = "Dynamic" + } + } + + dynamic "frontend_ip_configuration" { + for_each = var.private_networking_only ? [] : [1] + content { + name = "KubeApi" + public_ip_address_id = azurerm_public_ip.lbip.0.id + } } tags = { @@ -227,11 +244,23 @@ resource "azurerm_network_interface" "control_plane" { location = var.location resource_group_name = azurerm_resource_group.rg.name - ip_configuration { - name = "${var.cluster_name}-cp-${count.index}" - subnet_id = azurerm_subnet.subnet.id - private_ip_address_allocation = "Dynamic" - public_ip_address_id = element(azurerm_public_ip.control_plane.*.id, count.index) + dynamic "ip_configuration" { + for_each = var.private_networking_only ? [] : [1] + content { + name = "${var.cluster_name}-cp-${count.index}" + subnet_id = azurerm_subnet.subnet.id + private_ip_address_allocation = "Dynamic" + public_ip_address_id = element(azurerm_public_ip.control_plane.*.id, count.index) + } + } + + dynamic "ip_configuration" { + for_each = var.private_networking_only ? [1] : [] + content { + name = "${var.cluster_name}-cp-${count.index}" + subnet_id = azurerm_subnet.subnet.id + private_ip_address_allocation = "Dynamic" + } } } @@ -314,7 +343,116 @@ data "azurerm_public_ip" "control_plane" { depends_on = [ time_sleep.wait_30_seconds ] - count = var.control_plane_vm_count + count = var.private_networking_only ? 0 : var.control_plane_vm_count name = "${var.cluster_name}-cp-${count.index}" resource_group_name = azurerm_resource_group.rg.name } + +resource "azurerm_network_interface" "lb_vm_network" { + depends_on = [ + azurerm_lb.lb, + ] + count = var.private_networking_only ? local.lb_vm_count : 0 + + name = "${var.cluster_name}-lb-${count.index}" + location = var.location + resource_group_name = azurerm_resource_group.rg.name + ip_configuration { + name = "${var.cluster_name}-lb-${count.index}" + subnet_id = azurerm_subnet.subnet.id + private_ip_address_allocation = "Dynamic" + } + +} +resource "azurerm_virtual_machine" "lb_vm" { + depends_on = [ + azurerm_network_interface.lb_vm_network, + azurerm_virtual_machine.control_plane + ] + count = var.private_networking_only ? local.lb_vm_count : 0 + + name = "${var.cluster_name}-lb-${count.index}" + location = var.location + resource_group_name = azurerm_resource_group.rg.name + availability_set_id = azurerm_availability_set.avset.id + vm_size = var.lb_vm_size + network_interface_ids = [element(azurerm_network_interface.lb_vm_network.*.id, count.index)] + delete_os_disk_on_termination = true + delete_data_disks_on_termination = true + + dynamic "plan" { + for_each = var.image_references[var.os].plan + + content { + name = plan.value["name"] + publisher = plan.value["publisher"] + product = plan.value["product"] + } + } + storage_image_reference { + publisher = var.image_references[var.os].image.publisher + offer = var.image_references[var.os].image.offer + sku = var.image_references[var.os].image.sku + version = var.image_references[var.os].image.version + } + + storage_os_disk { + name = "${var.cluster_name}-lb-${count.index}" + caching = "ReadWrite" + create_option = "FromImage" + managed_disk_type = "Standard_LRS" + } + + os_profile { + computer_name = "${var.cluster_name}-lb-${count.index}" + admin_username = local.ssh_username + custom_data = file("gobetween.sh") + } + + os_profile_linux_config { + disable_password_authentication = true + + ssh_keys { + key_data = file(var.ssh_public_key_file) + path = "/home/${local.ssh_username}/.ssh/authorized_keys" + } + } + + tags = { + environment = "kubeone" + cluster = var.cluster_name + } + +} + + +resource "null_resource" "lb_config" { + depends_on = [ + azurerm_virtual_machine.lb_vm + ] + triggers = { + lb_instance_ids = join(",", azurerm_virtual_machine.lb_vm.*.id) + cluster_instance_ids = join(",", azurerm_virtual_machine.control_plane.*.id) + config = local.rendered_lb_config + } + + count = var.private_networking_only ? local.lb_vm_count : 0 + connection { + type = "ssh" + user = var.ssh_username + private_key = file(var.ssh_private_key_file) + host = element(azurerm_network_interface.lb_vm_network.*.private_ip_address, count.index) + } + + provisioner "file" { + content = local.rendered_lb_config + destination = "/tmp/gobetween.toml" + } + + provisioner "remote-exec" { + inline = [ + "sudo mv /tmp/gobetween.toml /etc/gobetween.toml", + "sudo systemctl restart gobetween", + ] + } +} diff --git a/examples/terraform/azure/output.tf b/examples/terraform/azure/output.tf index 7920213e2..e801e9ef9 100644 --- a/examples/terraform/azure/output.tf +++ b/examples/terraform/azure/output.tf @@ -95,7 +95,7 @@ output "kubeone_workers" { availabilitySet = azurerm_availability_set.avset_workers.name # assignAvailabilitySet = true/false securityGroupName = azurerm_network_security_group.sg.name - assignPublicIP = true + assignPublicIP = var.private_networking_only ? false : true imageReference = var.os != "rhel" ? var.image_references[var.os].image : null imagePlan = length(var.image_references[var.os].plan) > 0 && var.os != "rhel" ? var.image_references[var.os].plan[0] : null # Zones (optional) diff --git a/examples/terraform/azure/variables.tf b/examples/terraform/azure/variables.tf index f63e73d3f..1b8d7654d 100644 --- a/examples/terraform/azure/variables.tf +++ b/examples/terraform/azure/variables.tf @@ -24,6 +24,12 @@ variable "cluster_name" { } } +variable "private_networking_only" { + description = "Enable this to use private networking only" + default = false + type = bool +} + variable "apiserver_alternative_names" { description = "subject alternative names for the API Server signing cert." default = [] @@ -281,3 +287,9 @@ variable "disable_auto_update" { type = bool default = false } + +variable "lb_vm_size" { + description = "VM Size for Load Balancer machines" + type = string + default = "Standard_B2s" +}