From b6a7afe2819db645434f3e06124e0c327974c7b1 Mon Sep 17 00:00:00 2001 From: Damian Szymanski Date: Fri, 6 Mar 2020 09:00:43 +0100 Subject: [PATCH 1/7] Add us-east4 and us-east1 networks --- modules/gce_net_services/nat.tf | 68 +++++++++++++++++++++++++++ modules/gce_net_services/networks.tf | 28 +++++++++++ modules/gce_net_services/variables.tf | 8 ++++ 3 files changed, 104 insertions(+) diff --git a/modules/gce_net_services/nat.tf b/modules/gce_net_services/nat.tf index f3309600..8562851d 100644 --- a/modules/gce_net_services/nat.tf +++ b/modules/gce_net_services/nat.tf @@ -4,6 +4,20 @@ resource "google_compute_address" "services_nat" { project = "${var.project}" } +resource "google_compute_address" "services_nat_us_east1" { + count = "${var.nat_ip_count}" + name = "services-nat-ip-us-east1-${count.index}" + project = "${var.project}" + region = "us-east1" +} + +resource "google_compute_address" "services_nat_us_east4" { + count = "${var.nat_ip_count}" + name = "services-nat-ip-us-east4-${count.index}" + project = "${var.project}" + region = "us-east4" +} + resource "google_compute_router" "services_nat" { name = "router" project = "${var.project}" @@ -14,6 +28,28 @@ resource "google_compute_router" "services_nat" { } } +resource "google_compute_router" "services_nat_us_east1" { + name = "router" + project = "${var.project}" + network = "${google_compute_network.main.self_link}" + region = "us-east1" + + bgp { + asn = 64514 + } +} + +resource "google_compute_router" "services_nat_us_east4" { + name = "router" + project = "${var.project}" + network = "${google_compute_network.main.self_link}" + region = "us-east4" + + bgp { + asn = 64514 + } +} + resource "google_compute_router_nat" "services_nat" { name = "services-nat" project = "${var.project}" @@ -28,3 +64,35 @@ resource "google_compute_router_nat" "services_nat" { source_ip_ranges_to_nat = ["ALL_IP_RANGES"] } } + +resource "google_compute_router_nat" "services_nat_us_east1" { + name = "services-nat-us-east1" + project = "${var.project}" + region = "us-east1" + + nat_ip_allocate_option = "MANUAL_ONLY" + nat_ips = ["${google_compute_address.services_nat_us_east1.*.self_link}"] + router = "${google_compute_router.services_nat_us_east1.name}" + source_subnetwork_ip_ranges_to_nat = "LIST_OF_SUBNETWORKS" + + subnetwork { + name = "${google_compute_subnetwork.services-us-east1.self_link}" + source_ip_ranges_to_nat = ["ALL_IP_RANGES"] + } +} + +resource "google_compute_router_nat" "services_nat_us_east4" { + name = "services-nat-us-east4" + project = "${var.project}" + region = "us-east4" + + nat_ip_allocate_option = "MANUAL_ONLY" + nat_ips = ["${google_compute_address.services_nat_us_east4.*.self_link}"] + router = "${google_compute_router.services_nat_us_east4.name}" + source_subnetwork_ip_ranges_to_nat = "LIST_OF_SUBNETWORKS" + + subnetwork { + name = "${google_compute_subnetwork.services-us-east4.self_link}" + source_ip_ranges_to_nat = ["ALL_IP_RANGES"] + } +} diff --git a/modules/gce_net_services/networks.tf b/modules/gce_net_services/networks.tf index a5d49de0..53f01703 100644 --- a/modules/gce_net_services/networks.tf +++ b/modules/gce_net_services/networks.tf @@ -13,6 +13,26 @@ resource "google_compute_subnetwork" "services" { private_ip_google_access = "true" } +resource "google_compute_subnetwork" "services-us-east1" { + name = "services-us-east1" + region = "us-east1" + project = "${var.project}" + ip_cidr_range = "${var.services_subnet_cidr_range_us_east1}" + network = "${google_compute_network.main.self_link}" + enable_flow_logs = "true" + private_ip_google_access = "true" +} + +resource "google_compute_subnetwork" "services-us-east4" { + name = "services-us-east4" + region = "us-east4" + project = "${var.project}" + ip_cidr_range = "${var.services_subnet_cidr_range_us_east4}" + network = "${google_compute_network.main.self_link}" + enable_flow_logs = "true" + private_ip_google_access = "true" +} + output "main_network_name" { value = "${google_compute_network.main.name}" } @@ -20,3 +40,11 @@ output "main_network_name" { output "services_network_name" { value = "${google_compute_subnetwork.services.name}" } + +output "services_network_name_us_east1" { + value = "${google_compute_subnetwork.services-us-east1.name}" +} + +output "services_network_name_us_east4" { + value = "${google_compute_subnetwork.services-us-east4.name}" +} diff --git a/modules/gce_net_services/variables.tf b/modules/gce_net_services/variables.tf index 1a93a748..b4766904 100644 --- a/modules/gce_net_services/variables.tf +++ b/modules/gce_net_services/variables.tf @@ -7,3 +7,11 @@ variable "nat_ip_count" { variable "services_subnet_cidr_range" { default = "10.80.0.0/16" } + +variable "services_subnet_cidr_range_us_east4" { + default = "10.81.0.0/16" +} + +variable "services_subnet_cidr_range_us_east1" { + default = "10.82.0.0/16" +} From 76106e120fcb9cad760aab115476f64978dd2f54 Mon Sep 17 00:00:00 2001 From: Damian Szymanski Date: Fri, 6 Mar 2020 09:01:14 +0100 Subject: [PATCH 2/7] Initial node count from variable --- modules/gce_kubernetes/cluster.tf | 4 ++-- modules/gce_kubernetes/variables.tf | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/modules/gce_kubernetes/cluster.tf b/modules/gce_kubernetes/cluster.tf index 3dd7ca96..562212dd 100644 --- a/modules/gce_kubernetes/cluster.tf +++ b/modules/gce_kubernetes/cluster.tf @@ -7,7 +7,7 @@ resource "google_container_cluster" "gke_cluster" { min_master_version = "${var.min_master_version}" node_locations = "${var.node_locations}" - initial_node_count = 1 + initial_node_count = "1" remove_default_node_pool = true ip_allocation_policy = {} monitoring_service = "monitoring.googleapis.com/kubernetes" @@ -41,7 +41,7 @@ resource "google_container_node_pool" "node_pool" { location = "${var.region}" cluster = "${google_container_cluster.gke_cluster.name}" - initial_node_count = 1 + initial_node_count = "${var.initial_node_count}" node_config { machine_type = "${var.machine_type}" diff --git a/modules/gce_kubernetes/variables.tf b/modules/gce_kubernetes/variables.tf index d383f99c..942bd2a5 100644 --- a/modules/gce_kubernetes/variables.tf +++ b/modules/gce_kubernetes/variables.tf @@ -10,6 +10,10 @@ variable "min_master_version" { default = "1.13" } +variable "initial_node_count" { + default = "1" +} + variable "node_locations" { default = [] } From 0de0634eb9da5687c9f2340823ee101685a16d97 Mon Sep 17 00:00:00 2001 From: Damian Szymanski Date: Fri, 6 Mar 2020 09:01:37 +0100 Subject: [PATCH 3/7] Staging add us-east4 --- travis-ci-staging-services-1/modules.tf | 47 +++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/travis-ci-staging-services-1/modules.tf b/travis-ci-staging-services-1/modules.tf index 2c425b34..540086fe 100644 --- a/travis-ci-staging-services-1/modules.tf +++ b/travis-ci-staging-services-1/modules.tf @@ -23,6 +23,7 @@ module "kubernetes_cluster" { node_locations = ["us-central1-b", "us-central1-c"] node_pool_tags = ["services"] + min_node_count = 1 max_node_count = 10 machine_type = "c2-standard-4" enable_private_nodes = true @@ -52,3 +53,49 @@ output "client_key" { output "context" { value = "${module.kubernetes_cluster.context}" } + +module "kubernetes_cluster_us_east4" { + source = "../modules/gce_kubernetes" + + cluster_name = "travis-ci-services-1" + default_namespace = "default" + network = "${module.networking.main_network_name}" + pool_name = "pool1" + project = "${module.project.project_id}" + region = "us-east4" + subnetwork = "${module.networking.services_network_name_us_east4}" + + node_locations = ["us-east4-b", "us-east4-a", "us-east4-c"] + node_pool_tags = ["services"] + initial_node_count = 3 + min_node_count = 1 + max_node_count = 3 + machine_type = "n1-standard-4" + enable_private_nodes = true + private_master_ipv4_cidr_block = "172.16.0.16/28" + min_master_version = "1.15" +} + +// Use these outputs to be able to easily set up a context in kubectl on the local machine. +output "cluster_host_us_east4" { + value = "${module.kubernetes_cluster_us_east4.host}" +} + +output "cluster_ca_certificate_us_east4" { + value = "${module.kubernetes_cluster_us_east4.cluster_ca_certificate}" + sensitive = true +} + +output "client_certificate_us_east4" { + value = "${module.kubernetes_cluster_us_east4.client_certificate}" + sensitive = true +} + +output "client_key_us_east4" { + value = "${module.kubernetes_cluster_us_east4.client_key}" + sensitive = true +} + +output "context_us_east4" { + value = "${module.kubernetes_cluster_us_east4.context}" +} From da119183508547b3810d59421e14580a829cb3d7 Mon Sep 17 00:00:00 2001 From: Damian Szymanski Date: Fri, 6 Mar 2020 09:02:01 +0100 Subject: [PATCH 4/7] Production add us-east4 and us-east1 as default --- travis-ci-prod-services-1/modules.tf | 55 ++++++++++++++++++++++++-- travis-ci-prod-services-1/variables.tf | 2 +- 2 files changed, 52 insertions(+), 5 deletions(-) diff --git a/travis-ci-prod-services-1/modules.tf b/travis-ci-prod-services-1/modules.tf index 2c425b34..762b3dd3 100644 --- a/travis-ci-prod-services-1/modules.tf +++ b/travis-ci-prod-services-1/modules.tf @@ -19,14 +19,16 @@ module "kubernetes_cluster" { pool_name = "pool1" project = "${module.project.project_id}" region = "${var.region}" - subnetwork = "${module.networking.services_network_name}" + subnetwork = "${module.networking.services_network_name_us_east1}" - node_locations = ["us-central1-b", "us-central1-c"] + node_locations = ["us-east1-b", "us-east1-c", "us-east1-d"] node_pool_tags = ["services"] - max_node_count = 10 - machine_type = "c2-standard-4" + min_node_count = 2 + max_node_count = 50 + machine_type = "c2-standard-8" enable_private_nodes = true private_master_ipv4_cidr_block = "172.16.0.0/28" + min_master_version = "1.15" } // Use these outputs to be able to easily set up a context in kubectl on the local machine. @@ -52,3 +54,48 @@ output "client_key" { output "context" { value = "${module.kubernetes_cluster.context}" } + +module "kubernetes_cluster_us_east4" { + source = "../modules/gce_kubernetes" + + cluster_name = "travis-ci-services-1" + default_namespace = "default" + network = "${module.networking.main_network_name}" + pool_name = "pool1" + project = "${module.project.project_id}" + region = "us-east4" + subnetwork = "${module.networking.services_network_name_us_east4}" + + node_locations = ["us-east4-b", "us-east4-a", "us-east4-c"] + node_pool_tags = ["services"] + min_node_count = 4 + max_node_count = 50 + machine_type = "n1-standard-4" + enable_private_nodes = true + private_master_ipv4_cidr_block = "172.16.0.16/28" + min_master_version = "1.15" +} + +// Use these outputs to be able to easily set up a context in kubectl on the local machine. +output "cluster_host_us_east4" { + value = "${module.kubernetes_cluster_us_east4.host}" +} + +output "cluster_ca_certificate_us_east4" { + value = "${module.kubernetes_cluster_us_east4.cluster_ca_certificate}" + sensitive = true +} + +output "client_certificate_us_east4" { + value = "${module.kubernetes_cluster_us_east4.client_certificate}" + sensitive = true +} + +output "client_key_us_east4" { + value = "${module.kubernetes_cluster_us_east4.client_key}" + sensitive = true +} + +output "context_us_east4" { + value = "${module.kubernetes_cluster_us_east4.context}" +} diff --git a/travis-ci-prod-services-1/variables.tf b/travis-ci-prod-services-1/variables.tf index 98a012b7..ec1a78d1 100644 --- a/travis-ci-prod-services-1/variables.tf +++ b/travis-ci-prod-services-1/variables.tf @@ -7,5 +7,5 @@ variable "project_id" { } variable "region" { - default = "us-central1" + default = "us-east1" } From d52dfe5570e69bfffd83f73033a727db8f3045ab Mon Sep 17 00:00:00 2001 From: Damian Szymanski Date: Fri, 6 Mar 2020 09:10:21 +0100 Subject: [PATCH 5/7] Project services from variable --- modules/gce_project/project.tf | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/modules/gce_project/project.tf b/modules/gce_project/project.tf index fbc201de..40c3eaa6 100644 --- a/modules/gce_project/project.tf +++ b/modules/gce_project/project.tf @@ -1,13 +1,7 @@ variable "project_id" {} -data "google_project" "project" { - project_id = "${var.project_id}" -} - -resource "google_project_services" "project" { - project = "${data.google_project.project.project_id}" - - services = [ +variable "default_services" { + default = [ "bigquery-json.googleapis.com", "bigquerystorage.googleapis.com", "iam.googleapis.com", @@ -22,6 +16,16 @@ resource "google_project_services" "project" { ] } +data "google_project" "project" { + project_id = "${var.project_id}" +} + +resource "google_project_services" "project" { + project = "${data.google_project.project.project_id}" + + services = "${var.default_services}" +} + output "project_id" { value = "${data.google_project.project.project_id}" } From ea59816df2e2928bfa5d13a7317d66f1f7b1ca50 Mon Sep 17 00:00:00 2001 From: Damian Szymanski Date: Fri, 6 Mar 2020 09:10:41 +0100 Subject: [PATCH 6/7] Add services from variable --- travis-ci-prod-services-1/modules.tf | 3 ++- travis-ci-prod-services-1/variables.tf | 30 ++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/travis-ci-prod-services-1/modules.tf b/travis-ci-prod-services-1/modules.tf index 762b3dd3..34e8a128 100644 --- a/travis-ci-prod-services-1/modules.tf +++ b/travis-ci-prod-services-1/modules.tf @@ -1,5 +1,6 @@ module "project" { - source = "../modules/gce_project" + source = "../modules/gce_project" + default_services = "${var.default_services}" project_id = "${var.project_id}" } diff --git a/travis-ci-prod-services-1/variables.tf b/travis-ci-prod-services-1/variables.tf index ec1a78d1..a6d5d4a4 100644 --- a/travis-ci-prod-services-1/variables.tf +++ b/travis-ci-prod-services-1/variables.tf @@ -9,3 +9,33 @@ variable "project_id" { variable "region" { default = "us-east1" } + +variable "default_services" { + default = [ + "bigquery-json.googleapis.com", + "bigquerystorage.googleapis.com", + "iam.googleapis.com", + "iamcredentials.googleapis.com", + "oslogin.googleapis.com", + "pubsub.googleapis.com", + "storage-api.googleapis.com", + "compute.googleapis.com", + "container.googleapis.com", + "containerregistry.googleapis.com", + "storage-component.googleapis.com", + + "cloudtrace.googleapis.com", + "monitoring.googleapis.com", + "servicenetworking.googleapis.com", + "bigquery.googleapis.com", + "stackdriver.googleapis.com", + "logging.googleapis.com", + "cloudprofiler.googleapis.com", + "runtimeconfig.googleapis.com", + "deploymentmanager.googleapis.com", + "redis.googleapis.com", + "resourceviews.googleapis.com", + "firestore.googleapis.com", + "firebaserules.googleapis.com" + ] +} From 192893ce59c551d05d168b86a9feb8407096dd21 Mon Sep 17 00:00:00 2001 From: Damian Szymanski Date: Fri, 6 Mar 2020 09:10:54 +0100 Subject: [PATCH 7/7] Less nodes --- travis-ci-prod-services-1/modules.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/travis-ci-prod-services-1/modules.tf b/travis-ci-prod-services-1/modules.tf index 34e8a128..723b2deb 100644 --- a/travis-ci-prod-services-1/modules.tf +++ b/travis-ci-prod-services-1/modules.tf @@ -69,7 +69,7 @@ module "kubernetes_cluster_us_east4" { node_locations = ["us-east4-b", "us-east4-a", "us-east4-c"] node_pool_tags = ["services"] - min_node_count = 4 + min_node_count = 1 max_node_count = 50 machine_type = "n1-standard-4" enable_private_nodes = true