Skip to content

Commit 00c3b64

Browse files
gtoonstragtoonstra
authored
Merge pull request #78 from datafold/gerard-eng-3067-test-kubernetes-arc-for-builds
fix: Deploy new pools for ARC
2 parents bd7d629 + e259939 commit 00c3b64

File tree

4 files changed

+263
-130
lines changed

4 files changed

+263
-130
lines changed

‎main.tf‎

Lines changed: 179 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,14 @@ locals {
3232
vpc_private_subnets = module.networking.vpc_private_subnets
3333
azs = module.networking.azs
3434
vpc_cidr = module.networking.vpc_cidr
35+
36+
# Secondary subnet index for node groups 4/5/6 (e.g., ARC runners)
37+
# Falls back to private_subnet_index if not explicitly set
38+
secondary_subnet_index = coalesce(var.secondary_private_subnet_index, var.private_subnet_index)
39+
40+
# EKS cluster subnets - can be limited to avoid adding new AZs to existing cluster
41+
# Node groups can still use additional subnets beyond this list
42+
eks_cluster_subnets = var.eks_cluster_subnet_count != null ? slice(local.vpc_private_subnets, 0, var.eks_cluster_subnet_count) : local.vpc_private_subnets
3543
}
3644

3745
module "security" {
@@ -86,9 +94,9 @@ locals {
8694
subnet_ids = [local.vpc_private_subnets[var.private_subnet_index]]
8795
disk_size = var.default_node_disk_size
8896
tags = {
89-
"k8s.io/cluster-autoscaler/enabled" = "true"
90-
"k8s.io/cluster-autoscaler/${var.deployment_name}" = "owned"
91-
"k8s.io/cluster-autoscaler/node-template/label/role" = "${var.deployment_name}"
97+
"k8s.io/cluster-autoscaler/enabled" = "true"
98+
"k8s.io/cluster-autoscaler/${var.deployment_name}" = "owned"
99+
"k8s.io/cluster-autoscaler/node-template/label/role" = "${var.deployment_name}"
92100
}
93101
block_device_mappings = {
94102
xvda = {
@@ -107,15 +115,15 @@ locals {
107115
http_put_response_hop_limit = 2
108116
http_tokens = "required"
109117
}
110-
}, var.managed_node_grp1)
118+
}, var.managed_node_grp1)
111119
second_node_pool = merge(
112120
{
113121
subnet_ids = [local.vpc_private_subnets[var.private_subnet_index]]
114122
disk_size = var.default_node_disk_size
115123
tags = {
116-
"k8s.io/cluster-autoscaler/enabled" = "true"
117-
"k8s.io/cluster-autoscaler/${var.deployment_name}" = "owned"
118-
"k8s.io/cluster-autoscaler/node-template/label/role" = "${var.deployment_name}"
124+
"k8s.io/cluster-autoscaler/enabled" = "true"
125+
"k8s.io/cluster-autoscaler/${var.deployment_name}" = "owned"
126+
"k8s.io/cluster-autoscaler/node-template/label/role" = "${var.deployment_name}"
119127
}
120128
block_device_mappings = {
121129
xvda = {
@@ -134,15 +142,96 @@ locals {
134142
http_put_response_hop_limit = 2
135143
http_tokens = "required"
136144
}
137-
}, var.managed_node_grp2)
145+
}, var.managed_node_grp2)
138146
third_node_pool = merge(
139147
{
140148
subnet_ids = [local.vpc_private_subnets[var.private_subnet_index]]
141149
disk_size = var.default_node_disk_size
142150
tags = {
143-
"k8s.io/cluster-autoscaler/enabled" = "true"
144-
"k8s.io/cluster-autoscaler/${var.deployment_name}" = "owned"
145-
"k8s.io/cluster-autoscaler/node-template/label/role" = "${var.deployment_name}"
151+
"k8s.io/cluster-autoscaler/enabled" = "true"
152+
"k8s.io/cluster-autoscaler/${var.deployment_name}" = "owned"
153+
"k8s.io/cluster-autoscaler/node-template/label/role" = "${var.deployment_name}"
154+
}
155+
block_device_mappings = {
156+
xvda = {
157+
device_name = "/dev/xvda"
158+
ebs = {
159+
volume_size = var.default_node_disk_size
160+
volume_type = "gp3"
161+
iops = 3000
162+
throughput = 125
163+
encrypted = true
164+
delete_on_termination = true
165+
}
166+
}
167+
}
168+
metadata_options = {
169+
http_put_response_hop_limit = 2
170+
http_tokens = "required"
171+
}
172+
}, var.managed_node_grp3)
173+
fourth_node_pool = merge(
174+
{
175+
subnet_ids = [local.vpc_private_subnets[local.secondary_subnet_index]]
176+
disk_size = var.default_node_disk_size
177+
tags = {
178+
"k8s.io/cluster-autoscaler/enabled" = "true"
179+
"k8s.io/cluster-autoscaler/${var.deployment_name}" = "owned"
180+
"k8s.io/cluster-autoscaler/node-template/label/role" = "${var.deployment_name}"
181+
}
182+
block_device_mappings = {
183+
xvda = {
184+
device_name = "/dev/xvda"
185+
ebs = {
186+
volume_size = var.default_node_disk_size
187+
volume_type = "gp3"
188+
iops = 3000
189+
throughput = 125
190+
encrypted = true
191+
delete_on_termination = true
192+
}
193+
}
194+
}
195+
metadata_options = {
196+
http_put_response_hop_limit = 2
197+
http_tokens = "required"
198+
}
199+
}, var.managed_node_grp4)
200+
fifth_node_pool = merge(
201+
{
202+
subnet_ids = [local.vpc_private_subnets[local.secondary_subnet_index]]
203+
disk_size = var.default_node_disk_size
204+
tags = {
205+
"k8s.io/cluster-autoscaler/enabled" = "true"
206+
"k8s.io/cluster-autoscaler/${var.deployment_name}" = "owned"
207+
"k8s.io/cluster-autoscaler/node-template/label/role" = "${var.deployment_name}"
208+
}
209+
block_device_mappings = {
210+
xvda = {
211+
device_name = "/dev/xvda"
212+
ebs = {
213+
volume_size = var.default_node_disk_size
214+
volume_type = "gp3"
215+
iops = 3000
216+
throughput = 125
217+
encrypted = true
218+
delete_on_termination = true
219+
}
220+
}
221+
}
222+
metadata_options = {
223+
http_put_response_hop_limit = 2
224+
http_tokens = "required"
225+
}
226+
}, var.managed_node_grp5)
227+
sixth_node_pool = merge(
228+
{
229+
subnet_ids = [local.vpc_private_subnets[local.secondary_subnet_index]]
230+
disk_size = var.default_node_disk_size
231+
tags = {
232+
"k8s.io/cluster-autoscaler/enabled" = "true"
233+
"k8s.io/cluster-autoscaler/${var.deployment_name}" = "owned"
234+
"k8s.io/cluster-autoscaler/node-template/label/role" = "${var.deployment_name}"
146235
}
147236
block_device_mappings = {
148237
xvda = {
@@ -161,11 +250,14 @@ locals {
161250
http_put_response_hop_limit = 2
162251
http_tokens = "required"
163252
}
164-
}, var.managed_node_grp3)
253+
}, var.managed_node_grp6)
165254
managed_node_groups = merge(
166-
{"${var.deployment_name}-k8s": local.default_node_pool},
167-
var.managed_node_grp2 != null ? {"${var.deployment_name}-k8s-two" : local.second_node_pool} : {},
168-
var.managed_node_grp3 != null ? {"${var.deployment_name}-k8s-three" : local.third_node_pool} : {}
255+
{ "${var.deployment_name}-k8s" : local.default_node_pool },
256+
var.managed_node_grp2 != null ? { "${var.deployment_name}-k8s-two" : local.second_node_pool } : {},
257+
var.managed_node_grp3 != null ? { "${var.deployment_name}-k8s-three" : local.third_node_pool } : {},
258+
var.managed_node_grp4 != null ? { "${var.deployment_name}-k8s-four" : local.fourth_node_pool } : {},
259+
var.managed_node_grp5 != null ? { "${var.deployment_name}-k8s-five" : local.fifth_node_pool } : {},
260+
var.managed_node_grp6 != null ? { "${var.deployment_name}-k8s-six" : local.sixth_node_pool } : {}
169261
)
170262
}
171263

@@ -180,16 +272,17 @@ module "clickhouse_backup" {
180272
}
181273

182274
locals {
183-
clickhouse_backup_bucket_arn = module.clickhouse_backup.clickhouse_s3_bucket_arn
275+
clickhouse_backup_bucket_arn = module.clickhouse_backup.clickhouse_s3_bucket_arn
184276
}
185277

186278
module "eks" {
187279
source = "./modules/eks"
188280

189-
deployment_name = var.deployment_name
190-
k8s_vpc = local.vpc_id
281+
deployment_name = var.deployment_name
282+
k8s_vpc = local.vpc_id
191283
# https://aws.github.io/aws-eks-best-practices/networking/subnets/
192-
k8s_subnets = local.vpc_private_subnets
284+
# Use eks_cluster_subnets to limit subnets for control plane (avoids EKS AZ change errors)
285+
k8s_subnets = local.eks_cluster_subnets
193286
k8s_control_subnets = []
194287
k8s_module_version = var.k8s_module_version
195288
k8s_cluster_version = var.k8s_cluster_version
@@ -200,65 +293,65 @@ module "eks" {
200293
managed_node_grps = local.managed_node_groups
201294
k8s_api_access_roles = var.k8s_api_access_roles
202295

203-
tags = var.tags
204-
backend_app_port = var.backend_app_port
205-
rds_port = var.rds_port
206-
k8s_public_access_cidrs = var.k8s_public_access_cidrs
296+
tags = var.tags
297+
backend_app_port = var.backend_app_port
298+
rds_port = var.rds_port
299+
k8s_public_access_cidrs = var.k8s_public_access_cidrs
207300

208-
k8s_access_bedrock = var.k8s_access_bedrock
209-
clickhouse_backup_bucket_arn = local.clickhouse_backup_bucket_arn
210-
service_account_prefix = var.service_account_prefix
301+
k8s_access_bedrock = var.k8s_access_bedrock
302+
clickhouse_backup_bucket_arn = local.clickhouse_backup_bucket_arn
303+
service_account_prefix = var.service_account_prefix
211304
}
212305

213306
locals {
214-
cluster_name = module.eks.cluster_name
215-
control_plane_sg_id = module.eks.control_plane_security_group_id
307+
cluster_name = module.eks.cluster_name
308+
control_plane_sg_id = module.eks.control_plane_security_group_id
216309
}
217310

218311
module "database" {
219312
source = "./modules/database"
220313

221-
deployment_name = var.deployment_name
222-
rds_identifier = var.rds_identifier
223-
provider_region = var.provider_region
224-
vpc_private_subnets = local.vpc_private_subnets
225-
rds_username = var.rds_username
226-
rds_password_override = var.rds_password_override
227-
rds_instance = var.rds_instance
228-
rds_allocated_storage = var.rds_allocated_storage
229-
rds_max_allocated_storage = var.rds_max_allocated_storage
230-
rds_backups_replication_target_region = var.rds_backups_replication_target_region
231-
rds_backups_replication_retention_period = var.rds_backups_replication_retention_period
232-
rds_backup_window = var.rds_backup_window
233-
rds_maintenance_window = var.rds_maintenance_window
234-
create_rds_kms_key = var.create_rds_kms_key
235-
rds_kms_key_alias = var.rds_kms_key_alias
236-
use_default_rds_kms_key = var.use_default_rds_kms_key
237-
database_name = var.database_name
238-
db_subnet_group_name = var.db_subnet_group_name
239-
db_parameter_group_name = var.db_parameter_group_name
240-
rds_ro_username = var.rds_ro_username
241-
rds_version = var.rds_version
242-
rds_port = var.rds_port
243-
rds_param_group_family = var.rds_param_group_family
244-
apply_major_upgrade = var.apply_major_upgrade
245-
db_instance_tags = var.db_instance_tags
246-
db_parameter_group_tags = var.db_parameter_group_tags
247-
db_subnet_group_tags = var.db_subnet_group_tags
248-
rds_extra_tags = var.rds_extra_tags
249-
security_group_id = local.db_security_group_id
250-
db_extra_parameters = var.db_extra_parameters
251-
rds_multi_az = var.rds_multi_az
252-
rds_copy_tags_to_snapshot = var.rds_copy_tags_to_snapshot
253-
rds_performance_insights_enabled = var.rds_performance_insights_enabled
254-
rds_performance_insights_retention_period= var.rds_performance_insights_retention_period
255-
rds_monitoring_role_arn = var.rds_monitoring_role_arn
256-
rds_auto_minor_version_upgrade = var.rds_auto_minor_version_upgrade
257-
rds_monitoring_interval = var.rds_monitoring_interval
314+
deployment_name = var.deployment_name
315+
rds_identifier = var.rds_identifier
316+
provider_region = var.provider_region
317+
vpc_private_subnets = local.vpc_private_subnets
318+
rds_username = var.rds_username
319+
rds_password_override = var.rds_password_override
320+
rds_instance = var.rds_instance
321+
rds_allocated_storage = var.rds_allocated_storage
322+
rds_max_allocated_storage = var.rds_max_allocated_storage
323+
rds_backups_replication_target_region = var.rds_backups_replication_target_region
324+
rds_backups_replication_retention_period = var.rds_backups_replication_retention_period
325+
rds_backup_window = var.rds_backup_window
326+
rds_maintenance_window = var.rds_maintenance_window
327+
create_rds_kms_key = var.create_rds_kms_key
328+
rds_kms_key_alias = var.rds_kms_key_alias
329+
use_default_rds_kms_key = var.use_default_rds_kms_key
330+
database_name = var.database_name
331+
db_subnet_group_name = var.db_subnet_group_name
332+
db_parameter_group_name = var.db_parameter_group_name
333+
rds_ro_username = var.rds_ro_username
334+
rds_version = var.rds_version
335+
rds_port = var.rds_port
336+
rds_param_group_family = var.rds_param_group_family
337+
apply_major_upgrade = var.apply_major_upgrade
338+
db_instance_tags = var.db_instance_tags
339+
db_parameter_group_tags = var.db_parameter_group_tags
340+
db_subnet_group_tags = var.db_subnet_group_tags
341+
rds_extra_tags = var.rds_extra_tags
342+
security_group_id = local.db_security_group_id
343+
db_extra_parameters = var.db_extra_parameters
344+
rds_multi_az = var.rds_multi_az
345+
rds_copy_tags_to_snapshot = var.rds_copy_tags_to_snapshot
346+
rds_performance_insights_enabled = var.rds_performance_insights_enabled
347+
rds_performance_insights_retention_period = var.rds_performance_insights_retention_period
348+
rds_monitoring_role_arn = var.rds_monitoring_role_arn
349+
rds_auto_minor_version_upgrade = var.rds_auto_minor_version_upgrade
350+
rds_monitoring_interval = var.rds_monitoring_interval
258351
}
259352

260353
module "private_access" {
261-
count = var.deploy_private_access ? 1 : 0
354+
count = var.deploy_private_access ? 1 : 0
262355
source = "./modules/private_access"
263356

264357
allowed_principals = var.allowed_principals
@@ -281,7 +374,7 @@ resource "aws_ebs_volume" "clickhouse_data" {
281374

282375
tags = merge({
283376
Name = "${var.deployment_name}-clickhouse-data"
284-
}, var.ebs_extra_tags)
377+
}, var.ebs_extra_tags)
285378
}
286379

287380
resource "aws_ebs_volume" "clickhouse_logs" {
@@ -311,34 +404,34 @@ resource "aws_ebs_volume" "redis_data" {
311404
}
312405

313406
resource "random_password" "clickhouse_password" {
314-
length = 16
315-
min_upper = 2
316-
min_lower = 2
317-
min_numeric = 2
318-
special = false
407+
length = 16
408+
min_upper = 2
409+
min_lower = 2
410+
min_numeric = 2
411+
special = false
319412
}
320413

321414
resource "random_password" "redis_password" {
322-
length = 12
323-
special = false
415+
length = 12
416+
special = false
324417
}
325418

326419
module "github_reverse_proxy" {
327420
count = var.deploy_github_reverse_proxy ? 1 : 0
328421

329422
source = "./modules/github_reverse_proxy"
330423

331-
deployment_name = var.deployment_name
332-
environment = var.environment
333-
region = var.provider_region
334-
vpc_cidr = local.vpc_cidr
335-
vpc_id = local.vpc_id
336-
vpc_private_subnets = local.vpc_private_subnets
337-
github_cidrs = var.github_cidrs
338-
datadog_api_key = var.datadog_api_key
339-
use_private_egress = var.lb_internal
340-
341-
private_system_endpoint = module.load_balancer.load_balancer_dns
424+
deployment_name = var.deployment_name
425+
environment = var.environment
426+
region = var.provider_region
427+
vpc_cidr = local.vpc_cidr
428+
vpc_id = local.vpc_id
429+
vpc_private_subnets = local.vpc_private_subnets
430+
github_cidrs = var.github_cidrs
431+
datadog_api_key = var.datadog_api_key
432+
use_private_egress = var.lb_internal
433+
434+
private_system_endpoint = module.load_balancer.load_balancer_dns
342435
}
343436

344437
module "vpc_peering" {
@@ -368,7 +461,7 @@ resource "null_resource" "deployment_check" {
368461
}
369462

370463
provisioner "local-exec" {
371-
command = <<-EOT
464+
command = <<-EOT
372465
# Get the load balancer IPs value
373466
LB_IPS="${module.load_balancer.load_balancer_ips}"
374467

‎modules/eks/main.tf‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ module "eks" {
9191
configuration_values = jsonencode({
9292
"sidecars" : {
9393
"snapshotter" : {
94-
"forceEnable" : false
94+
"forceEnable" : true
9595
}
9696
}
9797
})

0 commit comments

Comments
 (0)