Skip to content

Commit 762aab1

Browse files
VeraQinVeraQin
committed
Add disk_io_scheduler and node_vfio_config to node config
1 parent 920bb6b commit 762aab1

6 files changed

Lines changed: 360 additions & 0 deletions

‎mmv1/third_party/terraform/services/container/node_config.go.tmpl‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1101,6 +1101,42 @@ func schemaNodeConfig() *schema.Schema {
11011101
},
11021102
},
11031103
},
1104+
"disk_io_scheduler": {
1105+
Type: schema.TypeList,
1106+
Optional: true,
1107+
MaxItems: 1,
1108+
Description: `Controls the configuration for the disk IO scheduler.`,
1109+
Elem: &schema.Resource{
1110+
Schema: map[string]*schema.Schema{
1111+
"node_system_io_scheduler": {
1112+
Type: schema.TypeString,
1113+
Optional: true,
1114+
Description: `Configures the IO scheduler for the boot disk or ephemeral lssd that runs node system workloads.`,
1115+
},
1116+
"node_attached_disk_io_scheduler": {
1117+
Type: schema.TypeString,
1118+
Optional: true,
1119+
Description: `Configures the IO scheduler for the attached disks.`,
1120+
},
1121+
},
1122+
},
1123+
},
1124+
"node_vfio_config": {
1125+
Type: schema.TypeList,
1126+
Optional: true,
1127+
MaxItems: 1,
1128+
Description: `Contains VFIO-related configurations for this node.`,
1129+
Elem: &schema.Resource{
1130+
Schema: map[string]*schema.Schema{
1131+
"dma_entry_limit": {
1132+
Type: schema.TypeInt,
1133+
Optional: true,
1134+
ValidateFunc: validation.IntBetween(65535, 4194304),
1135+
Description: `Specifies the maximum number of DMA entries (pages) that can be mapped by the VFIO IOMMU type 1 driver for a container.`,
1136+
},
1137+
},
1138+
},
1139+
},
11041140
"transparent_hugepage_enabled": {
11051141
Type: schema.TypeString,
11061142
Optional: true,
@@ -2335,6 +2371,14 @@ func expandLinuxNodeConfig(v interface{}) *container.LinuxNodeConfig {
23352371
linuxNodeConfig.CustomNodeInit = expandCustomNodeInit(v)
23362372
}
23372373

2374+
if v, ok := cfg["disk_io_scheduler"]; ok {
2375+
linuxNodeConfig.DiskIoScheduler = expandDiskIoScheduler(v)
2376+
}
2377+
2378+
if v, ok := cfg["node_vfio_config"]; ok {
2379+
linuxNodeConfig.NodeVfioConfig = expandNodeVfioConfig(v)
2380+
}
2381+
23382382
return linuxNodeConfig
23392383
}
23402384

@@ -2447,6 +2491,54 @@ func expandNodeKernelModuleLoading(v interface{}) *container.NodeKernelModuleLoa
24472491
return NodeKernelModuleLoading
24482492
}
24492493

2494+
func expandDiskIoScheduler(v interface{}) *container.DiskIoScheduler {
2495+
if v == nil {
2496+
return nil
2497+
}
2498+
ls := v.([]interface{})
2499+
if len(ls) == 0 {
2500+
return nil
2501+
}
2502+
if ls[0] == nil {
2503+
return &container.DiskIoScheduler{}
2504+
}
2505+
cfg := ls[0].(map[string]interface{})
2506+
2507+
diskIoScheduler := &container.DiskIoScheduler{}
2508+
2509+
if v, ok := cfg["node_system_io_scheduler"]; ok {
2510+
diskIoScheduler.NodeSystemIoScheduler = v.(string)
2511+
}
2512+
2513+
if v, ok := cfg["node_attached_disk_io_scheduler"]; ok {
2514+
diskIoScheduler.NodeAttachedDiskIoScheduler = v.(string)
2515+
}
2516+
2517+
return diskIoScheduler
2518+
}
2519+
2520+
func expandNodeVfioConfig(v interface{}) *container.NodeVfioConfig {
2521+
if v == nil {
2522+
return nil
2523+
}
2524+
ls := v.([]interface{})
2525+
if len(ls) == 0 {
2526+
return nil
2527+
}
2528+
if ls[0] == nil {
2529+
return &container.NodeVfioConfig{}
2530+
}
2531+
cfg := ls[0].(map[string]interface{})
2532+
2533+
nodeVfioConfig := &container.NodeVfioConfig{}
2534+
2535+
if v, ok := cfg["dma_entry_limit"]; ok {
2536+
nodeVfioConfig.DmaEntryLimit = int64(v.(int))
2537+
}
2538+
2539+
return nodeVfioConfig
2540+
}
2541+
24502542
func expandAccurateTimeConfig(v interface{}) *container.AccurateTimeConfig {
24512543
if v == nil {
24522544
return nil
@@ -3553,6 +3645,29 @@ func flattenLinuxNodeConfig(c *container.LinuxNodeConfig) []map[string]interface
35533645
"swap_config": flattenSwapConfig(c.SwapConfig),
35543646
"accurate_time_config": flattenAccurateTimeConfig(c.AccurateTimeConfig),
35553647
"custom_node_init": flattenCustomNodeInit(c.CustomNodeInit),
3648+
"disk_io_scheduler": flattenDiskIoScheduler(c.DiskIoScheduler),
3649+
"node_vfio_config": flattenNodeVfioConfig(c.NodeVfioConfig),
3650+
})
3651+
}
3652+
return result
3653+
}
3654+
3655+
func flattenDiskIoScheduler(c *container.DiskIoScheduler) []map[string]interface{} {
3656+
result := []map[string]interface{}{}
3657+
if c != nil {
3658+
result = append(result, map[string]interface{}{
3659+
"node_system_io_scheduler": c.NodeSystemIoScheduler,
3660+
"node_attached_disk_io_scheduler": c.NodeAttachedDiskIoScheduler,
3661+
})
3662+
}
3663+
return result
3664+
}
3665+
3666+
func flattenNodeVfioConfig(c *container.NodeVfioConfig) []map[string]interface{} {
3667+
result := []map[string]interface{}{}
3668+
if c != nil {
3669+
result = append(result, map[string]interface{}{
3670+
"dma_entry_limit": c.DmaEntryLimit,
35563671
})
35573672
}
35583673
return result

‎mmv1/third_party/terraform/services/container/resource_container_cluster_meta.yaml.tmpl‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -393,11 +393,14 @@ fields:
393393
- api_field: 'nodeConfig.linuxNodeConfig.customNodeInit.initScript.gcpSecretManagerSecretUri'
394394
- api_field: 'nodeConfig.linuxNodeConfig.customNodeInit.initScript.gcsGeneration'
395395
- api_field: 'nodeConfig.linuxNodeConfig.customNodeInit.initScript.gcsUri'
396+
- api_field: 'nodeConfig.linuxNodeConfig.diskIoScheduler.nodeAttachedDiskIoScheduler'
397+
- api_field: 'nodeConfig.linuxNodeConfig.diskIoScheduler.nodeSystemIoScheduler'
396398
- field: 'node_config.linux_node_config.hugepages_config.hugepage_size_1g'
397399
api_field: 'nodeConfig.linuxNodeConfig.hugepages.hugepageSize1g'
398400
- field: 'node_config.linux_node_config.hugepages_config.hugepage_size_2m'
399401
api_field: 'nodeConfig.linuxNodeConfig.hugepages.hugepageSize2m'
400402
- api_field: 'nodeConfig.linuxNodeConfig.nodeKernelModuleLoading.policy'
403+
- api_field: 'nodeConfig.linuxNodeConfig.nodeVfioConfig.dmaEntryLimit'
401404
- api_field: 'nodeConfig.linuxNodeConfig.sysctls'
402405
- api_field: 'nodeConfig.linuxNodeConfig.transparentHugepageDefrag'
403406
- api_field: 'nodeConfig.linuxNodeConfig.transparentHugepageEnabled'
@@ -695,12 +698,18 @@ fields:
695698
api_field: 'nodePools.config.linuxNodeConfig.customNodeInit.initScript.gcsGeneration'
696699
- field: 'node_pool.node_config.linux_node_config.custom_node_init.init_script.gcs_uri'
697700
api_field: 'nodePools.config.linuxNodeConfig.customNodeInit.initScript.gcsUri'
701+
- field: 'node_pool.node_config.linux_node_config.disk_io_scheduler.node_attached_disk_io_scheduler'
702+
api_field: 'nodePools.config.linuxNodeConfig.diskIoScheduler.nodeAttachedDiskIoScheduler'
703+
- field: 'node_pool.node_config.linux_node_config.disk_io_scheduler.node_system_io_scheduler'
704+
api_field: 'nodePools.config.linuxNodeConfig.diskIoScheduler.nodeSystemIoScheduler'
698705
- field: 'node_pool.node_config.linux_node_config.hugepages_config.hugepage_size_1g'
699706
api_field: 'nodePools.config.linuxNodeConfig.hugepages.hugepageSize1g'
700707
- field: 'node_pool.node_config.linux_node_config.hugepages_config.hugepage_size_2m'
701708
api_field: 'nodePools.config.linuxNodeConfig.hugepages.hugepageSize2m'
702709
- field: 'node_pool.node_config.linux_node_config.node_kernel_module_loading.policy'
703710
api_field: 'nodePools.config.linuxNodeConfig.nodeKernelModuleLoading.policy'
711+
- field: 'node_pool.node_config.linux_node_config.node_vfio_config.dma_entry_limit'
712+
api_field: 'nodePools.config.linuxNodeConfig.nodeVfioConfig.dmaEntryLimit'
704713
- field: 'node_pool.node_config.linux_node_config.sysctls'
705714
api_field: 'nodePools.config.linuxNodeConfig.sysctls'
706715
- field: 'node_pool.node_config.linux_node_config.transparent_hugepage_defrag'

‎mmv1/third_party/terraform/services/container/resource_container_cluster_test.go.tmpl‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2510,6 +2510,52 @@ func TestAccContainerCluster_withNodeConfigLinuxNodeConfig(t *testing.T) {
25102510
ImportStateVerify: true,
25112511
ImportStateVerifyIgnore: []string{"min_master_version", "deletion_protection"},
25122512
},
2513+
// Update node disk io scheduler config
2514+
{
2515+
Config: testAccContainerCluster_withNodeConfigLinuxNodeConfigDiskIoScheduler(clusterName, networkName, subnetworkName, "bfq", "mq-deadline"),
2516+
Check: resource.ComposeTestCheckFunc(
2517+
resource.TestCheckResourceAttr(
2518+
"google_container_cluster.with_linux_node_config",
2519+
"node_config.0.linux_node_config.0.disk_io_scheduler.0.node_system_io_scheduler", "bfq",
2520+
),
2521+
resource.TestCheckResourceAttr(
2522+
"google_container_cluster.with_linux_node_config",
2523+
"node_config.0.linux_node_config.0.disk_io_scheduler.0.node_attached_disk_io_scheduler", "mq-deadline",
2524+
),
2525+
),
2526+
ConfigPlanChecks: resource.ConfigPlanChecks{
2527+
PreApply: []plancheck.PlanCheck{
2528+
acctest.ExpectNoDelete(),
2529+
},
2530+
},
2531+
},
2532+
{
2533+
ResourceName: "google_container_cluster.with_linux_node_config",
2534+
ImportState: true,
2535+
ImportStateVerify: true,
2536+
ImportStateVerifyIgnore: []string{"min_master_version", "deletion_protection"},
2537+
},
2538+
// Update node vfio config
2539+
{
2540+
Config: testAccContainerCluster_withNodeConfigLinuxNodeConfigNodeVfioConfig(clusterName, networkName, subnetworkName, 100000),
2541+
Check: resource.ComposeTestCheckFunc(
2542+
resource.TestCheckResourceAttr(
2543+
"google_container_cluster.with_linux_node_config",
2544+
"node_config.0.linux_node_config.0.node_vfio_config.0.dma_entry_limit", "100000",
2545+
),
2546+
),
2547+
ConfigPlanChecks: resource.ConfigPlanChecks{
2548+
PreApply: []plancheck.PlanCheck{
2549+
acctest.ExpectNoDelete(),
2550+
},
2551+
},
2552+
},
2553+
{
2554+
ResourceName: "google_container_cluster.with_linux_node_config",
2555+
ImportState: true,
2556+
ImportStateVerify: true,
2557+
ImportStateVerifyIgnore: []string{"min_master_version", "deletion_protection"},
2558+
},
25132559
},
25142560
})
25152561
}
@@ -10412,6 +10458,67 @@ resource "google_container_cluster" "with_linux_node_config" {
1041210458
`, clusterName, linuxNodeConfig, networkName, subnetworkName)
1041310459
}
1041410460

10461+
func testAccContainerCluster_withNodeConfigLinuxNodeConfigDiskIoScheduler(clusterName, networkName, subnetworkName, sysScheduler, attachedScheduler string) string {
10462+
return fmt.Sprintf(`
10463+
data "google_container_engine_versions" "central1a" {
10464+
location = "us-central1-a"
10465+
}
10466+
resource "google_container_cluster" "with_linux_node_config" {
10467+
name = "%s"
10468+
location = "us-central1-f"
10469+
initial_node_count = 1
10470+
min_master_version = data.google_container_engine_versions.central1a.release_channel_latest_version["RAPID"]
10471+
10472+
release_channel {
10473+
channel = "RAPID"
10474+
}
10475+
10476+
node_config {
10477+
image_type = "COS_CONTAINERD"
10478+
linux_node_config {
10479+
disk_io_scheduler {
10480+
node_system_io_scheduler = "%s"
10481+
node_attached_disk_io_scheduler = "%s"
10482+
}
10483+
}
10484+
}
10485+
deletion_protection = false
10486+
network = "%s"
10487+
subnetwork = "%s"
10488+
}
10489+
`, clusterName, sysScheduler, attachedScheduler, networkName, subnetworkName)
10490+
}
10491+
10492+
func testAccContainerCluster_withNodeConfigLinuxNodeConfigNodeVfioConfig(clusterName, networkName, subnetworkName string, dmaEntryLimit int) string {
10493+
return fmt.Sprintf(`
10494+
data "google_container_engine_versions" "central1a" {
10495+
location = "us-central1-a"
10496+
}
10497+
resource "google_container_cluster" "with_linux_node_config" {
10498+
name = "%s"
10499+
location = "us-central1-f"
10500+
initial_node_count = 1
10501+
min_master_version = data.google_container_engine_versions.central1a.release_channel_latest_version["RAPID"]
10502+
10503+
release_channel {
10504+
channel = "RAPID"
10505+
}
10506+
10507+
node_config {
10508+
image_type = "COS_CONTAINERD"
10509+
linux_node_config {
10510+
node_vfio_config {
10511+
dma_entry_limit = %d
10512+
}
10513+
}
10514+
}
10515+
deletion_protection = false
10516+
network = "%s"
10517+
subnetwork = "%s"
10518+
}
10519+
`, clusterName, dmaEntryLimit, networkName, subnetworkName)
10520+
}
10521+
1041510522
func testAccContainerCluster_withNodeConfigScopeAlias(clusterName, networkName, subnetworkName string) string {
1041610523
return fmt.Sprintf(`
1041710524
resource "google_container_cluster" "with_node_config_scope_alias" {

‎mmv1/third_party/terraform/services/container/resource_container_node_pool_meta.yaml.tmpl‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -226,12 +226,18 @@ fields:
226226
api_field: 'config.linuxNodeConfig.customNodeInit.initScript.gcsGeneration'
227227
- field: 'node_config.linux_node_config.custom_node_init.init_script.gcs_uri'
228228
api_field: 'config.linuxNodeConfig.customNodeInit.initScript.gcsUri'
229+
- field: 'node_config.linux_node_config.disk_io_scheduler.node_attached_disk_io_scheduler'
230+
api_field: 'config.linuxNodeConfig.diskIoScheduler.nodeAttachedDiskIoScheduler'
231+
- field: 'node_config.linux_node_config.disk_io_scheduler.node_system_io_scheduler'
232+
api_field: 'config.linuxNodeConfig.diskIoScheduler.nodeSystemIoScheduler'
229233
- field: 'node_config.linux_node_config.hugepages_config.hugepage_size_1g'
230234
api_field: 'config.linuxNodeConfig.hugepages.hugepageSize1g'
231235
- field: 'node_config.linux_node_config.hugepages_config.hugepage_size_2m'
232236
api_field: 'config.linuxNodeConfig.hugepages.hugepageSize2m'
233237
- field: 'node_config.linux_node_config.node_kernel_module_loading.policy'
234238
api_field: 'config.linuxNodeConfig.nodeKernelModuleLoading.policy'
239+
- field: 'node_config.linux_node_config.node_vfio_config.dma_entry_limit'
240+
api_field: 'config.linuxNodeConfig.nodeVfioConfig.dmaEntryLimit'
235241
- field: 'node_config.linux_node_config.sysctls'
236242
api_field: 'config.linuxNodeConfig.sysctls'
237243
- field: 'node_config.linux_node_config.transparent_hugepage_defrag'

0 commit comments

Comments
 (0)