Skip to content

Commit 2807922

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #18159 from rifelpet/gce-apiserver
gce: Fix instance tags with role=apiserver
2 parents 22d1d2c + d83adc4 commit 2807922

6 files changed

Lines changed: 45 additions & 22 deletions

File tree

‎pkg/model/gcemodel/api_loadbalancer.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ func (b *APILoadBalancerBuilder) addFirewallRules(c *fi.CloudupModelBuilderConte
103103
Lifecycle: b.Lifecycle,
104104
Network: network,
105105
SourceRanges: b.Cluster.Spec.API.Access,
106-
TargetTags: []string{b.GCETagForRole(kops.InstanceGroupRoleControlPlane)},
106+
TargetTags: b.GCETagsForAPIServerTargets(),
107107
Allowed: []string{"tcp:" + strconv.Itoa(wellknownports.KubeAPIServer)},
108108
})
109109

@@ -114,7 +114,7 @@ func (b *APILoadBalancerBuilder) addFirewallRules(c *fi.CloudupModelBuilderConte
114114
Network: network,
115115
Family: gcetasks.AddressFamilyIPv4, // ip alias is always ipv4
116116
SourceRanges: []string{b.Cluster.Spec.Networking.PodCIDR},
117-
TargetTags: []string{b.GCETagForRole(kops.InstanceGroupRoleControlPlane)},
117+
TargetTags: b.GCETagsForAPIServerTargets(),
118118
Allowed: []string{"tcp:" + strconv.Itoa(wellknownports.KubeAPIServer)},
119119
})
120120
}

‎pkg/model/gcemodel/autoscalinggroup.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -216,7 +216,7 @@ func (b *AutoscalingGroupModelBuilder) buildInstanceTemplate(c *fi.CloudupModelB
216216

217217
case kops.InstanceGroupRoleAPIServer:
218218
t.Scopes = append(t.Scopes, "https://www.googleapis.com/auth/ndev.clouddns.readwrite")
219-
t.Tags = append(t.Tags, b.GCETagForRole(kops.InstanceGroupRoleControlPlane))
219+
t.Tags = append(t.Tags, b.GCETagForRole(kops.InstanceGroupRoleAPIServer))
220220

221221
case kops.InstanceGroupRoleNode:
222222
t.Tags = append(t.Tags, b.GCETagForRole(kops.InstanceGroupRoleNode))

‎pkg/model/gcemodel/context.go‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,28 @@ func (c *GCEModelContext) GCETagForRole(role kops.InstanceGroupRole) string {
9292
return gce.TagForRole(c.Cluster.ObjectMeta.Name, role)
9393
}
9494

95+
// HasAPIServerOnlyInstanceGroups returns true if the cluster has any APIServer-only instance groups.
96+
func (c *GCEModelContext) HasAPIServerOnlyInstanceGroups() bool {
97+
for _, ig := range c.InstanceGroups {
98+
if ig.Spec.Role == kops.InstanceGroupRoleAPIServer {
99+
return true
100+
}
101+
}
102+
return false
103+
}
104+
105+
// GCETagsForAPIServerTargets returns the network tags that should be used as firewall
106+
// targets for rules that need to reach API server instances. It always includes the
107+
// ControlPlane tag, and adds the APIServer tag only when the cluster has dedicated
108+
// APIServer instance groups.
109+
func (c *GCEModelContext) GCETagsForAPIServerTargets() []string {
110+
tags := []string{c.GCETagForRole(kops.InstanceGroupRoleControlPlane)}
111+
if c.HasAPIServerOnlyInstanceGroups() {
112+
tags = append(tags, c.GCETagForRole(kops.InstanceGroupRoleAPIServer))
113+
}
114+
return tags
115+
}
116+
95117
func (c *GCEModelContext) LinkToTargetPool(id string) *gcetasks.TargetPool {
96118
return &gcetasks.TargetPool{Name: s(c.NameForTargetPool(id))}
97119
}

‎pkg/model/gcemodel/external_access.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ func (b *ExternalAccessModelBuilder) Build(c *fi.CloudupModelBuilderContext) err
6464
})
6565
b.AddFirewallRulesTasks(c, "bastion-to-master-ssh", &gcetasks.FirewallRule{
6666
Lifecycle: b.Lifecycle,
67-
TargetTags: []string{b.GCETagForRole(kops.InstanceGroupRoleControlPlane), b.GCETagForRole("Master")},
67+
TargetTags: append(b.GCETagsForAPIServerTargets(), b.GCETagForRole("Master")),
6868
Allowed: []string{"tcp:22"},
6969
SourceTags: []string{b.GCETagForRole(kops.InstanceGroupRoleBastion)},
7070
Network: network,
@@ -83,7 +83,7 @@ func (b *ExternalAccessModelBuilder) Build(c *fi.CloudupModelBuilderContext) err
8383
}
8484
b.AddFirewallRulesTasks(c, "ssh-external-to-master", &gcetasks.FirewallRule{
8585
Lifecycle: b.Lifecycle,
86-
TargetTags: []string{b.GCETagForRole(kops.InstanceGroupRoleControlPlane), b.GCETagForRole("Master")},
86+
TargetTags: append(b.GCETagsForAPIServerTargets(), b.GCETagForRole("Master")),
8787
Allowed: []string{"tcp:22"},
8888
SourceRanges: b.Cluster.Spec.SSHAccess,
8989
Network: network,
@@ -135,7 +135,7 @@ func (b *ExternalAccessModelBuilder) Build(c *fi.CloudupModelBuilderContext) err
135135
}
136136
b.AddFirewallRulesTasks(c, "kubernetes-master-https", &gcetasks.FirewallRule{
137137
Lifecycle: b.Lifecycle,
138-
TargetTags: []string{b.GCETagForRole(kops.InstanceGroupRoleControlPlane), b.GCETagForRole("Master")},
138+
TargetTags: append(b.GCETagsForAPIServerTargets(), b.GCETagForRole("Master")),
139139
Allowed: []string{"tcp:443"},
140140
SourceRanges: b.Cluster.Spec.API.Access,
141141
Network: network,
@@ -148,7 +148,7 @@ func (b *ExternalAccessModelBuilder) Build(c *fi.CloudupModelBuilderContext) err
148148
Network: network,
149149
Family: gcetasks.AddressFamilyIPv4, // ip alias is always ipv4
150150
SourceRanges: []string{b.Cluster.Spec.Networking.PodCIDR},
151-
TargetTags: []string{b.GCETagForRole(kops.InstanceGroupRoleControlPlane)},
151+
TargetTags: b.GCETagsForAPIServerTargets(),
152152
Allowed: []string{"tcp:" + strconv.Itoa(wellknownports.KubeAPIServer)},
153153
})
154154
}

‎pkg/model/gcemodel/firewall.go‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ func (b *FirewallModelBuilder) Build(c *fi.CloudupModelBuilderContext) error {
6565
"209.85.204.0/22",
6666
"209.85.152.0/22",
6767
},
68-
TargetTags: []string{b.GCETagForRole(kops.InstanceGroupRoleControlPlane)},
68+
TargetTags: b.GCETagsForAPIServerTargets(),
6969
Allowed: []string{"tcp"},
7070
})
7171
}
@@ -93,12 +93,13 @@ func (b *FirewallModelBuilder) Build(c *fi.CloudupModelBuilderContext) error {
9393
if err != nil {
9494
return err
9595
}
96+
cpTags := append(b.GCETagsForAPIServerTargets(), b.GCETagForRole("Master"))
9697
t := &gcetasks.FirewallRule{
9798
Name: s(b.NameForFirewallRule("master-to-master")),
9899
Lifecycle: b.Lifecycle,
99100
Network: network,
100-
SourceTags: []string{b.GCETagForRole(kops.InstanceGroupRoleControlPlane), b.GCETagForRole("Master")},
101-
TargetTags: []string{b.GCETagForRole(kops.InstanceGroupRoleControlPlane), b.GCETagForRole("Master")},
101+
SourceTags: cpTags,
102+
TargetTags: cpTags,
102103
Allowed: allProtocols,
103104
}
104105
c.AddTask(t)
@@ -114,7 +115,7 @@ func (b *FirewallModelBuilder) Build(c *fi.CloudupModelBuilderContext) error {
114115
Name: s(b.NameForFirewallRule("master-to-node")),
115116
Lifecycle: b.Lifecycle,
116117
Network: network,
117-
SourceTags: []string{b.GCETagForRole(kops.InstanceGroupRoleControlPlane), b.GCETagForRole("Master")},
118+
SourceTags: append(b.GCETagsForAPIServerTargets(), b.GCETagForRole("Master")),
118119
TargetTags: []string{b.GCETagForRole(kops.InstanceGroupRoleNode)},
119120
Allowed: allProtocols,
120121
}
@@ -132,7 +133,7 @@ func (b *FirewallModelBuilder) Build(c *fi.CloudupModelBuilderContext) error {
132133
Lifecycle: b.Lifecycle,
133134
Network: network,
134135
SourceTags: []string{b.GCETagForRole(kops.InstanceGroupRoleNode)},
135-
TargetTags: []string{b.GCETagForRole(kops.InstanceGroupRoleControlPlane), b.GCETagForRole("Master")},
136+
TargetTags: append(b.GCETagsForAPIServerTargets(), b.GCETagForRole("Master")),
136137
Allowed: []string{
137138
fmt.Sprintf("tcp:%d", wellknownports.KubeAPIServer),
138139
fmt.Sprintf("tcp:%d", wellknownports.KubeletAPI),

‎tests/integration/update_cluster/minimal_gce_plb_apiserver/kubernetes.tf‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -222,7 +222,7 @@ resource "google_compute_firewall" "https-api-ipv6-minimal-gce-plb-apiserver-exa
222222
name = "https-api-ipv6-minimal-gce-plb-apiserver-example-com"
223223
network = google_compute_network.minimal-gce-plb-apiserver-example-com.name
224224
source_ranges = ["::/0"]
225-
target_tags = ["minimal-gce-plb-apiserver-example-com-k8s-io-role-control-plane"]
225+
target_tags = ["minimal-gce-plb-apiserver-example-com-k8s-io-role-control-plane", "minimal-gce-plb-apiserver-example-com-k8s-io-role-apiserver"]
226226
}
227227

228228
resource "google_compute_firewall" "https-api-minimal-gce-plb-apiserver-example-com" {
@@ -234,7 +234,7 @@ resource "google_compute_firewall" "https-api-minimal-gce-plb-apiserver-example-
234234
name = "https-api-minimal-gce-plb-apiserver-example-com"
235235
network = google_compute_network.minimal-gce-plb-apiserver-example-com.name
236236
source_ranges = ["0.0.0.0/0"]
237-
target_tags = ["minimal-gce-plb-apiserver-example-com-k8s-io-role-control-plane"]
237+
target_tags = ["minimal-gce-plb-apiserver-example-com-k8s-io-role-control-plane", "minimal-gce-plb-apiserver-example-com-k8s-io-role-apiserver"]
238238
}
239239

240240
resource "google_compute_firewall" "lb-health-checks-minimal-gce-plb-apiserver-example-com" {
@@ -245,7 +245,7 @@ resource "google_compute_firewall" "lb-health-checks-minimal-gce-plb-apiserver-e
245245
name = "lb-health-checks-minimal-gce-plb-apiserver-example-com"
246246
network = google_compute_network.minimal-gce-plb-apiserver-example-com.name
247247
source_ranges = ["35.191.0.0/16", "130.211.0.0/22", "209.85.204.0/22", "209.85.152.0/22"]
248-
target_tags = ["minimal-gce-plb-apiserver-example-com-k8s-io-role-control-plane"]
248+
target_tags = ["minimal-gce-plb-apiserver-example-com-k8s-io-role-control-plane", "minimal-gce-plb-apiserver-example-com-k8s-io-role-apiserver"]
249249
}
250250

251251
resource "google_compute_firewall" "master-to-master-minimal-gce-plb-apiserver-example-com" {
@@ -270,8 +270,8 @@ resource "google_compute_firewall" "master-to-master-minimal-gce-plb-apiserver-e
270270
disabled = false
271271
name = "master-to-master-minimal-gce-plb-apiserver-example-com"
272272
network = google_compute_network.minimal-gce-plb-apiserver-example-com.name
273-
source_tags = ["minimal-gce-plb-apiserver-example-com-k8s-io-role-control-plane", "minimal-gce-plb-apiserver-example-com-k8s-io-role-master"]
274-
target_tags = ["minimal-gce-plb-apiserver-example-com-k8s-io-role-control-plane", "minimal-gce-plb-apiserver-example-com-k8s-io-role-master"]
273+
source_tags = ["minimal-gce-plb-apiserver-example-com-k8s-io-role-control-plane", "minimal-gce-plb-apiserver-example-com-k8s-io-role-apiserver", "minimal-gce-plb-apiserver-example-com-k8s-io-role-master"]
274+
target_tags = ["minimal-gce-plb-apiserver-example-com-k8s-io-role-control-plane", "minimal-gce-plb-apiserver-example-com-k8s-io-role-apiserver", "minimal-gce-plb-apiserver-example-com-k8s-io-role-master"]
275275
}
276276

277277
resource "google_compute_firewall" "master-to-node-minimal-gce-plb-apiserver-example-com" {
@@ -296,7 +296,7 @@ resource "google_compute_firewall" "master-to-node-minimal-gce-plb-apiserver-exa
296296
disabled = false
297297
name = "master-to-node-minimal-gce-plb-apiserver-example-com"
298298
network = google_compute_network.minimal-gce-plb-apiserver-example-com.name
299-
source_tags = ["minimal-gce-plb-apiserver-example-com-k8s-io-role-control-plane", "minimal-gce-plb-apiserver-example-com-k8s-io-role-master"]
299+
source_tags = ["minimal-gce-plb-apiserver-example-com-k8s-io-role-control-plane", "minimal-gce-plb-apiserver-example-com-k8s-io-role-apiserver", "minimal-gce-plb-apiserver-example-com-k8s-io-role-master"]
300300
target_tags = ["minimal-gce-plb-apiserver-example-com-k8s-io-role-node"]
301301
}
302302

@@ -317,7 +317,7 @@ resource "google_compute_firewall" "node-to-master-minimal-gce-plb-apiserver-exa
317317
name = "node-to-master-minimal-gce-plb-apiserver-example-com"
318318
network = google_compute_network.minimal-gce-plb-apiserver-example-com.name
319319
source_tags = ["minimal-gce-plb-apiserver-example-com-k8s-io-role-node"]
320-
target_tags = ["minimal-gce-plb-apiserver-example-com-k8s-io-role-control-plane", "minimal-gce-plb-apiserver-example-com-k8s-io-role-master"]
320+
target_tags = ["minimal-gce-plb-apiserver-example-com-k8s-io-role-control-plane", "minimal-gce-plb-apiserver-example-com-k8s-io-role-apiserver", "minimal-gce-plb-apiserver-example-com-k8s-io-role-master"]
321321
}
322322

323323
resource "google_compute_firewall" "node-to-node-minimal-gce-plb-apiserver-example-com" {
@@ -387,7 +387,7 @@ resource "google_compute_firewall" "ssh-external-to-master-ipv6-minimal-gce-plb-
387387
name = "ssh-external-to-master-ipv6-minimal-gce-plb-apiserver-ex-eie140"
388388
network = google_compute_network.minimal-gce-plb-apiserver-example-com.name
389389
source_ranges = ["::/0"]
390-
target_tags = ["minimal-gce-plb-apiserver-example-com-k8s-io-role-control-plane", "minimal-gce-plb-apiserver-example-com-k8s-io-role-master"]
390+
target_tags = ["minimal-gce-plb-apiserver-example-com-k8s-io-role-control-plane", "minimal-gce-plb-apiserver-example-com-k8s-io-role-apiserver", "minimal-gce-plb-apiserver-example-com-k8s-io-role-master"]
391391
}
392392

393393
resource "google_compute_firewall" "ssh-external-to-master-minimal-gce-plb-apiserver-example-com" {
@@ -399,7 +399,7 @@ resource "google_compute_firewall" "ssh-external-to-master-minimal-gce-plb-apise
399399
name = "ssh-external-to-master-minimal-gce-plb-apiserver-example-com"
400400
network = google_compute_network.minimal-gce-plb-apiserver-example-com.name
401401
source_ranges = ["0.0.0.0/0"]
402-
target_tags = ["minimal-gce-plb-apiserver-example-com-k8s-io-role-control-plane", "minimal-gce-plb-apiserver-example-com-k8s-io-role-master"]
402+
target_tags = ["minimal-gce-plb-apiserver-example-com-k8s-io-role-control-plane", "minimal-gce-plb-apiserver-example-com-k8s-io-role-apiserver", "minimal-gce-plb-apiserver-example-com-k8s-io-role-master"]
403403
}
404404

405405
resource "google_compute_firewall" "ssh-external-to-node-ipv6-minimal-gce-plb-apiserver-example-com" {
@@ -582,7 +582,7 @@ resource "google_compute_instance_template" "apiserver-us-test1-a-minimal-gce-pl
582582
email = "default"
583583
scopes = ["https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/monitoring", "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/cloud-platform", "https://www.googleapis.com/auth/devstorage.read_only", "https://www.googleapis.com/auth/ndev.clouddns.readwrite"]
584584
}
585-
tags = ["minimal-gce-plb-apiserver-example-com-k8s-io-role-control-plane"]
585+
tags = ["minimal-gce-plb-apiserver-example-com-k8s-io-role-apiserver"]
586586
}
587587

588588
resource "google_compute_instance_template" "master-us-test1-a-minimal-gce-plb-apiserver-example-com" {

0 commit comments

Comments
 (0)