Make control plane split by role work.

channels/pkg/cmd/apply_channel.go

@@ -48,7 +48,7 @@ type ApplyChannelOptions struct {
 	Interval time.Duration
 	NodeName string
 
-	// Comma delimited label,value pairs to add to the node. Eg "kops.k8s.io/cloud-controller-manager,foo=bar"
+	// Comma delimited label,value pairs to add to the node. Eg "node-role.kops.k8s.io/cloud-controller-manager,foo=bar"
 	NodeLabels map[string]string
 }
 

cmd/kops-controller/controllers/node_controller.go

@@ -104,7 +104,12 @@ func (r *NodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.
 	for k := range node.Labels {
 		// If it is one of our managed labels, "prune" values we don't want to be there
 		switch k {
-		case nodelabels.RoleLabelAPIServer16, nodelabels.RoleLabelNode16, nodelabels.RoleLabelControlPlane20:
+		case nodelabels.RoleLabelAPIServer16,
+			nodelabels.RoleLabelEtcd,
+			nodelabels.RoleLabelScheduler,
+			nodelabels.RoleLabelKubeControllerManager,
+			nodelabels.RoleLabelNode16,
+			nodelabels.RoleLabelControlPlane20:
 			if _, found := labels[k]; !found {
 				deleteLabels[k] = struct{}{}
 			}

cmd/kops/create_cluster.go

@@ -193,6 +193,7 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command {
 	sshPublicKey := ""
 	associatePublicIP := false
 	encryptEtcdStorage := false
+	var controlPlaneCount int32
 
 	cmd := &cobra.Command{
 		Use:               "cluster [CLUSTER]",
@@ -212,6 +213,10 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command {
 				options.EncryptEtcdStorage = &encryptEtcdStorage
 			}
 
+			if cmd.Flag("control-plane-count").Changed || cmd.Flag("master-count").Changed {
+				options.ControlPlaneCount = &controlPlaneCount
+			}
+
 			if err := checkProjectFlag(cmd.Flag("project").Changed, options.Project); err != nil {
 				return err
 			}
@@ -295,9 +300,9 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command {
 		return []string{"pub"}, cobra.ShellCompDirectiveFilterFileExt
 	})
 
-	cmd.Flags().Int32Var(&options.ControlPlaneCount, "master-count", options.ControlPlaneCount, "Number of control-plane nodes. Defaults to one control-plane node per control-plane-zone")
+	cmd.Flags().Int32Var(&controlPlaneCount, "master-count", controlPlaneCount, "Number of control-plane nodes. Defaults to one control-plane node per control-plane-zone")
 	cmd.Flags().MarkDeprecated("master-count", "use --control-plane-count instead")
-	cmd.Flags().Int32Var(&options.ControlPlaneCount, "control-plane-count", options.ControlPlaneCount, "Number of control-plane nodes. Defaults to one control-plane node per control-plane-zone")
+	cmd.Flags().Int32Var(&controlPlaneCount, "control-plane-count", controlPlaneCount, "Number of control-plane nodes. Defaults to one control-plane node per control-plane-zone")
 	cmd.Flags().Int32Var(&options.NodeCount, "node-count", options.NodeCount, "Total number of worker nodes. Defaults to one node per zone")
 	if featureflag.APIServerNodes.Enabled() {
 		cmd.Flags().Int32Var(&options.APIServerCount, "api-server-count", options.APIServerCount, "Number of API server nodes. Defaults to 0.")

nodeup/pkg/model/bootstrap_client.go

@@ -45,7 +45,7 @@ type BootstrapClientBuilder struct {
 }
 
 func (b BootstrapClientBuilder) Build(c *fi.NodeupModelBuilderContext) error {
-	if b.IsMaster {
+	if b.IsMaster || b.BootConfig.InstanceGroupRole.IsControlPlaneType() {
 		return nil
 	}
 

nodeup/pkg/model/channels.go

@@ -46,7 +46,7 @@ type ChannelsBuilder struct {
 var _ fi.NodeupModelBuilder = &ChannelsBuilder{}
 
 func (b *ChannelsBuilder) Build(c *fi.NodeupModelBuilderContext) error {
-	if !b.IsMaster {
+	if !b.IsMaster && !b.HasAPIServer { // TODO Fix me, only include API Server if not control plane.
 		return nil
 	}
 

nodeup/pkg/model/discovery_service.go

@@ -42,7 +42,7 @@ var _ fi.NodeupModelBuilder = &DiscoveryService{}
 func (b *DiscoveryService) Build(c *fi.NodeupModelBuilderContext) error {
 	ctx := c.Context()
 
-	if !b.IsMaster {
+	if !b.IsMaster && !b.HasAPIServer {
 		return nil
 	}
 	discoveryServiceOptions := b.DiscoveryServiceOptions()

nodeup/pkg/model/etcd_manager_tls.go

@@ -32,7 +32,7 @@ var _ fi.NodeupModelBuilder = &EtcdManagerTLSBuilder{}
 
 // Build is responsible for TLS configuration for etcd-manager
 func (b *EtcdManagerTLSBuilder) Build(ctx *fi.NodeupModelBuilderContext) error {
-	if !b.IsMaster {
+	if !b.IsMaster && !b.BootConfig.InstanceGroupRole.HasEtcd() {
 		return nil
 	}
 

nodeup/pkg/model/kops_controller.go

@@ -34,7 +34,7 @@ var _ fi.NodeupModelBuilder = &KopsControllerBuilder{}
 
 // Build is responsible for configuring keys that will be used by kops-controller (via hostPath)
 func (b *KopsControllerBuilder) Build(c *fi.NodeupModelBuilderContext) error {
-	if !b.IsMaster {
+	if !b.IsMaster && !b.HasAPIServer {
 		return nil
 	}
 

nodeup/pkg/model/kube_controller_manager.go

@@ -46,7 +46,7 @@ var _ fi.NodeupModelBuilder = &KubeControllerManagerBuilder{}
 
 // Build is responsible for configuring the kube-controller-manager
 func (b *KubeControllerManagerBuilder) Build(c *fi.NodeupModelBuilderContext) error {
-	if !b.IsMaster {
+	if !b.IsMaster && !b.BootConfig.InstanceGroupRole.HasKubeControllerManager() {
 		return nil
 	}
 

nodeup/pkg/model/kube_proxy.go

@@ -45,6 +45,12 @@ func (b *KubeProxyBuilder) Build(c *fi.NodeupModelBuilderContext) error {
 		return nil
 	}
 
+	role := b.BootConfig.InstanceGroupRole
+	if role.HasEtcd() || role.HasKubeControllerManager() || role.HasScheduler() {
+		klog.V(2).Infof("Kube-proxy is disabled, role %s does not need kube-proxy.", role)
+		return nil
+	}
+
 	{
 		pod, err := b.buildPod()
 		if err != nil {

nodeup/pkg/model/kube_scheduler.go

@@ -62,7 +62,7 @@ var _ fi.NodeupModelBuilder = &KubeSchedulerBuilder{}
 
 // Build is responsible for building the manifest for the kube-scheduler
 func (b *KubeSchedulerBuilder) Build(c *fi.NodeupModelBuilderContext) error {
-	if !b.IsMaster {
+	if !b.IsMaster && !b.BootConfig.InstanceGroupRole.HasScheduler() {
 		return nil
 	}
 

nodeup/pkg/model/kubelet.go

@@ -131,6 +131,8 @@ func (b *KubeletBuilder) Build(c *fi.NodeupModelBuilderContext) error {
 				return fmt.Errorf("error querying Azure instance metadata: %v", err)
 			}
 			providerID = "azure://" + metadata.ResourceID
+		} else if b.CloudProvider() == kops.CloudProviderGCE {
+			// TODO
 		}
 
 		t, err := b.buildKubeletComponentConfig(kubeletConfig, providerID)
@@ -188,7 +190,7 @@ func (b *KubeletBuilder) Build(c *fi.NodeupModelBuilderContext) error {
 
 		{
 			var kubeconfig fi.Resource
-			if b.HasAPIServer {
+			if b.BootConfig.InstanceGroupRole.IsControlPlaneType() {
 				kubeconfig, err = b.buildControlPlaneKubeletKubeconfig(c)
 			} else {
 				kubeconfig, err = b.BuildBootstrapKubeconfig("kubelet", c)
@@ -863,7 +865,7 @@ func (b *KubeletBuilder) buildKubeletServingCertificate(c *fi.NodeupModelBuilder
 	dir := b.PathSrvKubernetes()
 
 	var cert, key fi.Resource
-	if !b.HasAPIServer {
+	if !b.BootConfig.InstanceGroupRole.IsControlPlaneType() {
 		var err error
 		cert, key, err = b.GetBootstrapCert(name, fi.CertificateIDCA)
 		if err != nil {

nodeup/pkg/model/manifests.go

@@ -38,7 +38,7 @@ func (b *ManifestsBuilder) Build(c *fi.NodeupModelBuilderContext) error {
 	ctx := c.Context()
 
 	// Write etcd manifests (currently etcd <=> master)
-	if b.IsMaster {
+	if b.IsMaster || b.BootConfig.InstanceGroupRole.HasEtcd() {
 		for _, manifest := range b.NodeupConfig.EtcdManifests {
 			p, err := vfs.Context.BuildVfsPath(manifest)
 			if err != nil {

pkg/apis/kops/instancegroup.go

@@ -394,6 +394,16 @@ func (g *InstanceGroup) IsControlPlane() bool {
 	}
 }
 
+// IsControlPlane checks if instanceGroup is a control-plane node.
+func (g *InstanceGroup) IsControlPlaneType() bool {
+	switch {
+	case g.Spec.Role.IsControlPlaneType():
+		return true
+	default:
+		return false
+	}
+}
+
 // IsAPIServerOnly checks if instanceGroup runs only the API Server
 func (g *InstanceGroup) IsAPIServerOnly() bool {
 	switch {

pkg/apis/kops/validation/instancegroup.go

@@ -73,6 +73,9 @@ func ValidateInstanceGroup(g *kops.InstanceGroup, cloud fi.Cloud, strict bool) f
 	case kops.InstanceGroupRoleNode:
 	case kops.InstanceGroupRoleBastion:
 	case kops.InstanceGroupRoleAPIServer:
+	case kops.InstanceGroupRoleEtcd:
+	case kops.InstanceGroupRoleScheduler:
+	case kops.InstanceGroupRoleKubeControllerManager:
 	default:
 		var supported []string
 		for _, role := range kops.AllInstanceGroupRoles {

pkg/apis/kops/validation/legacy.go

@@ -279,16 +279,41 @@ func DeepValidate(c *kops.Cluster, groups []*kops.InstanceGroup, strict bool, vf
 
 	controlPlaneGroupCount := 0
 	nodeGroupCount := 0
+	apiServerGroupCount := 0
+	etcdGroupCount := 0
+	kcmGroupCount := 0
+	schedulerGroupCount := 0
+	splitRoleCount := 0
 	for _, g := range groups {
 		if g.IsControlPlane() {
 			controlPlaneGroupCount++
-		} else {
+		}
+		if g.Spec.Role.HasNode() {
 			nodeGroupCount++
 		}
+		if g.RunsAPIServer() {
+			apiServerGroupCount++
+		}
+		if g.RunsEtcd() {
+			etcdGroupCount++
+		}
+		if g.RunsKubeControllerManager() {
+			kcmGroupCount++
+		}
+		if g.RunsScheduler() {
+			schedulerGroupCount++
+		}
+		if g.IsEtcdOnly() || g.IsSchedulerOnly() || g.IsKubeControllerManagerOnly() {
+			splitRoleCount++
+		}
+	}
+
+	if controlPlaneGroupCount > 0 && splitRoleCount > 0 {
+		return fmt.Errorf("cannot have both ControlPlane/Master InstanceGroups and split control plane InstanceGroups (Etcd, KubeControllerManager, Scheduler)")
 	}
 
-	if controlPlaneGroupCount == 0 {
-		return fmt.Errorf("must configure at least one ControlPlane InstanceGroup")
+	if controlPlaneGroupCount == 0 && (apiServerGroupCount == 0 || etcdGroupCount == 0 || kcmGroupCount == 0 || schedulerGroupCount == 0) {
+		return fmt.Errorf("must configure either a ControlPlane InstanceGroup or separate APIServer, Etcd, KubeControllerManager, and Scheduler InstanceGroups")
 	}
 
 	if nodeGroupCount == 0 {

pkg/apis/nodeup/config.go

@@ -391,7 +391,7 @@ func NewConfig(cluster *kops.Cluster, instanceGroup *kops.InstanceGroup) (*Confi
 		}
 	}
 
-	if instanceGroup.RunsAPIServer() {
+	if instanceGroup.RunsAPIServer() || instanceGroup.RunsEtcd() || instanceGroup.RunsKubeControllerManager() || instanceGroup.RunsScheduler() {
 		config.ConfigStore = &kops.ConfigStoreSpec{
 			Keypairs: cluster.Spec.ConfigStore.Keypairs,
 			Secrets:  cluster.Spec.ConfigStore.Secrets,
@@ -402,7 +402,7 @@ func NewConfig(cluster *kops.Cluster, instanceGroup *kops.InstanceGroup) (*Confi
 		config.Networking.EgressProxy = cluster.Spec.Networking.EgressProxy
 	}
 
-	if instanceGroup.IsControlPlane() {
+	if instanceGroup.RunsAPIServer() {
 		config.DNSZone = cluster.Spec.DNSZone
 	}
 
@@ -411,6 +411,20 @@ func NewConfig(cluster *kops.Cluster, instanceGroup *kops.InstanceGroup) (*Confi
 			KubeControllerManager: *cluster.Spec.KubeControllerManager,
 			KubeScheduler:         *cluster.Spec.KubeScheduler,
 		}
+	} else if instanceGroup.IsKubeControllerManagerOnly() {
+		config.ControlPlaneConfig = &ControlPlaneConfig{
+			KubeControllerManager: *cluster.Spec.KubeControllerManager,
+		}
+		config.APIServerConfig = &APIServerConfig{
+			ClusterDNSDomain: cluster.Spec.ClusterDNSDomain,
+		}
+	} else if instanceGroup.IsSchedulerOnly() {
+		config.ControlPlaneConfig = &ControlPlaneConfig{
+			KubeScheduler: *cluster.Spec.KubeScheduler,
+		}
+		config.APIServerConfig = &APIServerConfig{
+			ClusterDNSDomain: cluster.Spec.ClusterDNSDomain,
+		}
 	}
 
 	if len(instanceGroup.Spec.SysctlParameters) > 0 {

pkg/commands/toolbox_enroll.go

@@ -230,7 +230,7 @@ func enrollHost(ctx context.Context, ig *kops.InstanceGroup, bootstrapData *Boot
 
 	// We can't create the host resource in the API server for control-plane nodes,
 	// because the API server (likely) isn't running yet.
-	if !ig.IsControlPlane() {
+	if !ig.IsControlPlane() && !ig.IsEtcdOnly() && !ig.IsAPIServerOnly() {
 		if err := kubeClient.Create(ctx, hostData); err != nil {
 			return fmt.Errorf("failed to create host %s/%s: %w", hostData.Namespace, hostData.Name, err)
 		}

pkg/model/awsmodel/iam.go

@@ -274,6 +274,12 @@ func (b *IAMModelBuilder) roleKey(role iam.Subject) (string, bool) {
 		return strings.ToLower(string(kops.InstanceGroupRoleNode)), false
 	case *iam.NodeRoleBastion:
 		return strings.ToLower(string(kops.InstanceGroupRoleBastion)), false
+	case *iam.NodeRoleEtcd:
+		return strings.ToLower(string(kops.InstanceGroupRoleEtcd)), false
+	case *iam.NodeRoleScheduler:
+		return strings.ToLower(string(kops.InstanceGroupRoleScheduler)), false
+	case *iam.NodeRoleKubeControllerManager:
+		return strings.ToLower(string(kops.InstanceGroupRoleKubeControllerManager)), false
 
 	default:
 		klog.Fatalf("unknown node role type: %T", role)

pkg/model/bootstrapscript.go

@@ -166,7 +166,7 @@ func KeypairNamesForInstanceGroup(cluster *kops.Cluster, ig *kops.InstanceGroup)
 	keypairs := []string{"kubernetes-ca"}
 
 	// Add keypairs for default etcd clusters (main, events, and leases, not cilium)
-	if ig.IsControlPlane() {
+	if ig.IsControlPlane() || ig.IsEtcdOnly() {
 		for _, etcdCluster := range cluster.Spec.EtcdClusters {
 			k := etcdCluster.Name
 			if k != "events" && k != "main" && k != "leases" {
@@ -190,6 +190,8 @@ func KeypairNamesForInstanceGroup(cluster *kops.Cluster, ig *kops.InstanceGroup)
 
 	if ig.RunsAPIServer() {
 		keypairs = append(keypairs, "apiserver-aggregator-ca", "service-account", "etcd-clients-ca")
+	} else if ig.RunsKubeControllerManager() {
+		keypairs = append(keypairs, "service-account")
 	}
 
 	// Add keypairs for cilium etcd clusters (not the default etcd clusters)

pkg/model/components/channels/model.go

@@ -101,6 +101,7 @@ func (b *ChannelsBuilder) channelList() ([]string, error) {
 
 func (b *ChannelsBuilder) buildPod(channels []string) (*v1.Pod, error) {
 	image := b.AssetBuilder.RemapImage("registry.k8s.io/kops/channels:" + kopsroot.KopsVersionImageTag())
+	// image = "gcr.io/wfender-dev-20240625/kops/channels:1.37.0-alpha.1"
 
 	pod := &v1.Pod{
 		TypeMeta: metav1.TypeMeta{

pkg/model/defaults/volumes.go

@@ -38,6 +38,12 @@ func DefaultInstanceGroupVolumeSize(role kops.InstanceGroupRole) (int32, error)
 		return DefaultVolumeSizeMaster, nil
 	case kops.InstanceGroupRoleAPIServer:
 		return DefaultVolumeSizeNode, nil
+	case kops.InstanceGroupRoleEtcd:
+		return DefaultVolumeSizeMaster, nil
+	case kops.InstanceGroupRoleScheduler:
+		return DefaultVolumeSizeNode, nil
+	case kops.InstanceGroupRoleKubeControllerManager:
+		return DefaultVolumeSizeNode, nil
 	case kops.InstanceGroupRoleNode:
 		return DefaultVolumeSizeNode, nil
 	case kops.InstanceGroupRoleBastion: