[WIP] Initial work to make split control plane work.

Got the IG creation working.
Kubelet is coming up on the etcd IG.
Currently having issues with getting the etcd to attach to peers.

Author: cheftako

cmd/kops/create_cluster.go

@@ -193,6 +193,7 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command {
 	sshPublicKey := ""
 	associatePublicIP := false
 	encryptEtcdStorage := false
+	var controlPlaneCount int32
 
 	cmd := &cobra.Command{
 		Use:               "cluster [CLUSTER]",
@@ -212,6 +213,10 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command {
 				options.EncryptEtcdStorage = &encryptEtcdStorage
 			}
 
+			if cmd.Flag("control-plane-count").Changed || cmd.Flag("master-count").Changed {
+				options.ControlPlaneCount = &controlPlaneCount
+			}
+
 			if err := checkProjectFlag(cmd.Flag("project").Changed, options.Project); err != nil {
 				return err
 			}
@@ -295,9 +300,9 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command {
 		return []string{"pub"}, cobra.ShellCompDirectiveFilterFileExt
 	})
 
-	cmd.Flags().Int32Var(&options.ControlPlaneCount, "master-count", options.ControlPlaneCount, "Number of control-plane nodes. Defaults to one control-plane node per control-plane-zone")
+	cmd.Flags().Int32Var(&controlPlaneCount, "master-count", controlPlaneCount, "Number of control-plane nodes. Defaults to one control-plane node per control-plane-zone")
 	cmd.Flags().MarkDeprecated("master-count", "use --control-plane-count instead")
-	cmd.Flags().Int32Var(&options.ControlPlaneCount, "control-plane-count", options.ControlPlaneCount, "Number of control-plane nodes. Defaults to one control-plane node per control-plane-zone")
+	cmd.Flags().Int32Var(&controlPlaneCount, "control-plane-count", controlPlaneCount, "Number of control-plane nodes. Defaults to one control-plane node per control-plane-zone")
 	cmd.Flags().Int32Var(&options.NodeCount, "node-count", options.NodeCount, "Total number of worker nodes. Defaults to one node per zone")
 	if featureflag.APIServerNodes.Enabled() {
 		cmd.Flags().Int32Var(&options.APIServerCount, "api-server-count", options.APIServerCount, "Number of API server nodes. Defaults to 0.")

cmd/kops/create_cluster_integration_test.go

@@ -376,3 +376,10 @@ func runCreateClusterIntegrationTest(t *testing.T, srcDir string, version string
 	actualYAML := strings.Join(yamlAll, "\n\n---\n\n")
 	golden.AssertMatchesFile(t, actualYAML, path.Join(srcDir, expectedClusterPath))
 }
+
+// TestCreateClusterExperimentalRoles tests kops create cluster with ExperimentalRoles and control-plane-count=0
+func TestCreateClusterExperimentalRoles(t *testing.T) {
+	featureflag.ParseFlags("+APIServerNodes,+ExperimentalRoles")
+	defer featureflag.ParseFlags("-APIServerNodes,-ExperimentalRoles")
+	runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/experimental-roles", "v1alpha2")
+}

cmd/kops/create_instancegroup.go

@@ -142,7 +142,7 @@ func NewCmdCreateInstanceGroup(f *util.Factory, out io.Writer) *cobra.Command {
 			if r.HasCloudControllerManager() {
 				continue
 			}
-			if r.HasKubControllerManager() {
+			if r.HasKubeControllerManager() {
 				continue
 			}
 		}

nodeup/pkg/model/bootstrap_client.go

@@ -45,7 +45,7 @@ type BootstrapClientBuilder struct {
 }
 
 func (b BootstrapClientBuilder) Build(c *fi.NodeupModelBuilderContext) error {
-	if b.IsMaster {
+	if b.IsMaster || b.BootConfig.InstanceGroupRole.IsControlPlaneType() {
 		return nil
 	}
 

nodeup/pkg/model/channels.go

@@ -46,7 +46,7 @@ type ChannelsBuilder struct {
 var _ fi.NodeupModelBuilder = &ChannelsBuilder{}
 
 func (b *ChannelsBuilder) Build(c *fi.NodeupModelBuilderContext) error {
-	if !b.IsMaster {
+	if !b.IsMaster && !b.HasAPIServer {
 		return nil
 	}
 

nodeup/pkg/model/discovery_service.go

@@ -42,7 +42,7 @@ var _ fi.NodeupModelBuilder = &DiscoveryService{}
 func (b *DiscoveryService) Build(c *fi.NodeupModelBuilderContext) error {
 	ctx := c.Context()
 
-	if !b.IsMaster {
+	if !b.IsMaster && !b.HasAPIServer {
 		return nil
 	}
 	discoveryServiceOptions := b.DiscoveryServiceOptions()

nodeup/pkg/model/etcd_manager_tls.go

@@ -32,7 +32,7 @@ var _ fi.NodeupModelBuilder = &EtcdManagerTLSBuilder{}
 
 // Build is responsible for TLS configuration for etcd-manager
 func (b *EtcdManagerTLSBuilder) Build(ctx *fi.NodeupModelBuilderContext) error {
-	if !b.IsMaster {
+	if !b.IsMaster && !b.BootConfig.InstanceGroupRole.HasEtcd() {
 		return nil
 	}
 

nodeup/pkg/model/kops_controller.go

@@ -34,7 +34,7 @@ var _ fi.NodeupModelBuilder = &KopsControllerBuilder{}
 
 // Build is responsible for configuring keys that will be used by kops-controller (via hostPath)
 func (b *KopsControllerBuilder) Build(c *fi.NodeupModelBuilderContext) error {
-	if !b.IsMaster {
+	if !b.IsMaster && !b.HasAPIServer {
 		return nil
 	}
 

nodeup/pkg/model/kube_controller_manager.go

@@ -46,7 +46,7 @@ var _ fi.NodeupModelBuilder = &KubeControllerManagerBuilder{}
 
 // Build is responsible for configuring the kube-controller-manager
 func (b *KubeControllerManagerBuilder) Build(c *fi.NodeupModelBuilderContext) error {
-	if !b.IsMaster {
+	if !b.IsMaster && !b.BootConfig.InstanceGroupRole.HasKubeControllerManager() {
 		return nil
 	}
 

nodeup/pkg/model/kube_scheduler.go

@@ -62,7 +62,7 @@ var _ fi.NodeupModelBuilder = &KubeSchedulerBuilder{}
 
 // Build is responsible for building the manifest for the kube-scheduler
 func (b *KubeSchedulerBuilder) Build(c *fi.NodeupModelBuilderContext) error {
-	if !b.IsMaster {
+	if !b.IsMaster && !b.BootConfig.InstanceGroupRole.HasScheduler() {
 		return nil
 	}