[minikube]:windows node setup

cmd/minikube/cmd/start.go

@@ -187,6 +187,14 @@ func runStart(cmd *cobra.Command, _ []string) {
 		out.WarningT("Profile name '{{.name}}' is not valid", out.V{"name": ClusterFlagValue()})
 		exit.Message(reason.Usage, "Only alphanumeric and dashes '-' are permitted. Minimum 2 characters, starting with alphanumeric.")
 	}
+
+	// change the driver to hyperv, cni to flannel and container runtime to containerd if we have --node-os=windows
+	if cmd.Flags().Changed(nodeOS) {
+		viper.Set("driver", driver.HyperV)
+		viper.Set("cni", "flannel")
+		viper.Set(containerRuntime, constants.Containerd)
+
+	}
 	existing, err := config.Load(ClusterFlagValue())
 	if err != nil && !config.IsNotExist(err) {
 		kind := reason.HostConfigLoad
@@ -356,13 +364,6 @@ func provisionWithDriver(cmd *cobra.Command, ds registry.DriverState, existing *
 		os.Exit(0)
 	}
 
-	// If preloadWindowsIso flag is true then cache the windows ISO
-	if viper.GetBool(preloadWindowsIso) {
-		if err := download.WindowsISO(viper.GetString(windowsNodeVersion)); err != nil {
-			return node.Starter{}, errors.Wrap(err, "Failed to cache Windows ISO")
-		}
-	}
-
 	if driver.IsVM(driverName) && !driver.IsSSH(driverName) {
 		url, err := download.ISO(viper.GetStringSlice(isoURL), cmd.Flags().Changed(isoURL))
 		if err != nil {
@@ -474,6 +475,13 @@ func startWithDriver(cmd *cobra.Command, starter node.Starter, existing *config.
 	// target total and number of control-plane nodes
 	numCPNodes := 1
 	numNodes := viper.GetInt(nodes)
+	// if we have -node-os flag set, then nodes flag will be set to 2
+	//  it means one of the nodes is a control-plane node and the other is a windows worker node
+	// so we need to reduce the numNodes by 1
+	if cmd.Flags().Changed(nodeOS) {
+		numNodes--
+	}
+
 	if existing != nil {
 		numCPNodes = 0
 		for _, n := range existing.Nodes {
@@ -499,15 +507,46 @@ func startWithDriver(cmd *cobra.Command, starter node.Starter, existing *config.
 				KubernetesVersion: starter.Cfg.KubernetesConfig.KubernetesVersion,
 				ContainerRuntime:  starter.Cfg.KubernetesConfig.ContainerRuntime,
 				Worker:            true,
+				Guest: config.Guest{
+					Name:    "linux",
+					Version: "latest",
+					URL:     "",
+				},
 			}
 			if i < numCPNodes { // starter node is also counted as (primary) cp node
 				n.ControlPlane = true
 			}
 		}
 
 		out.Ln("") // extra newline for clarity on the command line
+		// 1st call
 		if err := node.Add(starter.Cfg, n, viper.GetBool(deleteOnFailure)); err != nil {
-			return nil, errors.Wrap(err, "adding node")
+			return nil, errors.Wrap(err, "adding linux node")
+		}
+	}
+
+	// we currently trigger the windows node start if the user has set the --windows-node-version or --node-os flag
+	// we might need to get rid of --windows-node-version in the future and just use --node-os flag
+	// start windows node. trigger windows node start  if windows node version or node node os is set at the time of minikube start
+	if cmd.Flags().Changed(windowsNodeVersion) || cmd.Flags().Changed(nodeOS) {
+		// TODO: if windows node version is set to windows server 2022 then the windows node name should be minikube-ws2022
+		nodeName := node.Name(numNodes + 1)
+		n := config.Node{
+			Name:              nodeName,
+			Port:              starter.Cfg.APIServerPort,
+			KubernetesVersion: starter.Cfg.KubernetesConfig.KubernetesVersion,
+			ContainerRuntime:  starter.Cfg.KubernetesConfig.ContainerRuntime,
+			Worker:            true,
+			Guest: config.Guest{
+				Name:    "windows",
+				Version: viper.GetString(windowsNodeVersion),
+				URL:     viper.GetString(windowsVhdURL),
+			},
+		}
+
+		out.Ln("") // extra newline for clarity on the command line
+		if err := node.Add(starter.Cfg, n, viper.GetBool(deleteOnFailure)); err != nil {
+			return nil, errors.Wrap(err, "adding windows node")
 		}
 	}
 
@@ -1316,11 +1355,32 @@ func validateFlags(cmd *cobra.Command, drvName string) { //nolint:gocyclo
 			exit.Message(reason.Usage, "{{.err}}", out.V{"err": err})
 		}
 
-		// set preloadWindowsIso to true since we need to download the windows ISO file
-		viper.Set(preloadWindowsIso, true)
+	}
 
+	if cmd.Flags().Changed(nodeOS) {
+		if err := validMultiNodeOS(viper.GetString(nodeOS)); err != nil {
+			exit.Message(reason.Usage, "{{.err}}", out.V{"err": err})
+		}
+
+		if viper.GetInt(nodes) != 2 {
+			exit.Message(reason.Usage, "The --nodes flag must be set to 2 when using --node-os")
+		}
 	}
 
+	if cmd.Flags().Changed(windowsVhdURL) {
+		if viper.GetString(windowsVhdURL) == "" {
+			// set a default URL if the user has not specified one
+			viper.Set(windowsVhdURL, constants.DefaultWindowsVhdURL)
+			exit.Message(reason.Usage, "The --windows-vhd-url flag must be set to a valid URL")
+		}
+
+		// add validation logic for the windows vhd URL
+		url := viper.GetString(windowsVhdURL)
+		if !strings.HasSuffix(url, ".vhd") && !strings.HasSuffix(url, ".vhdx") {
+			exit.Message(reason.Usage, "The --windows-vhd-url flag must point to a valid VHD or VHDX file")
+		}
+	} ////
+
 	if cmd.Flags().Changed(staticIP) {
 		if err := validateStaticIP(viper.GetString(staticIP), drvName, viper.GetString(subnet)); err != nil {
 			exit.Message(reason.Usage, "{{.err}}", out.V{"err": err})
@@ -1481,6 +1541,24 @@ func validateOSandVersion(os, version string) error {
 	return nil
 }
 
+// validateMultiNodeOS validates the supplied OS for multiple nodes
+func validMultiNodeOS(osString string) error {
+	if !strings.HasPrefix(osString, "[") || !strings.HasSuffix(osString, "]") {
+		return errors.Errorf("invalid OS string format: must be enclosed in [ ]")
+	}
+
+	osString = strings.Trim(osString, "[]")
+	osString = strings.ReplaceAll(osString, " ", "")
+
+	osValues := strings.Split(osString, ",")
+
+	if len(osValues) != 2 || osValues[0] != "linux" || osValues[1] != "windows" {
+		return errors.Errorf("invalid OS string format: must be [linux,windows]")
+	}
+
+	return nil
+}
+
 // validateRuntime validates the supplied runtime
 func validateRuntime(rtime string) error {
 	validOptions := cruntime.ValidRuntimes()

cmd/minikube/cmd/start_flags.go

@@ -118,7 +118,6 @@ const (
 	ha                      = "ha"
 	nodes                   = "nodes"
 	preload                 = "preload"
-	preloadWindowsIso       = "preload-windows-iso"
 	deleteOnFailure         = "delete-on-failure"
 	forceSystemd            = "force-systemd"
 	kicBaseImage            = "base-image"
@@ -146,6 +145,8 @@ const (
 	gpus                    = "gpus"
 	autoPauseInterval       = "auto-pause-interval"
 	windowsNodeVersion      = "windows-node-version"
+	nodeOS                  = "node-os"
+	windowsVhdURL           = "windows-vhd-url"
 )
 
 var (
@@ -196,7 +197,6 @@ func initMinikubeFlags() {
 	startCmd.Flags().Bool(ha, false, "Create Highly Available Multi-Control Plane Cluster with a minimum of three control-plane nodes that will also be marked for work.")
 	startCmd.Flags().IntP(nodes, "n", 1, "The total number of nodes to spin up. Defaults to 1.")
 	startCmd.Flags().Bool(preload, true, "If set, download tarball of preloaded images if available to improve start time. Defaults to true.")
-	startCmd.Flags().Bool(preloadWindowsIso, false, "If set, download the Windows ISO to improve start time of setting up a windows node. Defaults to false.")
 	startCmd.Flags().Bool(noKubernetes, false, "If set, minikube VM/container will start without starting or configuring Kubernetes. (only works on new clusters)")
 	startCmd.Flags().Bool(deleteOnFailure, false, "If set, delete the current cluster if start fails and try again. Defaults to false.")
 	startCmd.Flags().Bool(forceSystemd, false, "If set, force the container runtime to use systemd as cgroup manager. Defaults to false.")
@@ -211,7 +211,9 @@ func initMinikubeFlags() {
 	startCmd.Flags().String(staticIP, "", "Set a static IP for the minikube cluster, the IP must be: private, IPv4, and the last octet must be between 2 and 254, for example 192.168.200.200 (Docker and Podman drivers only)")
 	startCmd.Flags().StringP(gpus, "g", "", "Allow pods to use your NVIDIA GPUs. Options include: [all,nvidia] (Docker driver with Docker container-runtime only)")
 	startCmd.Flags().Duration(autoPauseInterval, time.Minute*1, "Duration of inactivity before the minikube VM is paused (default 1m0s)")
-	startCmd.Flags().String(windowsNodeVersion, constants.DefaultWindowsNodeVersion, "The version of Windows to use for the Windows node on a multi-node cluster (e.g., 2019, 2022). Defaults to Windows Server 2022")
+	startCmd.Flags().String(windowsNodeVersion, constants.DefaultWindowsNodeVersion, "The version of Windows to use for the windows node on a multi-node cluster (e.g., 2025). Currently support Windows Server 2025")
+	startCmd.Flags().String(nodeOS, "node-os", "The OS to use for the node. Currently support 'linux, windows'. If not set, it will be set to the same as the control plane node.")
+	startCmd.Flags().String(windowsVhdURL, constants.DefaultWindowsVhdURL, "The VHD URL to use for the windows node on a multi-node cluster. If not set, it will be set to the default Windows Server 2025 VHD URL.")
 
 }
 

cmd/minikube/cmd/start_test.go

@@ -483,16 +483,12 @@ func TestValidateWindowsOSVersion(t *testing.T) {
 		errorMsg  string
 	}{
 		{
-			osVersion: "2019",
-			errorMsg:  "",
-		},
-		{
-			osVersion: "2022",
+			osVersion: "2025",
 			errorMsg:  "",
 		},
 		{
 			osVersion: "2023",
-			errorMsg:  "Invalid Windows Server OS Version: 2023. Valid OS version are: [2019 2022]",
+			errorMsg:  "Invalid Windows Server OS Version: 2023. Valid OS version are: [2025]",
 		},
 	}
 	for _, test := range tests {
@@ -509,6 +505,51 @@ func TestValidateWindowsOSVersion(t *testing.T) {
 	}
 }
 
+func TestValidMultiNodeOS(t *testing.T) {
+	var tests = []struct {
+		osString string
+		errorMsg string
+	}{
+		{
+			osString: "[linux,windows]",
+			errorMsg: "",
+		},
+		{
+			osString: "[linux, windows]",
+			errorMsg: "",
+		},
+		{
+			osString: "[windows,linux]",
+			errorMsg: "invalid OS string format: must be [linux,windows]",
+		},
+		{
+			osString: "[linux]",
+			errorMsg: "invalid OS string format: must be [linux,windows]",
+		},
+		{
+			osString: "[linux,windows,mac]",
+			errorMsg: "invalid OS string format: must be [linux,windows]",
+		},
+		{
+			osString: "linux,windows",
+			errorMsg: "invalid OS string format: must be enclosed in [ ]",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.osString, func(t *testing.T) {
+			got := validMultiNodeOS(test.osString)
+			gotError := ""
+			if got != nil {
+				gotError = got.Error()
+			}
+			if gotError != test.errorMsg {
+				t.Errorf("validMultiNodeOS(osString=%v): got %v, expected %v", test.osString, gotError, test.errorMsg)
+			}
+		})
+	}
+}
+
 func TestIsTwoDigitSemver(t *testing.T) {
 	var tcs = []struct {
 		desc     string

pkg/minikube/bootstrapper/bootstrapper.go

@@ -19,6 +19,7 @@ package bootstrapper
 import (
 	"time"
 
+	"github.com/docker/machine/libmachine/host"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
@@ -41,8 +42,11 @@ type Bootstrapper interface {
 	UpdateCluster(config.ClusterConfig) error
 	DeleteCluster(config.KubernetesConfig) error
 	WaitForNode(config.ClusterConfig, config.Node, time.Duration) error
+	SetupMinikubeCert(*host.Host) (string, error)
+	JoinClusterWindows(*host.Host, config.ClusterConfig, config.Node, string, time.Duration) (string, error)
 	JoinCluster(config.ClusterConfig, config.Node, string) error
 	UpdateNode(config.ClusterConfig, config.Node, cruntime.Manager) error
+	GenerateTokenWindows(config.ClusterConfig) (string, error)
 	GenerateToken(config.ClusterConfig) (string, error)
 	// LogCommands returns a map of log type to a command which will display that log.
 	LogCommands(config.ClusterConfig, LogOptions) map[string]string

pkg/minikube/bootstrapper/certs.go

@@ -64,6 +64,11 @@ type sharedCACerts struct {
 
 // SetupCerts gets the generated credentials required to talk to the APIServer.
 func SetupCerts(k8s config.ClusterConfig, n config.Node, pcpCmd command.Runner, cmd command.Runner) error {
+	// no need to setup certs for windows worker nodes as the master node already took care of this
+	if n.Guest.Name == "windows" {
+		return nil
+	}
+
 	localPath := localpath.Profile(k8s.KubernetesConfig.ClusterName)
 	klog.Infof("Setting up %s for IP: %s", localPath, n.IP)