diff --git a/cmd/minikube/cmd/node_add.go b/cmd/minikube/cmd/node_add.go index d6952ccbc3f2..607ffe15b01a 100644 --- a/cmd/minikube/cmd/node_add.go +++ b/cmd/minikube/cmd/node_add.go @@ -17,6 +17,9 @@ limitations under the License. package cmd import ( + "strings" + + "github.com/pkg/errors" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -36,6 +39,14 @@ var ( cpNode bool workerNode bool deleteNodeOnFailure bool + osType string + + osTypeLong = "This flag should only be used when adding a windows node to a cluster.\n\n" + + "Specify the OS of the node to add in the format 'os=OS_TYPE,version=VERSION'.\n" + + "This means that the node to be added will be a Windows node and the version of Windows OS to use for that node is Windows Server 2022.\n" + + "Example: $ minikube node add --os='os=windows,version=2022'\n" + + "Valid options for OS_TYPE are: linux, windows. If not specified, the default value is linux.\n" + + "You do not need to specify the --os flag if you are adding a linux node." ) var nodeAddCmd = &cobra.Command{ @@ -43,6 +54,20 @@ var nodeAddCmd = &cobra.Command{ Short: "Adds a node to the given cluster.", Long: "Adds a node to the given cluster config, and starts it.", Run: func(cmd *cobra.Command, _ []string) { + + osType, windowsVersion, err := parseOSFlag(osType) + if err != nil { + exit.Message(reason.Usage, "{{.err}}", out.V{"err": err}) + } + + if err := validateOSandVersion(osType, windowsVersion); err != nil { + exit.Message(reason.Usage, "{{.err}}", out.V{"err": err}) + } + + if osType == "windows" && cpNode { + exit.Message(reason.Usage, "Windows node cannot be used as control-plane nodes.") + } + co := mustload.Healthy(ClusterFlagValue()) cc := co.Config @@ -109,6 +134,42 @@ func init() { nodeAddCmd.Flags().BoolVar(&cpNode, "control-plane", false, "If set, added node will become a control-plane. Defaults to false. Currently only supported for existing HA (multi-control plane) clusters.") nodeAddCmd.Flags().BoolVar(&workerNode, "worker", true, "If set, added node will be available as worker. Defaults to true.") nodeAddCmd.Flags().BoolVar(&deleteNodeOnFailure, "delete-on-failure", false, "If set, delete the current cluster if start fails and try again. Defaults to false.") + nodeAddCmd.Flags().StringVar(&osType, "os", "linux", osTypeLong) nodeCmd.AddCommand(nodeAddCmd) } + +// parseOSFlag parses the --os flag value , 'os=OS_TYPE,version=VERSION', and returns the os type and version +// For example, 'os=windows,version=2022' The output will be os: 'windows' and version: '2022' respectively +func parseOSFlag(osFlagValue string) (string, string, error) { + // Remove all spaces from the input string + osFlagValue = strings.ReplaceAll(osFlagValue, " ", "") + parts := strings.Split(osFlagValue, ",") + osInfo := map[string]string{ + "os": "linux", // default value + "version": "", + } + + for _, part := range parts { + kv := strings.Split(part, "=") + if len(kv) != 2 { + return "", "", errors.Errorf("Invalid format for --os flag: %s", osFlagValue) + } + osInfo[kv[0]] = kv[1] + } + + // if os is specified to linux, set the version to empty string as it is not required + if osInfo["os"] == "linux" { + if osInfo["version"] != "" { + out.WarningT("Ignoring version flag for linux os. You do not need to specify the version for linux os.") + } + osInfo["version"] = "" + } + + // if os is specified to windows and version is not specified, set the default version to 2022(Windows Server 2022) + if osInfo["os"] == "windows" && osInfo["version"] == "" { + osInfo["version"] = "2022" + } + + return osInfo["os"], osInfo["version"], nil +} diff --git a/cmd/minikube/cmd/node_add_test.go b/cmd/minikube/cmd/node_add_test.go new file mode 100644 index 000000000000..1cde9cba7d2e --- /dev/null +++ b/cmd/minikube/cmd/node_add_test.go @@ -0,0 +1,134 @@ +/* +Copyright 2024 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cmd + +import ( + "testing" + + "github.com/pkg/errors" +) + +func TestValidateOS(t *testing.T) { + tests := []struct { + osType string + errorMsg string + }{ + {"linux", ""}, + {"windows", ""}, + {"foo", "Invalid OS: foo. Valid OS are: linux, windows"}, + } + for _, test := range tests { + t.Run(test.osType, func(t *testing.T) { + got := validateOS(test.osType) + gotError := "" + if got != nil { + gotError = got.Error() + } + if gotError != test.errorMsg { + t.Errorf("validateOS(osType=%v): got %v, expected %v", test.osType, got, test.errorMsg) + } + }) + } +} + +// TestParseOSFlag is the main test function for parseOSFlag +func TestParseOSFlag(t *testing.T) { + tests := []struct { + name string + input string + expectedOS string + expectedVer string + expectedErr error + }{ + { + name: "Valid input with all fields", + input: "os=windows,version=2019", + expectedOS: "windows", + expectedVer: "2019", + expectedErr: nil, + }, + { + name: "Valid input with default version for windows", + input: "os=windows", + expectedOS: "windows", + expectedVer: "2022", + expectedErr: nil, + }, + { + name: "Valid input with linux and no version", + input: "os=linux", + expectedOS: "linux", + expectedVer: "", + expectedErr: nil, + }, + { + name: "Invalid input with missing version", + input: "os=linux,version=", + expectedOS: "linux", + expectedVer: "", + expectedErr: nil, + }, + { + name: "Invalid input with extra comma", + input: "os=linux,version=,", + expectedOS: "", + expectedVer: "", + expectedErr: errors.New("Invalid format for --os flag: os=linux,version=,"), + }, + { + name: "Invalid input with no key-value pair", + input: "linux,version=2022", + expectedOS: "", + expectedVer: "", + expectedErr: errors.New("Invalid format for --os flag: linux,version=2022"), + }, + { + name: "Valid input with extra spaces", + input: "os=linux , version=latest", + expectedOS: "linux", + expectedVer: "", + expectedErr: nil, + }, + { + name: "Valid input with capital letters in keys", + input: "OS=linux,Version=2022", + expectedOS: "linux", + expectedVer: "", + expectedErr: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotOS, gotVer, err := parseOSFlag(tt.input) + + if tt.expectedErr != nil && (err == nil || err.Error() != tt.expectedErr.Error()) { + t.Errorf("Expected error %v, got %v", tt.expectedErr, err) + } else if tt.expectedErr == nil && err != nil { + t.Errorf("Expected no error, but got %v", err) + } + + if gotOS != tt.expectedOS { + t.Errorf("Expected OS %s, got %s", tt.expectedOS, gotOS) + } + + if gotVer != tt.expectedVer { + t.Errorf("Expected version %s, got %s", tt.expectedVer, gotVer) + } + }) + } +} diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index add01e155567..84b208ce1c35 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -45,6 +45,7 @@ import ( gopshost "github.com/shirou/gopsutil/v3/host" "github.com/spf13/cobra" "github.com/spf13/viper" + "golang.org/x/exp/maps" "golang.org/x/text/cases" "golang.org/x/text/language" "k8s.io/minikube/pkg/minikube/command" @@ -186,6 +187,14 @@ func runStart(cmd *cobra.Command, _ []string) { out.WarningT("Profile name '{{.name}}' is not valid", out.V{"name": ClusterFlagValue()}) exit.Message(reason.Usage, "Only alphanumeric and dashes '-' are permitted. Minimum 2 characters, starting with alphanumeric.") } + + // change the driver to hyperv, cni to flannel and container runtime to containerd if we have --node-os=windows + if cmd.Flags().Changed(nodeOS) { + viper.Set("driver", driver.HyperV) + viper.Set("cni", "flannel") + viper.Set(containerRuntime, constants.Containerd) + + } existing, err := config.Load(ClusterFlagValue()) if err != nil && !config.IsNotExist(err) { kind := reason.HostConfigLoad @@ -466,6 +475,13 @@ func startWithDriver(cmd *cobra.Command, starter node.Starter, existing *config. // target total and number of control-plane nodes numCPNodes := 1 numNodes := viper.GetInt(nodes) + // if we have -node-os flag set, then nodes flag will be set to 2 + // it means one of the nodes is a control-plane node and the other is a windows worker node + // so we need to reduce the numNodes by 1 + if cmd.Flags().Changed(nodeOS) { + numNodes-- + } + if existing != nil { numCPNodes = 0 for _, n := range existing.Nodes { @@ -491,6 +507,11 @@ func startWithDriver(cmd *cobra.Command, starter node.Starter, existing *config. KubernetesVersion: starter.Cfg.KubernetesConfig.KubernetesVersion, ContainerRuntime: starter.Cfg.KubernetesConfig.ContainerRuntime, Worker: true, + Guest: config.Guest{ + Name: "linux", + Version: "latest", + URL: "", + }, } if i < numCPNodes { // starter node is also counted as (primary) cp node n.ControlPlane = true @@ -498,8 +519,34 @@ func startWithDriver(cmd *cobra.Command, starter node.Starter, existing *config. } out.Ln("") // extra newline for clarity on the command line + // 1st call if err := node.Add(starter.Cfg, n, viper.GetBool(deleteOnFailure)); err != nil { - return nil, errors.Wrap(err, "adding node") + return nil, errors.Wrap(err, "adding linux node") + } + } + + // we currently trigger the windows node start if the user has set the --windows-node-version or --node-os flag + // we might need to get rid of --windows-node-version in the future and just use --node-os flag + // start windows node. trigger windows node start if windows node version or node node os is set at the time of minikube start + if cmd.Flags().Changed(windowsNodeVersion) || cmd.Flags().Changed(nodeOS) { + // TODO: if windows node version is set to windows server 2022 then the windows node name should be minikube-ws2022 + nodeName := node.Name(numNodes + 1) + n := config.Node{ + Name: nodeName, + Port: starter.Cfg.APIServerPort, + KubernetesVersion: starter.Cfg.KubernetesConfig.KubernetesVersion, + ContainerRuntime: starter.Cfg.KubernetesConfig.ContainerRuntime, + Worker: true, + Guest: config.Guest{ + Name: "windows", + Version: viper.GetString(windowsNodeVersion), + URL: viper.GetString(windowsVhdURL), + }, + } + + out.Ln("") // extra newline for clarity on the command line + if err := node.Add(starter.Cfg, n, viper.GetBool(deleteOnFailure)); err != nil { + return nil, errors.Wrap(err, "adding windows node") } } @@ -1303,6 +1350,37 @@ func validateFlags(cmd *cobra.Command, drvName string) { //nolint:gocyclo validateCNI(cmd, viper.GetString(containerRuntime)) } + if cmd.Flags().Changed(windowsNodeVersion) { + if err := validateWindowsOSVersion(viper.GetString(windowsNodeVersion)); err != nil { + exit.Message(reason.Usage, "{{.err}}", out.V{"err": err}) + } + + } + + if cmd.Flags().Changed(nodeOS) { + if err := validMultiNodeOS(viper.GetString(nodeOS)); err != nil { + exit.Message(reason.Usage, "{{.err}}", out.V{"err": err}) + } + + if viper.GetInt(nodes) != 2 { + exit.Message(reason.Usage, "The --nodes flag must be set to 2 when using --node-os") + } + } + + if cmd.Flags().Changed(windowsVhdURL) { + if viper.GetString(windowsVhdURL) == "" { + // set a default URL if the user has not specified one + viper.Set(windowsVhdURL, constants.DefaultWindowsVhdURL) + exit.Message(reason.Usage, "The --windows-vhd-url flag must be set to a valid URL") + } + + // add validation logic for the windows vhd URL + url := viper.GetString(windowsVhdURL) + if !strings.HasSuffix(url, ".vhd") && !strings.HasSuffix(url, ".vhdx") { + exit.Message(reason.Usage, "The --windows-vhd-url flag must point to a valid VHD or VHDX file") + } + } //// + if cmd.Flags().Changed(staticIP) { if err := validateStaticIP(viper.GetString(staticIP), drvName, viper.GetString(subnet)); err != nil { exit.Message(reason.Usage, "{{.err}}", out.V{"err": err}) @@ -1424,6 +1502,63 @@ func validateDiskSize(diskSize string) error { return nil } +// validateWindowsOSVersion validates the supplied window server os version +func validateWindowsOSVersion(osVersion string) error { + validOptions := node.ValidWindowsOSVersions() + + if validOptions[osVersion] { + return nil + } + + return errors.Errorf("Invalid Windows Server OS Version: %s. Valid OS version are: %s", osVersion, maps.Keys(validOptions)) +} + +// validateOS validates the supplied OS +func validateOS(os string) error { + validOptions := node.ValidOS() + + for _, validOS := range validOptions { + if os == validOS { + return nil + } + } + + return errors.Errorf("Invalid OS: %s. Valid OS are: %s", os, strings.Join(validOptions, ", ")) +} + +// validateOSandVersion validates the supplied OS and version +func validateOSandVersion(os, version string) error { + + if err := validateOS(os); err != nil { + return err + } + + if version != "" { + if err := validateWindowsOSVersion(version); err != nil { + return err + } + } + return nil +} + +// validateMultiNodeOS validates the supplied OS for multiple nodes +func validMultiNodeOS(osString string) error { + if !strings.HasPrefix(osString, "[") || !strings.HasSuffix(osString, "]") { + return errors.Errorf("invalid OS string format: must be enclosed in [ ]") + } + + osString = strings.Trim(osString, "[]") + osString = strings.ReplaceAll(osString, " ", "") + + osValues := strings.Split(osString, ",") + + if len(osValues) != 2 || osValues[0] != "linux" || osValues[1] != "windows" { + return errors.Errorf("invalid OS string format: must be [linux,windows]") + } + + return nil +} + // validateRuntime validates the supplied runtime func validateRuntime(rtime string) error { validOptions := cruntime.ValidRuntimes() diff --git a/cmd/minikube/cmd/start_flags.go b/cmd/minikube/cmd/start_flags.go index 4f9359bb25b5..76e74111f72e 100644 --- a/cmd/minikube/cmd/start_flags.go +++ b/cmd/minikube/cmd/start_flags.go @@ -144,6 +144,9 @@ const ( staticIP = "static-ip" gpus = "gpus" autoPauseInterval = "auto-pause-interval" + windowsNodeVersion = "windows-node-version" + nodeOS = "node-os" + windowsVhdURL = "windows-vhd-url" ) var ( @@ -208,6 +211,10 @@ func initMinikubeFlags() { startCmd.Flags().String(staticIP, "", "Set a static IP for the minikube cluster, the IP must be: private, IPv4, and the last octet must be between 2 and 254, for example 192.168.200.200 (Docker and Podman drivers only)") startCmd.Flags().StringP(gpus, "g", "", "Allow pods to use your NVIDIA GPUs. Options include: [all,nvidia] (Docker driver with Docker container-runtime only)") startCmd.Flags().Duration(autoPauseInterval, time.Minute*1, "Duration of inactivity before the minikube VM is paused (default 1m0s)") + startCmd.Flags().String(windowsNodeVersion, constants.DefaultWindowsNodeVersion, "The version of Windows to use for the windows node on a multi-node cluster (e.g., 2025). Currently support Windows Server 2025") + startCmd.Flags().String(nodeOS, "node-os", "The OS to use for the node. Currently support 'linux, windows'. If not set, it will be set to the same as the control plane node.") + startCmd.Flags().String(windowsVhdURL, constants.DefaultWindowsVhdURL, "The VHD URL to use for the windows node on a multi-node cluster. If not set, it will be set to the default Windows Server 2025 VHD URL.") + } // initKubernetesFlags inits the commandline flags for Kubernetes related options @@ -586,6 +593,7 @@ func generateNewConfigFromFlags(cmd *cobra.Command, k8sVersion string, rtime str SocketVMnetClientPath: detect.SocketVMNetClientPath(), SocketVMnetPath: detect.SocketVMNetPath(), StaticIP: viper.GetString(staticIP), + WindowsNodeVersion: viper.GetString(windowsNodeVersion), KubernetesConfig: config.KubernetesConfig{ KubernetesVersion: k8sVersion, ClusterName: ClusterFlagValue(), @@ -837,6 +845,7 @@ func updateExistingConfigFromFlags(cmd *cobra.Command, existing *config.ClusterC updateStringFromFlag(cmd, &cc.SocketVMnetClientPath, socketVMnetClientPath) updateStringFromFlag(cmd, &cc.SocketVMnetPath, socketVMnetPath) updateDurationFromFlag(cmd, &cc.AutoPauseInterval, autoPauseInterval) + updateStringFromFlag(cmd, &cc.WindowsNodeVersion, windowsNodeVersion) if cmd.Flags().Changed(kubernetesVersion) { kubeVer, err := getKubernetesVersion(existing) diff --git a/cmd/minikube/cmd/start_test.go b/cmd/minikube/cmd/start_test.go index 6a5b799c304f..e388bcf7e4d7 100644 --- a/cmd/minikube/cmd/start_test.go +++ b/cmd/minikube/cmd/start_test.go @@ -477,6 +477,79 @@ func TestValidateRuntime(t *testing.T) { } } +func TestValidateWindowsOSVersion(t *testing.T) { + var tests = []struct { + osVersion string + errorMsg string + }{ + { + osVersion: "2025", + errorMsg: "", + }, + { + osVersion: "2023", + errorMsg: "Invalid Windows Server OS Version: 2023. Valid OS version are: [2025]", + }, + } + for _, test := range tests { + t.Run(test.osVersion, func(t *testing.T) { + got := validateWindowsOSVersion(test.osVersion) + gotError := "" + if got != nil { + gotError = got.Error() + } + if gotError != test.errorMsg { + t.Errorf("ValidateWindowsOSVersion(osVersion=%v): got %v, expected %v", test.osVersion, got, test.errorMsg) + } + }) + } +} + +func TestValidMultiNodeOS(t *testing.T) { + var tests = []struct { + osString string + errorMsg string + }{ + { + osString: "[linux,windows]", + errorMsg: "", + }, + { + osString: "[linux, windows]", + errorMsg: "", + }, + { + osString: "[windows,linux]", + errorMsg: "invalid OS string format: must be [linux,windows]", + }, + { + osString: "[linux]", + errorMsg: "invalid OS string format: must be [linux,windows]", + }, + { + osString: "[linux,windows,mac]", + errorMsg: "invalid OS string format: must be [linux,windows]", + }, + { + osString: "linux,windows", + errorMsg: "invalid OS string format: must be enclosed in [ ]", + }, + } + + for _, test := range tests { + t.Run(test.osString, func(t *testing.T) { + got := validMultiNodeOS(test.osString) + gotError := "" + if got != nil { + gotError = got.Error() + } + if gotError != test.errorMsg { + t.Errorf("validMultiNodeOS(osString=%v): got %v, expected %v", test.osString, gotError, test.errorMsg) + } + }) + } +} + func TestIsTwoDigitSemver(t *testing.T) { var tcs = []struct { desc string diff --git a/pkg/minikube/bootstrapper/bootstrapper.go b/pkg/minikube/bootstrapper/bootstrapper.go index a51ccc13e631..f8e0df568d9d 100644 --- a/pkg/minikube/bootstrapper/bootstrapper.go +++ b/pkg/minikube/bootstrapper/bootstrapper.go @@ -19,6 +19,7 @@ package bootstrapper import ( "time" + "github.com/docker/machine/libmachine/host" "k8s.io/minikube/pkg/minikube/bootstrapper/images" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/constants" @@ -41,8 +42,11 @@ type Bootstrapper interface { UpdateCluster(config.ClusterConfig) error DeleteCluster(config.KubernetesConfig) error WaitForNode(config.ClusterConfig, config.Node, time.Duration) error + SetupMinikubeCert(*host.Host) (string, error) + JoinClusterWindows(*host.Host, config.ClusterConfig, config.Node, string, time.Duration) (string, error) JoinCluster(config.ClusterConfig, config.Node, string) error UpdateNode(config.ClusterConfig, config.Node, cruntime.Manager) error + GenerateTokenWindows(config.ClusterConfig) (string, error) GenerateToken(config.ClusterConfig) (string, error) // LogCommands returns a map of log type to a command which will display that log. LogCommands(config.ClusterConfig, LogOptions) map[string]string diff --git a/pkg/minikube/bootstrapper/certs.go b/pkg/minikube/bootstrapper/certs.go index 23f2cb3788ec..c367d2c50a72 100644 --- a/pkg/minikube/bootstrapper/certs.go +++ b/pkg/minikube/bootstrapper/certs.go @@ -64,6 +64,11 @@ type sharedCACerts struct { // SetupCerts gets the generated credentials required to talk to the APIServer. func SetupCerts(k8s config.ClusterConfig, n config.Node, pcpCmd command.Runner, cmd command.Runner) error { + // no need to setup certs for windows worker nodes as the master node already took care of this + if n.Guest.Name == "windows" { + return nil + } + localPath := localpath.Profile(k8s.KubernetesConfig.ClusterName) klog.Infof("Setting up %s for IP: %s", localPath, n.IP) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 325ec0817657..bf4be19781dd 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -35,6 +35,7 @@ import ( "github.com/blang/semver/v4" "github.com/docker/machine/libmachine" + "github.com/docker/machine/libmachine/host" "github.com/docker/machine/libmachine/state" "github.com/pkg/errors" core "k8s.io/api/core/v1" @@ -751,6 +752,78 @@ func (k *Bootstrapper) restartPrimaryControlPlane(cfg config.ClusterConfig) erro return nil } +// +// + +func (k *Bootstrapper) SetupMinikubeCert(host *host.Host) (string, error) { + out.Step(style.Provisioning, "Setting up minikube certificates folder...") + + certsDir := `C:\var\lib\minikube\certs` + k8sPkiDir := `C:\etc\kubernetes\pki` + caCert := `ca.crt` + + script := fmt.Sprintf( + `mkdir %s; `+ + `Copy-Item %s\%s -Destination %s; `+ + `Remove-Item %s\%s`, + certsDir, k8sPkiDir, caCert, certsDir, k8sPkiDir, caCert, + ) + + script = strings.ReplaceAll(script, `"`, `\"`) + + command := fmt.Sprintf("powershell -NoProfile -NonInteractive -Command \"%s\"", script) + klog.Infof("[executing] : %v", command) + + host.RunSSHCommand(command) + + return "", nil +} + +func (k *Bootstrapper) JoinClusterWindows(host *host.Host, cc config.ClusterConfig, n config.Node, joinCmd string, timeout time.Duration) (string, error) { + setLocationPath := `Set-Location -Path "C:\k"` + + psScript := fmt.Sprintf("%s; %s", setLocationPath, joinCmd) + + psScript = strings.ReplaceAll(psScript, `"`, `\"`) + + command := fmt.Sprintf("powershell -NoProfile -NonInteractive -Command \"%s\"", psScript) + klog.Infof("[executing] : %v", command) + + // TODO: Explore how to make this better; channels for result and errors for now exist + resultChan := make(chan string, 1) + errorChan := make(chan error, 1) + + go func() { + output, err := host.RunSSHCommand(command) + if err != nil { + errorChan <- err + return + } + + resultChan <- output + }() + + if timeout > 0 { + // If timeout is set, enforce it + select { + case result := <-resultChan: + return result, nil + case err := <-errorChan: + return "", err + case <-time.After(timeout): + return "", fmt.Errorf("operation timed out after %s", timeout) + } + } else { + // If no timeout is set, just wait for result or error + select { + case result := <-resultChan: + return result, nil + case err := <-errorChan: + return "", err + } + } +} + // JoinCluster adds new node to an existing cluster. func (k *Bootstrapper) JoinCluster(cc config.ClusterConfig, n config.Node, joinCmd string) error { // Join the control plane by specifying its token @@ -778,6 +851,29 @@ func (k *Bootstrapper) JoinCluster(cc config.ClusterConfig, n config.Node, joinC return nil } +// GenerateTokenWindows creates a token and returns the appropriate kubeadm join command to run, or the already existing token +func (k *Bootstrapper) GenerateTokenWindows(cc config.ClusterConfig) (string, error) { + tokenCmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("%s token create --print-join-command --ttl=0", bsutil.InvokeKubeadm(cc.KubernetesConfig.KubernetesVersion))) + r, err := k.c.RunCmd(tokenCmd) + if err != nil { + return "", errors.Wrap(err, "generating join command") + } + + joinCmd := r.Stdout.String() + // log the join command for debugging purposes + klog.Infof("Generated join command ===: %s", joinCmd) + joinCmd = strings.Replace(joinCmd, "kubeadm", ".\\kubeadm.exe", 1) + joinCmd = fmt.Sprintf("%s --ignore-preflight-errors=all", strings.TrimSpace(joinCmd)) + + // append the cri-socket flag to the join command for windows + joinCmd = fmt.Sprintf("%s --cri-socket \"npipe:////./pipe/containerd-containerd\"", joinCmd) + + // append --v=5 to the join command for windows + joinCmd = fmt.Sprintf("%s --v=5", joinCmd) + + return joinCmd, nil +} + // GenerateToken creates a token and returns the appropriate kubeadm join command to run, or the already existing token func (k *Bootstrapper) GenerateToken(cc config.ClusterConfig) (string, error) { // Take that generated token and use it to get a kubeadm join command @@ -931,6 +1027,12 @@ func (k *Bootstrapper) UpdateCluster(cfg config.ClusterConfig) error { // UpdateNode updates new or existing node. func (k *Bootstrapper) UpdateNode(cfg config.ClusterConfig, n config.Node, r cruntime.Manager) error { + // skip if the node is a windows node + if n.Guest.Name == "windows" { + klog.Infof("skipping node %v update, as it is a windows node", n) + return nil + } + klog.Infof("updating node %v ...", n) kubeletCfg, err := bsutil.NewKubeletConfig(cfg, n, r) diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go index d8dc05281e78..4e4c763b4879 100644 --- a/pkg/minikube/config/types.go +++ b/pkg/minikube/config/types.go @@ -109,6 +109,7 @@ type ClusterConfig struct { SSHAgentPID int GPUs string AutoPauseInterval time.Duration // Specifies interval of time to wait before checking if cluster should be paused + WindowsNodeVersion string // OS version of windows node } // KubernetesConfig contains the parameters used to configure the VM Kubernetes. @@ -148,6 +149,7 @@ type Node struct { ContainerRuntime string ControlPlane bool Worker bool + Guest Guest } // VersionedExtraOption holds information on flags to apply to a specific range @@ -181,3 +183,9 @@ type ScheduledStopConfig struct { InitiationTime int64 Duration time.Duration } + +type Guest struct { + Name string + Version string + URL string +} diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index 32bc7bebe9e7..1c0626173ae6 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -30,6 +30,8 @@ import ( var ( // SupportedArchitectures is the list of supported architectures SupportedArchitectures = [5]string{"amd64", "arm", "arm64", "ppc64le", "s390x"} + // IP Address for the control plane + MasterNodeIP = "" ) const ( @@ -163,6 +165,13 @@ const ( // Mirror CN AliyunMirror = "registry.cn-hangzhou.aliyuncs.com/google_containers" + + // DefaultWindowsNodeVersion is the default version of Windows node + DefaultWindowsNodeVersion = "2025" + + // DefaultWindowsVhdURL is the VHD download URL for Windows Server 2025. + // This will be used whenever the user does NOT supply --windows-vhd-url. + DefaultWindowsVhdURL = "https://minikubevhdimagebuider.blob.core.windows.net/versions/hybrid-minikube-windows-server.vhdx" ) var ( diff --git a/pkg/minikube/download/iso.go b/pkg/minikube/download/iso.go index 223940dc77a4..2d726aa72831 100644 --- a/pkg/minikube/download/iso.go +++ b/pkg/minikube/download/iso.go @@ -117,6 +117,7 @@ func downloadISO(isoURL string, skipChecksum bool) error { // Lock before we check for existence to avoid thundering herd issues dst := localISOPath(u) + if err := os.MkdirAll(filepath.Dir(dst), 0777); err != nil { return errors.Wrapf(err, "making cache image directory: %s", dst) } diff --git a/pkg/minikube/machine/client.go b/pkg/minikube/machine/client.go index ff959365da42..9e3c7cc18356 100644 --- a/pkg/minikube/machine/client.go +++ b/pkg/minikube/machine/client.go @@ -86,14 +86,19 @@ type LocalClient struct { flock *fslock.Lock } +// DefineGuest sets/tracks the guest OS for the host +func (api *LocalClient) DefineGuest(h *host.Host) { + api.legacyClient.DefineGuest(h) +} + // NewHost creates a new Host -func (api *LocalClient) NewHost(drvName string, rawDriver []byte) (*host.Host, error) { +func (api *LocalClient) NewHost(drvName string, guest host.Guest, rawDriver []byte) (*host.Host, error) { def := registry.Driver(drvName) if def.Empty() { return nil, fmt.Errorf("driver %q does not exist", drvName) } if def.Init == nil { - return api.legacyClient.NewHost(drvName, rawDriver) + return api.legacyClient.NewHost(drvName, guest, rawDriver) } d := def.Init() err := json.Unmarshal(rawDriver, d) @@ -106,6 +111,7 @@ func (api *LocalClient) NewHost(drvName string, rawDriver []byte) (*host.Host, e Name: d.GetMachineName(), Driver: d, DriverName: d.DriverName(), + Guest: guest, HostOptions: &host.Options{ AuthOptions: &auth.Options{ CertDir: api.certsDir, @@ -229,10 +235,14 @@ func (api *LocalClient) Create(h *host.Host) error { { "provisioning", func() error { - // Skippable because we don't reconfigure Docker? + // Skipped because we don't reconfigure Docker? if driver.BareMetal(h.Driver.DriverName()) { return nil } + // Skipped because we don't reconfigure Docker for Windows Host + if h.Guest.Name == "windows" { + return nil + } return provisionDockerMachine(h) }, }, diff --git a/pkg/minikube/machine/client_test.go b/pkg/minikube/machine/client_test.go index 7ddd379babe2..30c3b8d666f4 100644 --- a/pkg/minikube/machine/client_test.go +++ b/pkg/minikube/machine/client_test.go @@ -24,6 +24,7 @@ import ( "testing" "github.com/docker/machine/libmachine/drivers/plugin/localbinary" + "github.com/docker/machine/libmachine/host" "k8s.io/minikube/pkg/minikube/driver" _ "k8s.io/minikube/pkg/minikube/registry/drvs/virtualbox" @@ -70,19 +71,30 @@ func TestLocalClientNewHost(t *testing.T) { var tests = []struct { description string driver string + guest host.Guest rawDriver []byte err bool }{ { description: "host vbox correct", driver: driver.VirtualBox, - rawDriver: []byte(vboxConfig), + guest: host.Guest{ + Name: "linux", + Version: "1.0.0", + URL: "https://example.com/linux.iso", + }, + rawDriver: []byte(vboxConfig), }, { description: "host vbox incorrect", driver: driver.VirtualBox, - rawDriver: []byte("?"), - err: true, + guest: host.Guest{ + Name: "linux", + Version: "1.0.0", + URL: "https://example.com/linux.iso", + }, + rawDriver: []byte("?"), + err: true, }, } @@ -90,7 +102,7 @@ func TestLocalClientNewHost(t *testing.T) { test := test t.Run(test.description, func(t *testing.T) { t.Parallel() - host, err := c.NewHost(test.driver, test.rawDriver) + host, err := c.NewHost(test.driver, test.guest, test.rawDriver) // A few sanity checks that we can do on the host if host != nil { if host.DriverName != test.driver { @@ -99,6 +111,9 @@ func TestLocalClientNewHost(t *testing.T) { if host.Name != host.Driver.GetMachineName() { t.Errorf("Host name is not correct. Expected :%s, got: %s", host.Driver.GetMachineName(), host.Name) } + if host.Guest.Name != test.guest.Name { + t.Errorf("Host's guest os is not correct. Expected :%s, got: %s", test.guest.Name, host.Guest.Name) + } } if err != nil && !test.err { t.Errorf("Unexpected error: %v", err) diff --git a/pkg/minikube/machine/start.go b/pkg/minikube/machine/start.go index d71fa5e14033..90a98e2aa8ea 100644 --- a/pkg/minikube/machine/start.go +++ b/pkg/minikube/machine/start.go @@ -145,7 +145,7 @@ func createHost(api libmachine.API, cfg *config.ClusterConfig, n *config.Node) ( return nil, errors.Wrap(err, "marshal") } - h, err := api.NewHost(cfg.Driver, data) + h, err := api.NewHost(cfg.Driver, host.Guest(n.Guest), data) if err != nil { return nil, errors.Wrap(err, "new host") } @@ -155,11 +155,17 @@ func createHost(api libmachine.API, cfg *config.ClusterConfig, n *config.Node) ( h.HostOptions.AuthOptions.StorePath = localpath.MiniPath() h.HostOptions.EngineOptions = engineOptions(*cfg) + api.DefineGuest(h) + cstart := time.Now() klog.Infof("libmachine.API.Create for %q (driver=%q)", cfg.Name, cfg.Driver) if cfg.StartHostTimeout == 0 { cfg.StartHostTimeout = 6 * time.Minute + // windows nodes take longer to start, so we increase the timeout + if n.Guest.Name == "windows" { + cfg.StartHostTimeout = 10 * time.Minute + } } if err := timedCreateHost(h, api, cfg.StartHostTimeout); err != nil { return nil, errors.Wrap(err, "creating host") @@ -183,6 +189,7 @@ func timedCreateHost(h *host.Host, api libmachine.API, t time.Duration) error { create := make(chan error, 1) go func() { defer close(create) + klog.Infof("libmachine.API.Create starting for %q (GuestOS=%q)", h.Name, h.Guest.Name) create <- api.Create(h) }() @@ -300,6 +307,12 @@ func postStartSetup(h *host.Host, mc config.ClusterConfig) error { return nil } + // skip postStartSetup for windows guest os + if h.Guest.Name == "windows" { + klog.Infof("skipping postStartSetup for windows guest os") + return nil + } + k8sVer, err := semver.ParseTolerant(mc.KubernetesConfig.KubernetesVersion) if err != nil { klog.Errorf("unable to parse Kubernetes version: %s", mc.KubernetesConfig.KubernetesVersion) @@ -427,3 +440,26 @@ func addHostAliasCommand(name string, record string, sudo bool, path string) *ex path) return exec.Command("/bin/bash", "-c", script) } + +func AddHostAliasWindows(host *host.Host, controlPlaneIP string) (string, error) { + out.Step(style.Provisioning, "Adding host alias for control plane ...") + + path := "C:\\Windows\\System32\\drivers\\etc\\hosts" + entry := fmt.Sprintf("\t%s\tcontrol-plane.minikube.internal", controlPlaneIP) + + psScript := fmt.Sprintf( + `$hostsContent = Get-Content -Path "%s" -Raw -ErrorAction SilentlyContinue; `+ + `if ($hostsContent -notmatch [regex]::Escape("%s")) { `+ + `Add-Content -Path "%s" -Value "%s" -Force | Out-Null }`, + path, entry, path, entry, + ) + + psScript = strings.ReplaceAll(psScript, `"`, `\"`) + + command := fmt.Sprintf("powershell -NoProfile -NonInteractive -Command \"%s\"", psScript) + klog.Infof("[executing] : %v", command) + + host.RunSSHCommand(command) + + return "", nil +} diff --git a/pkg/minikube/machine/stop.go b/pkg/minikube/machine/stop.go index 4397052c7462..c9ddbd28a5c6 100644 --- a/pkg/minikube/machine/stop.go +++ b/pkg/minikube/machine/stop.go @@ -57,6 +57,7 @@ func stop(h *host.Host) error { } if driver.NeedsShutdown(h.DriverName) { + klog.Infof("GuestOS: %s", h.Guest.Name) if err := trySSHPowerOff(h); err != nil { return errors.Wrap(err, "ssh power off") } diff --git a/pkg/minikube/node/powershell.go b/pkg/minikube/node/powershell.go new file mode 100644 index 000000000000..48e679531681 --- /dev/null +++ b/pkg/minikube/node/powershell.go @@ -0,0 +1,70 @@ +package node + +import ( + "bytes" + "errors" + "fmt" + "os/exec" + "strings" + + "golang.org/x/crypto/ssh" + + "k8s.io/klog/v2" +) + +var powershell string + +var ( + ErrPowerShellNotFound = errors.New("powershell was not found in the path") + ErrNotAdministrator = errors.New("hyper-v commands have to be run as an Administrator") + ErrNotInstalled = errors.New("hyper-V PowerShell Module is not available") +) + +func init() { + powershell, _ = exec.LookPath("powershell.exe") +} + +func cmdOut(args ...string) (string, error) { + args = append([]string{"-NoProfile", "-NonInteractive"}, args...) + cmd := exec.Command(powershell, args...) + klog.Infof("[executing ==>] : %v %v", powershell, strings.Join(args, " ")) + var stdout bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + err := cmd.Run() + klog.Infof("[stdout =====>] : %s", stdout.String()) + klog.Infof("[stderr =====>] : %s", stderr.String()) + return stdout.String(), err +} + +func cmd(args ...string) error { + _, err := cmdOut(args...) + return err +} + +func CmdOutSSH(client *ssh.Client, script string) (string, error) { + session, err := client.NewSession() + if err != nil { + return "", err + } + defer session.Close() + + command := fmt.Sprintf("powershell -NoProfile -NonInteractive -Command \"%s\"", script) + klog.Infof("[executing] : %v", command) + + var stdout, stderr bytes.Buffer + session.Stdout = &stdout + session.Stderr = &stderr + + err = session.Run(command) + klog.Infof("[stdout =====>] : %s", stdout.String()) + klog.Infof("[stderr =====>] : %s", stderr.String()) + return stdout.String(), err +} + +func cmdSSH(client *ssh.Client, args ...string) error { + script := strings.Join(args, " ") + _, err := CmdOutSSH(client, script) + return err +} diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index 0ca736708d16..bf41a9076758 100755 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -108,35 +108,60 @@ func Start(starter Starter) (*kubeconfig.Settings, error) { // nolint:gocyclo return nil, config.Write(viper.GetString(config.ProfileName), starter.Cfg) } - // wait for preloaded tarball to finish downloading before configuring runtimes - waitCacheRequiredImages(&cacheGroup) + // log starter.Node.OS here + klog.Infof("Node OS: %s", starter.Node.Guest.Name) + if starter.Node.Guest.Name != "windows" { + // wait for preloaded tarball to finish downloading before configuring runtimes + waitCacheRequiredImages(&cacheGroup) + } sv, err := util.ParseKubernetesVersion(starter.Node.KubernetesVersion) if err != nil { return nil, errors.Wrap(err, "Failed to parse Kubernetes version") } + klog.Infof("Kubernetes version: %s", sv) - // configure the runtime (docker, containerd, crio) - cr := configureRuntimes(starter.Runner, *starter.Cfg, sv) + var cr cruntime.Manager + if starter.Node.Guest.Name != "windows" { + // configure the runtime (docker, containerd, crio) only for windows nodes + cr = configureRuntimes(starter.Runner, *starter.Cfg, sv) - // check if installed runtime is compatible with current minikube code - if err = cruntime.CheckCompatibility(cr); err != nil { - return nil, err - } + // check if installed runtime is compatible with current minikube code + if err = cruntime.CheckCompatibility(cr); err != nil { + return nil, err + } - showVersionInfo(starter.Node.KubernetesVersion, cr) + showVersionInfo(starter.Node.KubernetesVersion, cr) + } + klog.Infof("configureRuntimes done: cr=%v", cr) // add "host.minikube.internal" dns alias (intentionally non-fatal) hostIP, err := cluster.HostIP(starter.Host, starter.Cfg.Name) if err != nil { klog.Errorf("Unable to get host IP: %v", err) - } else if err := machine.AddHostAlias(starter.Runner, constants.HostAlias, hostIP); err != nil { - klog.Errorf("Unable to add minikube host alias: %v", err) + } + + if starter.Node.Guest.Name != "windows" { + if err := machine.AddHostAlias(starter.Runner, constants.HostAlias, hostIP); err != nil { + klog.Warningf("Unable to add host alias: %v", err) + } + } else { + out.Step(style.Provisioning, "Configuring Windows node...") + if stdout, err := machine.AddHostAliasWindows(starter.Host, constants.MasterNodeIP); err != nil { + klog.Warningf("Unable to add host alias: %v", err) + } else { + klog.Infof("Host alias added: %s", stdout) + } } var kcs *kubeconfig.Settings var bs bootstrapper.Bootstrapper if config.IsPrimaryControlPlane(*starter.Cfg, *starter.Node) { + constants.MasterNodeIP, err = starter.Host.Driver.GetIP() + if err != nil { + klog.Errorf("Unable to get driver IP: %v", err) + } + klog.Infof("Driver IP: %s", constants.MasterNodeIP) // [re]start primary control-plane node kcs, bs, err = startPrimaryControlPlane(starter, cr) if err != nil { @@ -251,6 +276,18 @@ func Start(starter Starter) (*kubeconfig.Settings, error) { // nolint:gocyclo addons.UpdateConfigToDisable(starter.Cfg) } + // for windows node prepare the linux control plane node for windows-specific flannel CNI config + if config.IsPrimaryControlPlane(*starter.Cfg, *starter.Node) && starter.Cfg.WindowsNodeVersion == "2022" { + if err := prepareLinuxNode(starter.Runner); err != nil { + klog.Errorf("Failed to prepare Linux node for Windows-specific Flannel CNI config: %v", err) + } + + // set up flannel network issues + if err := configureFlannelCNI(); err != nil { + klog.Errorf("error configuring flannel CNI: %v", err) + } + } + // Write enabled addons to the config before completion klog.Infof("writing updated cluster config ...") return kcs, config.Write(viper.GetString(config.ProfileName), starter.Cfg) @@ -315,6 +352,7 @@ func startPrimaryControlPlane(starter Starter, cr cruntime.Manager) (*kubeconfig func joinCluster(starter Starter, cpBs bootstrapper.Bootstrapper, bs bootstrapper.Bootstrapper) error { start := time.Now() klog.Infof("joinCluster: %+v", starter.Cfg) + out.Step(style.Waiting, "Joining {{.name}} to the cluster", out.V{"name": starter.Node.Name}) defer func() { klog.Infof("duration metric: took %s to joinCluster", time.Since(start)) }() @@ -334,35 +372,93 @@ func joinCluster(starter Starter, cpBs bootstrapper.Bootstrapper, bs bootstrappe klog.Infof("successfully removed existing %s node %q from cluster: %+v", role, starter.Node.Name, starter.Node) } - joinCmd, err := cpBs.GenerateToken(*starter.Cfg) - if err != nil { - return fmt.Errorf("error generating join token: %w", err) + // declare joinCmd variable + var joinCmd string + var err error + + // if node is a windows node, generate the join command + if starter.Node.Guest.Name == "windows" { + joinCmd, err = cpBs.GenerateTokenWindows(*starter.Cfg) + if err != nil { + return fmt.Errorf("error generating join token: %w", err) + } + } else { + joinCmd, err = cpBs.GenerateToken(*starter.Cfg) + if err != nil { + return fmt.Errorf("error generating join token: %w", err) + } } + klog.Infof("join command: %s", joinCmd) + join := func() error { klog.Infof("trying to join %s node %q to cluster: %+v", role, starter.Node.Name, starter.Node) - if err := bs.JoinCluster(*starter.Cfg, *starter.Node, joinCmd); err != nil { - klog.Errorf("%s node failed to join cluster, will retry: %v", role, err) - - // reset node to revert any changes made by previous kubeadm init/join - klog.Infof("resetting %s node %q before attempting to rejoin cluster...", role, starter.Node.Name) - if _, err := starter.Runner.RunCmd(exec.Command("/bin/bash", "-c", fmt.Sprintf("%s reset --force", bsutil.InvokeKubeadm(starter.Cfg.KubernetesConfig.KubernetesVersion)))); err != nil { - klog.Infof("kubeadm reset failed, continuing anyway: %v", err) - } else { - klog.Infof("successfully reset %s node %q", role, starter.Node.Name) + if starter.Node.Guest.Name != "windows" { + if err := bs.JoinCluster(*starter.Cfg, *starter.Node, joinCmd); err != nil { + // log the error message and retry + klog.Errorf("%s node failed to join cluster, will retry: %v", role, err) + + // reset node to revert any changes made by previous kubeadm init/join + klog.Infof("resetting %s node %q before attempting to rejoin cluster...", role, starter.Node.Name) + if _, err := starter.Runner.RunCmd(exec.Command("/bin/bash", "-c", fmt.Sprintf("%s reset --force", bsutil.InvokeKubeadm(starter.Cfg.KubernetesConfig.KubernetesVersion)))); err != nil { + klog.Infof("kubeadm reset failed, continuing anyway: %v", err) + } else { + klog.Infof("successfully reset %s node %q", role, starter.Node.Name) + } + + return err + } + } else { + driverIP, err := starter.Host.Driver.GetIP() + if err != nil { + klog.Errorf("Unable to get driver IP: %v", err) } + klog.Infof("Driver IP: %s", driverIP) + + // Call with a timeout of 30 seconds + timeout := 20 * time.Second + + if commandResult, err := bs.JoinClusterWindows(starter.Host, *starter.Cfg, *starter.Node, joinCmd, timeout); err != nil { + klog.Infof("%s node failed to join cluster, will retry: %v", role, err) + klog.Infof("command result: %s", commandResult) + + // sort out the certificates issues + if cmd, err := bs.SetupMinikubeCert(starter.Host); err != nil { + klog.Errorf("error setting minikube folder error script: %v", err) + } else { + klog.Infof("command result: %s", cmd) + // retry the join command + if commandResult, err := bs.JoinClusterWindows(starter.Host, *starter.Cfg, *starter.Node, joinCmd, 0); err != nil { + klog.Errorf("error retrying join command: %v, command result: %s", err, commandResult) + return err + } - return err + // set up flannel network issues + if err := prepareWindowsNodeFlannel(); err != nil { + klog.Errorf("error preparing windows node flannel: %v", err) + } + + // set up kube-proxy issues + if err := prepareWindowsNodeKubeProxy(); err != nil { + klog.Errorf("error preparing windows node kube-proxy: %v", err) + } + } + // return err + + } } return nil } if err := retry.Expo(join, 10*time.Second, 3*time.Minute); err != nil { - return fmt.Errorf("error joining %s node %q to cluster: %w", role, starter.Node.Name, err) + if starter.Node.Guest.Name != "windows" { + return fmt.Errorf("error joining %s node %q to cluster: %w", role, starter.Node.Name, err) + } } if err := cpBs.LabelAndUntaintNode(*starter.Cfg, *starter.Node); err != nil { return fmt.Errorf("error applying %s node %q label: %w", role, starter.Node.Name, err) } + return nil } @@ -663,15 +759,21 @@ func startMachine(cfg *config.ClusterConfig, node *config.Node, delOnFail bool) if err != nil { return runner, preExists, m, host, errors.Wrap(err, "Failed to start host") } + // log that we managed to call startHostInternal + klog.Infof("startHostInternal returned: %v, %v, %v", host, preExists, err) runner, err = machine.CommandRunner(host) if err != nil { return runner, preExists, m, host, errors.Wrap(err, "Failed to get command runner") } + // log that we managed to get a command runner + klog.Infof("CommandRunner returned: %v", runner) ip, err := validateNetwork(host, runner, cfg.KubernetesConfig.ImageRepository) if err != nil { return runner, preExists, m, host, errors.Wrap(err, "Failed to validate network") } + // log that we managed to validate the network + klog.Infof("validateNetwork returned: %v", ip) if driver.IsQEMU(host.Driver.DriverName()) && network.IsBuiltinQEMU(cfg.Network) { apiServerPort, err := getPort() @@ -687,6 +789,12 @@ func startMachine(cfg *config.ClusterConfig, node *config.Node, delOnFail bool) out.FailureT("Failed to set NO_PROXY Env. Please use `export NO_PROXY=$NO_PROXY,{{.ip}}`.", out.V{"ip": ip}) } + // log that we managed to exclude the IP from the proxy + klog.Infof("Excluded IP from proxy: %v", ip) + + // log the result of the function + klog.Infof("startMachine returned: %v, %v, %v, %v", runner, preExists, m, host) + return runner, preExists, m, host, err } @@ -707,7 +815,9 @@ func getPort() (int, error) { // startHostInternal starts a new minikube host using a VM or None func startHostInternal(api libmachine.API, cc *config.ClusterConfig, n *config.Node, delOnFail bool) (*host.Host, bool, error) { + klog.Infof("StartHost: %s %+v", cc.Name, n) host, exists, err := machine.StartHost(api, cc, n) + klog.Infof("StartHost returned: %v, %v, %v", host, exists, err) if err == nil { return host, exists, nil } @@ -922,6 +1032,52 @@ func prepareNone() { } } +func configureFlannelCNI() error { + err := cmd("kubectl apply -f https://raw.githubusercontent.com/vrapolinario/MinikubeWindowsContainers/main/kube-flannel.yaml") + if err != nil { + klog.Errorf("failed to apply kube-flannel configuration: %v\n", err) + } + + roll_err := cmd("kubectl rollout restart ds kube-flannel-ds -n kube-flannel") + if roll_err != nil { + klog.Errorf("failed to restart kube-flannel daemonset: %v\n", roll_err) + } + klog.Infof("Successfully applied the configuration.") + + return nil +} + +// prepare windows node by flannel configuration +func prepareWindowsNodeFlannel() error { + err := cmd("kubectl apply -f https://raw.githubusercontent.com/vrapolinario/MinikubeWindowsContainers/main/flannel-overlay.yaml") + if err != nil { + klog.Errorf("failed to apply flannel configuration: %v\n", err) + } + klog.Infof("Successfully applied the configuration.") + return nil +} + +// prepare linux nodes for Windows-specific Flannel CNI config +func prepareLinuxNode(runner command.Runner) error { + c := exec.Command("sudo", "sysctl", "net.bridge.bridge-nf-call-iptables=1") + if rr, err := runner.RunCmd(c); err != nil { + klog.Infof("couldn't run %q command. error: %v", rr.Command(), err) + } + // log that we managed to run the command + klog.Infof("Successfully ran the command.") + return nil +} + +// prepare windows node kube-proxy yaml configuration +func prepareWindowsNodeKubeProxy() error { + err := cmd("kubectl apply -f https://raw.githubusercontent.com/vrapolinario/MinikubeWindowsContainers/main/kube-proxy.yaml") + if err != nil { + klog.Errorf("failed to apply kube-proxy configuration: %v\n", err) + } + klog.Infof("Successfully applied the configuration.") + return nil +} + // addCoreDNSEntry adds host name and IP record to the DNS by updating CoreDNS's ConfigMap. // ref: https://coredns.io/plugins/hosts/ // note: there can be only one 'hosts' block in CoreDNS's ConfigMap (avoid "plugin/hosts: this plugin can only be used once per Server Block" error) @@ -997,3 +1153,15 @@ To see benchmarks checkout https://minikube.sigs.k8s.io/docs/benchmarks/cpuusage `, out.V{"drivers": altDriverList.String()}) } } + +// ValidWindowsOSVersions lists the supported Windows OS versions +func ValidWindowsOSVersions() map[string]bool { + // TODO: add more versions as they are tested and supported + // return map[string]bool{"2019": true, "2022": true, "2025": true} + return map[string]bool{"2025": true} +} + +// ValidOS lists the supported OSes +func ValidOS() []string { + return []string{"linux", "windows"} +} diff --git a/pkg/minikube/tests/api_mock.go b/pkg/minikube/tests/api_mock.go index 022d1103c524..bcbc7fc8dc30 100644 --- a/pkg/minikube/tests/api_mock.go +++ b/pkg/minikube/tests/api_mock.go @@ -68,8 +68,13 @@ func (api *MockAPI) Close() error { return nil } +// DefineGuest sets/tracks the guest OS for the host +func (api *MockAPI) DefineGuest(h *host.Host) { + api.Logf("MockAPI.DefineGuest: guest=%q", h.Guest.Name) +} + // NewHost creates a new host.Host instance. -func (api *MockAPI) NewHost(drvName string, rawDriver []byte) (*host.Host, error) { +func (api *MockAPI) NewHost(drvName string, guest host.Guest, rawDriver []byte) (*host.Host, error) { var driver MockDriver if err := json.Unmarshal(rawDriver, &driver); err != nil { return nil, errors.Wrap(err, "error unmarshalling json") @@ -80,6 +85,7 @@ func (api *MockAPI) NewHost(drvName string, rawDriver []byte) (*host.Host, error RawDriver: rawDriver, Driver: &MockDriver{}, Name: fmt.Sprintf("mock-machine-%.8f", rand.Float64()), + Guest: guest, HostOptions: &host.Options{ AuthOptions: &auth.Options{}, SwarmOptions: &swarm.Options{}, diff --git a/site/content/en/docs/commands/start.md b/site/content/en/docs/commands/start.md index e092deff2a8f..fc600dc256fe 100644 --- a/site/content/en/docs/commands/start.md +++ b/site/content/en/docs/commands/start.md @@ -122,6 +122,7 @@ minikube start [flags] --vm-driver driver DEPRECATED, use driver instead. --wait strings comma separated list of Kubernetes components to verify and wait for after starting a cluster. defaults to "apiserver,system_pods", available options: "apiserver,system_pods,default_sa,apps_running,node_ready,kubelet" . other acceptable values are 'all' or 'none', 'true' and 'false' (default [apiserver,system_pods]) --wait-timeout duration max time to wait per Kubernetes or host to be healthy. (default 6m0s) + --windows-node-version string The version of Windows to use for the windows node on a multi-node cluster (e.g., 2025). Currently support Windows Server 2025 ``` ### Options inherited from parent commands diff --git a/test/integration/testdata/skaffold/leeroy-app/app.go b/test/integration/testdata/skaffold/leeroy-app/app.go index 40f9c08afaca..a9039a9ebfb6 100644 --- a/test/integration/testdata/skaffold/leeroy-app/app.go +++ b/test/integration/testdata/skaffold/leeroy-app/app.go @@ -1,3 +1,19 @@ +/* +Copyright 2024 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package main import ( diff --git a/test/integration/testdata/skaffold/leeroy-web/web.go b/test/integration/testdata/skaffold/leeroy-web/web.go index 69f927856513..ad4da9637883 100644 --- a/test/integration/testdata/skaffold/leeroy-web/web.go +++ b/test/integration/testdata/skaffold/leeroy-web/web.go @@ -1,3 +1,19 @@ +/* +Copyright 2024 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package main import (