From 08a4ee60beea3b1b210fa01a1f7fecb07843d7e7 Mon Sep 17 00:00:00 2001 From: Bob Sira Date: Tue, 13 Aug 2024 14:46:57 +0100 Subject: [PATCH 01/14] windows node init setup --- cmd/minikube/cmd/start.go | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index d9939b3f5622..b980126d4b45 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -506,8 +506,27 @@ func startWithDriver(cmd *cobra.Command, starter node.Starter, existing *config. } out.Ln("") // extra newline for clarity on the command line + // 1st call if err := node.Add(starter.Cfg, n, viper.GetBool(deleteOnFailure)); err != nil { - return nil, errors.Wrap(err, "adding node") + return nil, errors.Wrap(err, "adding linux node") + } + + // start windows node. trigger windows node start only if windows node version is set at the time of minikube start + if cmd.Flags().Changed(windowsNodeVersion) { + // TODO: if windows node version is set to windows server 2022 then the windows node name should be minikube-ws2022 + nodeName := node.Name(numNodes + 1) + n := config.Node{ + Name: nodeName, + Port: starter.Cfg.APIServerPort, + KubernetesVersion: starter.Cfg.KubernetesConfig.KubernetesVersion, + ContainerRuntime: starter.Cfg.KubernetesConfig.ContainerRuntime, + Worker: true, + } + + out.Ln("") // extra newline for clarity on the command line + if err := node.Add(starter.Cfg, n, viper.GetBool(deleteOnFailure)); err != nil { + return nil, errors.Wrap(err, "adding windows node") + } } } From df3e08a272e55026b0f16225adf916fc4913ea4b Mon Sep 17 00:00:00 2001 From: Bob Sira Date: Thu, 2 Jan 2025 12:09:20 +0000 Subject: [PATCH 02/14] logic for windows node initialization --- cmd/minikube/cmd/start.go | 32 +-- pkg/minikube/bootstrapper/bootstrapper.go | 3 + pkg/minikube/bootstrapper/certs.go | 5 + pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 95 ++++++++ pkg/minikube/config/types.go | 1 + pkg/minikube/constants/constants.go | 2 + pkg/minikube/machine/client.go | 16 +- pkg/minikube/machine/start.go | 79 ++++++- pkg/minikube/node/powershell.go | 41 ++++ pkg/minikube/node/start.go | 219 ++++++++++++++++--- 10 files changed, 447 insertions(+), 46 deletions(-) create mode 100644 pkg/minikube/node/powershell.go diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index b980126d4b45..102db830e0dc 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -499,6 +499,7 @@ func startWithDriver(cmd *cobra.Command, starter node.Starter, existing *config. KubernetesVersion: starter.Cfg.KubernetesConfig.KubernetesVersion, ContainerRuntime: starter.Cfg.KubernetesConfig.ContainerRuntime, Worker: true, + OS: "linux", } if i < numCPNodes { // starter node is also counted as (primary) cp node n.ControlPlane = true @@ -510,23 +511,24 @@ func startWithDriver(cmd *cobra.Command, starter node.Starter, existing *config. if err := node.Add(starter.Cfg, n, viper.GetBool(deleteOnFailure)); err != nil { return nil, errors.Wrap(err, "adding linux node") } + } - // start windows node. trigger windows node start only if windows node version is set at the time of minikube start - if cmd.Flags().Changed(windowsNodeVersion) { - // TODO: if windows node version is set to windows server 2022 then the windows node name should be minikube-ws2022 - nodeName := node.Name(numNodes + 1) - n := config.Node{ - Name: nodeName, - Port: starter.Cfg.APIServerPort, - KubernetesVersion: starter.Cfg.KubernetesConfig.KubernetesVersion, - ContainerRuntime: starter.Cfg.KubernetesConfig.ContainerRuntime, - Worker: true, - } + // start windows node. trigger windows node start only if windows node version is set at the time of minikube start + if cmd.Flags().Changed(windowsNodeVersion) { + // TODO: if windows node version is set to windows server 2022 then the windows node name should be minikube-ws2022 + nodeName := node.Name(numNodes + 1) + n := config.Node{ + Name: nodeName, + Port: starter.Cfg.APIServerPort, + KubernetesVersion: starter.Cfg.KubernetesConfig.KubernetesVersion, + ContainerRuntime: starter.Cfg.KubernetesConfig.ContainerRuntime, + Worker: true, + OS: "windows", + } - out.Ln("") // extra newline for clarity on the command line - if err := node.Add(starter.Cfg, n, viper.GetBool(deleteOnFailure)); err != nil { - return nil, errors.Wrap(err, "adding windows node") - } + out.Ln("") // extra newline for clarity on the command line + if err := node.Add(starter.Cfg, n, viper.GetBool(deleteOnFailure)); err != nil { + return nil, errors.Wrap(err, "adding windows node") } } diff --git a/pkg/minikube/bootstrapper/bootstrapper.go b/pkg/minikube/bootstrapper/bootstrapper.go index a51ccc13e631..120d0ecdc71a 100644 --- a/pkg/minikube/bootstrapper/bootstrapper.go +++ b/pkg/minikube/bootstrapper/bootstrapper.go @@ -41,8 +41,11 @@ type Bootstrapper interface { UpdateCluster(config.ClusterConfig) error DeleteCluster(config.KubernetesConfig) error WaitForNode(config.ClusterConfig, config.Node, time.Duration) error + SetMinikubeFolderErrorScript(string) (string, error) + JoinClusterWindows(string, config.ClusterConfig, config.Node, string) (string, error) JoinCluster(config.ClusterConfig, config.Node, string) error UpdateNode(config.ClusterConfig, config.Node, cruntime.Manager) error + GenerateTokenWindows(config.ClusterConfig) (string, error) GenerateToken(config.ClusterConfig) (string, error) // LogCommands returns a map of log type to a command which will display that log. LogCommands(config.ClusterConfig, LogOptions) map[string]string diff --git a/pkg/minikube/bootstrapper/certs.go b/pkg/minikube/bootstrapper/certs.go index 23f2cb3788ec..27ecd0602cad 100644 --- a/pkg/minikube/bootstrapper/certs.go +++ b/pkg/minikube/bootstrapper/certs.go @@ -64,6 +64,11 @@ type sharedCACerts struct { // SetupCerts gets the generated credentials required to talk to the APIServer. func SetupCerts(k8s config.ClusterConfig, n config.Node, pcpCmd command.Runner, cmd command.Runner) error { + // no need to setup certs for windows worker nodes as the master node already took care of this + if n.OS == "windows" { + return nil + } + localPath := localpath.Profile(k8s.KubernetesConfig.ClusterName) klog.Infof("Setting up %s for IP: %s", localPath, n.IP) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 325ec0817657..5b1c8728a354 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -37,6 +37,7 @@ import ( "github.com/docker/machine/libmachine" "github.com/docker/machine/libmachine/state" "github.com/pkg/errors" + "golang.org/x/crypto/ssh" core "k8s.io/api/core/v1" meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" @@ -751,6 +752,73 @@ func (k *Bootstrapper) restartPrimaryControlPlane(cfg config.ClusterConfig) erro return nil } +func (k *Bootstrapper) SetMinikubeFolderErrorScript(hostDriverIP string) (string, error) { + script := fmt.Sprintf( + `mkdir c:\var\lib\minikube\certs; Copy-Item C:\etc\kubernetes\pki\ca.crt -Destination C:\var\lib\Minikube\Certs; Remove-Item C:\etc\kubernetes\pki\ca.crt`) + + // Escape double quotes for remote execution + script = strings.ReplaceAll(script, `"`, `\"`) + + config := &ssh.ClientConfig{ + User: "Administrator", + Auth: []ssh.AuthMethod{ + ssh.Password("password"), + }, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + } + + // Add port 22 to the IP + hostDriverIP = hostDriverIP + ":22" + // Log the IP + klog.Infof("hostDriverIP: %s", hostDriverIP) + + client, err := ssh.Dial("tcp", hostDriverIP, config) + if err != nil { + klog.Warningf("Failed to connect: %v", err) + return "", err + } + defer client.Close() + + // Execute the script on the remote Windows node + return machine.CmdOut(client, script) +} + +// JoinCluster adds new node to an existing cluster. +func (k *Bootstrapper) JoinClusterWindows(hostDriverIP string, cc config.ClusterConfig, n config.Node, joinCmd string) (string, error) { + // define the path to set location + setLocationPath := `Set-Location -Path "C:\k"` + + // Form the script to be executed + psScript := fmt.Sprintf("%s; %s", setLocationPath, joinCmd) + + // Escape double quotes inside the script + psScript = strings.ReplaceAll(psScript, `"`, `\"`) + + config := &ssh.ClientConfig{ + User: "Administrator", + Auth: []ssh.AuthMethod{ + ssh.Password("password"), + }, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + } + + // Add port 22 to the IP + hostDriverIP = hostDriverIP + ":22" + // Log the IP + klog.Infof("hostDriverIP: %s", hostDriverIP) + + client, err := ssh.Dial("tcp", hostDriverIP, config) + if err != nil { + klog.Warningf("Failed to connect: %v", err) + return "", err + } + defer client.Close() + + // Execute the script on the remote Windows node + return machine.CmdOut(client, psScript) + +} + // JoinCluster adds new node to an existing cluster. func (k *Bootstrapper) JoinCluster(cc config.ClusterConfig, n config.Node, joinCmd string) error { // Join the control plane by specifying its token @@ -778,6 +846,27 @@ func (k *Bootstrapper) JoinCluster(cc config.ClusterConfig, n config.Node, joinC return nil } +// GenerateToken creates a token and returns the appropriate kubeadm join command to run, or the already existing token +func (k *Bootstrapper) GenerateTokenWindows(cc config.ClusterConfig) (string, error) { + tokenCmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("%s token create --print-join-command --ttl=0", bsutil.InvokeKubeadm(cc.KubernetesConfig.KubernetesVersion))) + r, err := k.c.RunCmd(tokenCmd) + if err != nil { + return "", errors.Wrap(err, "generating join command") + } + + joinCmd := r.Stdout.String() + // log the join command for debugging purposes + klog.Infof("Generated join command ===: %s", joinCmd) + joinCmd = strings.Replace(joinCmd, "kubeadm", ".\\kubeadm.exe", 1) + joinCmd = fmt.Sprintf("%s --ignore-preflight-errors=all", strings.TrimSpace(joinCmd)) + + // append the cri-socket flag to the join command for windows + joinCmd = fmt.Sprintf("%s --cri-socket \"npipe:////./pipe/containerd-containerd\"", joinCmd) + + return joinCmd, nil + +} + // GenerateToken creates a token and returns the appropriate kubeadm join command to run, or the already existing token func (k *Bootstrapper) GenerateToken(cc config.ClusterConfig) (string, error) { // Take that generated token and use it to get a kubeadm join command @@ -931,6 +1020,12 @@ func (k *Bootstrapper) UpdateCluster(cfg config.ClusterConfig) error { // UpdateNode updates new or existing node. func (k *Bootstrapper) UpdateNode(cfg config.ClusterConfig, n config.Node, r cruntime.Manager) error { + // skip if the node is a windows node + if n.OS == "windows" { + klog.Infof("skipping node %v update, as it is a windows node", n) + return nil + } + klog.Infof("updating node %v ...", n) kubeletCfg, err := bsutil.NewKubeletConfig(cfg, n, r) diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go index a5059e74bf64..3951b6ab6a51 100644 --- a/pkg/minikube/config/types.go +++ b/pkg/minikube/config/types.go @@ -149,6 +149,7 @@ type Node struct { ContainerRuntime string ControlPlane bool Worker bool + OS string } // VersionedExtraOption holds information on flags to apply to a specific range diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index eb957be288dd..a7c55f369477 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -30,6 +30,8 @@ import ( var ( // SupportedArchitectures is the list of supported architectures SupportedArchitectures = [5]string{"amd64", "arm", "arm64", "ppc64le", "s390x"} + // IP Address for the control plane + MasterNodeIP = "" ) const ( diff --git a/pkg/minikube/machine/client.go b/pkg/minikube/machine/client.go index ff959365da42..bd8f922b7934 100644 --- a/pkg/minikube/machine/client.go +++ b/pkg/minikube/machine/client.go @@ -86,14 +86,19 @@ type LocalClient struct { flock *fslock.Lock } +// DefineGuest is a method required by the libmachine.API interface. +func (api *LocalClient) DefineGuest(h *host.Host) { + api.legacyClient.DefineGuest(h) +} + // NewHost creates a new Host -func (api *LocalClient) NewHost(drvName string, rawDriver []byte) (*host.Host, error) { +func (api *LocalClient) NewHost(drvName string, guestOS string, rawDriver []byte) (*host.Host, error) { def := registry.Driver(drvName) if def.Empty() { return nil, fmt.Errorf("driver %q does not exist", drvName) } if def.Init == nil { - return api.legacyClient.NewHost(drvName, rawDriver) + return api.legacyClient.NewHost(drvName, guestOS, rawDriver) } d := def.Init() err := json.Unmarshal(rawDriver, d) @@ -106,6 +111,7 @@ func (api *LocalClient) NewHost(drvName string, rawDriver []byte) (*host.Host, e Name: d.GetMachineName(), Driver: d, DriverName: d.DriverName(), + GuestOS: guestOS, HostOptions: &host.Options{ AuthOptions: &auth.Options{ CertDir: api.certsDir, @@ -229,10 +235,14 @@ func (api *LocalClient) Create(h *host.Host) error { { "provisioning", func() error { - // Skippable because we don't reconfigure Docker? + // Skipped because we don't reconfigure Docker? if driver.BareMetal(h.Driver.DriverName()) { return nil } + // Skipped because we don't reconfigure Docker for Windows Host + if h.GuestOS == "windows" { + return nil + } return provisionDockerMachine(h) }, }, diff --git a/pkg/minikube/machine/start.go b/pkg/minikube/machine/start.go index d71fa5e14033..59fb75696552 100644 --- a/pkg/minikube/machine/start.go +++ b/pkg/minikube/machine/start.go @@ -17,6 +17,7 @@ limitations under the License. package machine import ( + "bytes" "encoding/json" "fmt" "net" @@ -28,6 +29,8 @@ import ( "strings" "time" + "golang.org/x/crypto/ssh" + "github.com/blang/semver/v4" "github.com/docker/machine/libmachine" "github.com/docker/machine/libmachine/drivers" @@ -145,7 +148,7 @@ func createHost(api libmachine.API, cfg *config.ClusterConfig, n *config.Node) ( return nil, errors.Wrap(err, "marshal") } - h, err := api.NewHost(cfg.Driver, data) + h, err := api.NewHost(cfg.Driver, n.OS, data) if err != nil { return nil, errors.Wrap(err, "new host") } @@ -155,6 +158,8 @@ func createHost(api libmachine.API, cfg *config.ClusterConfig, n *config.Node) ( h.HostOptions.AuthOptions.StorePath = localpath.MiniPath() h.HostOptions.EngineOptions = engineOptions(*cfg) + api.DefineGuest(h) + cstart := time.Now() klog.Infof("libmachine.API.Create for %q (driver=%q)", cfg.Name, cfg.Driver) @@ -183,6 +188,7 @@ func timedCreateHost(h *host.Host, api libmachine.API, t time.Duration) error { create := make(chan error, 1) go func() { defer close(create) + klog.Infof("libmachine.API.Create starting for %q (GuestOS=%q)", h.Name, h.GuestOS) create <- api.Create(h) }() @@ -300,6 +306,12 @@ func postStartSetup(h *host.Host, mc config.ClusterConfig) error { return nil } + // skip postStartSetup for windows guest os + if h.GuestOS == "windows" { + klog.Infof("skipping postStartSetup for windows guest os") + return nil + } + k8sVer, err := semver.ParseTolerant(mc.KubernetesConfig.KubernetesVersion) if err != nil { klog.Errorf("unable to parse Kubernetes version: %s", mc.KubernetesConfig.KubernetesVersion) @@ -427,3 +439,68 @@ func addHostAliasCommand(name string, record string, sudo bool, path string) *ex path) return exec.Command("/bin/bash", "-c", script) } + +func AddHostAliasWindows(controlPlaneIP string, hostDriverIP string) (string, error) { + // log controlPlaneIP + klog.Infof("controlPlaneIP: %s", controlPlaneIP) + path := "C:\\Windows\\System32\\drivers\\etc\\hosts" + psScript := handleHostScriptCommand(controlPlaneIP, path) + + // Escape double quotes inside the script + psScript = strings.ReplaceAll(psScript, `"`, `\"`) + + config := &ssh.ClientConfig{ + User: "Administrator", + Auth: []ssh.AuthMethod{ + ssh.Password("password"), + }, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + } + + // add 22 port to ip + hostDriverIP = hostDriverIP + ":22" + // log the ip + klog.Infof("hostDriverIP: %s", hostDriverIP) + + client, err := ssh.Dial("tcp", hostDriverIP, config) + if err != nil { + klog.Warningf("Failed to connect: %v", err) + } + defer client.Close() + + return CmdOut(client, psScript) +} + +func handleHostScriptCommand(ip string, path string) string { + entry := fmt.Sprintf("\t%s\tcontrol-plane.minikube.internal", ip) + script := fmt.Sprintf( + `$hostsContent = Get-Content -Path "%s" -Raw -ErrorAction SilentlyContinue; if ($hostsContent -notmatch [regex]::Escape("%s")) { Add-Content -Path "%s" -Value "%s" -Force | Out-Null }`, + path, entry, path, entry) + return script +} + +func CmdOut(client *ssh.Client, script string) (string, error) { + session, err := client.NewSession() + if err != nil { + return "", err + } + defer session.Close() + + command := fmt.Sprintf("powershell -NoProfile -NonInteractive -Command \"%s\"", script) + klog.Infof("[executing] : %v", command) + + var stdout, stderr bytes.Buffer + session.Stdout = &stdout + session.Stderr = &stderr + + err = session.Run(command) + klog.Infof("[stdout =====>] : %s", stdout.String()) + klog.Infof("[stderr =====>] : %s", stderr.String()) + return stdout.String(), err +} + +func cmd(client *ssh.Client, args ...string) error { + script := strings.Join(args, " ") + _, err := CmdOut(client, script) + return err +} diff --git a/pkg/minikube/node/powershell.go b/pkg/minikube/node/powershell.go new file mode 100644 index 000000000000..c8417e7fb100 --- /dev/null +++ b/pkg/minikube/node/powershell.go @@ -0,0 +1,41 @@ +package node + +import ( + "bytes" + "errors" + "os/exec" + "strings" + + "k8s.io/klog/v2" +) + +var powershell string + +var ( + ErrPowerShellNotFound = errors.New("powershell was not found in the path") + ErrNotAdministrator = errors.New("hyper-v commands have to be run as an Administrator") + ErrNotInstalled = errors.New("hyper-V PowerShell Module is not available") +) + +func init() { + powershell, _ = exec.LookPath("powershell.exe") +} + +func cmdOut(args ...string) (string, error) { + args = append([]string{"-NoProfile", "-NonInteractive"}, args...) + cmd := exec.Command(powershell, args...) + klog.Infof("[executing ==>] : %v %v", powershell, strings.Join(args, " ")) + var stdout bytes.Buffer + var stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + err := cmd.Run() + klog.Infof("[stdout =====>] : %s", stdout.String()) + klog.Infof("[stderr =====>] : %s", stderr.String()) + return stdout.String(), err +} + +func cmd(args ...string) error { + _, err := cmdOut(args...) + return err +} diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index dba655fcbc74..1b1d20d1a6ae 100755 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -108,35 +108,69 @@ func Start(starter Starter) (*kubeconfig.Settings, error) { // nolint:gocyclo return nil, config.Write(viper.GetString(config.ProfileName), starter.Cfg) } - // wait for preloaded tarball to finish downloading before configuring runtimes - waitCacheRequiredImages(&cacheGroup) + // log starter.Node.OS here + klog.Infof("Node OS: %s", starter.Node.OS) + if starter.Node.OS != "windows" { + // wait for preloaded tarball to finish downloading before configuring runtimes + waitCacheRequiredImages(&cacheGroup) + } sv, err := util.ParseKubernetesVersion(starter.Node.KubernetesVersion) if err != nil { return nil, errors.Wrap(err, "Failed to parse Kubernetes version") } + klog.Infof("Kubernetes version: %s", sv) - // configure the runtime (docker, containerd, crio) - cr := configureRuntimes(starter.Runner, *starter.Cfg, sv) + var cr cruntime.Manager + if starter.Node.OS != "windows" { + // configure the runtime (docker, containerd, crio) only for windows nodes + cr = configureRuntimes(starter.Runner, *starter.Cfg, sv) - // check if installed runtime is compatible with current minikube code - if err = cruntime.CheckCompatibility(cr); err != nil { - return nil, err - } + // check if installed runtime is compatible with current minikube code + if err = cruntime.CheckCompatibility(cr); err != nil { + return nil, err + } - showVersionInfo(starter.Node.KubernetesVersion, cr) + showVersionInfo(starter.Node.KubernetesVersion, cr) + } + klog.Infof("configureRuntimes done: cr=%v", cr) // add "host.minikube.internal" dns alias (intentionally non-fatal) hostIP, err := cluster.HostIP(starter.Host, starter.Cfg.Name) if err != nil { klog.Errorf("Unable to get host IP: %v", err) - } else if err := machine.AddHostAlias(starter.Runner, constants.HostAlias, hostIP); err != nil { - klog.Errorf("Unable to add minikube host alias: %v", err) + } + + if starter.Node.OS != "windows" { + if err := machine.AddHostAlias(starter.Runner, constants.HostAlias, hostIP); err != nil { + klog.Warningf("Unable to add host alias: %v", err) + } + } else { + out.Step(style.Provisioning, "Configuring Windows node...") + // log starter.Host.Driver.GetIP() + driverIP, err := starter.Host.Driver.GetIP() + if err != nil { + klog.Errorf("Unable to get driver IP: %v", err) + } + klog.Infof("Driver IP: %s", driverIP) + // log + // add "control-plane.minikube.internal" + // hack for windows node to find the primary control-plane node via it's fully qualified domain name + if stdout, err := machine.AddHostAliasWindows(constants.MasterNodeIP, driverIP); err != nil { + klog.Warningf("Unable to add host alias: %v", err) + } else { + klog.Infof("Host alias added: %s", stdout) + } } var kcs *kubeconfig.Settings var bs bootstrapper.Bootstrapper if config.IsPrimaryControlPlane(*starter.Cfg, *starter.Node) { + constants.MasterNodeIP, err = starter.Host.Driver.GetIP() + if err != nil { + klog.Errorf("Unable to get driver IP: %v", err) + } + klog.Infof("Driver IP: %s", constants.MasterNodeIP) // [re]start primary control-plane node kcs, bs, err = startPrimaryControlPlane(starter, cr) if err != nil { @@ -251,6 +285,18 @@ func Start(starter Starter) (*kubeconfig.Settings, error) { // nolint:gocyclo addons.UpdateConfigToDisable(starter.Cfg) } + // for windows node prepare the linux control plane node for windows-specific flannel CNI config + if config.IsPrimaryControlPlane(*starter.Cfg, *starter.Node) && starter.Cfg.WindowsNodeVersion == "2022" { + if err := prepareLinuxNode(starter.Runner); err != nil { + klog.Errorf("Failed to prepare Linux node for Windows-specific Flannel CNI config: %v", err) + } + + // set up flannel network issues + if err := configureFlannelCNI(); err != nil { + klog.Errorf("error configuring flannel CNI: %v", err) + } + } + // Write enabled addons to the config before completion klog.Infof("writing updated cluster config ...") return kcs, config.Write(viper.GetString(config.ProfileName), starter.Cfg) @@ -334,35 +380,94 @@ func joinCluster(starter Starter, cpBs bootstrapper.Bootstrapper, bs bootstrappe klog.Infof("successfully removed existing %s node %q from cluster: %+v", role, starter.Node.Name, starter.Node) } - joinCmd, err := cpBs.GenerateToken(*starter.Cfg) - if err != nil { - return fmt.Errorf("error generating join token: %w", err) + // declare joinCmd variable + var joinCmd string + var err error + + // if node is a windows node, generate the join command + if starter.Node.OS == "windows" { + joinCmd, err = cpBs.GenerateTokenWindows(*starter.Cfg) + if err != nil { + return fmt.Errorf("error generating join token: %w", err) + } + } else { + joinCmd, err = cpBs.GenerateToken(*starter.Cfg) + if err != nil { + return fmt.Errorf("error generating join token: %w", err) + } } + // log the join command + klog.Infof("join command: %s", joinCmd) + // making the to test the machine code + join := func() error { klog.Infof("trying to join %s node %q to cluster: %+v", role, starter.Node.Name, starter.Node) - if err := bs.JoinCluster(*starter.Cfg, *starter.Node, joinCmd); err != nil { - klog.Errorf("%s node failed to join cluster, will retry: %v", role, err) - - // reset node to revert any changes made by previous kubeadm init/join - klog.Infof("resetting %s node %q before attempting to rejoin cluster...", role, starter.Node.Name) - if _, err := starter.Runner.RunCmd(exec.Command("/bin/bash", "-c", fmt.Sprintf("%s reset --force", bsutil.InvokeKubeadm(starter.Cfg.KubernetesConfig.KubernetesVersion)))); err != nil { - klog.Infof("kubeadm reset failed, continuing anyway: %v", err) - } else { - klog.Infof("successfully reset %s node %q", role, starter.Node.Name) + if starter.Node.OS != "windows" { + if err := bs.JoinCluster(*starter.Cfg, *starter.Node, joinCmd); err != nil { + // log the error message and retry + klog.Errorf("%s node failed to join cluster, will retry: %v", role, err) + + // reset node to revert any changes made by previous kubeadm init/join + klog.Infof("resetting %s node %q before attempting to rejoin cluster...", role, starter.Node.Name) + if _, err := starter.Runner.RunCmd(exec.Command("/bin/bash", "-c", fmt.Sprintf("%s reset --force", bsutil.InvokeKubeadm(starter.Cfg.KubernetesConfig.KubernetesVersion)))); err != nil { + klog.Infof("kubeadm reset failed, continuing anyway: %v", err) + } else { + klog.Infof("successfully reset %s node %q", role, starter.Node.Name) + } + + return err } + } else { + driverIP, err := starter.Host.Driver.GetIP() + if err != nil { + klog.Errorf("Unable to get driver IP: %v", err) + } + klog.Infof("Driver IP: %s", driverIP) + + if commandResult, err := bs.JoinClusterWindows(driverIP, *starter.Cfg, *starter.Node, joinCmd); err != nil { + klog.Infof("%s node failed to join cluster, will retry: %v", role, err) + klog.Infof("command result: %s", commandResult) + + // sort out the certificates issues + if cmd, err := bs.SetMinikubeFolderErrorScript(driverIP); err != nil { + klog.Errorf("error setting minikube folder error script: %v", err) + } else { + klog.Infof("command result: %s", cmd) + // retry the join command + if commandResult, err := bs.JoinClusterWindows(driverIP, *starter.Cfg, *starter.Node, joinCmd); err != nil { + klog.Errorf("error retrying join command: %v, command result: %s", err, commandResult) + return err + } - return err + // set up flannel network issues + if err := prepareWindowsNodeFlannel(); err != nil { + klog.Errorf("error preparing windows node flannel: %v", err) + } + + // set up kube-proxy issues + if err := prepareWindowsNodeKubeProxy(); err != nil { + klog.Errorf("error preparing windows node kube-proxy: %v", err) + } + } + // return err + + } } return nil } if err := retry.Expo(join, 10*time.Second, 3*time.Minute); err != nil { - return fmt.Errorf("error joining %s node %q to cluster: %w", role, starter.Node.Name, err) + if starter.Node.OS != "windows" { + return fmt.Errorf("error joining %s node %q to cluster: %w", role, starter.Node.Name, err) + } } - if err := cpBs.LabelAndUntaintNode(*starter.Cfg, *starter.Node); err != nil { - return fmt.Errorf("error applying %s node %q label: %w", role, starter.Node.Name, err) + if starter.Cfg.WindowsNodeVersion != "2022" { + if err := cpBs.LabelAndUntaintNode(*starter.Cfg, *starter.Node); err != nil { + return fmt.Errorf("error applying %s node %q label: %w", role, starter.Node.Name, err) + } } + return nil } @@ -663,15 +768,21 @@ func startMachine(cfg *config.ClusterConfig, node *config.Node, delOnFail bool) if err != nil { return runner, preExists, m, host, errors.Wrap(err, "Failed to start host") } + // log that we managed to call startHostInternal + klog.Infof("startHostInternal returned: %v, %v, %v", host, preExists, err) runner, err = machine.CommandRunner(host) if err != nil { return runner, preExists, m, host, errors.Wrap(err, "Failed to get command runner") } + // log that we managed to get a command runner + klog.Infof("CommandRunner returned: %v", runner) ip, err := validateNetwork(host, runner, cfg.KubernetesConfig.ImageRepository) if err != nil { return runner, preExists, m, host, errors.Wrap(err, "Failed to validate network") } + // log that we managed to validate the network + klog.Infof("validateNetwork returned: %v", ip) if driver.IsQEMU(host.Driver.DriverName()) && network.IsBuiltinQEMU(cfg.Network) { apiServerPort, err := getPort() @@ -687,6 +798,12 @@ func startMachine(cfg *config.ClusterConfig, node *config.Node, delOnFail bool) out.FailureT("Failed to set NO_PROXY Env. Please use `export NO_PROXY=$NO_PROXY,{{.ip}}`.", out.V{"ip": ip}) } + // log that we managed to exclude the IP from the proxy + klog.Infof("Excluded IP from proxy: %v", ip) + + // log the result of the function + klog.Infof("startMachine returned: %v, %v, %v, %v", runner, preExists, m, host) + return runner, preExists, m, host, err } @@ -707,7 +824,9 @@ func getPort() (int, error) { // startHostInternal starts a new minikube host using a VM or None func startHostInternal(api libmachine.API, cc *config.ClusterConfig, n *config.Node, delOnFail bool) (*host.Host, bool, error) { + klog.Infof("StartHost: %s %+v", cc.Name, n) host, exists, err := machine.StartHost(api, cc, n) + klog.Infof("StartHost returned: %v, %v, %v", host, exists, err) if err == nil { return host, exists, nil } @@ -922,6 +1041,52 @@ func prepareNone() { } } +func configureFlannelCNI() error { + err := cmd("kubectl apply -f https://raw.githubusercontent.com/vrapolinario/MinikubeWindowsContainers/main/kube-flannel.yaml") + if err != nil { + klog.Errorf("failed to apply kube-flannel configuration: %v\n", err) + } + + roll_err := cmd("kubectl rollout restart ds kube-flannel-ds -n kube-flannel") + if roll_err != nil { + klog.Errorf("failed to restart kube-flannel daemonset: %v\n", roll_err) + } + klog.Infof("Successfully applied the configuration.") + + return nil +} + +// prepare windows node by flannel configuration +func prepareWindowsNodeFlannel() error { + err := cmd("kubectl apply -f https://raw.githubusercontent.com/vrapolinario/MinikubeWindowsContainers/main/flannel-overlay.yaml") + if err != nil { + klog.Errorf("failed to apply flannel configuration: %v\n", err) + } + klog.Infof("Successfully applied the configuration.") + return nil +} + +// prepare linux nodes for Windows-specific Flannel CNI config +func prepareLinuxNode(runner command.Runner) error { + c := exec.Command("sudo", "sysctl", "net.bridge.bridge-nf-call-iptables=1") + if rr, err := runner.RunCmd(c); err != nil { + klog.Infof("couldn't run %q command. error: %v", rr.Command(), err) + } + // log that we managed to run the command + klog.Infof("Successfully ran the command.") + return nil +} + +// prepare windows node kube-proxy yaml configuration +func prepareWindowsNodeKubeProxy() error { + err := cmd("kubectl apply -f https://raw.githubusercontent.com/vrapolinario/MinikubeWindowsContainers/main/kube-proxy.yaml") + if err != nil { + klog.Errorf("failed to apply kube-proxy configuration: %v\n", err) + } + klog.Infof("Successfully applied the configuration.") + return nil +} + // addCoreDNSEntry adds host name and IP record to the DNS by updating CoreDNS's ConfigMap. // ref: https://coredns.io/plugins/hosts/ // note: there can be only one 'hosts' block in CoreDNS's ConfigMap (avoid "plugin/hosts: this plugin can only be used once per Server Block" error) From f6ff78c7f2cca59b25a2bf3e9abca43ed408d449 Mon Sep 17 00:00:00 2001 From: Bob Sira Date: Thu, 30 Jan 2025 18:00:28 +0000 Subject: [PATCH 03/14] efforts to reduce node join command time --- pkg/minikube/bootstrapper/bootstrapper.go | 2 +- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 58 ++++++++++++++++---- pkg/minikube/machine/start.go | 1 + pkg/minikube/node/powershell.go | 29 ++++++++++ pkg/minikube/node/start.go | 12 ++-- 5 files changed, 85 insertions(+), 17 deletions(-) diff --git a/pkg/minikube/bootstrapper/bootstrapper.go b/pkg/minikube/bootstrapper/bootstrapper.go index 120d0ecdc71a..9bd5598284ea 100644 --- a/pkg/minikube/bootstrapper/bootstrapper.go +++ b/pkg/minikube/bootstrapper/bootstrapper.go @@ -42,7 +42,7 @@ type Bootstrapper interface { DeleteCluster(config.KubernetesConfig) error WaitForNode(config.ClusterConfig, config.Node, time.Duration) error SetMinikubeFolderErrorScript(string) (string, error) - JoinClusterWindows(string, config.ClusterConfig, config.Node, string) (string, error) + JoinClusterWindows(string, config.ClusterConfig, config.Node, string, time.Duration) (string, error) JoinCluster(config.ClusterConfig, config.Node, string) error UpdateNode(config.ClusterConfig, config.Node, cruntime.Manager) error GenerateTokenWindows(config.ClusterConfig) (string, error) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 5b1c8728a354..f2f0cca4c3f1 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -753,6 +753,7 @@ func (k *Bootstrapper) restartPrimaryControlPlane(cfg config.ClusterConfig) erro } func (k *Bootstrapper) SetMinikubeFolderErrorScript(hostDriverIP string) (string, error) { + out.Step(style.Provisioning, "Setting up minikube certificates folder...") script := fmt.Sprintf( `mkdir c:\var\lib\minikube\certs; Copy-Item C:\etc\kubernetes\pki\ca.crt -Destination C:\var\lib\Minikube\Certs; Remove-Item C:\etc\kubernetes\pki\ca.crt`) @@ -784,8 +785,8 @@ func (k *Bootstrapper) SetMinikubeFolderErrorScript(hostDriverIP string) (string } // JoinCluster adds new node to an existing cluster. -func (k *Bootstrapper) JoinClusterWindows(hostDriverIP string, cc config.ClusterConfig, n config.Node, joinCmd string) (string, error) { - // define the path to set location +func (k *Bootstrapper) JoinClusterWindows(hostDriverIP string, cc config.ClusterConfig, n config.Node, joinCmd string, timeout time.Duration) (string, error) { + // Define the path to set location setLocationPath := `Set-Location -Path "C:\k"` // Form the script to be executed @@ -804,19 +805,51 @@ func (k *Bootstrapper) JoinClusterWindows(hostDriverIP string, cc config.Cluster // Add port 22 to the IP hostDriverIP = hostDriverIP + ":22" - // Log the IP klog.Infof("hostDriverIP: %s", hostDriverIP) - client, err := ssh.Dial("tcp", hostDriverIP, config) - if err != nil { - klog.Warningf("Failed to connect: %v", err) - return "", err - } - defer client.Close() + // this need to be explored more to see if we can make it better + // Create channels for result and errors + resultChan := make(chan string, 1) + errorChan := make(chan error, 1) - // Execute the script on the remote Windows node - return machine.CmdOut(client, psScript) + go func() { + client, err := ssh.Dial("tcp", hostDriverIP, config) + if err != nil { + klog.Warningf("Failed to connect: %v", err) + errorChan <- err + return + } + defer client.Close() + // Execute the script on the remote Windows node + output, err := machine.CmdOut(client, psScript) + if err != nil { + errorChan <- err + return + } + + resultChan <- output + }() + + if timeout > 0 { + // If timeout is set, enforce it + select { + case result := <-resultChan: + return result, nil + case err := <-errorChan: + return "", err + case <-time.After(timeout): + return "", fmt.Errorf("operation timed out after %s", timeout) + } + } else { + // If no timeout is set, just wait for result or error + select { + case result := <-resultChan: + return result, nil + case err := <-errorChan: + return "", err + } + } } // JoinCluster adds new node to an existing cluster. @@ -863,6 +896,9 @@ func (k *Bootstrapper) GenerateTokenWindows(cc config.ClusterConfig) (string, er // append the cri-socket flag to the join command for windows joinCmd = fmt.Sprintf("%s --cri-socket \"npipe:////./pipe/containerd-containerd\"", joinCmd) + // append --v=5 to the join command for windows + joinCmd = fmt.Sprintf("%s --v=5", joinCmd) + return joinCmd, nil } diff --git a/pkg/minikube/machine/start.go b/pkg/minikube/machine/start.go index 59fb75696552..3dfaf0ea6177 100644 --- a/pkg/minikube/machine/start.go +++ b/pkg/minikube/machine/start.go @@ -442,6 +442,7 @@ func addHostAliasCommand(name string, record string, sudo bool, path string) *ex func AddHostAliasWindows(controlPlaneIP string, hostDriverIP string) (string, error) { // log controlPlaneIP + out.Step(style.Provisioning, "Adding host alias for control plane ...") klog.Infof("controlPlaneIP: %s", controlPlaneIP) path := "C:\\Windows\\System32\\drivers\\etc\\hosts" psScript := handleHostScriptCommand(controlPlaneIP, path) diff --git a/pkg/minikube/node/powershell.go b/pkg/minikube/node/powershell.go index c8417e7fb100..48e679531681 100644 --- a/pkg/minikube/node/powershell.go +++ b/pkg/minikube/node/powershell.go @@ -3,9 +3,12 @@ package node import ( "bytes" "errors" + "fmt" "os/exec" "strings" + "golang.org/x/crypto/ssh" + "k8s.io/klog/v2" ) @@ -39,3 +42,29 @@ func cmd(args ...string) error { _, err := cmdOut(args...) return err } + +func CmdOutSSH(client *ssh.Client, script string) (string, error) { + session, err := client.NewSession() + if err != nil { + return "", err + } + defer session.Close() + + command := fmt.Sprintf("powershell -NoProfile -NonInteractive -Command \"%s\"", script) + klog.Infof("[executing] : %v", command) + + var stdout, stderr bytes.Buffer + session.Stdout = &stdout + session.Stderr = &stderr + + err = session.Run(command) + klog.Infof("[stdout =====>] : %s", stdout.String()) + klog.Infof("[stderr =====>] : %s", stderr.String()) + return stdout.String(), err +} + +func cmdSSH(client *ssh.Client, args ...string) error { + script := strings.Join(args, " ") + _, err := CmdOutSSH(client, script) + return err +} diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index 1b1d20d1a6ae..cc707d35842a 100755 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -361,6 +361,7 @@ func startPrimaryControlPlane(starter Starter, cr cruntime.Manager) (*kubeconfig func joinCluster(starter Starter, cpBs bootstrapper.Bootstrapper, bs bootstrapper.Bootstrapper) error { start := time.Now() klog.Infof("joinCluster: %+v", starter.Cfg) + out.Step(style.Waiting, "Joining {{.name}} to the cluster", out.V{"name": starter.Node.Name}) defer func() { klog.Infof("duration metric: took %s to joinCluster", time.Since(start)) }() @@ -397,9 +398,7 @@ func joinCluster(starter Starter, cpBs bootstrapper.Bootstrapper, bs bootstrappe } } - // log the join command klog.Infof("join command: %s", joinCmd) - // making the to test the machine code join := func() error { klog.Infof("trying to join %s node %q to cluster: %+v", role, starter.Node.Name, starter.Node) @@ -425,7 +424,10 @@ func joinCluster(starter Starter, cpBs bootstrapper.Bootstrapper, bs bootstrappe } klog.Infof("Driver IP: %s", driverIP) - if commandResult, err := bs.JoinClusterWindows(driverIP, *starter.Cfg, *starter.Node, joinCmd); err != nil { + // Call with a timeout of 30 seconds + timeout := 30 * time.Second + + if commandResult, err := bs.JoinClusterWindows(driverIP, *starter.Cfg, *starter.Node, joinCmd, timeout); err != nil { klog.Infof("%s node failed to join cluster, will retry: %v", role, err) klog.Infof("command result: %s", commandResult) @@ -435,7 +437,7 @@ func joinCluster(starter Starter, cpBs bootstrapper.Bootstrapper, bs bootstrappe } else { klog.Infof("command result: %s", cmd) // retry the join command - if commandResult, err := bs.JoinClusterWindows(driverIP, *starter.Cfg, *starter.Node, joinCmd); err != nil { + if commandResult, err := bs.JoinClusterWindows(driverIP, *starter.Cfg, *starter.Node, joinCmd, 0); err != nil { klog.Errorf("error retrying join command: %v, command result: %s", err, commandResult) return err } @@ -1165,7 +1167,7 @@ To see benchmarks checkout https://minikube.sigs.k8s.io/docs/benchmarks/cpuusage // ValidWindowsOSVersions lists the supported Windows OS versions func ValidWindowsOSVersions() map[string]bool { - return map[string]bool{"2019": true, "2022": true} + return map[string]bool{"2019": true, "2022": true, "2025": true} } // ValidOS lists the supported OSes From 83dc84ff546a5c07f0bf35763cee3ba43d030279 Mon Sep 17 00:00:00 2001 From: Bob Sira Date: Tue, 11 Feb 2025 02:01:34 +0000 Subject: [PATCH 04/14] logic for only server 2025 --- cmd/minikube/cmd/start_test.go | 6 +----- pkg/minikube/constants/constants.go | 2 +- pkg/minikube/node/start.go | 8 +++++++- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/cmd/minikube/cmd/start_test.go b/cmd/minikube/cmd/start_test.go index aaa859dee7af..330cf5da7e1d 100644 --- a/cmd/minikube/cmd/start_test.go +++ b/cmd/minikube/cmd/start_test.go @@ -483,11 +483,7 @@ func TestValidateWindowsOSVersion(t *testing.T) { errorMsg string }{ { - osVersion: "2019", - errorMsg: "", - }, - { - osVersion: "2022", + osVersion: "2025", errorMsg: "", }, { diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index a7c55f369477..2ac418360cfd 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -167,7 +167,7 @@ const ( AliyunMirror = "registry.cn-hangzhou.aliyuncs.com/google_containers" // DefaultWindowsNodeVersion is the default version of Windows node - DefaultWindowsNodeVersion = "2022" + DefaultWindowsNodeVersion = "2025" // Windows Server ISO URL DefaultWindowsServerIsoURL = "https://go.microsoft.com/fwlink/p/?LinkID=2195280&clcid=0x409&culture=en-us&country=US" diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index cc707d35842a..3fd3c9eb8856 100755 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -1167,10 +1167,16 @@ To see benchmarks checkout https://minikube.sigs.k8s.io/docs/benchmarks/cpuusage // ValidWindowsOSVersions lists the supported Windows OS versions func ValidWindowsOSVersions() map[string]bool { - return map[string]bool{"2019": true, "2022": true, "2025": true} + // TODO: add more versions as they are tested and supported + // return map[string]bool{"2019": true, "2022": true, "2025": true} + return map[string]bool{"2025": true} } // ValidOS lists the supported OSes func ValidOS() []string { return []string{"linux", "windows"} } + +/// +// +// From 3991998fd80fd60f63c6e5d7c6c806a400c4b322 Mon Sep 17 00:00:00 2001 From: Bob Sira Date: Mon, 17 Feb 2025 16:56:00 +0000 Subject: [PATCH 05/14] short-circuit for minikube delete for windows --- pkg/minikube/machine/stop.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/minikube/machine/stop.go b/pkg/minikube/machine/stop.go index 4397052c7462..821b827c1224 100644 --- a/pkg/minikube/machine/stop.go +++ b/pkg/minikube/machine/stop.go @@ -57,6 +57,11 @@ func stop(h *host.Host) error { } if driver.NeedsShutdown(h.DriverName) { + klog.Infof("GuestOS: %s", h.GuestOS) + if h.GuestOS == "windows" { + // short circuit for windows and exit this branch + return nil + } if err := trySSHPowerOff(h); err != nil { return errors.Wrap(err, "ssh power off") } From 293f56cf399a91547a83991428cb4cb232847e7e Mon Sep 17 00:00:00 2001 From: Bob Sira Date: Thu, 20 Feb 2025 22:52:43 +0000 Subject: [PATCH 06/14] ssh authnetication refactor --- cmd/minikube/cmd/start_test.go | 2 +- pkg/minikube/bootstrapper/bootstrapper.go | 5 +- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 76 +++++++------------- pkg/minikube/machine/start.go | 72 ++++--------------- pkg/minikube/machine/stop.go | 4 -- pkg/minikube/node/start.go | 23 ++---- 6 files changed, 46 insertions(+), 136 deletions(-) diff --git a/cmd/minikube/cmd/start_test.go b/cmd/minikube/cmd/start_test.go index 330cf5da7e1d..b87ea06a97f4 100644 --- a/cmd/minikube/cmd/start_test.go +++ b/cmd/minikube/cmd/start_test.go @@ -488,7 +488,7 @@ func TestValidateWindowsOSVersion(t *testing.T) { }, { osVersion: "2023", - errorMsg: "Invalid Windows Server OS Version: 2023. Valid OS version are: [2019 2022]", + errorMsg: "Invalid Windows Server OS Version: 2023. Valid OS version are: [2025]", }, } for _, test := range tests { diff --git a/pkg/minikube/bootstrapper/bootstrapper.go b/pkg/minikube/bootstrapper/bootstrapper.go index 9bd5598284ea..f8e0df568d9d 100644 --- a/pkg/minikube/bootstrapper/bootstrapper.go +++ b/pkg/minikube/bootstrapper/bootstrapper.go @@ -19,6 +19,7 @@ package bootstrapper import ( "time" + "github.com/docker/machine/libmachine/host" "k8s.io/minikube/pkg/minikube/bootstrapper/images" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/constants" @@ -41,8 +42,8 @@ type Bootstrapper interface { UpdateCluster(config.ClusterConfig) error DeleteCluster(config.KubernetesConfig) error WaitForNode(config.ClusterConfig, config.Node, time.Duration) error - SetMinikubeFolderErrorScript(string) (string, error) - JoinClusterWindows(string, config.ClusterConfig, config.Node, string, time.Duration) (string, error) + SetupMinikubeCert(*host.Host) (string, error) + JoinClusterWindows(*host.Host, config.ClusterConfig, config.Node, string, time.Duration) (string, error) JoinCluster(config.ClusterConfig, config.Node, string) error UpdateNode(config.ClusterConfig, config.Node, cruntime.Manager) error GenerateTokenWindows(config.ClusterConfig) (string, error) diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index f2f0cca4c3f1..0e2f5000aae8 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -35,9 +35,9 @@ import ( "github.com/blang/semver/v4" "github.com/docker/machine/libmachine" + "github.com/docker/machine/libmachine/host" "github.com/docker/machine/libmachine/state" "github.com/pkg/errors" - "golang.org/x/crypto/ssh" core "k8s.io/api/core/v1" meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" @@ -752,77 +752,49 @@ func (k *Bootstrapper) restartPrimaryControlPlane(cfg config.ClusterConfig) erro return nil } -func (k *Bootstrapper) SetMinikubeFolderErrorScript(hostDriverIP string) (string, error) { +// +// + +func (k *Bootstrapper) SetupMinikubeCert(host *host.Host) (string, error) { out.Step(style.Provisioning, "Setting up minikube certificates folder...") + + certsDir := `C:\var\lib\minikube\certs` + k8sPkiDir := `C:\etc\kubernetes\pki` + caCert := `ca.crt` + script := fmt.Sprintf( - `mkdir c:\var\lib\minikube\certs; Copy-Item C:\etc\kubernetes\pki\ca.crt -Destination C:\var\lib\Minikube\Certs; Remove-Item C:\etc\kubernetes\pki\ca.crt`) + `mkdir %s; `+ + `Copy-Item %s\%s -Destination %s; `+ + `Remove-Item %s\%s`, + certsDir, k8sPkiDir, caCert, certsDir, k8sPkiDir, caCert, + ) - // Escape double quotes for remote execution script = strings.ReplaceAll(script, `"`, `\"`) - config := &ssh.ClientConfig{ - User: "Administrator", - Auth: []ssh.AuthMethod{ - ssh.Password("password"), - }, - HostKeyCallback: ssh.InsecureIgnoreHostKey(), - } - - // Add port 22 to the IP - hostDriverIP = hostDriverIP + ":22" - // Log the IP - klog.Infof("hostDriverIP: %s", hostDriverIP) + command := fmt.Sprintf("powershell -NoProfile -NonInteractive -Command \"%s\"", script) + klog.Infof("[executing] : %v", command) - client, err := ssh.Dial("tcp", hostDriverIP, config) - if err != nil { - klog.Warningf("Failed to connect: %v", err) - return "", err - } - defer client.Close() + host.RunSSHCommand(command) - // Execute the script on the remote Windows node - return machine.CmdOut(client, script) + return "", nil } -// JoinCluster adds new node to an existing cluster. -func (k *Bootstrapper) JoinClusterWindows(hostDriverIP string, cc config.ClusterConfig, n config.Node, joinCmd string, timeout time.Duration) (string, error) { - // Define the path to set location +func (k *Bootstrapper) JoinClusterWindows(host *host.Host, cc config.ClusterConfig, n config.Node, joinCmd string, timeout time.Duration) (string, error) { setLocationPath := `Set-Location -Path "C:\k"` - // Form the script to be executed psScript := fmt.Sprintf("%s; %s", setLocationPath, joinCmd) - // Escape double quotes inside the script psScript = strings.ReplaceAll(psScript, `"`, `\"`) - config := &ssh.ClientConfig{ - User: "Administrator", - Auth: []ssh.AuthMethod{ - ssh.Password("password"), - }, - HostKeyCallback: ssh.InsecureIgnoreHostKey(), - } - - // Add port 22 to the IP - hostDriverIP = hostDriverIP + ":22" - klog.Infof("hostDriverIP: %s", hostDriverIP) + command := fmt.Sprintf("powershell -NoProfile -NonInteractive -Command \"%s\"", psScript) + klog.Infof("[executing] : %v", command) - // this need to be explored more to see if we can make it better - // Create channels for result and errors + // TODO: Explore how to make this better; channels for result and errors for now exist resultChan := make(chan string, 1) errorChan := make(chan error, 1) go func() { - client, err := ssh.Dial("tcp", hostDriverIP, config) - if err != nil { - klog.Warningf("Failed to connect: %v", err) - errorChan <- err - return - } - defer client.Close() - - // Execute the script on the remote Windows node - output, err := machine.CmdOut(client, psScript) + output, err := host.RunSSHCommand(command) if err != nil { errorChan <- err return diff --git a/pkg/minikube/machine/start.go b/pkg/minikube/machine/start.go index 3dfaf0ea6177..03a0c564db4c 100644 --- a/pkg/minikube/machine/start.go +++ b/pkg/minikube/machine/start.go @@ -17,7 +17,6 @@ limitations under the License. package machine import ( - "bytes" "encoding/json" "fmt" "net" @@ -29,8 +28,6 @@ import ( "strings" "time" - "golang.org/x/crypto/ssh" - "github.com/blang/semver/v4" "github.com/docker/machine/libmachine" "github.com/docker/machine/libmachine/drivers" @@ -440,68 +437,25 @@ func addHostAliasCommand(name string, record string, sudo bool, path string) *ex return exec.Command("/bin/bash", "-c", script) } -func AddHostAliasWindows(controlPlaneIP string, hostDriverIP string) (string, error) { - // log controlPlaneIP +func AddHostAliasWindows(host *host.Host, controlPlaneIP string) (string, error) { out.Step(style.Provisioning, "Adding host alias for control plane ...") - klog.Infof("controlPlaneIP: %s", controlPlaneIP) - path := "C:\\Windows\\System32\\drivers\\etc\\hosts" - psScript := handleHostScriptCommand(controlPlaneIP, path) - - // Escape double quotes inside the script - psScript = strings.ReplaceAll(psScript, `"`, `\"`) - - config := &ssh.ClientConfig{ - User: "Administrator", - Auth: []ssh.AuthMethod{ - ssh.Password("password"), - }, - HostKeyCallback: ssh.InsecureIgnoreHostKey(), - } - - // add 22 port to ip - hostDriverIP = hostDriverIP + ":22" - // log the ip - klog.Infof("hostDriverIP: %s", hostDriverIP) - - client, err := ssh.Dial("tcp", hostDriverIP, config) - if err != nil { - klog.Warningf("Failed to connect: %v", err) - } - defer client.Close() - return CmdOut(client, psScript) -} + path := "C:\\Windows\\System32\\drivers\\etc\\hosts" + entry := fmt.Sprintf("\t%s\tcontrol-plane.minikube.internal", controlPlaneIP) -func handleHostScriptCommand(ip string, path string) string { - entry := fmt.Sprintf("\t%s\tcontrol-plane.minikube.internal", ip) - script := fmt.Sprintf( - `$hostsContent = Get-Content -Path "%s" -Raw -ErrorAction SilentlyContinue; if ($hostsContent -notmatch [regex]::Escape("%s")) { Add-Content -Path "%s" -Value "%s" -Force | Out-Null }`, - path, entry, path, entry) - return script -} + psScript := fmt.Sprintf( + `$hostsContent = Get-Content -Path "%s" -Raw -ErrorAction SilentlyContinue; `+ + `if ($hostsContent -notmatch [regex]::Escape("%s")) { `+ + `Add-Content -Path "%s" -Value "%s" -Force | Out-Null }`, + path, entry, path, entry, + ) -func CmdOut(client *ssh.Client, script string) (string, error) { - session, err := client.NewSession() - if err != nil { - return "", err - } - defer session.Close() + psScript = strings.ReplaceAll(psScript, `"`, `\"`) - command := fmt.Sprintf("powershell -NoProfile -NonInteractive -Command \"%s\"", script) + command := fmt.Sprintf("powershell -NoProfile -NonInteractive -Command \"%s\"", psScript) klog.Infof("[executing] : %v", command) - var stdout, stderr bytes.Buffer - session.Stdout = &stdout - session.Stderr = &stderr - - err = session.Run(command) - klog.Infof("[stdout =====>] : %s", stdout.String()) - klog.Infof("[stderr =====>] : %s", stderr.String()) - return stdout.String(), err -} + host.RunSSHCommand(command) -func cmd(client *ssh.Client, args ...string) error { - script := strings.Join(args, " ") - _, err := CmdOut(client, script) - return err + return "", nil } diff --git a/pkg/minikube/machine/stop.go b/pkg/minikube/machine/stop.go index 821b827c1224..5c6178c943ef 100644 --- a/pkg/minikube/machine/stop.go +++ b/pkg/minikube/machine/stop.go @@ -58,10 +58,6 @@ func stop(h *host.Host) error { if driver.NeedsShutdown(h.DriverName) { klog.Infof("GuestOS: %s", h.GuestOS) - if h.GuestOS == "windows" { - // short circuit for windows and exit this branch - return nil - } if err := trySSHPowerOff(h); err != nil { return errors.Wrap(err, "ssh power off") } diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index 3fd3c9eb8856..9efc013ff3f8 100755 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -147,16 +147,7 @@ func Start(starter Starter) (*kubeconfig.Settings, error) { // nolint:gocyclo } } else { out.Step(style.Provisioning, "Configuring Windows node...") - // log starter.Host.Driver.GetIP() - driverIP, err := starter.Host.Driver.GetIP() - if err != nil { - klog.Errorf("Unable to get driver IP: %v", err) - } - klog.Infof("Driver IP: %s", driverIP) - // log - // add "control-plane.minikube.internal" - // hack for windows node to find the primary control-plane node via it's fully qualified domain name - if stdout, err := machine.AddHostAliasWindows(constants.MasterNodeIP, driverIP); err != nil { + if stdout, err := machine.AddHostAliasWindows(starter.Host, constants.MasterNodeIP); err != nil { klog.Warningf("Unable to add host alias: %v", err) } else { klog.Infof("Host alias added: %s", stdout) @@ -425,19 +416,19 @@ func joinCluster(starter Starter, cpBs bootstrapper.Bootstrapper, bs bootstrappe klog.Infof("Driver IP: %s", driverIP) // Call with a timeout of 30 seconds - timeout := 30 * time.Second + timeout := 20 * time.Second - if commandResult, err := bs.JoinClusterWindows(driverIP, *starter.Cfg, *starter.Node, joinCmd, timeout); err != nil { + if commandResult, err := bs.JoinClusterWindows(starter.Host, *starter.Cfg, *starter.Node, joinCmd, timeout); err != nil { klog.Infof("%s node failed to join cluster, will retry: %v", role, err) klog.Infof("command result: %s", commandResult) // sort out the certificates issues - if cmd, err := bs.SetMinikubeFolderErrorScript(driverIP); err != nil { + if cmd, err := bs.SetupMinikubeCert(starter.Host); err != nil { klog.Errorf("error setting minikube folder error script: %v", err) } else { klog.Infof("command result: %s", cmd) // retry the join command - if commandResult, err := bs.JoinClusterWindows(driverIP, *starter.Cfg, *starter.Node, joinCmd, 0); err != nil { + if commandResult, err := bs.JoinClusterWindows(starter.Host, *starter.Cfg, *starter.Node, joinCmd, 0); err != nil { klog.Errorf("error retrying join command: %v, command result: %s", err, commandResult) return err } @@ -1176,7 +1167,3 @@ func ValidWindowsOSVersions() map[string]bool { func ValidOS() []string { return []string{"linux", "windows"} } - -/// -// -// From 4ebe11c6c7e3c5e21243e5f63b0b3dc7177199c8 Mon Sep 17 00:00:00 2001 From: Bob Sira Date: Thu, 20 Feb 2025 23:15:56 +0000 Subject: [PATCH 07/14] reverted Download and cache windows server ISO to preloads folder PR 330156e -> 59b3417 --- cmd/minikube/cmd/start.go | 10 ------- cmd/minikube/cmd/start_flags.go | 2 -- pkg/minikube/constants/constants.go | 3 --- pkg/minikube/download/iso.go | 42 ----------------------------- 4 files changed, 57 deletions(-) diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index 102db830e0dc..78c4f71cd463 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -356,13 +356,6 @@ func provisionWithDriver(cmd *cobra.Command, ds registry.DriverState, existing * os.Exit(0) } - // If preloadWindowsIso flag is true then cache the windows ISO - if viper.GetBool(preloadWindowsIso) { - if err := download.WindowsISO(viper.GetString(windowsNodeVersion)); err != nil { - return node.Starter{}, errors.Wrap(err, "Failed to cache Windows ISO") - } - } - if driver.IsVM(driverName) && !driver.IsSSH(driverName) { url, err := download.ISO(viper.GetStringSlice(isoURL), cmd.Flags().Changed(isoURL)) if err != nil { @@ -1337,9 +1330,6 @@ func validateFlags(cmd *cobra.Command, drvName string) { //nolint:gocyclo exit.Message(reason.Usage, "{{.err}}", out.V{"err": err}) } - // set preloadWindowsIso to true since we need to download the windows ISO file - viper.Set(preloadWindowsIso, true) - } if cmd.Flags().Changed(staticIP) { diff --git a/cmd/minikube/cmd/start_flags.go b/cmd/minikube/cmd/start_flags.go index 6ccc00b41240..b2527536ff1e 100644 --- a/cmd/minikube/cmd/start_flags.go +++ b/cmd/minikube/cmd/start_flags.go @@ -118,7 +118,6 @@ const ( ha = "ha" nodes = "nodes" preload = "preload" - preloadWindowsIso = "preload-windows-iso" deleteOnFailure = "delete-on-failure" forceSystemd = "force-systemd" kicBaseImage = "base-image" @@ -196,7 +195,6 @@ func initMinikubeFlags() { startCmd.Flags().Bool(ha, false, "Create Highly Available Multi-Control Plane Cluster with a minimum of three control-plane nodes that will also be marked for work.") startCmd.Flags().IntP(nodes, "n", 1, "The total number of nodes to spin up. Defaults to 1.") startCmd.Flags().Bool(preload, true, "If set, download tarball of preloaded images if available to improve start time. Defaults to true.") - startCmd.Flags().Bool(preloadWindowsIso, false, "If set, download the Windows ISO to improve start time of setting up a windows node. Defaults to false.") startCmd.Flags().Bool(noKubernetes, false, "If set, minikube VM/container will start without starting or configuring Kubernetes. (only works on new clusters)") startCmd.Flags().Bool(deleteOnFailure, false, "If set, delete the current cluster if start fails and try again. Defaults to false.") startCmd.Flags().Bool(forceSystemd, false, "If set, force the container runtime to use systemd as cgroup manager. Defaults to false.") diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index 2ac418360cfd..0b55f41abb1a 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -168,9 +168,6 @@ const ( // DefaultWindowsNodeVersion is the default version of Windows node DefaultWindowsNodeVersion = "2025" - - // Windows Server ISO URL - DefaultWindowsServerIsoURL = "https://go.microsoft.com/fwlink/p/?LinkID=2195280&clcid=0x409&culture=en-us&country=US" ) var ( diff --git a/pkg/minikube/download/iso.go b/pkg/minikube/download/iso.go index 4c18b3a6707a..2d726aa72831 100644 --- a/pkg/minikube/download/iso.go +++ b/pkg/minikube/download/iso.go @@ -18,8 +18,6 @@ package download import ( "fmt" - "mime" - "net/http" "net/url" "os" "path" @@ -31,7 +29,6 @@ import ( "github.com/juju/mutex/v2" "github.com/pkg/errors" "k8s.io/klog/v2" - "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/detect" "k8s.io/minikube/pkg/minikube/out" "k8s.io/minikube/pkg/minikube/style" @@ -39,8 +36,6 @@ import ( "k8s.io/minikube/pkg/version" ) -var isWindowsISO bool - const fileScheme = "file" // DefaultISOURLs returns a list of ISO URL's to consult by default, in priority order @@ -55,22 +50,6 @@ func DefaultISOURLs() []string { } } -// WindowsISOURL retrieves the ISO URL for the Windows version specified -func WindowsISOURL(version string) string { - versionToIsoURL := map[string]string{ - "2022": constants.DefaultWindowsServerIsoURL, - // Add more versions here when we support them - } - - url, exists := versionToIsoURL[version] - if !exists { - klog.Warningf("Windows version %s is not supported. Using default Windows Server ISO URL", version) - return constants.DefaultWindowsServerIsoURL - } - - return url -} - // LocalISOResource returns a local file:// URI equivalent for a local or remote ISO path func LocalISOResource(isoURL string) string { u, err := url.Parse(isoURL) @@ -124,12 +103,6 @@ func ISO(urls []string, skipChecksum bool) (string, error) { return "", fmt.Errorf(msg.String()) } -func WindowsISO(windowsVersion string) error { - isWindowsISO = true - isoURL := WindowsISOURL(windowsVersion) - return downloadISO(isoURL, false) -} - // downloadISO downloads an ISO URL func downloadISO(isoURL string, skipChecksum bool) error { u, err := url.Parse(isoURL) @@ -144,21 +117,6 @@ func downloadISO(isoURL string, skipChecksum bool) error { // Lock before we check for existence to avoid thundering herd issues dst := localISOPath(u) - if isWindowsISO { - resp, err := http.Head(isoURL) - if err != nil { - return errors.Wrapf(err, "HEAD %s", isoURL) - } - - _, params, err := mime.ParseMediaType(resp.Header.Get("Content-Disposition")) - if err != nil { - return errors.Wrapf(err, "ParseMediaType %s", resp.Header.Get("Content-Disposition")) - } - - dst = filepath.Join(detect.ISOCacheDir(), params["filename"]) - - isWindowsISO = false - } if err := os.MkdirAll(filepath.Dir(dst), 0777); err != nil { return errors.Wrapf(err, "making cache image directory: %s", dst) From 26f42a489d2907efaa110a5edbdcb3ed71e32f56 Mon Sep 17 00:00:00 2001 From: Bob Sira Date: Fri, 21 Feb 2025 19:45:03 +0000 Subject: [PATCH 08/14] modified the tests for NewHost --- pkg/minikube/machine/client.go | 2 +- pkg/minikube/machine/client_test.go | 8 +++++++- pkg/minikube/tests/api_mock.go | 8 +++++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/pkg/minikube/machine/client.go b/pkg/minikube/machine/client.go index bd8f922b7934..2ebf57e8c66d 100644 --- a/pkg/minikube/machine/client.go +++ b/pkg/minikube/machine/client.go @@ -86,7 +86,7 @@ type LocalClient struct { flock *fslock.Lock } -// DefineGuest is a method required by the libmachine.API interface. +// DefineGuest sets/tracks the guest OS for the host func (api *LocalClient) DefineGuest(h *host.Host) { api.legacyClient.DefineGuest(h) } diff --git a/pkg/minikube/machine/client_test.go b/pkg/minikube/machine/client_test.go index 7ddd379babe2..e2713f4ba197 100644 --- a/pkg/minikube/machine/client_test.go +++ b/pkg/minikube/machine/client_test.go @@ -70,17 +70,20 @@ func TestLocalClientNewHost(t *testing.T) { var tests = []struct { description string driver string + guestOS string rawDriver []byte err bool }{ { description: "host vbox correct", driver: driver.VirtualBox, + guestOS: "linux", rawDriver: []byte(vboxConfig), }, { description: "host vbox incorrect", driver: driver.VirtualBox, + guestOS: "linux", rawDriver: []byte("?"), err: true, }, @@ -90,7 +93,7 @@ func TestLocalClientNewHost(t *testing.T) { test := test t.Run(test.description, func(t *testing.T) { t.Parallel() - host, err := c.NewHost(test.driver, test.rawDriver) + host, err := c.NewHost(test.driver, test.guestOS, test.rawDriver) // A few sanity checks that we can do on the host if host != nil { if host.DriverName != test.driver { @@ -99,6 +102,9 @@ func TestLocalClientNewHost(t *testing.T) { if host.Name != host.Driver.GetMachineName() { t.Errorf("Host name is not correct. Expected :%s, got: %s", host.Driver.GetMachineName(), host.Name) } + if host.GuestOS != test.guestOS { + t.Errorf("Host guest os is not correct. Expected :%s, got: %s", test.guestOS, host.GuestOS) + } } if err != nil && !test.err { t.Errorf("Unexpected error: %v", err) diff --git a/pkg/minikube/tests/api_mock.go b/pkg/minikube/tests/api_mock.go index 022d1103c524..eb6b6b9d4861 100644 --- a/pkg/minikube/tests/api_mock.go +++ b/pkg/minikube/tests/api_mock.go @@ -68,8 +68,13 @@ func (api *MockAPI) Close() error { return nil } +// DefineGuest sets/tracks the guest OS for the host +func (api *MockAPI) DefineGuest(h *host.Host) { + api.Logf("MockAPI.DefineGuest: guest=%q", h.GuestOS) +} + // NewHost creates a new host.Host instance. -func (api *MockAPI) NewHost(drvName string, rawDriver []byte) (*host.Host, error) { +func (api *MockAPI) NewHost(drvName string, guestOS string, rawDriver []byte) (*host.Host, error) { var driver MockDriver if err := json.Unmarshal(rawDriver, &driver); err != nil { return nil, errors.Wrap(err, "error unmarshalling json") @@ -80,6 +85,7 @@ func (api *MockAPI) NewHost(drvName string, rawDriver []byte) (*host.Host, error RawDriver: rawDriver, Driver: &MockDriver{}, Name: fmt.Sprintf("mock-machine-%.8f", rand.Float64()), + GuestOS: guestOS, HostOptions: &host.Options{ AuthOptions: &auth.Options{}, SwarmOptions: &swarm.Options{}, From 361c0b300287c177e6074c41d82f1241fc930d96 Mon Sep 17 00:00:00 2001 From: Bob Sira Date: Fri, 4 Apr 2025 14:19:03 +0100 Subject: [PATCH 09/14] node-os flag implementation --- cmd/minikube/cmd/start.go | 49 +++++++++++++++++++- cmd/minikube/cmd/start_flags.go | 4 +- cmd/minikube/cmd/start_test.go | 45 ++++++++++++++++++ pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 1 - site/content/en/docs/commands/start.md | 2 +- 5 files changed, 96 insertions(+), 5 deletions(-) diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index 78c4f71cd463..d66f2f17e5c4 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -187,6 +187,14 @@ func runStart(cmd *cobra.Command, _ []string) { out.WarningT("Profile name '{{.name}}' is not valid", out.V{"name": ClusterFlagValue()}) exit.Message(reason.Usage, "Only alphanumeric and dashes '-' are permitted. Minimum 2 characters, starting with alphanumeric.") } + + // change the driver to hyperv, cni to flannel and container runtime to containerd if we have --node-os=windows + if cmd.Flags().Changed(nodeOS) { + viper.Set("driver", driver.HyperV) + viper.Set("cni", "flannel") + viper.Set(containerRuntime, constants.Containerd) + + } existing, err := config.Load(ClusterFlagValue()) if err != nil && !config.IsNotExist(err) { kind := reason.HostConfigLoad @@ -467,6 +475,13 @@ func startWithDriver(cmd *cobra.Command, starter node.Starter, existing *config. // target total and number of control-plane nodes numCPNodes := 1 numNodes := viper.GetInt(nodes) + // if we have -node-os flag set, then nodes flag will be set to 2 + // it means one of the nodes is a control-plane node and the other is a windows worker node + // so we need to reduce the numNodes by 1 + if cmd.Flags().Changed(nodeOS) { + numNodes-- + } + if existing != nil { numCPNodes = 0 for _, n := range existing.Nodes { @@ -506,8 +521,10 @@ func startWithDriver(cmd *cobra.Command, starter node.Starter, existing *config. } } - // start windows node. trigger windows node start only if windows node version is set at the time of minikube start - if cmd.Flags().Changed(windowsNodeVersion) { + // we currently trigger the windows node start if the user has set the --windows-node-version or --node-os flag + // we might need to get rid of --windows-node-version in the future and just use --node-os flag + // start windows node. trigger windows node start if windows node version or node node os is set at the time of minikube start + if cmd.Flags().Changed(windowsNodeVersion) || cmd.Flags().Changed(nodeOS) { // TODO: if windows node version is set to windows server 2022 then the windows node name should be minikube-ws2022 nodeName := node.Name(numNodes + 1) n := config.Node{ @@ -1332,6 +1349,16 @@ func validateFlags(cmd *cobra.Command, drvName string) { //nolint:gocyclo } + if cmd.Flags().Changed(nodeOS) { + if err := validMultiNodeOS(viper.GetString(nodeOS)); err != nil { + exit.Message(reason.Usage, "{{.err}}", out.V{"err": err}) + } + + if viper.GetInt(nodes) != 2 { + exit.Message(reason.Usage, "The --nodes flag must be set to 2 when using --node-os") + } + } + if cmd.Flags().Changed(staticIP) { if err := validateStaticIP(viper.GetString(staticIP), drvName, viper.GetString(subnet)); err != nil { exit.Message(reason.Usage, "{{.err}}", out.V{"err": err}) @@ -1492,6 +1519,24 @@ func validateOSandVersion(os, version string) error { return nil } +// validateMultiNodeOS validates the supplied OS for multiple nodes +func validMultiNodeOS(osString string) error { + if !strings.HasPrefix(osString, "[") || !strings.HasSuffix(osString, "]") { + return errors.Errorf("invalid OS string format: must be enclosed in [ ]") + } + + osString = strings.Trim(osString, "[]") + osString = strings.ReplaceAll(osString, " ", "") + + osValues := strings.Split(osString, ",") + + if len(osValues) != 2 || osValues[0] != "linux" || osValues[1] != "windows" { + return errors.Errorf("invalid OS string format: must be [linux,windows]") + } + + return nil +} + // validateRuntime validates the supplied runtime func validateRuntime(rtime string) error { validOptions := cruntime.ValidRuntimes() diff --git a/cmd/minikube/cmd/start_flags.go b/cmd/minikube/cmd/start_flags.go index b2527536ff1e..0b717f1d3e7e 100644 --- a/cmd/minikube/cmd/start_flags.go +++ b/cmd/minikube/cmd/start_flags.go @@ -145,6 +145,7 @@ const ( gpus = "gpus" autoPauseInterval = "auto-pause-interval" windowsNodeVersion = "windows-node-version" + nodeOS = "node-operating-system" ) var ( @@ -209,7 +210,8 @@ func initMinikubeFlags() { startCmd.Flags().String(staticIP, "", "Set a static IP for the minikube cluster, the IP must be: private, IPv4, and the last octet must be between 2 and 254, for example 192.168.200.200 (Docker and Podman drivers only)") startCmd.Flags().StringP(gpus, "g", "", "Allow pods to use your NVIDIA GPUs. Options include: [all,nvidia] (Docker driver with Docker container-runtime only)") startCmd.Flags().Duration(autoPauseInterval, time.Minute*1, "Duration of inactivity before the minikube VM is paused (default 1m0s)") - startCmd.Flags().String(windowsNodeVersion, constants.DefaultWindowsNodeVersion, "The version of Windows to use for the Windows node on a multi-node cluster (e.g., 2019, 2022). Defaults to Windows Server 2022") + startCmd.Flags().String(windowsNodeVersion, constants.DefaultWindowsNodeVersion, "The version of Windows to use for the windows node on a multi-node cluster (e.g., 2025). Currently support Windows Server 2025") + startCmd.Flags().StringP(nodeOS, "node-os", "", "The OS to use for the node. Currently support 'linux, windows'. If not set, it will be set to the same as the control plane node.") } diff --git a/cmd/minikube/cmd/start_test.go b/cmd/minikube/cmd/start_test.go index b87ea06a97f4..e388bcf7e4d7 100644 --- a/cmd/minikube/cmd/start_test.go +++ b/cmd/minikube/cmd/start_test.go @@ -505,6 +505,51 @@ func TestValidateWindowsOSVersion(t *testing.T) { } } +func TestValidMultiNodeOS(t *testing.T) { + var tests = []struct { + osString string + errorMsg string + }{ + { + osString: "[linux,windows]", + errorMsg: "", + }, + { + osString: "[linux, windows]", + errorMsg: "", + }, + { + osString: "[windows,linux]", + errorMsg: "invalid OS string format: must be [linux,windows]", + }, + { + osString: "[linux]", + errorMsg: "invalid OS string format: must be [linux,windows]", + }, + { + osString: "[linux,windows,mac]", + errorMsg: "invalid OS string format: must be [linux,windows]", + }, + { + osString: "linux,windows", + errorMsg: "invalid OS string format: must be enclosed in [ ]", + }, + } + + for _, test := range tests { + t.Run(test.osString, func(t *testing.T) { + got := validMultiNodeOS(test.osString) + gotError := "" + if got != nil { + gotError = got.Error() + } + if gotError != test.errorMsg { + t.Errorf("validMultiNodeOS(osString=%v): got %v, expected %v", test.osString, gotError, test.errorMsg) + } + }) + } +} + func TestIsTwoDigitSemver(t *testing.T) { var tcs = []struct { desc string diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 0e2f5000aae8..eb598f590748 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -872,7 +872,6 @@ func (k *Bootstrapper) GenerateTokenWindows(cc config.ClusterConfig) (string, er joinCmd = fmt.Sprintf("%s --v=5", joinCmd) return joinCmd, nil - } // GenerateToken creates a token and returns the appropriate kubeadm join command to run, or the already existing token diff --git a/site/content/en/docs/commands/start.md b/site/content/en/docs/commands/start.md index 863d8c0148e5..fc600dc256fe 100644 --- a/site/content/en/docs/commands/start.md +++ b/site/content/en/docs/commands/start.md @@ -122,7 +122,7 @@ minikube start [flags] --vm-driver driver DEPRECATED, use driver instead. --wait strings comma separated list of Kubernetes components to verify and wait for after starting a cluster. defaults to "apiserver,system_pods", available options: "apiserver,system_pods,default_sa,apps_running,node_ready,kubelet" . other acceptable values are 'all' or 'none', 'true' and 'false' (default [apiserver,system_pods]) --wait-timeout duration max time to wait per Kubernetes or host to be healthy. (default 6m0s) - --windows-node-version string The version of Windows to use for the windows node on a multi-node cluster (e.g., 2019, 2022). Defaults to Windows Server 2022 + --windows-node-version string The version of Windows to use for the windows node on a multi-node cluster (e.g., 2025). Currently support Windows Server 2025 ``` ### Options inherited from parent commands From 4b4c79dc0cf44296a039c55972e821af54065c06 Mon Sep 17 00:00:00 2001 From: Bob Sira Date: Sat, 5 Apr 2025 13:43:16 +0100 Subject: [PATCH 10/14] node-os shorthand is more than one ASCII character panic fix --- cmd/minikube/cmd/start_flags.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/minikube/cmd/start_flags.go b/cmd/minikube/cmd/start_flags.go index 0b717f1d3e7e..2a605986f866 100644 --- a/cmd/minikube/cmd/start_flags.go +++ b/cmd/minikube/cmd/start_flags.go @@ -145,7 +145,7 @@ const ( gpus = "gpus" autoPauseInterval = "auto-pause-interval" windowsNodeVersion = "windows-node-version" - nodeOS = "node-operating-system" + nodeOS = "node-os" ) var ( @@ -211,7 +211,7 @@ func initMinikubeFlags() { startCmd.Flags().StringP(gpus, "g", "", "Allow pods to use your NVIDIA GPUs. Options include: [all,nvidia] (Docker driver with Docker container-runtime only)") startCmd.Flags().Duration(autoPauseInterval, time.Minute*1, "Duration of inactivity before the minikube VM is paused (default 1m0s)") startCmd.Flags().String(windowsNodeVersion, constants.DefaultWindowsNodeVersion, "The version of Windows to use for the windows node on a multi-node cluster (e.g., 2025). Currently support Windows Server 2025") - startCmd.Flags().StringP(nodeOS, "node-os", "", "The OS to use for the node. Currently support 'linux, windows'. If not set, it will be set to the same as the control plane node.") + startCmd.Flags().String(nodeOS, "node-os", "The OS to use for the node. Currently support 'linux, windows'. If not set, it will be set to the same as the control plane node.") } From b5ef9b0502fc992209347eaed8f8133c0e1145fd Mon Sep 17 00:00:00 2001 From: Bob Sira Date: Mon, 16 Jun 2025 11:58:20 +0100 Subject: [PATCH 11/14] user personalized server vhd logic --- cmd/minikube/cmd/start.go | 26 ++++++++++++++++++-- cmd/minikube/cmd/start_flags.go | 2 ++ pkg/minikube/bootstrapper/certs.go | 2 +- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 4 +-- pkg/minikube/config/types.go | 8 +++++- pkg/minikube/constants/constants.go | 4 +++ pkg/minikube/machine/client.go | 8 +++--- pkg/minikube/machine/start.go | 10 +++++--- pkg/minikube/machine/stop.go | 2 +- pkg/minikube/node/start.go | 14 +++++------ 10 files changed, 59 insertions(+), 21 deletions(-) diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index d66f2f17e5c4..84b208ce1c35 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -507,7 +507,11 @@ func startWithDriver(cmd *cobra.Command, starter node.Starter, existing *config. KubernetesVersion: starter.Cfg.KubernetesConfig.KubernetesVersion, ContainerRuntime: starter.Cfg.KubernetesConfig.ContainerRuntime, Worker: true, - OS: "linux", + Guest: config.Guest{ + Name: "linux", + Version: "latest", + URL: "", + }, } if i < numCPNodes { // starter node is also counted as (primary) cp node n.ControlPlane = true @@ -533,7 +537,11 @@ func startWithDriver(cmd *cobra.Command, starter node.Starter, existing *config. KubernetesVersion: starter.Cfg.KubernetesConfig.KubernetesVersion, ContainerRuntime: starter.Cfg.KubernetesConfig.ContainerRuntime, Worker: true, - OS: "windows", + Guest: config.Guest{ + Name: "windows", + Version: viper.GetString(windowsNodeVersion), + URL: viper.GetString(windowsVhdURL), + }, } out.Ln("") // extra newline for clarity on the command line @@ -1359,6 +1367,20 @@ func validateFlags(cmd *cobra.Command, drvName string) { //nolint:gocyclo } } + if cmd.Flags().Changed(windowsVhdURL) { + if viper.GetString(windowsVhdURL) == "" { + // set a default URL if the user has not specified one + viper.Set(windowsVhdURL, constants.DefaultWindowsVhdURL) + exit.Message(reason.Usage, "The --windows-vhd-url flag must be set to a valid URL") + } + + // add validation logic for the windows vhd URL + url := viper.GetString(windowsVhdURL) + if !strings.HasSuffix(url, ".vhd") && !strings.HasSuffix(url, ".vhdx") { + exit.Message(reason.Usage, "The --windows-vhd-url flag must point to a valid VHD or VHDX file") + } + } //// + if cmd.Flags().Changed(staticIP) { if err := validateStaticIP(viper.GetString(staticIP), drvName, viper.GetString(subnet)); err != nil { exit.Message(reason.Usage, "{{.err}}", out.V{"err": err}) diff --git a/cmd/minikube/cmd/start_flags.go b/cmd/minikube/cmd/start_flags.go index 2a605986f866..76e74111f72e 100644 --- a/cmd/minikube/cmd/start_flags.go +++ b/cmd/minikube/cmd/start_flags.go @@ -146,6 +146,7 @@ const ( autoPauseInterval = "auto-pause-interval" windowsNodeVersion = "windows-node-version" nodeOS = "node-os" + windowsVhdURL = "windows-vhd-url" ) var ( @@ -212,6 +213,7 @@ func initMinikubeFlags() { startCmd.Flags().Duration(autoPauseInterval, time.Minute*1, "Duration of inactivity before the minikube VM is paused (default 1m0s)") startCmd.Flags().String(windowsNodeVersion, constants.DefaultWindowsNodeVersion, "The version of Windows to use for the windows node on a multi-node cluster (e.g., 2025). Currently support Windows Server 2025") startCmd.Flags().String(nodeOS, "node-os", "The OS to use for the node. Currently support 'linux, windows'. If not set, it will be set to the same as the control plane node.") + startCmd.Flags().String(windowsVhdURL, constants.DefaultWindowsVhdURL, "The VHD URL to use for the windows node on a multi-node cluster. If not set, it will be set to the default Windows Server 2025 VHD URL.") } diff --git a/pkg/minikube/bootstrapper/certs.go b/pkg/minikube/bootstrapper/certs.go index 27ecd0602cad..c367d2c50a72 100644 --- a/pkg/minikube/bootstrapper/certs.go +++ b/pkg/minikube/bootstrapper/certs.go @@ -65,7 +65,7 @@ type sharedCACerts struct { // SetupCerts gets the generated credentials required to talk to the APIServer. func SetupCerts(k8s config.ClusterConfig, n config.Node, pcpCmd command.Runner, cmd command.Runner) error { // no need to setup certs for windows worker nodes as the master node already took care of this - if n.OS == "windows" { + if n.Guest.Name == "windows" { return nil } diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index eb598f590748..bf4be19781dd 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -851,7 +851,7 @@ func (k *Bootstrapper) JoinCluster(cc config.ClusterConfig, n config.Node, joinC return nil } -// GenerateToken creates a token and returns the appropriate kubeadm join command to run, or the already existing token +// GenerateTokenWindows creates a token and returns the appropriate kubeadm join command to run, or the already existing token func (k *Bootstrapper) GenerateTokenWindows(cc config.ClusterConfig) (string, error) { tokenCmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("%s token create --print-join-command --ttl=0", bsutil.InvokeKubeadm(cc.KubernetesConfig.KubernetesVersion))) r, err := k.c.RunCmd(tokenCmd) @@ -1028,7 +1028,7 @@ func (k *Bootstrapper) UpdateCluster(cfg config.ClusterConfig) error { // UpdateNode updates new or existing node. func (k *Bootstrapper) UpdateNode(cfg config.ClusterConfig, n config.Node, r cruntime.Manager) error { // skip if the node is a windows node - if n.OS == "windows" { + if n.Guest.Name == "windows" { klog.Infof("skipping node %v update, as it is a windows node", n) return nil } diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go index 3951b6ab6a51..4e4c763b4879 100644 --- a/pkg/minikube/config/types.go +++ b/pkg/minikube/config/types.go @@ -149,7 +149,7 @@ type Node struct { ContainerRuntime string ControlPlane bool Worker bool - OS string + Guest Guest } // VersionedExtraOption holds information on flags to apply to a specific range @@ -183,3 +183,9 @@ type ScheduledStopConfig struct { InitiationTime int64 Duration time.Duration } + +type Guest struct { + Name string + Version string + URL string +} diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index 0b55f41abb1a..1c0626173ae6 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -168,6 +168,10 @@ const ( // DefaultWindowsNodeVersion is the default version of Windows node DefaultWindowsNodeVersion = "2025" + + // DefaultWindowsVhdURL is the VHD download URL for Windows Server 2025. + // This will be used whenever the user does NOT supply --windows-vhd-url. + DefaultWindowsVhdURL = "https://minikubevhdimagebuider.blob.core.windows.net/versions/hybrid-minikube-windows-server.vhdx" ) var ( diff --git a/pkg/minikube/machine/client.go b/pkg/minikube/machine/client.go index 2ebf57e8c66d..9e3c7cc18356 100644 --- a/pkg/minikube/machine/client.go +++ b/pkg/minikube/machine/client.go @@ -92,13 +92,13 @@ func (api *LocalClient) DefineGuest(h *host.Host) { } // NewHost creates a new Host -func (api *LocalClient) NewHost(drvName string, guestOS string, rawDriver []byte) (*host.Host, error) { +func (api *LocalClient) NewHost(drvName string, guest host.Guest, rawDriver []byte) (*host.Host, error) { def := registry.Driver(drvName) if def.Empty() { return nil, fmt.Errorf("driver %q does not exist", drvName) } if def.Init == nil { - return api.legacyClient.NewHost(drvName, guestOS, rawDriver) + return api.legacyClient.NewHost(drvName, guest, rawDriver) } d := def.Init() err := json.Unmarshal(rawDriver, d) @@ -111,7 +111,7 @@ func (api *LocalClient) NewHost(drvName string, guestOS string, rawDriver []byte Name: d.GetMachineName(), Driver: d, DriverName: d.DriverName(), - GuestOS: guestOS, + Guest: guest, HostOptions: &host.Options{ AuthOptions: &auth.Options{ CertDir: api.certsDir, @@ -240,7 +240,7 @@ func (api *LocalClient) Create(h *host.Host) error { return nil } // Skipped because we don't reconfigure Docker for Windows Host - if h.GuestOS == "windows" { + if h.Guest.Name == "windows" { return nil } return provisionDockerMachine(h) diff --git a/pkg/minikube/machine/start.go b/pkg/minikube/machine/start.go index 03a0c564db4c..90a98e2aa8ea 100644 --- a/pkg/minikube/machine/start.go +++ b/pkg/minikube/machine/start.go @@ -145,7 +145,7 @@ func createHost(api libmachine.API, cfg *config.ClusterConfig, n *config.Node) ( return nil, errors.Wrap(err, "marshal") } - h, err := api.NewHost(cfg.Driver, n.OS, data) + h, err := api.NewHost(cfg.Driver, host.Guest(n.Guest), data) if err != nil { return nil, errors.Wrap(err, "new host") } @@ -162,6 +162,10 @@ func createHost(api libmachine.API, cfg *config.ClusterConfig, n *config.Node) ( if cfg.StartHostTimeout == 0 { cfg.StartHostTimeout = 6 * time.Minute + // windows nodes take longer to start, so we increase the timeout + if n.Guest.Name == "windows" { + cfg.StartHostTimeout = 10 * time.Minute + } } if err := timedCreateHost(h, api, cfg.StartHostTimeout); err != nil { return nil, errors.Wrap(err, "creating host") @@ -185,7 +189,7 @@ func timedCreateHost(h *host.Host, api libmachine.API, t time.Duration) error { create := make(chan error, 1) go func() { defer close(create) - klog.Infof("libmachine.API.Create starting for %q (GuestOS=%q)", h.Name, h.GuestOS) + klog.Infof("libmachine.API.Create starting for %q (GuestOS=%q)", h.Name, h.Guest.Name) create <- api.Create(h) }() @@ -304,7 +308,7 @@ func postStartSetup(h *host.Host, mc config.ClusterConfig) error { } // skip postStartSetup for windows guest os - if h.GuestOS == "windows" { + if h.Guest.Name == "windows" { klog.Infof("skipping postStartSetup for windows guest os") return nil } diff --git a/pkg/minikube/machine/stop.go b/pkg/minikube/machine/stop.go index 5c6178c943ef..c9ddbd28a5c6 100644 --- a/pkg/minikube/machine/stop.go +++ b/pkg/minikube/machine/stop.go @@ -57,7 +57,7 @@ func stop(h *host.Host) error { } if driver.NeedsShutdown(h.DriverName) { - klog.Infof("GuestOS: %s", h.GuestOS) + klog.Infof("GuestOS: %s", h.Guest.Name) if err := trySSHPowerOff(h); err != nil { return errors.Wrap(err, "ssh power off") } diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index 9efc013ff3f8..cb78ad206b5e 100755 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -109,8 +109,8 @@ func Start(starter Starter) (*kubeconfig.Settings, error) { // nolint:gocyclo } // log starter.Node.OS here - klog.Infof("Node OS: %s", starter.Node.OS) - if starter.Node.OS != "windows" { + klog.Infof("Node OS: %s", starter.Node.Guest.Name) + if starter.Node.Guest.Name != "windows" { // wait for preloaded tarball to finish downloading before configuring runtimes waitCacheRequiredImages(&cacheGroup) } @@ -122,7 +122,7 @@ func Start(starter Starter) (*kubeconfig.Settings, error) { // nolint:gocyclo klog.Infof("Kubernetes version: %s", sv) var cr cruntime.Manager - if starter.Node.OS != "windows" { + if starter.Node.Guest.Name != "windows" { // configure the runtime (docker, containerd, crio) only for windows nodes cr = configureRuntimes(starter.Runner, *starter.Cfg, sv) @@ -141,7 +141,7 @@ func Start(starter Starter) (*kubeconfig.Settings, error) { // nolint:gocyclo klog.Errorf("Unable to get host IP: %v", err) } - if starter.Node.OS != "windows" { + if starter.Node.Guest.Name != "windows" { if err := machine.AddHostAlias(starter.Runner, constants.HostAlias, hostIP); err != nil { klog.Warningf("Unable to add host alias: %v", err) } @@ -377,7 +377,7 @@ func joinCluster(starter Starter, cpBs bootstrapper.Bootstrapper, bs bootstrappe var err error // if node is a windows node, generate the join command - if starter.Node.OS == "windows" { + if starter.Node.Guest.Name == "windows" { joinCmd, err = cpBs.GenerateTokenWindows(*starter.Cfg) if err != nil { return fmt.Errorf("error generating join token: %w", err) @@ -393,7 +393,7 @@ func joinCluster(starter Starter, cpBs bootstrapper.Bootstrapper, bs bootstrappe join := func() error { klog.Infof("trying to join %s node %q to cluster: %+v", role, starter.Node.Name, starter.Node) - if starter.Node.OS != "windows" { + if starter.Node.Guest.Name != "windows" { if err := bs.JoinCluster(*starter.Cfg, *starter.Node, joinCmd); err != nil { // log the error message and retry klog.Errorf("%s node failed to join cluster, will retry: %v", role, err) @@ -450,7 +450,7 @@ func joinCluster(starter Starter, cpBs bootstrapper.Bootstrapper, bs bootstrappe return nil } if err := retry.Expo(join, 10*time.Second, 3*time.Minute); err != nil { - if starter.Node.OS != "windows" { + if starter.Node.Guest.Name != "windows" { return fmt.Errorf("error joining %s node %q to cluster: %w", role, starter.Node.Name, err) } } From 20058719d34746daae25467eeb52a3c844d90e5d Mon Sep 17 00:00:00 2001 From: Bob Sira Date: Mon, 16 Jun 2025 15:15:05 +0100 Subject: [PATCH 12/14] fixed failing tests --- pkg/minikube/machine/client_test.go | 27 ++++++++++++++++++--------- pkg/minikube/tests/api_mock.go | 6 +++--- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/pkg/minikube/machine/client_test.go b/pkg/minikube/machine/client_test.go index e2713f4ba197..30c3b8d666f4 100644 --- a/pkg/minikube/machine/client_test.go +++ b/pkg/minikube/machine/client_test.go @@ -24,6 +24,7 @@ import ( "testing" "github.com/docker/machine/libmachine/drivers/plugin/localbinary" + "github.com/docker/machine/libmachine/host" "k8s.io/minikube/pkg/minikube/driver" _ "k8s.io/minikube/pkg/minikube/registry/drvs/virtualbox" @@ -70,22 +71,30 @@ func TestLocalClientNewHost(t *testing.T) { var tests = []struct { description string driver string - guestOS string + guest host.Guest rawDriver []byte err bool }{ { description: "host vbox correct", driver: driver.VirtualBox, - guestOS: "linux", - rawDriver: []byte(vboxConfig), + guest: host.Guest{ + Name: "linux", + Version: "1.0.0", + URL: "https://example.com/linux.iso", + }, + rawDriver: []byte(vboxConfig), }, { description: "host vbox incorrect", driver: driver.VirtualBox, - guestOS: "linux", - rawDriver: []byte("?"), - err: true, + guest: host.Guest{ + Name: "linux", + Version: "1.0.0", + URL: "https://example.com/linux.iso", + }, + rawDriver: []byte("?"), + err: true, }, } @@ -93,7 +102,7 @@ func TestLocalClientNewHost(t *testing.T) { test := test t.Run(test.description, func(t *testing.T) { t.Parallel() - host, err := c.NewHost(test.driver, test.guestOS, test.rawDriver) + host, err := c.NewHost(test.driver, test.guest, test.rawDriver) // A few sanity checks that we can do on the host if host != nil { if host.DriverName != test.driver { @@ -102,8 +111,8 @@ func TestLocalClientNewHost(t *testing.T) { if host.Name != host.Driver.GetMachineName() { t.Errorf("Host name is not correct. Expected :%s, got: %s", host.Driver.GetMachineName(), host.Name) } - if host.GuestOS != test.guestOS { - t.Errorf("Host guest os is not correct. Expected :%s, got: %s", test.guestOS, host.GuestOS) + if host.Guest.Name != test.guest.Name { + t.Errorf("Host's guest os is not correct. Expected :%s, got: %s", test.guest.Name, host.Guest.Name) } } if err != nil && !test.err { diff --git a/pkg/minikube/tests/api_mock.go b/pkg/minikube/tests/api_mock.go index eb6b6b9d4861..bcbc7fc8dc30 100644 --- a/pkg/minikube/tests/api_mock.go +++ b/pkg/minikube/tests/api_mock.go @@ -70,11 +70,11 @@ func (api *MockAPI) Close() error { // DefineGuest sets/tracks the guest OS for the host func (api *MockAPI) DefineGuest(h *host.Host) { - api.Logf("MockAPI.DefineGuest: guest=%q", h.GuestOS) + api.Logf("MockAPI.DefineGuest: guest=%q", h.Guest.Name) } // NewHost creates a new host.Host instance. -func (api *MockAPI) NewHost(drvName string, guestOS string, rawDriver []byte) (*host.Host, error) { +func (api *MockAPI) NewHost(drvName string, guest host.Guest, rawDriver []byte) (*host.Host, error) { var driver MockDriver if err := json.Unmarshal(rawDriver, &driver); err != nil { return nil, errors.Wrap(err, "error unmarshalling json") @@ -85,7 +85,7 @@ func (api *MockAPI) NewHost(drvName string, guestOS string, rawDriver []byte) (* RawDriver: rawDriver, Driver: &MockDriver{}, Name: fmt.Sprintf("mock-machine-%.8f", rand.Float64()), - GuestOS: guestOS, + Guest: guest, HostOptions: &host.Options{ AuthOptions: &auth.Options{}, SwarmOptions: &swarm.Options{}, From 71e77301b78614f06162ae8b569aaf9ed8bd2bfe Mon Sep 17 00:00:00 2001 From: Bob Sira Date: Tue, 2 Sep 2025 00:49:28 +0100 Subject: [PATCH 13/14] attempts to fix trySigKillProcess test --- cmd/minikube/cmd/delete_test.go | 2 +- pkg/minikube/node/start.go | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/cmd/minikube/cmd/delete_test.go b/cmd/minikube/cmd/delete_test.go index 6301ab9cf88c..f7dc2456517b 100644 --- a/cmd/minikube/cmd/delete_test.go +++ b/cmd/minikube/cmd/delete_test.go @@ -242,7 +242,7 @@ func main() { done := make(chan struct{}) defer close(ch) - signal.Notify(ch, syscall.SIGHUP) + signal.Notify(ch, syscall.SIGTERM) defer signal.Stop(ch) go func() { diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index cb78ad206b5e..bf41a9076758 100755 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -455,10 +455,8 @@ func joinCluster(starter Starter, cpBs bootstrapper.Bootstrapper, bs bootstrappe } } - if starter.Cfg.WindowsNodeVersion != "2022" { - if err := cpBs.LabelAndUntaintNode(*starter.Cfg, *starter.Node); err != nil { - return fmt.Errorf("error applying %s node %q label: %w", role, starter.Node.Name, err) - } + if err := cpBs.LabelAndUntaintNode(*starter.Cfg, *starter.Node); err != nil { + return fmt.Errorf("error applying %s node %q label: %w", role, starter.Node.Name, err) } return nil From 3489fbf979ebb2e9b33e815cd5e039f22dbdf5aa Mon Sep 17 00:00:00 2001 From: Bob Sira Date: Tue, 2 Sep 2025 14:16:21 +0100 Subject: [PATCH 14/14] reverting test changes --- cmd/minikube/cmd/delete_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/minikube/cmd/delete_test.go b/cmd/minikube/cmd/delete_test.go index f7dc2456517b..6301ab9cf88c 100644 --- a/cmd/minikube/cmd/delete_test.go +++ b/cmd/minikube/cmd/delete_test.go @@ -242,7 +242,7 @@ func main() { done := make(chan struct{}) defer close(ch) - signal.Notify(ch, syscall.SIGTERM) + signal.Notify(ch, syscall.SIGHUP) defer signal.Stop(ch) go func() {