vz: add SSH over AF_VSOCK

Since systemd v256 (Ubuntu 24.10), SSH is bound to AF_VSOCK port 22.

https://github.com/systemd/systemd/releases/tag/v256
> - If the system is run in a VM providing AF_VSOCK support, it automatically
binds sshd to AF_VSOCK port 22.

https://discourse.ubuntu.com/t/oracular-oriole-release-notes/44878
> - When sshd is installed on a system, a new systemd generator, systemd-ssh-generator
binds a socket-activated SSH server to local AF_VSOCK and AF_UNIX sockets under certain conditions.

This changes to delay starting SSH port forwarding until the SSH server on the VM becomes ready.
If AF_VSOCK port 22 can be connected, start a local SSH port as a proxy for AF_VSOCK port 22,
instead of starting gvisor's port forwarder.

SSH over VSOCK is faster than SSH over gvisor's port forwarder.

This change is opt-out because it requires VZ and VM with systemd v256+,
setting `LIMA_SSH_OVER_VSOCK=true` does not mean it works.
To disable, set `LIMA_SSH_OVER_VSOCK=false`.

Signed-off-by: Norio Nomura <[email protected]>

Author: norio-nomura

cmd/limactl/usernet.go

@@ -80,12 +80,13 @@ func usernetAction(cmd *cobra.Command, _ []string) error {
 	// Environment Variables
 	// LIMA_USERNET_RESOLVE_IP_ADDRESS_TIMEOUT: Specifies the timeout duration for resolving IP addresses in minutes. Default is 2 minutes.
 
-	return usernet.StartGVisorNetstack(cmd.Context(), &usernet.GVisorNetstackOpts{
+	_, err = usernet.StartGVisorNetstack(cmd.Context(), &usernet.GVisorNetstackOpts{
 		MTU:           mtu,
 		Endpoint:      endpoint,
 		QemuSocket:    qemuSocket,
 		FdSocket:      fdSocket,
 		Subnet:        subnet,
 		DefaultLeases: leases,
 	})
+	return err
 }

pkg/driver/vz/vm_darwin.go

@@ -108,10 +108,35 @@ func startVM(ctx context.Context, inst *limatype.Instance, sshLocalPort int) (*v
 					filesToRemove[pidFile] = struct{}{}
 					logrus.Info("[VZ] - vm state change: running")
 
-					err := usernetClient.ConfigureDriver(ctx, inst, sshLocalPort)
-					if err != nil {
-						errCh <- err
-					}
+					go func() {
+						sshLocalPort := sshLocalPort
+						useSSHOverVsock := true
+						if envVar := os.Getenv("LIMA_SSH_OVER_VSOCK"); envVar != "" {
+							b, err := strconv.ParseBool(envVar)
+							if err != nil {
+								logrus.WithError(err).Warnf("invalid LIMA_SSH_OVER_VSOCK value %q", envVar)
+							} else {
+								useSSHOverVsock = b
+							}
+						}
+						if !useSSHOverVsock {
+							logrus.Info("LIMA_SSH_OVER_VSOCK is false, skipping detection of SSH server on vsock port")
+						} else if err := usernetClient.WaitOpeningSSHPort(ctx, inst, sshLocalPort); err == nil {
+							if err := wrapper.startVsockForwarder(ctx, 22, uint32(sshLocalPort)); err == nil {
+								logrus.Infof("Detected SSH server is listening on the vsock port; changed localhost:%d to proxy for the vsock port", sshLocalPort)
+								sshLocalPort = 0 // disable gvisor ssh port forwarding
+							} else {
+								logrus.WithError(err).Warn("Failed to detect SSH server on vsock port, falling back to gvisor's forwarder")
+							}
+						} else {
+							logrus.WithError(err).Warn("Failed to wait for the guest SSH server to become available, falling back to gvisor's forwarder")
+						}
+						err := usernetClient.ConfigureDriver(ctx, inst, sshLocalPort)
+						if err != nil {
+							errCh <- err
+						}
+					}()
+
 				case vz.VirtualMachineStateStopped:
 					logrus.Info("[VZ] - vm state change: stopped")
 					wrapper.mu.Lock()
@@ -147,7 +172,7 @@ func startUsernet(ctx context.Context, inst *limatype.Instance) (*usernet.Client
 	os.RemoveAll(endpointSock)
 	os.RemoveAll(vzSock)
 	ctx, cancel := context.WithCancel(ctx)
-	err = usernet.StartGVisorNetstack(ctx, &usernet.GVisorNetstackOpts{
+	vn, err := usernet.StartGVisorNetstack(ctx, &usernet.GVisorNetstackOpts{
 		MTU:      1500,
 		Endpoint: endpointSock,
 		FdSocket: vzSock,
@@ -162,7 +187,7 @@ func startUsernet(ctx context.Context, inst *limatype.Instance) (*usernet.Client
 		return nil, nil, err
 	}
 	subnetIP, _, err := net.ParseCIDR(networks.SlirpNetwork)
-	return usernet.NewClient(endpointSock, subnetIP), cancel, err
+	return usernet.NewClient(endpointSock, subnetIP, vn), cancel, err
 }
 
 func createVM(ctx context.Context, inst *limatype.Instance) (*vz.VirtualMachine, error) {

pkg/driver/vz/vsock_forwarder.go

@@ -0,0 +1,64 @@
+//go:build darwin && !no_vz
+
+// SPDX-FileCopyrightText: Copyright The Lima Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package vz
+
+import (
+	"context"
+	"fmt"
+	"net"
+
+	"github.com/containers/gvisor-tap-vsock/pkg/tcpproxy"
+	"github.com/sirupsen/logrus"
+)
+
+func (m *virtualMachineWrapper) startVsockForwarder(ctx context.Context, vsockPort, hostPort uint32) error {
+	// Test if the vsock port is open
+	conn, err := m.dialVsock(ctx, vsockPort)
+	if err != nil {
+		return err
+	}
+	conn.Close()
+	// Start listening on localhost:hostPort and forward to vsock:vsockPort
+	var lc net.ListenConfig
+	l, err := lc.Listen(ctx, "tcp", fmt.Sprintf("127.0.0.1:%d", hostPort))
+	if err != nil {
+		return err
+	}
+	logrus.Infof("started vsock forwarder: localhost:%d -> vsock:%d on VM", hostPort, vsockPort)
+	go func() {
+		defer l.Close()
+		for {
+			conn, err := l.Accept()
+			if err != nil {
+				logrus.WithError(err).Errorf("vsock forwarder accept error: %v", err)
+				return
+			}
+			p := tcpproxy.DialProxy{
+				DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
+					return m.dialVsock(ctx, vsockPort)
+				},
+			}
+			go p.HandleConn(conn)
+			select {
+			case <-ctx.Done():
+				return
+			default:
+				continue
+			}
+		}
+	}()
+	return nil
+}
+
+func (m *virtualMachineWrapper) dialVsock(_ context.Context, port uint32) (conn net.Conn, err error) {
+	for _, socket := range m.SocketDevices() {
+		conn, err = socket.Connect(port)
+		if err == nil {
+			return conn, nil
+		}
+	}
+	return nil, err
+}

pkg/networks/usernet/client.go

@@ -16,6 +16,8 @@ import (
 
 	gvproxyclient "github.com/containers/gvisor-tap-vsock/pkg/client"
 	"github.com/containers/gvisor-tap-vsock/pkg/types"
+	"github.com/containers/gvisor-tap-vsock/pkg/virtualnetwork"
+	"github.com/sirupsen/logrus"
 
 	"github.com/lima-vm/lima/v2/pkg/httpclientutil"
 	"github.com/lima-vm/lima/v2/pkg/limatype"
@@ -30,6 +32,8 @@ type Client struct {
 	delegate *gvproxyclient.Client
 	base     string
 	subnet   net.IP
+
+	vn *virtualnetwork.VirtualNetwork
 }
 
 func (c *Client) ConfigureDriver(ctx context.Context, inst *limatype.Instance, sshLocalPort int) error {
@@ -38,9 +42,11 @@ func (c *Client) ConfigureDriver(ctx context.Context, inst *limatype.Instance, s
 	if err != nil {
 		return err
 	}
-	err = c.ResolveAndForwardSSH(ipAddress, sshLocalPort)
-	if err != nil {
-		return err
+	if sshLocalPort != 0 {
+		err = c.ResolveAndForwardSSH(ipAddress, sshLocalPort)
+		if err != nil {
+			return err
+		}
 	}
 	hosts := inst.Config.HostResolver.Hosts
 	if hosts == nil {
@@ -127,6 +133,37 @@ func (c *Client) Leases(ctx context.Context) (map[string]string, error) {
 	return leases, nil
 }
 
+// Wait until the guest ssh server is available.
+func (c *Client) WaitOpeningSSHPort(ctx context.Context, inst *limatype.Instance, blockingPort int) error {
+	if c.vn == nil {
+		return errors.New("internal error: vn is nil")
+	}
+	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+	macAddress := limayaml.MACAddress(inst.Dir)
+	ipAddr, err := c.ResolveIPAddress(ctx, macAddress)
+	if err != nil {
+		return err
+	}
+	// Wait until the guest ssh server is available.
+	for {
+		conn, err := c.vn.DialContextTCP(ctx, fmt.Sprintf("%s:22", ipAddr))
+		if err == nil {
+			conn.Close()
+			logrus.Infof("Guest SSH server is available on %s:22", ipAddr)
+			break
+		}
+		select {
+		case <-ctx.Done():
+			return errors.New("timed out waiting for SSH server to be available")
+		default:
+		}
+		logrus.Infof("Waiting for guest ssh server to be available on %s:22", ipAddr)
+		time.Sleep(500 * time.Millisecond)
+	}
+	return nil
+}
+
 func NewClientByName(nwName string) *Client {
 	endpointSock, err := Sock(nwName, EndpointSock)
 	if err != nil {
@@ -136,14 +173,14 @@ func NewClientByName(nwName string) *Client {
 	if err != nil {
 		return nil
 	}
-	return NewClient(endpointSock, subnet)
+	return NewClient(endpointSock, subnet, nil)
 }
 
-func NewClient(endpointSock string, subnet net.IP) *Client {
-	return create(endpointSock, subnet, "http://lima")
+func NewClient(endpointSock string, subnet net.IP, vn *virtualnetwork.VirtualNetwork) *Client {
+	return create(endpointSock, subnet, "http://lima", vn)
 }
 
-func create(sock string, subnet net.IP, base string) *Client {
+func create(sock string, subnet net.IP, base string, vn *virtualnetwork.VirtualNetwork) *Client {
 	client := &http.Client{
 		Transport: &http.Transport{
 			DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
@@ -158,5 +195,6 @@ func create(sock string, subnet net.IP, base string) *Client {
 		delegate: delegate,
 		base:     base,
 		subnet:   subnet,
+		vn:       vn,
 	}
 }

pkg/networks/usernet/gvproxy.go

@@ -40,12 +40,12 @@ var opts *GVisorNetstackOpts
 
 const gatewayMacAddr = "5a:94:ef:e4:0c:dd"
 
-func StartGVisorNetstack(ctx context.Context, gVisorOpts *GVisorNetstackOpts) error {
+func StartGVisorNetstack(ctx context.Context, gVisorOpts *GVisorNetstackOpts) (*virtualnetwork.VirtualNetwork, error) {
 	opts = gVisorOpts
 
 	ip, ipNet, err := net.ParseCIDR(opts.Subnet)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	gatewayIP := GatewayIP(ip)
 
@@ -82,41 +82,41 @@ func StartGVisorNetstack(ctx context.Context, gVisorOpts *GVisorNetstackOpts) er
 	}
 
 	groupErrs, ctx := errgroup.WithContext(ctx)
-	err = run(ctx, groupErrs, &config)
+	vn, err := run(ctx, groupErrs, &config)
 	if err != nil {
-		return err
+		return nil, err
 	}
 	if opts.Async {
-		return err
+		return vn, nil
 	}
-	return groupErrs.Wait()
+	return vn, groupErrs.Wait()
 }
 
-func run(ctx context.Context, g *errgroup.Group, configuration *types.Configuration) error {
+func run(ctx context.Context, g *errgroup.Group, configuration *types.Configuration) (*virtualnetwork.VirtualNetwork, error) {
 	vn, err := virtualnetwork.New(configuration)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	ln, err := transport.Listen(fmt.Sprintf("unix://%s", opts.Endpoint))
 	if err != nil {
-		return err
+		return nil, err
 	}
 	httpServe(ctx, g, ln, vn.Mux())
 
 	if opts.QemuSocket != "" {
 		err = listenQEMU(ctx, vn)
 		if err != nil {
-			return err
+			return nil, err
 		}
 	}
 	if opts.FdSocket != "" {
 		err = listenFD(ctx, vn)
 		if err != nil {
-			return err
+			return nil, err
 		}
 	}
-	return nil
+	return vn, nil
 }
 
 func listenQEMU(ctx context.Context, vn *virtualnetwork.VirtualNetwork) error {

website/content/en/docs/config/environment-variables.md

@@ -106,6 +106,15 @@ This page documents the environment variables used in Lima.
   lima
   ```
 
+### `LIMA_SSH_OVER_VSOCK`
+- **Description**: Specifies to use vsock for SSH connection instead of port forwarding.
+- **Default**: `true`
+- **Usage**: 
+  ```sh
+  export LIMA_SSH_OVER_VSOCK=true
+  ```
+- **Note**: This variable is effective only if the VM is VZ based and systemd is v256 or later (e.g. Ubuntu 24.10+).
+
 ### `LIMA_SSH_PORT_FORWARDER`
 
 - **Description**: Specifies to use the SSH port forwarder (slow) instead of gRPC (fast, previously unstable)

website/content/en/docs/config/port.md

@@ -36,6 +36,20 @@ LIMA_SSH_PORT_FORWARDER=true limactl start
 - Doesn't support UDP based port forwarding
 - Spawns child process on host for running SSH master.
 
+#### SSH over AF_VSOCK
+
+| ⚡ Requirement | Lima >= 2.0 |
+|---------------|-------------|
+
+If VM is VZ based and systemd is v256 or later (e.g. Ubuntu 24.10+), Lima uses AF_VSOCK for communication between host and guest.
+SSH based port forwarding is much faster when using AF_VSOCK compared to traditional virtual network based port forwarding.
+
+To disable this feature, set `LIMA_SSH_OVER_VSOCK` to `false`:
+
+```bash
+export LIMA_SSH_OVER_VSOCK=false
+```
+
 ### Using GRPC
 
 | ⚡ Requirement | Lima >= 1.0 |