diff --git a/go.mod b/go.mod index ee01504d4..5faf229dd 100644 --- a/go.mod +++ b/go.mod @@ -11,10 +11,11 @@ require ( github.com/containerd/log v0.1.0 github.com/containerd/ttrpc v1.2.8 github.com/creack/pty v1.1.24 + github.com/cyphar/filepath-securejoin v0.5.2 github.com/elastic/go-seccomp-bpf v1.6.0 github.com/hashicorp/go-version v1.9.0 github.com/jackpal/gateway v1.2.0 - github.com/moby/sys/mount v0.3.5 + github.com/moby/sys/mountinfo v0.7.2 github.com/moby/sys/userns v0.1.0 github.com/nubificus/hedge_cli v0.0.3 github.com/onsi/ginkgo/v2 v2.28.1 @@ -62,7 +63,6 @@ require ( github.com/klauspost/compress v1.18.6 // indirect github.com/mattn/go-colorable v0.1.15 // indirect github.com/mattn/go-isatty v0.0.22 // indirect - github.com/moby/sys/mountinfo v0.7.2 // indirect github.com/moby/sys/sequential v0.7.0 // indirect github.com/moby/sys/user v0.4.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect diff --git a/go.sum b/go.sum index 3f4290407..f7d7bb0de 100644 --- a/go.sum +++ b/go.sum @@ -145,8 +145,6 @@ github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3v github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= github.com/moby/sys/capability v0.4.0 h1:4D4mI6KlNtWMCM1Z/K0i7RV1FkX+DBDHKVJpCndZoHk= github.com/moby/sys/capability v0.4.0/go.mod h1:4g9IK291rVkms3LKCDOoYlnV8xKwoDTpIrNEE35Wq0I= -github.com/moby/sys/mount v0.3.5 h1:eS3fsZTjHaBihwjp4/+5Z3jxqLXYsbwxqpVSfFv3M00= -github.com/moby/sys/mount v0.3.5/go.mod h1:WUQDO+/uCiCIkIztx8SrwIDVn2dtMFRBebRhpDFT71M= github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= github.com/moby/sys/sequential v0.7.0 h1:ASQNGNROJSuOO6LL6bPHbKvuZu6NU8P4ldPWk31zj/8= diff --git a/pkg/unikontainers/block.go b/pkg/unikontainers/block.go index 1b7bf7892..0598904b9 100644 --- a/pkg/unikontainers/block.go +++ b/pkg/unikontainers/block.go @@ -22,12 +22,12 @@ import ( "path/filepath" "strings" - "golang.org/x/sys/unix" - - "github.com/moby/sys/mount" + "github.com/containerd/containerd/mount" + "github.com/moby/sys/mountinfo" "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" "github.com/urunc-dev/urunc/pkg/unikontainers/types" + "golang.org/x/sys/unix" ) // TODO: Find and set the correct size for the tmpfs in the host @@ -157,7 +157,7 @@ func copyMountfiles(targetPath string, mounts []specs.Mount) error { if m.Type != "bind" { continue } - err := fileFromHost(targetPath, m.Source, m.Destination, 0, true) + err := fileFromHost(targetPath, m.Source, m.Destination) if (err != nil) && !errors.Is(err, ErrCopyDir) { return err } @@ -189,7 +189,7 @@ func handleExplicitBlockImage(blockImg string, mountPoint string) (types.BlockDe // Search all the mount entries in the container's config and // find the ones that come from a block. -func getBlockVolumes(monRootfs string, mounts []specs.Mount, ukernel types.Unikernel) ([]types.BlockDevParams, error) { +func getBlockVolumes(mounts []specs.Mount, ukernel types.Unikernel) ([]types.BlockDevParams, error) { blkImgs := []types.BlockDevParams{} for i, m := range mounts { // We check only bind mounts @@ -208,15 +208,26 @@ func getBlockVolumes(monRootfs string, mounts []specs.Mount, ukernel types.Unike return nil, err } if ukernel.SupportsFS(mInfo.FsType) { - err = mount.Unmount(mInfo.MountPoint) + // So, there was an issue which was manifested from the testing. + // If we have a file (e.g. ext2) and mount it, then we use + // a loop device for the mount and this is what is shown + // in the mount list. However, since we perform the unmount + // the device might also get removed. See + // https://www.kernel.org/doc/Documentation/ABI/testing/sysfs-block-loop + // If the device gets removed, then we attach nothing to the + // sandbox. TO resolve this we remove the autoclear flag + // and therefore the device will persist. + cleared, err := setLoopAutoclear(mInfo.Source, false) if err != nil { return nil, err } - err = setupDev(monRootfs, mInfo.Source) + mInfo.LoopAutoclear = cleared + err = mount.Unmount(mInfo.MountPoint, 0) if err != nil { return nil, err } mInfo.ID = fmt.Sprintf("vol%d", i) + mInfo.HostMountPoint = mInfo.MountPoint mInfo.MountPoint = m.Destination blkImgs = append(blkImgs, mInfo) } @@ -225,6 +236,105 @@ func getBlockVolumes(monRootfs string, mounts []specs.Mount, ukernel types.Unike return blkImgs, nil } +// restoreBlockVolumes mounts the block volume sources that were unmounted +// during create +func restoreBlockVolumes(blockArgs []types.BlockDevParams) error { + for _, b := range blockArgs { + // Only volumes gathered from the container's mounts carry the + // host mountpoint where their source was originally mounted. + if b.HostMountPoint == "" { + continue + } + mounted, err := mountinfo.Mounted(b.HostMountPoint) + if err != nil { + return fmt.Errorf("failed to check if %s is a mountpoint: %w", b.HostMountPoint, err) + } + if mounted { + continue + } + + err = unix.Mount(b.Source, b.HostMountPoint, b.FsType, 0, "") + if err != nil { + return fmt.Errorf("failed to remount %s at %s: %w", b.Source, b.HostMountPoint, err) + } + + if b.LoopAutoclear { + _, err = setLoopAutoclear(b.Source, true) + if err != nil { + return err + } + } + } + + return nil +} + +// setLoopAutoclear sets or clears the autoclear flag of a loop device and +// returns true when the flag was changed. It returns false without an error +// when the path is not a loop device or the flag already has the wanted value. +func setLoopAutoclear(devPath string, autoclear bool) (bool, error) { + _, err := os.Stat(filepath.Join("/sys/class/block", filepath.Base(devPath), "loop")) + if os.IsNotExist(err) { + // Not a loop device + return false, nil + } + if err != nil { + return false, err + } + + dev, err := os.OpenFile(devPath, os.O_RDWR, 0) + if err != nil { + return false, fmt.Errorf("failed to open %s: %w", devPath, err) + } + defer dev.Close() + + info, err := unix.IoctlLoopGetStatus64(int(dev.Fd())) + if err != nil { + return false, fmt.Errorf("failed to get the status of %s: %w", devPath, err) + } + if autoclear == (info.Flags&unix.LO_FLAGS_AUTOCLEAR != 0) { + return false, nil + } + + if autoclear { + info.Flags |= unix.LO_FLAGS_AUTOCLEAR + } else { + info.Flags &^= unix.LO_FLAGS_AUTOCLEAR + } + err = unix.IoctlLoopSetStatus64(int(dev.Fd()), info) + if err != nil { + return false, fmt.Errorf("failed to set the status of %s: %w", devPath, err) + } + + return true, nil +} + +// blockDevNodes transforms a list of types.BlockDevParams to a list of +// specs.LinuxDevice which cna be then used for replicating these block +// devices form the host to the monitor's execution environment. +func blockDevNodes(blockArgs []types.BlockDevParams, rootfs types.RootfsParams) ([]specs.LinuxDevice, error) { + if rootfs.Type != "block" { + return nil, nil + } + + var blockDevs []specs.LinuxDevice + for _, b := range blockArgs { + // The rootfs device is a real host device only when a container rootfs + // was converted to a block device (MountedPath set). When it is an + // explicit block image referenced by an annotation, no node is created. + if b.Source == rootfs.Path && rootfs.MountedPath == "" { + continue + } + bDev, err := deviceFromHost(b.Source) + if err != nil { + return nil, err + } + blockDevs = append(blockDevs, bDev) + } + + return blockDevs, nil +} + func (b blockRootfs) preSetup() error { if b.mountedPath == "" { return nil @@ -242,7 +352,7 @@ func (b blockRootfs) preSetup() error { return fmt.Errorf("failed to extract boot files from rootfs: %w", err) } - err = mount.Unmount(b.mountedPath) + err = mount.Unmount(b.mountedPath, 0) if err != nil { return fmt.Errorf("failed to unmount rootfs: %w", err) } @@ -251,21 +361,11 @@ func (b blockRootfs) preSetup() error { } func (b blockRootfs) postSetup() error { - if b.mountedPath != "" { - err := setupDev(b.monRootfs, b.path) - if err != nil { - return err - } - } - - err := createTmpfs(b.monRootfs, "/tmp", - unix.MS_NOSUID|unix.MS_NOEXEC|unix.MS_STRICTATIME, - "1777", tmpfsSizeForBlockRootfs) - if err != nil { - err = fmt.Errorf("failed to create tmpfs for monitor's execution environment: %w", err) - } + return nil +} - return err +func (b blockRootfs) getMounts() ([]specs.Mount, error) { + return []specs.Mount{tmpfsMount("/tmp", tmpfsSizeForBlockRootfs)}, nil } func (b blockRootfs) getBlockDevs() ([]types.BlockDevParams, error) { @@ -285,7 +385,7 @@ func (b blockRootfs) getBlockDevs() ([]types.BlockDevParams, error) { } blockArgs = append(blockArgs, rootfsBlock) - blockFromMounts, err := getBlockVolumes(b.monRootfs, b.mounts, b.guest) + blockFromMounts, err := getBlockVolumes(b.mounts, b.guest) if err != nil { return nil, err } diff --git a/pkg/unikontainers/initrd_rootfs.go b/pkg/unikontainers/initrd_rootfs.go index 3f899fd3d..ee70d9da2 100644 --- a/pkg/unikontainers/initrd_rootfs.go +++ b/pkg/unikontainers/initrd_rootfs.go @@ -17,8 +17,6 @@ package unikontainers import ( "fmt" - "golang.org/x/sys/unix" - "github.com/opencontainers/runtime-spec/specs-go" "github.com/urunc-dev/urunc/pkg/unikontainers/initrd" "github.com/urunc-dev/urunc/pkg/unikontainers/types" @@ -43,14 +41,11 @@ func (i initrdRootfs) postSetup() error { return fmt.Errorf("failed to update guest's initrd: %w", err) } - err = createTmpfs(i.monRootfs, "/tmp", - unix.MS_NOSUID|unix.MS_NOEXEC|unix.MS_STRICTATIME, - "1777", tmpfsSizeForInitrdRootfs) - if err != nil { - err = fmt.Errorf("failed to create tmpfs for monitor's execution environment: %w", err) - } + return nil +} - return err +func (i initrdRootfs) getMounts() ([]specs.Mount, error) { + return []specs.Mount{tmpfsMount("/tmp", tmpfsSizeForInitrdRootfs)}, nil } func (i initrdRootfs) getBlockDevs() ([]types.BlockDevParams, error) { diff --git a/pkg/unikontainers/mount.go b/pkg/unikontainers/mount.go index a4be7cd5e..914276147 100644 --- a/pkg/unikontainers/mount.go +++ b/pkg/unikontainers/mount.go @@ -22,9 +22,13 @@ package unikontainers import ( "errors" "fmt" + "math" "os" "path/filepath" + "slices" + "github.com/containerd/containerd/mount" + securejoin "github.com/cyphar/filepath-securejoin" "github.com/moby/sys/userns" "golang.org/x/sys/unix" @@ -33,88 +37,53 @@ import ( var ErrCopyDir = errors.New("can not copy a directory") -type mountFlagStruct struct { - clear bool - flag int -} - -// createTmpfs creates a new tmpfs at path inside monRootfs -// In particular, it is used for the creation of /tmp and /dev. -// This is necessary to create the required devices for the monitor execution, -// such as KVM, null, urandom etc. -func createTmpfs(monRootfs string, path string, flags uint64, mode string, size string) error { - dstPath := filepath.Join(monRootfs, path) - mountType := "tmpfs" - data := "mode=" + mode + ",size=" + size - - err := os.MkdirAll(dstPath, 0755) - if err != nil { - return fmt.Errorf("failed to create %s dir: %w", path, err) - } - - err = unix.Mount(mountType, dstPath, mountType, uintptr(flags), data) - if err != nil { - return fmt.Errorf("failed to mount %s tmpfs: %w", path, err) - } - - // Remove propagation - err = unix.Mount("", dstPath, "", unix.MS_PRIVATE, "") - if err != nil { - return fmt.Errorf("failed to create %s tmpfs: %w", path, err) - } - - if mode == "1777" { - // sonarcloud:go:S2612 -- This is a tmpfs mount point, sticky bit 1777 is required (like /tmp), controlled path, safe by design - err := os.Chmod(dstPath, 01777) // NOSONAR +// setupDevices creates every device from the list inside monitor's rootfs +func setupDevices(monRootfs string, devices []specs.LinuxDevice, needsTAP bool) error { + for _, dev := range devices { + if !needsTAP && dev.Path == "/dev/net/tun" { + continue + } + err := setupDev(monRootfs, dev) if err != nil { - return fmt.Errorf("failed to chmod %s: %w", path, err) + return err } } + return nil } -// SetupDev set ups one new device in the container's rootfs. +// setupDev set ups one new device in the container's rootfs. // This function will get the major and minor number of // the device from the host's rootfs and it will replicate the device // inside the container's rootfs. -// When running in a user namespace, urunc bind-mounts the host device node to avoid EPERM from mknod. -// When not running in a user namespace, urunc creates the device node with mknod and adds rw for other -// users so non-root monitors can access it. -func setupDev(monRootfs string, devPath string) error { +// When running in a user namespace, urunc bind-mounts the host device node +// instead of using mknod. +func setupDev(monRootfs string, dev specs.LinuxDevice) error { // In a user namespace, always bind-mount the existing host device node. // Only MS_BIND is used here (no extra flags) to mirror runc's device handling. if userns.RunningInUserNS() { - return fileFromHost(monRootfs, devPath, "", unix.MS_BIND, false) - } - - // Get info of the original file - var devStat unix.Stat_t - err := unix.Stat(devPath, &devStat) - if err != nil { - return fmt.Errorf("failed to stat dev %s: %w", devPath, err) + return applyMount(monRootfs, bindMount(dev.Path, dev.Path, false)) } - // mask file's mode - mode := devStat.Mode & unix.S_IFMT - if mode != unix.S_IFCHR && mode != unix.S_IFBLK { - return fmt.Errorf("%s is not a device node", devPath) + var devType uint32 + switch dev.Type { + case "c": + devType = unix.S_IFCHR + case "b": + devType = unix.S_IFBLK + default: + return fmt.Errorf("%s is not a device node", dev.Path) } - // Get minor,major numbers - rdev := devStat.Rdev - major := unix.Major(uint64(rdev)) - minor := unix.Minor(uint64(rdev)) - - newDev := unix.Mkdev(major, minor) // Set the correct target path - relHostPath, err := filepath.Rel("/", devPath) + relHostPath, err := filepath.Rel("/", dev.Path) if err != nil { - return fmt.Errorf("failed to get relative path of %s to /: %w", devPath, err) + return fmt.Errorf("failed to get relative path of %s to /: %w", dev.Path, err) } dstPath := filepath.Join(monRootfs, relHostPath) // If the device is not at /dev but further down the tree, create // the necessary directories - if filepath.Dir(devPath) != "/dev" { + if filepath.Dir(dev.Path) != "/dev" { dstDir := filepath.Dir(dstPath) err = os.MkdirAll(dstDir, 0755) if err != nil { @@ -122,8 +91,20 @@ func setupDev(monRootfs string, devPath string) error { } } + if dev.FileMode == nil { + return fmt.Errorf("reference of %s FileMode is nil", dev.Path) + } // Create the new device node - err = unix.Mknod(dstPath, devStat.Mode, int(newDev)) //nolint: gosec + permBits := uint32(*dev.FileMode) & 0o777 + if dev.Major < 0 || dev.Major > math.MaxUint32 || + dev.Minor < 0 || dev.Minor > math.MaxUint32 { + return fmt.Errorf("device number out of range for %s: major=%d minor=%d", dstPath, dev.Major, dev.Minor) + } + newDev := unix.Mkdev(uint32(dev.Major), uint32(dev.Minor)) //#nosec G115 -- device numbers validated previously + if newDev > uint64(math.MaxInt) { + return fmt.Errorf("device number for %s too large: %d", dstPath, newDev) + } + err = unix.Mknod(dstPath, devType|permBits, int(newDev)) //#nosec G115 -- device numbers validated previously if err != nil { return fmt.Errorf("failed to make device node %s: %w", dstPath, err) } @@ -131,15 +112,20 @@ func setupDev(monRootfs string, devPath string) error { // Set up permissions, adding rw for others to ensure that any user can // read/write them. This is helpful for non-root monitor execution and // removes the burdain of getting kvm/block group id - permBits := devStat.Mode & 0o777 permBits |= 0o006 err = unix.Chmod(dstPath, permBits) if err != nil { return fmt.Errorf("failed to chmod %s: %w", dstPath, err) } + if dev.UID == nil { + return fmt.Errorf("reference of %s UID is nil", dev.Path) + } + if dev.GID == nil { + return fmt.Errorf("reference of %s GID is nil", dev.Path) + } // Set the owner as in the original file - err = os.Chown(dstPath, int(devStat.Uid), int(devStat.Gid)) + err = os.Chown(dstPath, int(*dev.UID), int(*dev.GID)) if err != nil { return fmt.Errorf("failed to chown %s: %w", dstPath, err) } @@ -147,23 +133,24 @@ func setupDev(monRootfs string, devPath string) error { return nil } -// fileFromHost set ups a mirror of file from the host's rootfs inside the -// container's rootfs. Also, it preserves the permissions and ownership of the -// file in the host's rootfs. -// if withCopy is set then copy the file, otherwise -// bind mount it. -// In the context of monitor binaries a copy is considered safer, since -// none of the monitor processes will share memory with other processes -// of the same monitor. On the other hand, a copy is slower and consumes -// more space. -func fileFromHost(monRootfs string, hostPath string, target string, mFlags int, withCopy bool) error { +// fileFromHost copies a file from the host's rootfs into monRootfs, preserving +// the original file's permissions and ownership. If target is empty, the file +// is placed at the same path it has on the host. +// A copy is preferred for monitor binaries: none of the monitor processes share +// memory with other processes of the same monitor, at the cost of being slower +// and consuming more space. Copying a directory is not supported and returns +// ErrCopyDir. +func fileFromHost(monRootfs string, hostPath string, target string) error { // Get the info of the original file var fileInfo unix.Stat_t err := unix.Stat(hostPath, &fileInfo) if err != nil { return err } - mode := fileInfo.Mode + + if (fileInfo.Mode & unix.S_IFMT) == unix.S_IFDIR { + return ErrCopyDir + } if target == "" { // Set the correct path @@ -174,81 +161,20 @@ func fileFromHost(monRootfs string, hostPath string, target string, mFlags int, } dstPath := filepath.Join(monRootfs, target) - if (mode & unix.S_IFMT) != unix.S_IFDIR { - dstDir := filepath.Dir(dstPath) - if withCopy { - err = copyFile(hostPath, dstPath) - if err != nil { - return fmt.Errorf("failed to copy file %s: %w", hostPath, err) - } - } else { - err = bindMountFile(hostPath, dstDir, dstPath, fileInfo.Mode, mFlags, false) - if err != nil { - return fmt.Errorf("failed to bind mount file %s: %w", hostPath, err) - } - } - } else { - if withCopy { - return ErrCopyDir - } - err = bindMountFile(hostPath, dstPath, "", 0, mFlags, true) - if err != nil { - return fmt.Errorf("failed to bind mount file %s: %w", hostPath, err) - } - } - - // If a copy is created, set up the permissions and ownership to match the original file. - // For bind mounts the host inode attributes remain unchanged. - if withCopy { - err = unix.Chmod(dstPath, fileInfo.Mode) - if err != nil { - return fmt.Errorf("failed to chmod %s: %w", dstPath, err) - } - - err = os.Chown(dstPath, int(fileInfo.Uid), int(fileInfo.Gid)) - if err != nil { - return fmt.Errorf("failed to chown %s: %w", dstPath, err) - } - } - - // The initial MS_BIND won't change the mount options, we need to do a - // separate MS_BIND|MS_REMOUNT to apply the mount options. We skip - // doing this if the user has not specified any mount flags at all - // (including cleared flags) -- in which case we just keep the original - // mount flags. - if mFlags & ^(unix.MS_BIND|unix.MS_REC|unix.MS_REMOUNT) != 0 { - flags := mFlags | unix.MS_BIND | unix.MS_REMOUNT - err = unix.Mount(dstPath, dstPath, "", uintptr(flags), "") - if err != nil { - return fmt.Errorf("failed to set mount flags for %s: %w", dstPath, err) - } - } - - return nil -} - -// bindMountFile bind mounts a file/directory to a new path -func bindMountFile(hostPath string, dstDir string, dstPath string, perm uint32, mFlags int, isDir bool) error { - var mountTarget string - err := os.MkdirAll(dstDir, 0755) + err = copyFile(hostPath, dstPath) if err != nil { - return fmt.Errorf("failed to create directory %s: %w", dstDir, err) + return fmt.Errorf("failed to copy file %s: %w", hostPath, err) } - if !isDir { - dstFile, err1 := unix.Open(dstPath, unix.O_CREAT, perm) - if err1 != nil { - return fmt.Errorf("failed to create file %s: %w", dstPath, err1) - } - unix.Close(dstFile) - mountTarget = dstPath - } else { - mountTarget = dstDir + // Set up the permissions and ownership to match the original file. + err = unix.Chmod(dstPath, fileInfo.Mode) + if err != nil { + return fmt.Errorf("failed to chmod %s: %w", dstPath, err) } - err = unix.Mount(hostPath, mountTarget, "", uintptr(mFlags), "") + err = os.Chown(dstPath, int(fileInfo.Uid), int(fileInfo.Gid)) if err != nil { - return fmt.Errorf("failed to bind mount %s: %w", mountTarget, err) + return fmt.Errorf("failed to chown %s: %w", dstPath, err) } return nil @@ -327,94 +253,121 @@ func prepareRoot(path string, rootfsPropagation string) error { return unix.Mount(path, path, "bind", unix.MS_BIND|unix.MS_REC, "") } -func mountVolumes(rootfsPath string, mounts []specs.Mount) error { +// applyMounts sets up every mount specified in the mounts argument inside the +// directory of rootfsPath. +func applyMounts(rootfsPath string, mounts []specs.Mount) error { for _, m := range mounts { - // Skip non-bind mounts - // TODO handle other types of mounts too - if m.Type != "bind" { - continue + err := applyMount(rootfsPath, m) + if err != nil { + // NOTE: We do not cleanup here because we assume that + // a mount namespace was declared and we are inside one. + // Therefore, after a fail, we will exit reexec (the only + // process inside the mount namespace and therefore the + // namespace will get removed and the kernel will cleanup + // the mounts. Revisit this in the future and check if it + // is urunc's responsibility to cleanup. If it is we need + // to account for cases where a mount namespace was not + // specified and therefore we need to maually unmount everything. + return fmt.Errorf("failed to mount %s at %s: %w", m.Source, m.Destination, err) } - var mountFlags int - var mountClearedFlags int - var propFlag []int - mountFlags = 0 - mountClearedFlags = 0 - for _, o := range m.Options { - f, exists := mapMountFlag(o) - if exists { - if f.clear { - mountFlags &= ^f.flag - mountClearedFlags |= f.flag - } else { - mountFlags |= f.flag - mountClearedFlags &= ^f.flag - } - continue - } - fprop, err := mapRootfsPropagationFlag(o) - if err == nil { - propFlag = append(propFlag, fprop) - } - // Ignore unknown flags - // TODO: Handle unknown flags. These can be mount attribute flags - // or specific flags for a particular fs type. + } + + return nil +} + +// applyMount mounts a single entry under rootfsPath. +func applyMount(rootfsPath string, m specs.Mount) error { + target, err := securejoin.SecureJoin(rootfsPath, m.Destination) + if err != nil { + return fmt.Errorf("failed to resolve mount target %s: %w", m.Destination, err) + } + + err = createMountPoint(target, m) + if err != nil { + return fmt.Errorf("failed to create mount target %s: %w", target, err) + } + + // containerd's option parser does not translate propagation flags (e.g. + // private, rslave) into flags, so we strip them here and apply them + // manually + opts, propFlags := splitPropagationOptions(m.Options) + + cm := mount.Mount{ + Type: m.Type, + Source: m.Source, + Options: opts, + } + err = cm.Mount(target) + if err != nil { + return fmt.Errorf("failed to mount %s at %s: %w", cm.Source, target, err) + } + + for _, pFlag := range propFlags { + if err := unix.Mount("", target, "", uintptr(pFlag), ""); err != nil { + return fmt.Errorf("failed to set propagation flag for %s: %w", m.Destination, err) } - err := fileFromHost(rootfsPath, m.Source, m.Destination, mountFlags, false) + } + + if m.Type == "tmpfs" && slices.Contains(m.Options, "mode=1777") { + err = os.Chmod(target, 01777) if err != nil { - return err + return fmt.Errorf("failed to chmod %s: %w", target, err) } + } + + return nil +} - dstPath := filepath.Join(rootfsPath, m.Destination) - for _, pFlag := range propFlag { - err = unix.Mount(dstPath, dstPath, "", uintptr(pFlag), "") +// createMountPoint creates the mountpoint of a mount. For file mounts a file is +// created ad for directories a directory is created. +func createMountPoint(target string, m specs.Mount) error { + if m.Type == "bind" { + var st unix.Stat_t + err := unix.Stat(m.Source, &st) + if err != nil { + return fmt.Errorf("failed to stat mount source %s: %w", m.Source, err) + } + if st.Mode&unix.S_IFMT != unix.S_IFDIR { + dstDir := filepath.Dir(target) + err := os.MkdirAll(dstDir, 0755) + if err != nil { + return fmt.Errorf("failed to create directory %s: %w", dstDir, err) + } + fd, err := unix.Open(target, unix.O_CREAT, 0644) if err != nil { - return fmt.Errorf("failed to set propagation flag for %s: %w", m.Source, err) + return fmt.Errorf("failed to create file %s: %w", target, err) } + err = unix.Close(fd) + if err != nil { + return fmt.Errorf("failed to close file %s: %w", target, err) + } + + return nil } } + err := os.MkdirAll(target, 0755) + if err != nil { + return fmt.Errorf("failed to create directory %s: %w", target, err) + } + return nil } -// mapMountFlag retrieves the mount flags of a mount entry -// from the container's configuration -func mapMountFlag(value string) (mountFlagStruct, bool) { - mountFlagsMapping := map[string]mountFlagStruct{ - "async": {true, unix.MS_SYNCHRONOUS}, - "atime": {true, unix.MS_NOATIME}, - "bind": {false, unix.MS_BIND}, - "defaults": {false, 0}, - "dev": {true, unix.MS_NODEV}, - "diratime": {true, unix.MS_NODIRATIME}, - "dirsync": {false, unix.MS_DIRSYNC}, - "exec": {true, unix.MS_NOEXEC}, - "iversion": {false, unix.MS_I_VERSION}, - "lazytime": {false, unix.MS_LAZYTIME}, - "loud": {true, unix.MS_SILENT}, - "mand": {false, unix.MS_MANDLOCK}, - "noatime": {false, unix.MS_NOATIME}, - "nodev": {false, unix.MS_NODEV}, - "nodiratime": {false, unix.MS_NODIRATIME}, - "noexec": {false, unix.MS_NOEXEC}, - "noiversion": {true, unix.MS_I_VERSION}, - "nolazytime": {true, unix.MS_LAZYTIME}, - "nomand": {true, unix.MS_MANDLOCK}, - "norelatime": {true, unix.MS_RELATIME}, - "nostrictatime": {true, unix.MS_STRICTATIME}, - "nosuid": {false, unix.MS_NOSUID}, - "nosymfollow": {false, unix.MS_NOSYMFOLLOW}, // since kernel 5.10 - "rbind": {false, unix.MS_BIND | unix.MS_REC}, - "relatime": {false, unix.MS_RELATIME}, - "remount": {false, unix.MS_REMOUNT}, - "ro": {false, unix.MS_RDONLY}, - "rw": {true, unix.MS_RDONLY}, - "silent": {false, unix.MS_SILENT}, - "strictatime": {false, unix.MS_STRICTATIME}, - "suid": {true, unix.MS_NOSUID}, - "sync": {false, unix.MS_SYNCHRONOUS}, - "symfollow": {true, unix.MS_NOSYMFOLLOW}, // since kernel 5.10 - } - - f, e := mountFlagsMapping[value] - return f, e +// splitPropagationOptions separates mount propagation tokens from the rest of +// the mount options. containerd's mount package does not translate propagation +// tokens into flags, so they are returned separately to be applied after the +// mount. +func splitPropagationOptions(options []string) ([]string, []int) { + var opts []string + var propFlags []int + for _, o := range options { + if pFlag, err := mapRootfsPropagationFlag(o); err == nil { + propFlags = append(propFlags, pFlag) + continue + } + opts = append(opts, o) + } + + return opts, propFlags } diff --git a/pkg/unikontainers/rootfs.go b/pkg/unikontainers/rootfs.go index 0d7c80c0f..881ea50f3 100644 --- a/pkg/unikontainers/rootfs.go +++ b/pkg/unikontainers/rootfs.go @@ -20,6 +20,7 @@ import ( "path/filepath" "strconv" + "github.com/opencontainers/runtime-spec/specs-go" "golang.org/x/sys/unix" "github.com/urunc-dev/urunc/pkg/unikontainers/types" @@ -35,11 +36,78 @@ const annotRootfsParams = "com.urunc.internal.rootfs.params" type rootfsBuilder interface { preSetup() error postSetup() error + getMounts() ([]specs.Mount, error) getBlockDevs() ([]types.BlockDevParams, error) getSharedDirs() (types.SharedfsParams, error) preStart() error } +// tmpfsMount creates a mount for a tmpfs in the form of "/tmp" at target +func tmpfsMount(target string, size string) specs.Mount { + return specs.Mount{ + Type: "tmpfs", + Source: "tmpfs", + Destination: target, + Options: []string{ + "nosuid", + "noexec", + "strictatime", + "mode=1777", + "size=" + size, + }, + } +} + +// bindMount builds a private, non-recursive bind mount of source at target +func bindMount(source string, target string, private bool) specs.Mount { + m := specs.Mount{ + Type: "bind", + Source: source, + Destination: target, + Options: []string{"bind"}, + } + + if private { + m.Options = append(m.Options, "private") + } + + return m +} + +// deviceFromHost finds a device in the host from the path and returns its info +// in the form of specs.LinuxDevice +func deviceFromHost(path string) (specs.LinuxDevice, error) { + var devStat unix.Stat_t + err := unix.Stat(path, &devStat) + if err != nil { + return specs.LinuxDevice{}, fmt.Errorf("failed to stat dev %s: %w", path, err) + } + + var devType string + switch devStat.Mode & unix.S_IFMT { + case unix.S_IFCHR: + devType = "c" + case unix.S_IFBLK: + devType = "b" + default: + return specs.LinuxDevice{}, fmt.Errorf("%s is not a device node", path) + } + + mode := os.FileMode(devStat.Mode & 0o777) + uid := devStat.Uid + gid := devStat.Gid + + return specs.LinuxDevice{ + Path: path, + Type: devType, + Major: int64(unix.Major(uint64(devStat.Rdev))), + Minor: int64(unix.Minor(uint64(devStat.Rdev))), + FileMode: &mode, + UID: &uid, + GID: &gid, + }, nil +} + // rootfsSelector encapsulates the context for rootfs selection type rootfsSelector struct { bundle string @@ -61,14 +129,11 @@ func (n noRootfs) preSetup() error { } func (n noRootfs) postSetup() error { - err := createTmpfs(n.monRootfs, "/tmp", - unix.MS_NOSUID|unix.MS_NOEXEC|unix.MS_STRICTATIME, - "1777", tmpfsSizeForNoRootfs) - if err != nil { - err = fmt.Errorf("failed to create tmpfs for monitor's execution environment: %w", err) - } + return nil +} - return err +func (n noRootfs) getMounts() ([]specs.Mount, error) { + return []specs.Mount{tmpfsMount("/tmp", tmpfsSizeForNoRootfs)}, nil } func (n noRootfs) getBlockDevs() ([]types.BlockDevParams, error) { @@ -326,136 +391,45 @@ func changeRoot(rootfsDir string, pivot bool) error { return nil } -// nolint:gocyclo -// prepareMonRootfs prepares the rootfs where the monitor will execute. It -// essentially sets up the devices (KVM, snapshotter block device) that are required -// for the guest execution and any other files (e.g. binaries). -func prepareMonRootfs(monRootfs string, monitorPath string, monitorDataPath string, needsKVM bool, needsTAP bool) error { - err := fileFromHost(monRootfs, monitorPath, "", unix.MS_BIND|unix.MS_PRIVATE, false) +// getMonitorDevices returns the devices which are needed for the execution of +// the monitor process +func getMonitorDevices(needsKVM bool) ([]specs.LinuxDevice, error) { + nullDev, err := deviceFromHost("/dev/null") if err != nil { - return err + return nil, fmt.Errorf("could not get host device /dev/null: %w", err) } - - // TODO: Remove these when we switch to static binaries - monitorName := filepath.Base(monitorPath) - if monitorName != "firecracker" { - err = fileFromHost(monRootfs, "/lib", "", unix.MS_BIND|unix.MS_PRIVATE, false) - if err != nil { - return err - } - - err = fileFromHost(monRootfs, "/lib64", "", unix.MS_BIND|unix.MS_PRIVATE, false) - if err != nil { - // If the file does not exist, just ignore it - if !os.IsNotExist(err) { - return err - } - } - - err = fileFromHost(monRootfs, "/usr/lib", "", unix.MS_BIND|unix.MS_PRIVATE, false) - if err != nil { - return err - } - } - - // TODO: Remove these when we switch to static binaries - if len(monitorName) >= 4 && monitorName[:4] == "qemu" { - var qDataPath string - var sBiosPath string - var err error - if monitorDataPath == "" { - qDataPath, err = findQemuDataDir("qemu") - } else { - qDataPath = filepath.Join(monitorDataPath, "qemu") - err = nil - } - if err != nil { - return err - } - - err = fileFromHost(monRootfs, qDataPath, "/usr/share/qemu", unix.MS_BIND|unix.MS_PRIVATE, false) - if err != nil { - return err - } - - if monitorDataPath == "" { - sBiosPath, err = findQemuDataDir("seabios") - } else { - sBiosPath = filepath.Join(monitorDataPath, "seabios") - err = nil - } - if err != nil { - return fmt.Errorf("failed to get info of seabios directory: %w", err) - } - err = fileFromHost(monRootfs, sBiosPath, "/usr/share/seabios", unix.MS_BIND|unix.MS_PRIVATE, false) - if err != nil { - // In urunc-deploy and in some distros seabios does not exist and - // we do not need it. So if we could not find it, just ignore it. - if !os.IsNotExist(err) { - return err - } - } - } - - newProcDir := filepath.Join(monRootfs, "/proc") - err = os.MkdirAll(newProcDir, 0555) + randomDev, err := deviceFromHost("/dev/urandom") if err != nil { - return err - } - - err = unix.Mount("proc", newProcDir, "proc", 0, "") - if err != nil { - return err - } - - err = createTmpfs(monRootfs, "/dev", unix.MS_NOSUID|unix.MS_STRICTATIME, "755", "65536k") + return nil, fmt.Errorf("could not get host device /dev/urandom: %w", err) + } + // The tun device is always included because in urunc create we can not know + // if there will be a virtual ethernet device or not, since the CNI hooks + // have not executed yet. Therefore, the decision about whether it is actually + // created in the monitor rootfs is decided in Exec, which checks the + // network configuration. + tunDev, err := deviceFromHost("/dev/net/tun") if err != nil { - return err - } - - err = setupDev(monRootfs, "/dev/null") - if err != nil { - return err - } - - err = setupDev(monRootfs, "/dev/urandom") - if err != nil { - return err - } - - if needsTAP { - err = setupDev(monRootfs, "/dev/net/tun") - if err != nil { - return err - } + return nil, fmt.Errorf("could not get host device /dev/net/tun: %w", err) } + devices := []specs.LinuxDevice{nullDev, randomDev, tunDev} if needsKVM { - err = setupDev(monRootfs, "/dev/kvm") + kvmDev, err := deviceFromHost("/dev/kvm") if err != nil { - return err + return nil, fmt.Errorf("could not get host device /dev/kvm: %w", err) } + devices = append(devices, kvmDev) } - // Setup /dev/pts for PTY support (needed for console and debugging tools like cntr) - // This allows tools like cntr to attach to the container with a shell - devPtsDir := filepath.Join(monRootfs, "/dev/pts") - err = os.MkdirAll(devPtsDir, 0755) - if err != nil { - return fmt.Errorf("failed to create /dev/pts directory: %w", err) - } - - // Mount devpts filesystem - // Using newinstance creates an isolated pts namespace for this container - err = unix.Mount("devpts", devPtsDir, "devpts", unix.MS_NOSUID|unix.MS_NOEXEC, "newinstance,ptmxmode=0666,mode=0620,gid=5") - if err != nil { - return fmt.Errorf("failed to mount devpts: %w", err) - } + return devices, nil +} +// setupConsole setups a /dev/console file for the monitor's execution environment +func setupConsole(monRootfs string) error { // Create /dev/ptmx as a symlink to /dev/pts/ptmx // This is the standard way to provide the PTY master device ptmxPath := filepath.Join(monRootfs, "/dev/ptmx") - err = os.Symlink("pts/ptmx", ptmxPath) + err := os.Symlink("pts/ptmx", ptmxPath) if err != nil && !os.IsExist(err) { return fmt.Errorf("failed to create /dev/ptmx symlink: %w", err) } @@ -475,3 +449,73 @@ func prepareMonRootfs(monRootfs string, monitorPath string, monitorDataPath stri return nil } + +// mountsForMonitor returns all the required mounts from the host for the +// monitor execution: the binary, supporting libraries and data directories. +// These are the host files that need to be mirrored inside the monitor rootfs. +// In addition, it also includes the special filesystems required for the monitor +// execution (e.g. procfs) +func mountsForMonitor(monitorPath string, monitorDataPath string) ([]specs.Mount, error) { + procMount := specs.Mount{ + Type: "proc", + Source: "proc", + Destination: "/proc", + } + devMount := specs.Mount{ + Type: "tmpfs", + Source: "tmpfs", + Destination: "/dev", + Options: []string{"nosuid", "strictatime", "mode=755", "size=65536k"}, + } + devPtsMount := specs.Mount{ + Type: "devpts", + Source: "devpts", + Destination: "/dev/pts", + Options: []string{"nosuid", "noexec", "newinstance", "ptmxmode=0666", "mode=0620"}, + } + + mounts := []specs.Mount{procMount, devMount, devPtsMount, bindMount(monitorPath, monitorPath, true)} + + monitorName := filepath.Base(monitorPath) + // TODO: Remove most of these when we switch to static binaries. + if monitorName != "firecracker" { + mounts = append(mounts, bindMount("/lib", "/lib", true)) + + // If /lib64 does not exist, just ignore it + if _, err := os.Stat("/lib64"); err == nil { + mounts = append(mounts, bindMount("/lib64", "/lib64", true)) + } else if !os.IsNotExist(err) { + return nil, err + } + + mounts = append(mounts, bindMount("/usr/lib", "/usr/lib", true)) + } + + if len(monitorName) >= 4 && monitorName[:4] == "qemu" { + qDataPath := filepath.Join(monitorDataPath, "qemu") + sBiosPath := filepath.Join(monitorDataPath, "seabios") + if monitorDataPath == "" { + var err error + qDataPath, err = findQemuDataDir("qemu") + if err != nil { + return nil, err + } + sBiosPath, err = findQemuDataDir("seabios") + if err != nil { + return nil, fmt.Errorf("failed to get info of seabios directory: %w", err) + } + } + + mounts = append(mounts, bindMount(qDataPath, "/usr/share/qemu", true)) + + // In urunc-deploy and in some distros seabios does not exist and + // we do not need it. So if we could not find it, just ignore it. + if _, err := os.Stat(sBiosPath); err == nil { + mounts = append(mounts, bindMount(sBiosPath, "/usr/share/seabios", true)) + } else if !os.IsNotExist(err) { + return nil, err + } + } + + return mounts, nil +} diff --git a/pkg/unikontainers/shared_fs.go b/pkg/unikontainers/shared_fs.go index 9ac86e8b0..9441f39a7 100644 --- a/pkg/unikontainers/shared_fs.go +++ b/pkg/unikontainers/shared_fs.go @@ -19,8 +19,6 @@ import ( "path/filepath" "strings" - "golang.org/x/sys/unix" - "github.com/opencontainers/runtime-spec/specs-go" "github.com/urunc-dev/urunc/pkg/unikontainers/hypervisors" "github.com/urunc-dev/urunc/pkg/unikontainers/types" @@ -44,35 +42,24 @@ func (s sharedfsRootfs) preSetup() error { } func (s sharedfsRootfs) postSetup() error { - // Mount the container's rootfs inside the monitor rootfs - err := fileFromHost(s.monRootfs, s.mountedPath, containerRootfsMountPath, unix.MS_BIND|unix.MS_PRIVATE, false) - if err != nil { - return fmt.Errorf("failed to mount container's rootfs in monitor rootfs; %w", err) - } + return nil +} - newCntrRootfs := filepath.Join(s.monRootfs, containerRootfsMountPath) - err = mountVolumes(newCntrRootfs, s.mounts) - if err != nil { - return fmt.Errorf("failed to mount volumes in container's rootfs; %w", err) - } +func (s sharedfsRootfs) getMounts() ([]specs.Mount, error) { + // Mount the container's rootfs inside the monitor rootfs and then the + // container's volumes on top of it. + mounts := []specs.Mount{bindMount(s.mountedPath, containerRootfsMountPath, true)} if s.sfsType == "virtiofs" { // Get the virtiofsd binary from host in monRootfs - err = fileFromHost(s.monRootfs, s.vfsdConfig.Path, "", unix.MS_BIND|unix.MS_PRIVATE, false) - if err != nil { - return fmt.Errorf("could not bind mount %s: %w", s.vfsdConfig.Path, err) - } + mounts = append(mounts, bindMount(s.vfsdConfig.Path, s.vfsdConfig.Path, true)) } tmpfsSize := chooseTmpfsSize(s.sfsType, s.memory) - err = createTmpfs(s.monRootfs, "/tmp", - unix.MS_NOSUID|unix.MS_NOEXEC|unix.MS_STRICTATIME, - "1777", tmpfsSize) - if err != nil { - err = fmt.Errorf("failed to create tmpfs for monitor's execution environment: %w", err) - } + mounts = append(mounts, tmpfsMount("/tmp", tmpfsSize)) + mounts = append(mounts, filterBindMounts(s.mounts)...) - return err + return mounts, nil } func (s sharedfsRootfs) getBlockDevs() ([]types.BlockDevParams, error) { @@ -134,3 +121,25 @@ func adjustPathsForSharedfs(path string) string { return path } + +// filterBindMounts filters the mounts form the container's spec keeping only the +// bind mounts and adjusts the Destination path to the mountpoint of the +// container's rootfs inside the monitor rootfs. +func filterBindMounts(mounts []specs.Mount) []specs.Mount { + var result []specs.Mount + for _, m := range mounts { + // Skip non-bind mounts + // TODO handle other types of mounts too + if m.Type != "bind" { + continue + } + result = append(result, specs.Mount{ + Type: "bind", + Source: m.Source, + Destination: filepath.Join(containerRootfsMountPath, m.Destination), + Options: m.Options, + }) + } + + return result +} diff --git a/pkg/unikontainers/types/types.go b/pkg/unikontainers/types/types.go index c6388e2cc..91519f452 100644 --- a/pkg/unikontainers/types/types.go +++ b/pkg/unikontainers/types/types.go @@ -54,10 +54,12 @@ type NetDevParams struct { } type BlockDevParams struct { - Source string - MountPoint string - FsType string - ID string + Source string + MountPoint string + FsType string + ID string + HostMountPoint string + LoopAutoclear bool } type SharedfsParams struct { diff --git a/pkg/unikontainers/unikontainers.go b/pkg/unikontainers/unikontainers.go index 84172aca4..d8d422738 100644 --- a/pkg/unikontainers/unikontainers.go +++ b/pkg/unikontainers/unikontainers.go @@ -197,22 +197,24 @@ func (u *Unikontainer) InitialSetup() error { "mountedPath": rootfsParams.MountedPath, }).Debug("guest rootfs params") - if rootfsParams.Type == "block" { - rfsBuilder := blockRootfs{ - mounts: u.Spec.Mounts, - monRootfs: rootfsParams.MonRootfs, - mountedPath: rootfsParams.MountedPath, - path: rootfsParams.Path, - kernelPath: unikernelPath, - initrdPath: initrdPath, - uruncJSONPath: uruncJSONFilename, - guestType: unikernelType, - guest: unikernel, - } - err := rfsBuilder.preSetup() - if err != nil { - return fmt.Errorf("pre setup step for rootfs failed: %w", err) - } + vmmType := u.State.Annotations[annotHypervisor] + vmm, err := hypervisors.NewVMM(hypervisors.VmmType(vmmType), u.UruncCfg.Monitors) + if err != nil { + return err + } + + defaultMemSizeMB := u.UruncCfg.Monitors[vmmType].DefaultMemoryMB + memory := monitorMemoryBytes(defaultMemSizeMB, u.Spec.Linux.Resources) + rfsBuilder := u.newRootfsBuilder(rootfsParams, unikernel, unikernelPath, initrdPath, memory) + + err = rfsBuilder.preSetup() + if err != nil { + return fmt.Errorf("pre setup step for rootfs failed: %w", err) + } + + monRes, err := getMonitorResources(rfsBuilder, rootfsParams, vmm, u.UruncCfg.Monitors[vmmType].DataPath) + if err != nil { + return err } u.State.Status = specs.StateCreating @@ -221,6 +223,12 @@ func (u *Unikontainer) InitialSetup() error { if err != nil { return err } + + err = saveMonitorResources(u.BaseDir, monRes) + if err != nil { + return fmt.Errorf("failed to store monitor resources: %w", err) + } + return u.saveContainerState() } @@ -350,6 +358,94 @@ func ChooseRootfs(bundle, specRoot string, annot map[string]string, cfg *UruncCo return result, nil } +// getMonitorResources collects every mount and device required for the monitor's +// execution environment, plus the block parameters passed to the guest. +func getMonitorResources(rfs rootfsBuilder, rootfsParams types.RootfsParams, vmm types.VMM, monitorDataPath string) (monitorResources, error) { + var res monitorResources + var err error + + res.Mounts, err = mountsForMonitor(vmm.Path(), monitorDataPath) + if err != nil { + return res, err + } + rootfsMounts, err := rfs.getMounts() + if err != nil { + return res, fmt.Errorf("failed to get mounts for monitor rootfs: %w", err) + } + res.Mounts = append(res.Mounts, rootfsMounts...) + + res.Devices, err = getMonitorDevices(vmm.UsesKVM()) + if err != nil { + return res, fmt.Errorf("failed to get host devices for monitor: %w", err) + } + res.BlockArgs, err = rfs.getBlockDevs() + if err != nil { + return res, fmt.Errorf("failed to get block devices to attach in sandbox: %w", err) + } + blockDevs, err := blockDevNodes(res.BlockArgs, rootfsParams) + if err != nil { + return res, fmt.Errorf("failed to get block devices which should be replicated inside monitor's execution environment: %w", err) + } + res.Devices = append(res.Devices, blockDevs...) + + return res, nil +} + +// newRootfsBuilder constructs the rootfsBuilder matching the selected guest +// rootfs. It is shared by InitialSetup (which gathers the monitor resources) and +// Exec (which performs the per-rootfs actions). +func (u *Unikontainer) newRootfsBuilder(rootfsParams types.RootfsParams, unikernel types.Unikernel, unikernelPath string, initrdPath string, memory uint64) rootfsBuilder { + switch rootfsParams.Type { + case "block": + return blockRootfs{ + mounts: u.Spec.Mounts, + monRootfs: rootfsParams.MonRootfs, + mountedPath: rootfsParams.MountedPath, + path: rootfsParams.Path, + kernelPath: unikernelPath, + initrdPath: initrdPath, + uruncJSONPath: uruncJSONFilename, + guestType: u.State.Annotations[annotType], + guest: unikernel, + } + case "initrd": + return initrdRootfs{ + mounts: u.Spec.Mounts, + initrdHostFullPath: filepath.Join(rootfsParams.MonRootfs, rootfsParams.Path), + monRootfs: rootfsParams.MonRootfs, + } + case "virtiofs", "9pfs": + return sharedfsRootfs{ + mounts: u.Spec.Mounts, + monRootfs: rootfsParams.MonRootfs, + mountedPath: rootfsParams.MountedPath, + sfsType: rootfsParams.Type, + vfsdConfig: u.UruncCfg.ExtraBins["virtiofsd"], + sharedPath: containerRootfsMountPath, + memory: memory, + } + default: + return noRootfs{ + monRootfs: rootfsParams.MonRootfs, + annotBlockPath: u.State.Annotations[annotBlock], + annotBlockMountPoint: u.State.Annotations[annotBlockMntPoint], + } + } +} + +// monitorMemoryBytes returns the guest memory size in bytes, honoring a memory +// limit from the OCI spec and falling back to the monitor's configured default. +func monitorMemoryBytes(defaultMem uint, resources *specs.LinuxResources) uint64 { + mem := uint64(defaultMem * 1024 * 1024) + if resources != nil && resources.Memory != nil { + if resources.Memory.Limit != nil && *resources.Memory.Limit > 0 { + mem = uint64(*resources.Memory.Limit) // nolint:gosec + } + } + + return mem +} + // nolint:gocyclo func (u *Unikontainer) Exec(metrics m.Writer) error { metrics.Capture(m.TS15) @@ -409,21 +505,11 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { UnikernelPath: unikernelPath, InitrdPath: initrdPath, Seccomp: true, // Enable Seccomp by default - MemSizeB: uint64(defaultMemSizeMB * 1024 * 1024), + MemSizeB: monitorMemoryBytes(defaultMemSizeMB, u.Spec.Linux.Resources), VCPUs: uint(defaultVCPUs), Environment: os.Environ(), } - // ExecArgs - // If memory limit is set in spec, use it instead of the config default value - if u.Spec.Linux.Resources != nil && u.Spec.Linux.Resources.Memory != nil { - if u.Spec.Linux.Resources.Memory.Limit != nil { - if *u.Spec.Linux.Resources.Memory.Limit > 0 { - vmmArgs.MemSizeB = uint64(*u.Spec.Linux.Resources.Memory.Limit) // nolint:gosec - } - } - } - // ExecArgs // Check if container is set to unconfined -- disable seccomp if u.Spec.Linux.Seccomp == nil { @@ -464,9 +550,6 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { // ExecArgs vmmArgs.Net = netArgs - // virtiofsd config - virtiofsdConfig := u.UruncCfg.ExtraBins["virtiofsd"] - // guest rootfs // block // handle guest's rootfs. @@ -497,51 +580,11 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { "mon_rootfs": rootfsParams.MonRootfs, }).Debug("guest rootfs params") - // TODO: Add support for using both an existing - // block based snapshot of the container's rootfs - // and an auxiliary block image placed in the container's image - // Currently if a block Image is present in the container's image, then - // we will just use this image. - var rfsBuilder rootfsBuilder - switch rootfsParams.Type { - case "block": - rfsBuilder = blockRootfs{ - mounts: u.Spec.Mounts, - monRootfs: rootfsParams.MonRootfs, - mountedPath: rootfsParams.MountedPath, - path: rootfsParams.Path, - kernelPath: unikernelPath, - initrdPath: initrdPath, - uruncJSONPath: uruncJSONFilename, - guestType: unikernelType, - guest: unikernel, - } - case "initrd": - rfsBuilder = initrdRootfs{ - mounts: u.Spec.Mounts, - initrdHostFullPath: filepath.Join(rootfsParams.MonRootfs, rootfsParams.Path), - monRootfs: rootfsParams.MonRootfs, - } - case "virtiofs", "9pfs": - rfsBuilder = sharedfsRootfs{ - mounts: u.Spec.Mounts, - monRootfs: rootfsParams.MonRootfs, - mountedPath: rootfsParams.MountedPath, - sfsType: rootfsParams.Type, - vfsdConfig: virtiofsdConfig, - sharedPath: containerRootfsMountPath, - memory: vmmArgs.MemSizeB, - } + rfsBuilder := u.newRootfsBuilder(rootfsParams, unikernel, unikernelPath, initrdPath, vmmArgs.MemSizeB) + if rootfsParams.Type == "virtiofs" || rootfsParams.Type == "9pfs" { // Update the paths of the files we need to pass in the monitor process. vmmArgs.UnikernelPath = adjustPathsForSharedfs(vmmArgs.UnikernelPath) vmmArgs.InitrdPath = adjustPathsForSharedfs(vmmArgs.InitrdPath) - default: - uniklog.Debug("No rootfs for guest") - rfsBuilder = noRootfs{ - monRootfs: rootfsParams.MonRootfs, - annotBlockPath: u.State.Annotations[annotBlock], - annotBlockMountPoint: u.State.Annotations[annotBlockMntPoint], - } } if err = os.MkdirAll(rootfsParams.MonRootfs, 0o755); err != nil { @@ -556,39 +599,36 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { return err } - // Setup the rootfs for the monitor execution, creating necessary - // devices and the monitor's binary. - err = prepareMonRootfs(rootfsParams.MonRootfs, vmm.Path(), u.UruncCfg.Monitors[vmmType].DataPath, vmm.UsesKVM(), withTUNTAP) + // The monitor mounts and devices were gathered and stored in a file during + // InitialSetup; here we just apply them. postSetup applies the container's own + // bind mounts on top of the shared rootfs, and setupDevices decides whether to + // create the TUN/TAP device based on the container's network configuration. + monRes, err := loadMonitorResources(u.BaseDir) if err != nil { - return err + return fmt.Errorf("failed to load monitor resources: %w", err) + } + + err = applyMounts(rootfsParams.MonRootfs, monRes.Mounts) + if err != nil { + return fmt.Errorf("failed to apply rootfs mounts: %w", err) } err = rfsBuilder.postSetup() if err != nil { - return fmt.Errorf("post setup step for block based rootfs failed: %w", err) + return fmt.Errorf("post setup step for rootfs failed: %w", err) } - blockArgs, err := rfsBuilder.getBlockDevs() + err = setupDevices(rootfsParams.MonRootfs, monRes.Devices, withTUNTAP) if err != nil { - return fmt.Errorf("failed to get block devices to attach in sandbox: %w", err) + return fmt.Errorf("failed to create devices in monitor rootfs: %w", err) } - sharedfsArgs, err := rfsBuilder.getSharedDirs() + err = setupConsole(rootfsParams.MonRootfs) if err != nil { - uniklog.Errorf("failed to get directories to share with sandbox: %v", err) return err } - unikernelParams.Rootfs = rootfsParams - metrics.Capture(m.TS17) - - // unikernelParams - unikernelParams.Block = blockArgs - - // ExecArgs - vmmArgs.Sharedfs = sharedfsArgs - // vAccel setup vAccelType, vsockSocketPath, rpcAddress, err := resolveVAccelConfig(u.State.Annotations[annotHypervisor], u.Spec.Annotations) if err != nil { @@ -608,9 +648,13 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { unikernelParams.EnvVars = append(unikernelParams.EnvVars, "VACCEL_RPC_ADDRESS="+rpcAddress) // Prepare the guest environment for vAccel vsock communication - err = prepareVSockEnvironment(rootfsParams.MonRootfs, u.State.Annotations[annotHypervisor], vsockSocketPath) + vaccelDevices, err := prepareVSockEnvironment(rootfsParams.MonRootfs, u.State.Annotations[annotHypervisor], vsockSocketPath) if err != nil { - uniklog.Debugf("failed to prepare all required vsock mounts: %v", err) + uniklog.Debugf("failed to prepare get required vsock devices: %v", err) + } + err = setupDevices(rootfsParams.MonRootfs, vaccelDevices, false) + if err != nil { + return fmt.Errorf("failed to create devices in monitor rootfs: %w", err) } vmmArgs.VAccelType = vAccelType @@ -618,6 +662,20 @@ func (u *Unikontainer) Exec(metrics m.Writer) error { vmmArgs.VSockDevID = idToGuestCID(u.State.ID) } + unikernelParams.Rootfs = rootfsParams + + // unikernelParams + // The block parameters were gathered in InitialSetup and stored in the + // monitor resources file. + unikernelParams.Block = monRes.BlockArgs + + // ExecArgs + sharedfsArgs, err := rfsBuilder.getSharedDirs() + if err != nil { + return fmt.Errorf("failed to get directories to share with sandbox: %w", err) + } + vmmArgs.Sharedfs = sharedfsArgs + // unikernel err = unikernel.Init(unikernelParams) if errors.Is(err, unikernels.ErrUndefinedVersion) || @@ -786,6 +844,20 @@ func (u *Unikontainer) Delete() error { return fmt.Errorf("cannot delete running container: %s", u.State.ID) } + // Restore the block volume mounts that were unmounted during create, + // so their sources become discoverable by future containers. Do it in + // a best-effort way, since a failure to restore a mount should not + // prevent the deletion of the container. + monRes, err := loadMonitorResources(u.BaseDir) + if err != nil { + uniklog.Errorf("failed to load monitor resources: %v", err) + } + + err = restoreBlockVolumes(monRes.BlockArgs) + if err != nil { + uniklog.Errorf("failed to restore block volume mounts: %v", err) + } + // get a monitor instance of the running monitor vmmType := u.State.Annotations[annotHypervisor] vmm, err := hypervisors.NewVMM(hypervisors.VmmType(vmmType), u.UruncCfg.Monitors) diff --git a/pkg/unikontainers/utils.go b/pkg/unikontainers/utils.go index dd9fd0b58..7d5ac5490 100644 --- a/pkg/unikontainers/utils.go +++ b/pkg/unikontainers/utils.go @@ -32,16 +32,54 @@ import ( "github.com/opencontainers/runtime-spec/specs-go" "github.com/urunc-dev/urunc/internal/constants" + "github.com/urunc-dev/urunc/pkg/unikontainers/types" ) const ( configFilename = "config.json" stateFilename = "state.json" + monitorFilename = "monitor.json" initPidFilename = "init.pid" uruncJSONFilename = "urunc.json" rootfsDirName = "rootfs" ) +// monitorResources holds the mounts and devices that must be replicated inside +// the monitor's rootfs, along with the block parameters the guest needs. +type monitorResources struct { + Mounts []specs.Mount `json:"mounts"` + Devices []specs.LinuxDevice `json:"devices"` + BlockArgs []types.BlockDevParams `json:"blockArgs"` +} + +// saveMonitorResources persists stores res as a JSON file under the container's base +// directory, so that the reexec's Exec can read it back. +func saveMonitorResources(baseDir string, res monitorResources) error { + data, err := json.Marshal(res) + if err != nil { + return err + } + + path := filepath.Join(baseDir, monitorFilename) + + return os.WriteFile(path, data, 0o644) //nolint: gosec +} + +// loadMonitorResources reads the monitor resources file stored by InitialSetup. +func loadMonitorResources(baseDir string) (monitorResources, error) { + var res monitorResources + + path := filepath.Join(baseDir, monitorFilename) + data, err := os.ReadFile(path) + if err != nil { + return res, err + } + + err = json.Unmarshal(data, &res) + + return res, err +} + // copy sourceFile to targetDir // creates targetDir and all necessary parent directories func copyFile(sourceFile string, targetPath string) error { diff --git a/pkg/unikontainers/vaccel.go b/pkg/unikontainers/vaccel.go index f6905696f..cc22cb17f 100644 --- a/pkg/unikontainers/vaccel.go +++ b/pkg/unikontainers/vaccel.go @@ -19,7 +19,7 @@ import ( "fmt" "regexp" - "golang.org/x/sys/unix" + "github.com/opencontainers/runtime-spec/specs-go" ) // ErrVAccelDisabled is returned by resolveVAccelConfig when the vAccel @@ -109,23 +109,24 @@ func resolveVAccelConfig(hypervisor string, annotations map[string]string) (stri // prepareVSockEnvironment prepares all required vsock devices and mounts // for vAccel execution inside the guest. This includes /dev/vsock, // /dev/vhost-vsock, and (for Firecracker) binding the host unix socket. -func prepareVSockEnvironment(monRootfs string, hypervisor string, vsockSocketPath string) error { - err := setupDev(monRootfs, "/dev/vsock") +func prepareVSockEnvironment(monRootfs string, hypervisor string, vsockSocketPath string) ([]specs.LinuxDevice, error) { + vsockDev, err := deviceFromHost("/dev/vsock") if err != nil { - return err + return nil, fmt.Errorf("could not get host device /dev/vsock: %w", err) } - - err = setupDev(monRootfs, "/dev/vhost-vsock") + vhostSockDev, err := deviceFromHost("/dev/vhost-vsock") if err != nil { - return err + return nil, fmt.Errorf("could not get host device /dev/vhost-vsock: %w", err) } + devices := []specs.LinuxDevice{vsockDev, vhostSockDev} // bind mount the unix socket directory if hypervisor == "firecracker" { - err = fileFromHost(monRootfs, vsockSocketPath, "", unix.MS_BIND|unix.MS_PRIVATE, false) + err = applyMount(monRootfs, bindMount(vsockSocketPath, vsockSocketPath, true)) if err != nil { - return err + return nil, err } } - return nil + + return devices, nil }