use clone3 for exec process creation to reduce cgroup lock contention

Currently, the runc exec process creates child processes by first cloning the child process and then writing its PID into cgroup.procs. This approach leads to high lock contention on the cgroup_threadgroup_rwsem read-write lock under conditions of high container density and numerous exec probes, potentially causing system hang.

This change introduces the usage of the clone3 system call within the setnsProcess.start function to merge the application of the cgroup into the clone operation (assuming cgroup v2 is in use). By doing so, it avoids the need to write PIDs to cgroup.procs directly, thereby bypassing the requirement for taking the write lock and reducing the risk of lock contention.

Author: jinda.ljd

libcontainer/process_linux.go

@@ -13,6 +13,7 @@ import (
 	"runtime"
 	"strconv"
 	"sync"
+	"syscall"
 	"time"
 
 	"github.com/opencontainers/runtime-spec/specs-go"
@@ -203,6 +204,28 @@ func (p *setnsProcess) start() (retErr error) {
 
 	// Get the "before" value of oom kill count.
 	oom, _ := p.manager.OOMKillCount()
+	useClone3 := false
+	if cgroups.IsCgroup2UnifiedMode() && p.initProcessPid != 0 {
+		initProcCgroupFile := fmt.Sprintf("/proc/%d/cgroup", p.initProcessPid)
+		initCg, initCgErr := cgroups.ParseCgroupFile(initProcCgroupFile)
+		if initCgErr == nil {
+			if initCgPath, ok := initCg[""]; ok {
+				useClone3 = true
+				initCgDirpath := filepath.Join(fs2.UnifiedMountpoint, initCgPath)
+				fd, err := os.Open(initCgDirpath)
+				if err != nil {
+					return fmt.Errorf("error opening cgroup dir %q: %w", initCgDirpath, err)
+				}
+				defer fd.Close()
+				if p.cmd.SysProcAttr == nil {
+					p.cmd.SysProcAttr = &syscall.SysProcAttr{}
+				}
+				p.cmd.SysProcAttr.UseCgroupFD = true
+				p.cmd.SysProcAttr.CgroupFD = int(fd.Fd())
+			}
+		}
+	}
+
 	err := p.startWithCPUAffinity()
 	// Close the child-side of the pipes (controlled by child).
 	p.comm.closeChild()
@@ -232,7 +255,11 @@ func (p *setnsProcess) start() (retErr error) {
 		return fmt.Errorf("error executing setns process: %w", err)
 	}
 	for _, path := range p.cgroupPaths {
-		if err := cgroups.WriteCgroupProc(path, p.pid()); err != nil && !p.rootlessCgroups {
+		procPid := p.pid()
+		if useClone3 {
+			procPid = -1
+		}
+		if err := cgroups.WriteCgroupProc(path, procPid); err != nil && !p.rootlessCgroups {
 			// On cgroup v2 + nesting + domain controllers, WriteCgroupProc may fail with EBUSY.
 			// https://github.com/opencontainers/runc/issues/2356#issuecomment-621277643
 			// Try to join the cgroup of InitProcessPid.