libcontainer/intelrdt: add support for EnableMonitoring field

The linux.intelRdt.enableMonitoring field enables the creation of
a per-container monitoring group. The monitoring group is removed when
the container is destroyed.

Signed-off-by: Markus Lehtonen <[email protected]>

Author: marquiz

features.go

@@ -56,7 +56,8 @@ var featuresCommand = cli.Command{
 					Enabled: &t,
 				},
 				IntelRdt: &features.IntelRdt{
-					Enabled: &t,
+					Enabled:    &t,
+					Monitoring: &t,
 				},
 				MountExtensions: &features.MountExtensions{
 					IDMap: &features.IDMap{

libcontainer/configs/intelrdt.go

@@ -13,4 +13,7 @@ type IntelRdt struct {
 	// The unit of memory bandwidth is specified in "percentages" by
 	// default, and in "MBps" if MBA Software Controller is enabled.
 	MemBwSchema string `json:"memBwSchema,omitempty"`
+
+	// Create a monitoring group for the container.
+	EnableMonitoring bool `json:"enableMonitoring,omitempty"`
 }

libcontainer/container_linux.go

@@ -73,6 +73,11 @@ type State struct {
 
 	// Intel RDT "resource control" filesystem path.
 	IntelRdtPath string `json:"intel_rdt_path,omitempty"`
+
+	// Path of the container specific monitoring group in resctrl filesystem.
+	// Empty if the container does not have aindividual dedicated monitoring
+	// group.
+	IntelRdtMonPath string `json:"intel_rdt_mon_path,omitempty"`
 }
 
 // ID returns the container's unique ID
@@ -938,8 +943,10 @@ func (c *Container) currentState() *State {
 	}
 
 	intelRdtPath := ""
+	intelRdtMonPath := ""
 	if c.intelRdtManager != nil {
 		intelRdtPath = c.intelRdtManager.GetPath()
+		intelRdtMonPath = c.intelRdtManager.GetMonPath()
 	}
 	state := &State{
 		BaseState: BaseState{
@@ -952,6 +959,7 @@ func (c *Container) currentState() *State {
 		Rootless:            c.config.RootlessEUID && c.config.RootlessCgroups,
 		CgroupPaths:         c.cgroupManager.GetPaths(),
 		IntelRdtPath:        intelRdtPath,
+		IntelRdtMonPath:     intelRdtMonPath,
 		NamespacePaths:      make(map[configs.NamespaceType]string),
 		ExternalDescriptors: externalDescriptors,
 	}

libcontainer/intelrdt/intelrdt.go

@@ -458,6 +458,7 @@ func (m *Manager) Apply(pid int) (err error) {
 	}
 
 	path := m.GetPath()
+	fmt.Println("intelrdt: applying intelrdt path:", path, "pid:", pid)
 
 	m.mu.Lock()
 	defer m.mu.Unlock()
@@ -478,6 +479,16 @@ func (m *Manager) Apply(pid int) (err error) {
 		return newLastCmdError(err)
 	}
 
+	// Create MON group
+	if monPath := m.GetMonPath(); monPath != "" {
+		if err := os.Mkdir(monPath, 0o755); err != nil && !os.IsExist(err) {
+			return newLastCmdError(err)
+		}
+		if err := WriteIntelRdtTasks(monPath, pid); err != nil {
+			return newLastCmdError(err)
+		}
+	}
+
 	m.path = path
 	return nil
 }
@@ -487,13 +498,21 @@ func (m *Manager) Destroy() error {
 	// Don't remove resctrl group if closid has been explicitly specified. The
 	// group is likely externally managed, i.e. by some other entity than us.
 	// There are probably other containers/tasks sharing the same group.
-	if m.config.IntelRdt != nil && m.config.IntelRdt.ClosID == "" {
-		m.mu.Lock()
-		defer m.mu.Unlock()
-		if err := os.Remove(m.GetPath()); err != nil && !os.IsNotExist(err) {
-			return err
+	if m.config.IntelRdt != nil {
+		if m.config.IntelRdt.ClosID == "" {
+			m.mu.Lock()
+			defer m.mu.Unlock()
+			if err := os.Remove(m.GetPath()); err != nil && !os.IsNotExist(err) {
+				return err
+			}
+			m.path = ""
+		} else if monPath := m.GetMonPath(); monPath != "" {
+			// If ClosID is not specified the possible monintoring group was
+			// removed with the CLOS above.
+			if err := os.Remove(monPath); err != nil && !os.IsNotExist(err) {
+				return err
+			}
 		}
-		m.path = ""
 	}
 	return nil
 }
@@ -504,6 +523,20 @@ func (m *Manager) GetPath() string {
 	return m.path
 }
 
+// GetMonPath returns path of the monitoring group of the container. Returns an
+// empty string if the container does not have a individual dedicated
+// monitoring group.
+func (m *Manager) GetMonPath() string {
+	if closPath := m.GetPath(); closPath != "" && m.config.IntelRdt.EnableMonitoring {
+		path, err := securejoin.SecureJoin(filepath.Join(closPath, "mon_groups"), m.id)
+		if err != nil {
+			return ""
+		}
+		return path
+	}
+	return ""
+}
+
 // GetStats returns statistics for Intel RDT.
 func (m *Manager) GetStats() (*Stats, error) {
 	// If intelRdt is not specified in config
@@ -581,7 +614,16 @@ func (m *Manager) GetStats() (*Stats, error) {
 	}
 
 	if IsMBMEnabled() || IsCMTEnabled() {
-		err = getMonitoringStats(containerPath, stats)
+		monPath := m.GetMonPath()
+		if monPath == "" {
+			// NOTE: If per-container monitoring is not enabled, the monitoring
+			// data we get here might have little to do with this container as
+			// there might be anything from this single container to the half
+			// of the system assigned in the group. Should consider not
+			// exposing stats in this case(?)
+			monPath = containerPath
+		}
+		err = getMonitoringStats(monPath, stats)
 		if err != nil {
 			return nil, err
 		}

libcontainer/intelrdt/intelrdt_test.go

@@ -5,6 +5,8 @@ import (
 	"path/filepath"
 	"strings"
 	"testing"
+
+	"github.com/opencontainers/runc/libcontainer/configs"
 )
 
 func TestIntelRdtSetL3CacheSchema(t *testing.T) {
@@ -98,31 +100,157 @@ func TestIntelRdtSetMemBwScSchema(t *testing.T) {
 }
 
 func TestApply(t *testing.T) {
-	helper := NewIntelRdtTestUtil(t)
-
 	const closID = "test-clos"
-	closPath := filepath.Join(helper.IntelRdtPath, closID)
+	// TC-1: failure because non-pre-existing CLOS
+	{
+		helper := NewIntelRdtTestUtil(t)
+		helper.config.IntelRdt = &configs.IntelRdt{
+			ClosID: closID,
+		}
 
-	helper.config.IntelRdt.ClosID = closID
-	intelrdt := newManager(helper.config, "container-1", closPath)
-	if err := intelrdt.Apply(1234); err == nil {
-		t.Fatal("unexpected success when applying pid")
+		closPath := filepath.Join(intelRdtRoot, closID)
+		intelrdt := newManager(helper.config, "", closPath)
+		if err := intelrdt.Apply(1234); err == nil {
+			t.Fatal("unexpected success when applying pid")
+		}
+		if _, err := os.Stat(closPath); err == nil {
+			t.Fatal("closid dir should not exist")
+		}
 	}
-	if _, err := os.Stat(closPath); err == nil {
-		t.Fatal("closid dir should not exist")
+	// TC-2: CLOS dir should be created if some schema has been specified
+	{
+		helper := NewIntelRdtTestUtil(t)
+		helper.config.IntelRdt = &configs.IntelRdt{
+			ClosID:        closID,
+			L3CacheSchema: "L3:0=f",
+		}
+
+		closPath := filepath.Join(intelRdtRoot, closID)
+		intelrdt := newManager(helper.config, "", closPath)
+		if err := intelrdt.Apply(1235); err != nil {
+			t.Fatalf("Apply() failed: %v", err)
+		}
+
+		pids, err := getIntelRdtParamString(closPath, "tasks")
+		if err != nil {
+			t.Fatalf("failed to read tasks file: %v", err)
+		}
+		if pids != "1235" {
+			t.Fatalf("unexpected tasks file, expected '1235', got %q", pids)
+		}
 	}
+	// TC-3: clos and monitoring group should be created if EnableMonitoring is true
+	{
+		helper := NewIntelRdtTestUtil(t)
+		helper.config.IntelRdt = &configs.IntelRdt{
+			EnableMonitoring: true,
+		}
+		id := "aaaa-bbbb"
 
-	// Dir should be created if some schema has been specified
-	intelrdt.config.IntelRdt.L3CacheSchema = "L3:0=f"
-	if err := intelrdt.Apply(1235); err != nil {
-		t.Fatalf("Apply() failed: %v", err)
+		closPath := filepath.Join(intelRdtRoot, id)
+		intelrdt := newManager(helper.config, id, closPath)
+		// We need to pre-create the CLOS/mon_groups directory
+		if err := os.MkdirAll(filepath.Join(closPath, "mon_groups"), 0o755); err != nil {
+			t.Fatal(err)
+		}
+
+		if err := intelrdt.Apply(1236); err != nil {
+			t.Fatalf("Apply() failed: %v", err)
+		}
+
+		pids, err := getIntelRdtParamString(closPath, "tasks")
+		if err != nil {
+			t.Fatalf("failed to read tasks file: %v", err)
+		}
+		if pids != "1236" {
+			t.Fatalf("unexpected tasks file, expected '1236', got %q", pids)
+		}
 	}
+}
 
-	pids, err := getIntelRdtParamString(intelrdt.GetPath(), "tasks")
-	if err != nil {
-		t.Fatalf("failed to read tasks file: %v", err)
+func TestDestroy(t *testing.T) {
+	const closID = "test-clos"
+
+	// TC-1: per-container CLOS dir should be removed
+	{
+		helper := NewIntelRdtTestUtil(t)
+		id := "abcd-efgh"
+
+		intelrdt := newManager(helper.config, id, "")
+		if err := intelrdt.Apply(1234); err != nil {
+			t.Fatalf("Apply() failed: %v", err)
+		}
+		closPath := filepath.Join(intelRdtRoot, id)
+		if _, err := os.Stat(closPath); err != nil {
+			t.Fatal("CLOS dir should exist")
+		}
+		// Need to delete the tasks file so that the dir is empty
+		os.Remove(filepath.Join(closPath, "tasks"))
+		if err := intelrdt.Destroy(); err != nil {
+			t.Fatalf("Destroy() failed: %v", err)
+		}
+		if _, err := os.Stat(closPath); err == nil {
+			t.Fatal("CLOS dir should not exist")
+		}
+	}
+	// TC-2: pre-existing CLOS should not be removed
+	{
+		helper := NewIntelRdtTestUtil(t)
+		helper.config.IntelRdt = &configs.IntelRdt{
+			ClosID: closID,
+		}
+
+		closPath := filepath.Join(intelRdtRoot, closID)
+		if err := os.MkdirAll(closPath, 0o755); err != nil {
+			t.Fatal(err)
+		}
+
+		intelrdt := newManager(helper.config, "", "")
+		if err := intelrdt.Apply(1234); err != nil {
+			t.Fatalf("Apply() failed: %v", err)
+		}
+		if _, err := os.Stat(closPath); err != nil {
+			t.Fatal("CLOS dir should exist")
+		}
+		if err := intelrdt.Destroy(); err != nil {
+			t.Fatalf("Destroy() failed: %v", err)
+		}
+		if _, err := os.Stat(closPath); err != nil {
+			t.Fatal("CLOS dir should exist")
+		}
 	}
-	if pids != "1235" {
-		t.Fatalf("unexpected tasks file, expected '1235', got %q", pids)
+	// TC-3: per-container MON dir in pre-existing CLOS should be removed
+	{
+		helper := NewIntelRdtTestUtil(t)
+		helper.config.IntelRdt = &configs.IntelRdt{
+			ClosID:           closID,
+			EnableMonitoring: true,
+		}
+		id := "abcd-efgh"
+
+		closPath := filepath.Join(intelRdtRoot, closID)
+		if err := os.MkdirAll(filepath.Join(closPath, "mon_groups"), 0o755); err != nil {
+			t.Fatal(err)
+		}
+
+		intelrdt := newManager(helper.config, id, "")
+		if err := intelrdt.Apply(1234); err != nil {
+			t.Fatalf("Apply() failed: %v", err)
+		}
+		monPath := filepath.Join(closPath, "mon_groups", id)
+		if _, err := os.Stat(monPath); err != nil {
+			t.Fatal("MON dir should exist")
+		}
+		// Need to delete the tasks file so that the dir is empty
+		os.Remove(filepath.Join(monPath, "tasks"))
+		if err := intelrdt.Destroy(); err != nil {
+			t.Fatalf("Destroy() failed: %v", err)
+		}
+		if _, err := os.Stat(closPath); err != nil {
+			t.Fatalf("CLOS dir should exist: %f", err)
+		}
+		if _, err := os.Stat(monPath); err == nil {
+			t.Fatal("MON dir should not exist")
+		}
 	}
 }

libcontainer/specconv/spec_linux.go

@@ -462,9 +462,10 @@ func CreateLibcontainerConfig(opts *CreateOpts) (*configs.Config, error) {
 		}
 		if spec.Linux.IntelRdt != nil {
 			config.IntelRdt = &configs.IntelRdt{
-				ClosID:        spec.Linux.IntelRdt.ClosID,
-				L3CacheSchema: spec.Linux.IntelRdt.L3CacheSchema,
-				MemBwSchema:   spec.Linux.IntelRdt.MemBwSchema,
+				ClosID:           spec.Linux.IntelRdt.ClosID,
+				L3CacheSchema:    spec.Linux.IntelRdt.L3CacheSchema,
+				MemBwSchema:      spec.Linux.IntelRdt.MemBwSchema,
+				EnableMonitoring: spec.Linux.IntelRdt.EnableMonitoring,
 			}
 		}
 		if spec.Linux.Personality != nil {