From 7aa4e1a63df265d62478313e7b8ff40d6e10783b Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Fri, 1 Aug 2025 11:07:31 +0300 Subject: [PATCH 1/2] libcontainer/intelrdt: add support for EnableMonitoring field The linux.intelRdt.enableMonitoring field enables the creation of a per-container monitoring group. The monitoring group is removed when the container is destroyed. Signed-off-by: Markus Lehtonen --- features.go | 5 +- libcontainer/configs/intelrdt.go | 3 + .../configs/validate/intelrdt_test.go | 17 +- libcontainer/container_linux.go | 8 + libcontainer/intelrdt/intelrdt.go | 47 ++++- libcontainer/intelrdt/intelrdt_test.go | 186 ++++++++++++++++-- libcontainer/specconv/spec_linux.go | 9 +- 7 files changed, 234 insertions(+), 41 deletions(-) diff --git a/features.go b/features.go index 72aef5c3c51..292f9bde26e 100644 --- a/features.go +++ b/features.go @@ -56,8 +56,9 @@ var featuresCommand = cli.Command{ Enabled: &t, }, IntelRdt: &features.IntelRdt{ - Enabled: &t, - Schemata: &t, + Enabled: &t, + Schemata: &t, + Monitoring: &t, }, MountExtensions: &features.MountExtensions{ IDMap: &features.IDMap{ diff --git a/libcontainer/configs/intelrdt.go b/libcontainer/configs/intelrdt.go index ddf1bb3a855..11ccc2d1f59 100644 --- a/libcontainer/configs/intelrdt.go +++ b/libcontainer/configs/intelrdt.go @@ -17,4 +17,7 @@ type IntelRdt struct { // The unit of memory bandwidth is specified in "percentages" by // default, and in "MBps" if MBA Software Controller is enabled. MemBwSchema string `json:"memBwSchema,omitempty"` + + // Create a monitoring group for the container. + EnableMonitoring bool `json:"enableMonitoring,omitempty"` } diff --git a/libcontainer/configs/validate/intelrdt_test.go b/libcontainer/configs/validate/intelrdt_test.go index d8672225fda..ddee2562559 100644 --- a/libcontainer/configs/validate/intelrdt_test.go +++ b/libcontainer/configs/validate/intelrdt_test.go @@ -19,22 +19,17 @@ func TestValidateIntelRdt(t *testing.T) { isErr bool }{ { - name: "rdt not supported, no config", - rdtEnabled: false, - config: nil, - isErr: false, + name: "rdt not supported, no config", }, { - name: "rdt not supported, with config", - rdtEnabled: false, - config: &configs.IntelRdt{}, - isErr: true, + name: "rdt not supported, with config", + config: &configs.IntelRdt{}, + isErr: true, }, { name: "empty config", rdtEnabled: true, config: &configs.IntelRdt{}, - isErr: false, }, { name: "root clos", @@ -42,7 +37,6 @@ func TestValidateIntelRdt(t *testing.T) { config: &configs.IntelRdt{ ClosID: "/", }, - isErr: false, }, { name: "invalid ClosID (.)", @@ -71,7 +65,6 @@ func TestValidateIntelRdt(t *testing.T) { { name: "cat not supported", rdtEnabled: true, - catEnabled: false, config: &configs.IntelRdt{ L3CacheSchema: "0=ff", }, @@ -80,7 +73,6 @@ func TestValidateIntelRdt(t *testing.T) { { name: "mba not supported", rdtEnabled: true, - mbaEnabled: false, config: &configs.IntelRdt{ MemBwSchema: "0=100", }, @@ -96,7 +88,6 @@ func TestValidateIntelRdt(t *testing.T) { L3CacheSchema: "0=ff", MemBwSchema: "0=100", }, - isErr: false, }, } for _, tc := range testCases { diff --git a/libcontainer/container_linux.go b/libcontainer/container_linux.go index 7f0c51f31cf..ec417b66d08 100644 --- a/libcontainer/container_linux.go +++ b/libcontainer/container_linux.go @@ -73,6 +73,11 @@ type State struct { // Intel RDT "resource control" filesystem path. IntelRdtPath string `json:"intel_rdt_path,omitempty"` + + // Path of the container specific monitoring group in resctrl filesystem. + // Empty if the container does not have aindividual dedicated monitoring + // group. + IntelRdtMonPath string `json:"intel_rdt_mon_path,omitempty"` } // ID returns the container's unique ID @@ -942,8 +947,10 @@ func (c *Container) currentState() *State { } intelRdtPath := "" + intelRdtMonPath := "" if c.intelRdtManager != nil { intelRdtPath = c.intelRdtManager.GetPath() + intelRdtMonPath = c.intelRdtManager.GetMonPath() } state := &State{ BaseState: BaseState{ @@ -956,6 +963,7 @@ func (c *Container) currentState() *State { Rootless: c.config.RootlessEUID && c.config.RootlessCgroups, CgroupPaths: c.cgroupManager.GetPaths(), IntelRdtPath: intelRdtPath, + IntelRdtMonPath: intelRdtMonPath, NamespacePaths: make(map[configs.NamespaceType]string), ExternalDescriptors: externalDescriptors, } diff --git a/libcontainer/intelrdt/intelrdt.go b/libcontainer/intelrdt/intelrdt.go index c3cd9ef2372..aa935231154 100644 --- a/libcontainer/intelrdt/intelrdt.go +++ b/libcontainer/intelrdt/intelrdt.go @@ -468,22 +468,41 @@ func (m *Manager) Apply(pid int) (err error) { return newLastCmdError(err) } + // Create MON group + if monPath := m.GetMonPath(); monPath != "" { + if err := os.Mkdir(monPath, 0o755); err != nil && !os.IsExist(err) { + return newLastCmdError(err) + } + if err := WriteIntelRdtTasks(monPath, pid); err != nil { + return newLastCmdError(err) + } + } + m.path = path return nil } // Destroy destroys the Intel RDT container-specific container_id group. func (m *Manager) Destroy() error { + if m.config.IntelRdt == nil { + return nil + } // Don't remove resctrl group if closid has been explicitly specified. The // group is likely externally managed, i.e. by some other entity than us. // There are probably other containers/tasks sharing the same group. - if m.config.IntelRdt != nil && m.config.IntelRdt.ClosID == "" { + if m.config.IntelRdt.ClosID == "" { m.mu.Lock() defer m.mu.Unlock() if err := os.Remove(m.GetPath()); err != nil && !os.IsNotExist(err) { return err } m.path = "" + } else if monPath := m.GetMonPath(); monPath != "" { + // If ClosID is not specified the possible monintoring group was + // removed with the CLOS above. + if err := os.Remove(monPath); err != nil && !os.IsNotExist(err) { + return err + } } return nil } @@ -494,6 +513,21 @@ func (m *Manager) GetPath() string { return m.path } +// GetMonPath returns path of the monitoring group of the container. Returns an +// empty string if the container does not have a individual dedicated +// monitoring group. +func (m *Manager) GetMonPath() string { + if !m.config.IntelRdt.EnableMonitoring { + return "" + } + closPath := m.GetPath() + if closPath == "" { + return "" + } + + return filepath.Join(closPath, "mon_groups", m.id) +} + // GetStats returns statistics for Intel RDT. func (m *Manager) GetStats() (*Stats, error) { // If intelRdt is not specified in config @@ -573,7 +607,16 @@ func (m *Manager) GetStats() (*Stats, error) { } if IsMBMEnabled() || IsCMTEnabled() { - err = getMonitoringStats(containerPath, stats) + monPath := m.GetMonPath() + if monPath == "" { + // NOTE: If per-container monitoring is not enabled, the monitoring + // data we get here might have little to do with this container as + // there might be anything from this single container to the half + // of the system assigned in the group. Should consider not + // exposing stats in this case(?) + monPath = containerPath + } + err = getMonitoringStats(monPath, stats) if err != nil { return nil, err } diff --git a/libcontainer/intelrdt/intelrdt_test.go b/libcontainer/intelrdt/intelrdt_test.go index 209424d554c..1ed05860e38 100644 --- a/libcontainer/intelrdt/intelrdt_test.go +++ b/libcontainer/intelrdt/intelrdt_test.go @@ -4,6 +4,7 @@ import ( "os" "path/filepath" "slices" + "strconv" "strings" "testing" @@ -126,31 +127,176 @@ func TestIntelRdtSet(t *testing.T) { } func TestApply(t *testing.T) { - helper := NewIntelRdtTestUtil(t) + const pid = 1234 + tests := []struct { + name string + config configs.IntelRdt + precreateClos bool + isError bool + postApplyAssert func(*Manager) + }{ + { + name: "failure because non-pre-existing CLOS", + config: configs.IntelRdt{ + ClosID: "non-existing-clos", + }, + isError: true, + postApplyAssert: func(m *Manager) { + if _, err := os.Stat(m.path); err == nil { + t.Fatal("closid dir should not exist") + } + }, + }, + { + name: "CLOS dir should be created if some schema has been specified", + config: configs.IntelRdt{ + ClosID: "clos-to-be-created", + L3CacheSchema: "L3:0=f", + }, + postApplyAssert: func(m *Manager) { + pids, err := getIntelRdtParamString(m.path, "tasks") + if err != nil { + t.Fatalf("failed to read tasks file: %v", err) + } + if pids != strconv.Itoa(pid) { + t.Fatalf("unexpected tasks file, expected '%d', got %q", pid, pids) + } + }, + }, + { + name: "clos and monitoring group should be created if EnableMonitoring is true", + config: configs.IntelRdt{ + EnableMonitoring: true, + }, + precreateClos: true, + postApplyAssert: func(m *Manager) { + pids, err := getIntelRdtParamString(m.path, "tasks") + if err != nil { + t.Fatalf("failed to read tasks file: %v", err) + } + if pids != strconv.Itoa(pid) { + t.Fatalf("unexpected tasks file, expected '%d', got %q", pid, pids) + } + }, + }, + } - const closID = "test-clos" - closPath := filepath.Join(helper.IntelRdtPath, closID) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + NewIntelRdtTestUtil(t) + id := "abcd-1234" + closPath := filepath.Join(intelRdtRoot, id) + if tt.config.ClosID != "" { + closPath = filepath.Join(intelRdtRoot, tt.config.ClosID) + } - helper.config.IntelRdt.ClosID = closID - intelrdt := newManager(helper.config, "container-1", closPath) - if err := intelrdt.Apply(1234); err == nil { - t.Fatal("unexpected success when applying pid") - } - if _, err := os.Stat(closPath); err == nil { - t.Fatal("closid dir should not exist") + if tt.precreateClos { + if err := os.MkdirAll(filepath.Join(closPath, "mon_groups"), 0o755); err != nil { + t.Fatal(err) + } + } + m := newManager(&configs.Config{IntelRdt: &tt.config}, id, closPath) + err := m.Apply(pid) + if tt.isError && err == nil { + t.Fatal("expected error, got nil") + } else if !tt.isError && err != nil { + t.Fatalf("unexpected error: %v", err) + } + tt.postApplyAssert(m) + }) } +} - // Dir should be created if some schema has been specified - intelrdt.config.IntelRdt.L3CacheSchema = "L3:0=f" - if err := intelrdt.Apply(1235); err != nil { - t.Fatalf("Apply() failed: %v", err) - } +func TestDestroy(t *testing.T) { + tests := []struct { + name string + config configs.IntelRdt + testFunc func(*Manager) + }{ + { + name: "per-container CLOS dir should be removed", + testFunc: func(m *Manager) { + closPath := m.path + if _, err := os.Stat(closPath); err != nil { + t.Fatal("CLOS dir should exist") + } + // Need to delete the tasks file so that the dir is empty + if err := os.Remove(filepath.Join(closPath, "tasks")); err != nil { + t.Fatalf("failed to remove tasks file: %v", err) + } + if err := m.Destroy(); err != nil { + t.Fatalf("Destroy() failed: %v", err) + } + if _, err := os.Stat(closPath); err == nil { + t.Fatal("CLOS dir should not exist") + } + }, + }, + { + name: "pre-existing CLOS should not be removed", + config: configs.IntelRdt{ + ClosID: "pre-existing-clos", + }, + testFunc: func(m *Manager) { + closPath := m.path - pids, err := getIntelRdtParamString(intelrdt.GetPath(), "tasks") - if err != nil { - t.Fatalf("failed to read tasks file: %v", err) + if _, err := os.Stat(closPath); err != nil { + t.Fatal("CLOS dir should exist") + } + if err := m.Destroy(); err != nil { + t.Fatalf("Destroy() failed: %v", err) + } + if _, err := os.Stat(closPath); err != nil { + t.Fatal("CLOS dir should exist") + } + }, + }, + { + name: "per-container MON dir in pre-existing CLOS should be removed", + config: configs.IntelRdt{ + ClosID: "pre-existing-clos", + EnableMonitoring: true, + }, + testFunc: func(m *Manager) { + closPath := m.path + + monPath := filepath.Join(closPath, "mon_groups", m.id) + if _, err := os.Stat(monPath); err != nil { + t.Fatal("MON dir should exist") + } + // Need to delete the tasks file so that the dir is empty + os.Remove(filepath.Join(monPath, "tasks")) + if err := m.Destroy(); err != nil { + t.Fatalf("Destroy() failed: %v", err) + } + if _, err := os.Stat(closPath); err != nil { + t.Fatalf("CLOS dir should exist: %f", err) + } + if _, err := os.Stat(monPath); err == nil { + t.Fatal("MON dir should not exist") + } + }, + }, } - if pids != "1235" { - t.Fatalf("unexpected tasks file, expected '1235', got %q", pids) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + NewIntelRdtTestUtil(t) + + id := "abcd-1234" + closPath := filepath.Join(intelRdtRoot, id) + if tt.config.ClosID != "" { + closPath = filepath.Join(intelRdtRoot, tt.config.ClosID) + // Pre-create the CLOS directory + if err := os.MkdirAll(filepath.Join(closPath, "mon_groups"), 0o755); err != nil { + t.Fatal(err) + } + } + m := newManager(&configs.Config{IntelRdt: &tt.config}, id, closPath) + if err := m.Apply(1234); err != nil { + t.Fatalf("Apply() failed: %v", err) + } + tt.testFunc(m) + }) } } diff --git a/libcontainer/specconv/spec_linux.go b/libcontainer/specconv/spec_linux.go index b19ba358bc5..d06ed756463 100644 --- a/libcontainer/specconv/spec_linux.go +++ b/libcontainer/specconv/spec_linux.go @@ -462,10 +462,11 @@ func CreateLibcontainerConfig(opts *CreateOpts) (*configs.Config, error) { } if spec.Linux.IntelRdt != nil { config.IntelRdt = &configs.IntelRdt{ - ClosID: spec.Linux.IntelRdt.ClosID, - Schemata: spec.Linux.IntelRdt.Schemata, - L3CacheSchema: spec.Linux.IntelRdt.L3CacheSchema, - MemBwSchema: spec.Linux.IntelRdt.MemBwSchema, + ClosID: spec.Linux.IntelRdt.ClosID, + Schemata: spec.Linux.IntelRdt.Schemata, + L3CacheSchema: spec.Linux.IntelRdt.L3CacheSchema, + MemBwSchema: spec.Linux.IntelRdt.MemBwSchema, + EnableMonitoring: spec.Linux.IntelRdt.EnableMonitoring, } } if spec.Linux.Personality != nil { From f352280b940b77734da367bbeaf9776ed86b4755 Mon Sep 17 00:00:00 2001 From: Markus Lehtonen Date: Mon, 15 Sep 2025 18:47:54 +0300 Subject: [PATCH 2/2] libcontainer/intelrdt: make mock clos name an arg of test helper Signed-off-by: Markus Lehtonen --- libcontainer/intelrdt/intelrdt_test.go | 32 ++++++++++++-------------- libcontainer/intelrdt/util_test.go | 13 ++++++----- 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/libcontainer/intelrdt/intelrdt_test.go b/libcontainer/intelrdt/intelrdt_test.go index 1ed05860e38..95580a5f46c 100644 --- a/libcontainer/intelrdt/intelrdt_test.go +++ b/libcontainer/intelrdt/intelrdt_test.go @@ -105,7 +105,7 @@ func TestIntelRdtSet(t *testing.T) { for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { - helper := NewIntelRdtTestUtil(t) + helper := NewIntelRdtTestUtil(t, "clos-1") helper.config.IntelRdt = tc.config intelrdt := newManager(helper.config, "", helper.IntelRdtPath) @@ -183,19 +183,19 @@ func TestApply(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - NewIntelRdtTestUtil(t) id := "abcd-1234" - closPath := filepath.Join(intelRdtRoot, id) + closName := id if tt.config.ClosID != "" { - closPath = filepath.Join(intelRdtRoot, tt.config.ClosID) + closName = tt.config.ClosID } + preConfiguredClos := "" if tt.precreateClos { - if err := os.MkdirAll(filepath.Join(closPath, "mon_groups"), 0o755); err != nil { - t.Fatal(err) - } + preConfiguredClos = closName } - m := newManager(&configs.Config{IntelRdt: &tt.config}, id, closPath) + NewIntelRdtTestUtil(t, preConfiguredClos) + + m := newManager(&configs.Config{IntelRdt: &tt.config}, id, filepath.Join(intelRdtRoot, closName)) err := m.Apply(pid) if tt.isError && err == nil { t.Fatal("expected error, got nil") @@ -281,18 +281,16 @@ func TestDestroy(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - NewIntelRdtTestUtil(t) - id := "abcd-1234" - closPath := filepath.Join(intelRdtRoot, id) + closName := id + preConfiguredClos := "" if tt.config.ClosID != "" { - closPath = filepath.Join(intelRdtRoot, tt.config.ClosID) - // Pre-create the CLOS directory - if err := os.MkdirAll(filepath.Join(closPath, "mon_groups"), 0o755); err != nil { - t.Fatal(err) - } + closName = tt.config.ClosID + preConfiguredClos = closName } - m := newManager(&configs.Config{IntelRdt: &tt.config}, id, closPath) + NewIntelRdtTestUtil(t, preConfiguredClos) + + m := newManager(&configs.Config{IntelRdt: &tt.config}, id, filepath.Join(intelRdtRoot, closName)) if err := m.Apply(1234); err != nil { t.Fatalf("Apply() failed: %v", err) } diff --git a/libcontainer/intelrdt/util_test.go b/libcontainer/intelrdt/util_test.go index 1771a231b4a..b23b5c6c26b 100644 --- a/libcontainer/intelrdt/util_test.go +++ b/libcontainer/intelrdt/util_test.go @@ -15,14 +15,14 @@ import ( type intelRdtTestUtil struct { config *configs.Config - // Path to the mock Intel RDT "resource control" filesystem directory + // Path to the mock pre-existing CLOS (or resctrl root if no CLOS is specified) IntelRdtPath string t *testing.T } -// Creates a new test util -func NewIntelRdtTestUtil(t *testing.T) *intelRdtTestUtil { +// Creates a new test util. If mockClosName is non-empty, a mock CLOS with that name will be created. +func NewIntelRdtTestUtil(t *testing.T, mockClosName string) *intelRdtTestUtil { config := &configs.Config{ IntelRdt: &configs.IntelRdt{}, } @@ -32,11 +32,12 @@ func NewIntelRdtTestUtil(t *testing.T) *intelRdtTestUtil { // Make sure Root() won't even try to parse mountinfo. rootOnce.Do(func() {}) - testIntelRdtPath := filepath.Join(intelRdtRoot, "resctrl") + testIntelRdtPath := filepath.Join(intelRdtRoot, mockClosName) - // Ensure the full mock Intel RDT "resource control" filesystem path exists - if err := os.MkdirAll(testIntelRdtPath, 0o755); err != nil { + // Ensure the mocked CLOS exists + if err := os.MkdirAll(filepath.Join(testIntelRdtPath, "mon_groups"), 0o755); err != nil { t.Fatal(err) } + return &intelRdtTestUtil{config: config, IntelRdtPath: testIntelRdtPath, t: t} }