From 1aa7ca804641ebd22b3abea68bf834a80d259cda Mon Sep 17 00:00:00 2001 From: Daniel Dao Date: Fri, 28 Jan 2022 17:07:36 +0000 Subject: [PATCH] libct/cg/stats: support PSI for cgroup v2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We read output from the following files if they exists: - cpu.pressure - memory.pressure - io.pressure Each are in format: ``` some avg10=0.00 avg60=0.00 avg300=0.00 total=0 full avg10=0.00 avg60=0.00 avg300=0.00 total=0 ``` Signed-off-by: Daniel Dao Co-authored-by: Sandor Szücs Co-authored-by: Kir Kolyshkin Signed-off-by: Kir Kolyshkin --- events.go | 3 + libcontainer/cgroups/fs2/fs2.go | 11 ++++ libcontainer/cgroups/fs2/psi.go | 89 ++++++++++++++++++++++++++++ libcontainer/cgroups/fs2/psi_test.go | 47 +++++++++++++++ libcontainer/cgroups/stats.go | 15 +++++ tests/integration/events.bats | 29 +++++++++ tests/integration/helpers.bash | 7 +++ types/events.go | 12 +++- 8 files changed, 212 insertions(+), 1 deletion(-) create mode 100644 libcontainer/cgroups/fs2/psi.go create mode 100644 libcontainer/cgroups/fs2/psi_test.go mode change 100644 => 100755 tests/integration/helpers.bash diff --git a/events.go b/events.go index 6cdc01cdd3a..47aa1abe22b 100644 --- a/events.go +++ b/events.go @@ -129,6 +129,7 @@ func convertLibcontainerStats(ls *libcontainer.Stats) *types.Stats { s.CPU.Throttling.Periods = cg.CpuStats.ThrottlingData.Periods s.CPU.Throttling.ThrottledPeriods = cg.CpuStats.ThrottlingData.ThrottledPeriods s.CPU.Throttling.ThrottledTime = cg.CpuStats.ThrottlingData.ThrottledTime + s.CPU.PSI = cg.CpuStats.PSI s.CPUSet = types.CPUSet(cg.CPUSetStats) @@ -138,6 +139,7 @@ func convertLibcontainerStats(ls *libcontainer.Stats) *types.Stats { s.Memory.Swap = convertMemoryEntry(cg.MemoryStats.SwapUsage) s.Memory.Usage = convertMemoryEntry(cg.MemoryStats.Usage) s.Memory.Raw = cg.MemoryStats.Stats + s.Memory.PSI = cg.MemoryStats.PSI s.Blkio.IoServiceBytesRecursive = convertBlkioEntry(cg.BlkioStats.IoServiceBytesRecursive) s.Blkio.IoServicedRecursive = convertBlkioEntry(cg.BlkioStats.IoServicedRecursive) @@ -147,6 +149,7 @@ func convertLibcontainerStats(ls *libcontainer.Stats) *types.Stats { s.Blkio.IoMergedRecursive = convertBlkioEntry(cg.BlkioStats.IoMergedRecursive) s.Blkio.IoTimeRecursive = convertBlkioEntry(cg.BlkioStats.IoTimeRecursive) s.Blkio.SectorsRecursive = convertBlkioEntry(cg.BlkioStats.SectorsRecursive) + s.Blkio.PSI = cg.BlkioStats.PSI s.Hugetlb = make(map[string]types.Hugetlb) for k, v := range cg.HugetlbStats { diff --git a/libcontainer/cgroups/fs2/fs2.go b/libcontainer/cgroups/fs2/fs2.go index 42b5bcb60cf..47b67afc2a1 100644 --- a/libcontainer/cgroups/fs2/fs2.go +++ b/libcontainer/cgroups/fs2/fs2.go @@ -114,6 +114,17 @@ func (m *Manager) GetStats() (*cgroups.Stats, error) { if err := statCpu(m.dirPath, st); err != nil && !os.IsNotExist(err) { errs = append(errs, err) } + // PSI (since kernel 4.20). + var err error + if st.CpuStats.PSI, err = statPSI(m.dirPath, "cpu.pressure"); err != nil { + errs = append(errs, err) + } + if st.MemoryStats.PSI, err = statPSI(m.dirPath, "memory.pressure"); err != nil { + errs = append(errs, err) + } + if st.BlkioStats.PSI, err = statPSI(m.dirPath, "io.pressure"); err != nil { + errs = append(errs, err) + } // hugetlb (since kernel 5.6) if err := statHugeTlb(m.dirPath, st); err != nil && !os.IsNotExist(err) { errs = append(errs, err) diff --git a/libcontainer/cgroups/fs2/psi.go b/libcontainer/cgroups/fs2/psi.go new file mode 100644 index 00000000000..09f34888516 --- /dev/null +++ b/libcontainer/cgroups/fs2/psi.go @@ -0,0 +1,89 @@ +package fs2 + +import ( + "bufio" + "errors" + "fmt" + "os" + "strconv" + "strings" + + "golang.org/x/sys/unix" + + "github.com/opencontainers/runc/libcontainer/cgroups" +) + +func statPSI(dirPath string, file string) (*cgroups.PSIStats, error) { + f, err := cgroups.OpenFile(dirPath, file, os.O_RDONLY) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + // Kernel < 4.20, or CONFIG_PSI is not set, + // or PSI stats are turned off for the cgroup + // ("echo 0 > cgroup.pressure", kernel >= 6.1). + return nil, nil + } + return nil, err + } + defer f.Close() + + var psistats cgroups.PSIStats + sc := bufio.NewScanner(f) + for sc.Scan() { + parts := strings.Fields(sc.Text()) + var pv *cgroups.PSIData + switch parts[0] { + case "some": + pv = &psistats.Some + case "full": + pv = &psistats.Full + } + if pv != nil { + *pv, err = parsePSIData(parts[1:]) + if err != nil { + return nil, &parseError{Path: dirPath, File: file, Err: err} + } + } + } + if err := sc.Err(); err != nil { + if errors.Is(err, unix.ENOTSUP) { + // Some kernels (e.g. CS9) may return ENOTSUP on read + // if psi=1 kernel cmdline parameter is required. + return nil, nil + } + return nil, &parseError{Path: dirPath, File: file, Err: err} + } + return &psistats, nil +} + +func parsePSIData(psi []string) (cgroups.PSIData, error) { + data := cgroups.PSIData{} + for _, f := range psi { + kv := strings.SplitN(f, "=", 2) + if len(kv) != 2 { + return data, fmt.Errorf("invalid psi data: %q", f) + } + var pv *float64 + switch kv[0] { + case "avg10": + pv = &data.Avg10 + case "avg60": + pv = &data.Avg60 + case "avg300": + pv = &data.Avg300 + case "total": + v, err := strconv.ParseUint(kv[1], 10, 64) + if err != nil { + return data, fmt.Errorf("invalid %s PSI value: %w", kv[0], err) + } + data.Total = v + } + if pv != nil { + v, err := strconv.ParseFloat(kv[1], 64) + if err != nil { + return data, fmt.Errorf("invalid %s PSI value: %w", kv[0], err) + } + *pv = v + } + } + return data, nil +} diff --git a/libcontainer/cgroups/fs2/psi_test.go b/libcontainer/cgroups/fs2/psi_test.go new file mode 100644 index 00000000000..f70c19f7835 --- /dev/null +++ b/libcontainer/cgroups/fs2/psi_test.go @@ -0,0 +1,47 @@ +package fs2 + +import ( + "os" + "path/filepath" + "reflect" + "testing" + + "github.com/opencontainers/runc/libcontainer/cgroups" +) + +func TestStatCPUPSI(t *testing.T) { + const examplePSIData = `some avg10=1.71 avg60=2.36 avg300=2.57 total=230548833 +full avg10=1.00 avg60=1.01 avg300=1.00 total=157622356` + + // We're using a fake cgroupfs. + cgroups.TestMode = true + + fakeCgroupDir := t.TempDir() + statPath := filepath.Join(fakeCgroupDir, "cpu.pressure") + + if err := os.WriteFile(statPath, []byte(examplePSIData), 0o644); err != nil { + t.Fatal(err) + } + + st, err := statPSI(fakeCgroupDir, "cpu.pressure") + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(*st, cgroups.PSIStats{ + Some: cgroups.PSIData{ + Avg10: 1.71, + Avg60: 2.36, + Avg300: 2.57, + Total: 230548833, + }, + Full: cgroups.PSIData{ + Avg10: 1.00, + Avg60: 1.01, + Avg300: 1.00, + Total: 157622356, + }, + }) { + t.Errorf("unexpected PSI result: %+v", st) + } +} diff --git a/libcontainer/cgroups/stats.go b/libcontainer/cgroups/stats.go index 40a81dd5a86..8ff1fbb52bb 100644 --- a/libcontainer/cgroups/stats.go +++ b/libcontainer/cgroups/stats.go @@ -32,9 +32,22 @@ type CpuUsage struct { UsageInUsermode uint64 `json:"usage_in_usermode"` } +type PSIData struct { + Avg10 float64 `json:"avg10"` + Avg60 float64 `json:"avg60"` + Avg300 float64 `json:"avg300"` + Total uint64 `json:"total"` +} + +type PSIStats struct { + Some PSIData `json:"some,omitempty"` + Full PSIData `json:"full,omitempty"` +} + type CpuStats struct { CpuUsage CpuUsage `json:"cpu_usage,omitempty"` ThrottlingData ThrottlingData `json:"throttling_data,omitempty"` + PSI *PSIStats `json:"psi,omitempty"` } type CPUSetStats struct { @@ -89,6 +102,7 @@ type MemoryStats struct { UseHierarchy bool `json:"use_hierarchy"` Stats map[string]uint64 `json:"stats,omitempty"` + PSI *PSIStats `json:"psi,omitempty"` } type PageUsageByNUMA struct { @@ -133,6 +147,7 @@ type BlkioStats struct { IoMergedRecursive []BlkioStatEntry `json:"io_merged_recursive,omitempty"` IoTimeRecursive []BlkioStatEntry `json:"io_time_recursive,omitempty"` SectorsRecursive []BlkioStatEntry `json:"sectors_recursive,omitempty"` + PSI *PSIStats `json:"psi,omitempty"` } type HugetlbStats struct { diff --git a/tests/integration/events.bats b/tests/integration/events.bats index 051c81e4e28..707fe4e73fc 100644 --- a/tests/integration/events.bats +++ b/tests/integration/events.bats @@ -27,6 +27,35 @@ function teardown() { [[ "${lines[0]}" == *"data"* ]] } +# shellcheck disable=SC2030 +@test "events --stats with psi data" { + requires root cgroups_v2 psi + init_cgroup_paths + + update_config '.linux.resources.cpu |= { "quota": 1000 }' + + runc run -d --console-socket "$CONSOLE_SOCKET" test_busybox + [ "$status" -eq 0 ] + + # Stress the CPU a bit. Need something that runs for more than 10s. + runc exec test_busybox dd if=/dev/zero bs=1 count=128K of=/dev/null + [ "$status" -eq 0 ] + + runc exec test_busybox sh -c 'tail /sys/fs/cgroup/*.pressure' + + runc events --stats test_busybox + [ "$status" -eq 0 ] + + # Check PSI metrics. + jq '.data.cpu.psi' <<<"${lines[0]}" + for psi_type in some full; do + for psi_metric in avg10 avg60 avg300 total; do + echo -n "checking .data.cpu.psi.$psi_type.$psi_metric != 0: " + jq -e '.data.cpu.psi.'$psi_type.$psi_metric' != 0' <<<"${lines[0]}" + done + done +} + function test_events() { # XXX: currently cgroups require root containers. requires root diff --git a/tests/integration/helpers.bash b/tests/integration/helpers.bash old mode 100644 new mode 100755 index 7b1cc4a088f..fb3d0e50e72 --- a/tests/integration/helpers.bash +++ b/tests/integration/helpers.bash @@ -477,6 +477,13 @@ function requires() { skip_me=1 fi ;; + psi) + # If PSI is not compiled in the kernel, the file will not exist. + # If PSI is compiled, but not enabled, read will fail with ENOTSUPP. + if ! cat /sys/fs/cgroup/cpu.pressure &>/dev/null; then + skip_me=1 + fi + ;; *) fail "BUG: Invalid requires $var." ;; diff --git a/types/events.go b/types/events.go index 81bde829da5..e28ac8c3836 100644 --- a/types/events.go +++ b/types/events.go @@ -1,6 +1,9 @@ package types -import "github.com/opencontainers/runc/libcontainer/intelrdt" +import ( + "github.com/opencontainers/runc/libcontainer/cgroups" + "github.com/opencontainers/runc/libcontainer/intelrdt" +) // Event struct for encoding the event data to json. type Event struct { @@ -21,6 +24,10 @@ type Stats struct { NetworkInterfaces []*NetworkInterface `json:"network_interfaces"` } +type PSIData = cgroups.PSIData + +type PSIStats = cgroups.PSIStats + type Hugetlb struct { Usage uint64 `json:"usage,omitempty"` Max uint64 `json:"max,omitempty"` @@ -43,6 +50,7 @@ type Blkio struct { IoMergedRecursive []BlkioEntry `json:"ioMergedRecursive,omitempty"` IoTimeRecursive []BlkioEntry `json:"ioTimeRecursive,omitempty"` SectorsRecursive []BlkioEntry `json:"sectorsRecursive,omitempty"` + PSI *PSIStats `json:"psi,omitempty"` } type Pids struct { @@ -69,6 +77,7 @@ type CpuUsage struct { type Cpu struct { Usage CpuUsage `json:"usage,omitempty"` Throttling Throttling `json:"throttling,omitempty"` + PSI *PSIStats `json:"psi,omitempty"` } type CPUSet struct { @@ -99,6 +108,7 @@ type Memory struct { Kernel MemoryEntry `json:"kernel,omitempty"` KernelTCP MemoryEntry `json:"kernelTCP,omitempty"` Raw map[string]uint64 `json:"raw,omitempty"` + PSI *PSIStats `json:"psi,omitempty"` } type L3CacheInfo struct {