libcontainer: cgroups: add intel_rdt support in runc

This PR fixes issue #433
#433

About Intel RDT/CAT feature:
Intel platforms with new Xeon CPU support Resource Director Technology (RDT).
Intel Cache Allocation Technology (CAT) is a sub-feature of RDT. Currently L3
Cache is the only resource that is supported in RDT.

This feature provides a way for the software to restrict cache allocation to a
defined 'subset' of L3 cache which may be overlapping with other 'subsets'.
The different subsets are identified by class of service (CLOS) and each CLOS
has a capacity bitmask (CBM).

More information can be found in the section 17.16 of Intel Software Developer
Manual.

About intel_rdt cgroup:
Linux kernel 4.6 (or later) will introduce new cgroup subsystem 'intel_rdt'
with kernel config CONFIG_INTEL_RDT.

The 'intel_rdt' cgroup manages L3 cache allocation. It has a file 'l3_cbm'
which represents the L3 cache capacity bitmask (CBM). The CBM needs to have
only *contiguous bits set* and number of bits that can be set is less than the
max bits. The max bits in the CBM is varied among supported Intel platforms.
The tasks belonging to a cgroup get to fill in the L3 cache represented by
the CBM. For example, if the max bits in the CBM is 10 and the L3 cache size
is 10MB, each bit represents 1MB of the L3 cache capacity.

Root cgroup always has all the bits set in the l3_cbm. User can create more
cgroups with mkdir syscall. By default the child cgroups inherit the CBM from
parent. User can change the CBM specified in hex for each cgroup.

For more information about intel_rdt cgroup:
https://lkml.org/lkml/2015/10/2/74

An example:
    Root cgroup: intel_rdt.l3_cbm == 0xfffff, the max bits of CBM is 20
    L3 cache size: 55 MB
This assigns 11 MB (1/5) of L3 cache to the child group:
    $ /bin/echo 0xf > intel_rdt.l3_cbm

Signed-off-by: Xiaochen Shen <xiaochen.shen@intel.com>

libcontainer/SPEC.md

@@ -142,6 +142,7 @@ system resources like cpu, memory, and device access.
 | perf_event | 1       |
 | freezer    | 1       |
 | hugetlb    | 1       |
+| intel_rdt  | 1       |
 
 
 All cgroup subsystem are joined so that statistics can be collected from

libcontainer/cgroups/fs/apply_raw.go

@@ -29,6 +29,7 @@ var (
 		&NetPrioGroup{},
 		&PerfEventGroup{},
 		&FreezerGroup{},
+		&IntelRdtGroup{},
 	}
 	CgroupProcesses  = "cgroup.procs"
 	HugePageSizes, _ = cgroups.GetHugePageSize()

libcontainer/cgroups/fs/intel_rdt.go

@@ -0,0 +1,75 @@
+// +build linux
+
+package fs
+
+import (
+	"fmt"
+	"strconv"
+
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+	"github.com/opencontainers/runc/libcontainer/configs"
+)
+
+type IntelRdtGroup struct {
+}
+
+func (s *IntelRdtGroup) Name() string {
+	return "intel_rdt"
+}
+
+func (s *IntelRdtGroup) Apply(d *cgroupData) error {
+	dir, err := d.join("intel_rdt")
+	if err != nil {
+		if !cgroups.IsNotFound(err) {
+			return err
+		}
+		// We will not return err here when:
+		// 1. The h/w platform doesn't support Intel RDT/CAT feature,
+		//    intel_rdt cgroup is not enabled in kernel.
+		// 2. intel_rdt cgroup is not mounted
+		return nil
+	}
+
+	if err := s.Set(dir, d.config); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (s *IntelRdtGroup) Set(path string, cgroup *configs.Cgroup) error {
+	// The valid CBM (capacity bitmask) is a *contiguous bits set* and
+	// number of bits that can be set is less than the max bit. The max
+	// bits in the CBM is varied among supported Intel platforms.
+	//
+	// By default the child cgroups inherit the CBM from parent. The CBM
+	// in a child cgroup should be a subset of the CBM in parent. Kernel
+	// will check if it is valid when writing.
+	//
+	// e.g., 0xfffff in root cgroup indicates the max bits of CBM is 20
+	// bits, which mapping to entire L3 cache capacity. Some valid CBM
+	// values to Set in children cgroup: 0xf, 0xf0, 0x3ff, 0x1f00 and etc.
+	if cgroup.Resources.IntelRdtL3Cbm != 0 {
+		l3CbmStr := fmt.Sprintf("0x%s", strconv.FormatUint(cgroup.Resources.IntelRdtL3Cbm, 16))
+		if err := writeFile(path, "intel_rdt.l3_cbm", l3CbmStr); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (s *IntelRdtGroup) Remove(d *cgroupData) error {
+	return removePath(d.path("intel_rdt"))
+}
+
+func (s *IntelRdtGroup) GetStats(path string, stats *cgroups.Stats) error {
+	value, err := getCgroupParamUintHex(path, "intel_rdt.l3_cbm")
+	if err != nil {
+		return fmt.Errorf("failed to parse intel_rdt.l3_cbm - %s", err)
+	}
+
+	stats.IntelRdtStats.L3Cbm = value
+
+	return nil
+}

libcontainer/cgroups/fs/intel_rdt_test.go

@@ -0,0 +1,62 @@
+// +build linux
+
+package fs
+
+import (
+	"strconv"
+	"testing"
+
+	"github.com/opencontainers/runc/libcontainer/cgroups"
+)
+
+func TestIntelRdtSetL3Cbm(t *testing.T) {
+	helper := NewCgroupTestUtil("intel_rdt", t)
+	defer helper.cleanup()
+
+	const (
+		l3CbmBefore = 0xf
+		l3CbmAfter  = 0xf0
+	)
+
+	helper.writeFileContents(map[string]string{
+		"intel_rdt.l3_cbm": strconv.FormatUint(l3CbmBefore, 16),
+	})
+
+	helper.CgroupData.config.Resources.IntelRdtL3Cbm = l3CbmAfter
+	intelrdt := &IntelRdtGroup{}
+	if err := intelrdt.Set(helper.CgroupPath, helper.CgroupData.config); err != nil {
+		t.Fatal(err)
+	}
+
+	value, err := getCgroupParamUintHex(helper.CgroupPath, "intel_rdt.l3_cbm")
+	if err != nil {
+		t.Fatalf("Failed to parse intel_rdt.l3_cbm - %s", err)
+	}
+
+	if value != l3CbmAfter {
+		t.Fatal("Got the wrong value, set intel_rdt.l3_cbm failed.")
+	}
+}
+
+func TestIntelRdtStats(t *testing.T) {
+	helper := NewCgroupTestUtil("intel_rdt", t)
+	defer helper.cleanup()
+
+	const (
+		l3CbmContents = 0x1f00
+	)
+
+	helper.writeFileContents(map[string]string{
+		"intel_rdt.l3_cbm": strconv.FormatUint(l3CbmContents, 16),
+	})
+
+	intelrdt := &IntelRdtGroup{}
+	stats := *cgroups.NewStats()
+	if err := intelrdt.GetStats(helper.CgroupPath, &stats); err != nil {
+		t.Fatal(err)
+	}
+
+	if stats.IntelRdtStats.L3Cbm != l3CbmContents {
+		t.Fatalf("Expected '0x%x', got '0x%x' for intel_rdt.l3_cbm", l3CbmContents, stats.IntelRdtStats.L3Cbm)
+	}
+}

libcontainer/cgroups/fs/utils.go

@@ -68,6 +68,22 @@ func getCgroupParamUint(cgroupPath, cgroupFile string) (uint64, error) {
 	return res, nil
 }
 
+// Gets a single hex uint64 value from the specified cgroup file.
+func getCgroupParamUintHex(cgroupPath, cgroupFile string) (uint64, error) {
+	fileName := filepath.Join(cgroupPath, cgroupFile)
+	contents, err := ioutil.ReadFile(fileName)
+	if err != nil {
+		return 0, err
+	}
+
+	hexStr := strings.TrimSpace(strings.TrimPrefix(string(contents), "0x"))
+	res, err := parseUint(hexStr, 16, 64)
+	if err != nil {
+		return res, fmt.Errorf("unable to parse %q as a uint from Cgroup file %q", string(contents), fileName)
+	}
+	return res, nil
+}
+
 // Gets a string value from the specified cgroup file
 func getCgroupParamString(cgroupPath, cgroupFile string) (string, error) {
 	contents, err := ioutil.ReadFile(filepath.Join(cgroupPath, cgroupFile))

libcontainer/cgroups/stats.go

@@ -77,12 +77,17 @@ type HugetlbStats struct {
 	Failcnt uint64 `json:"failcnt"`
 }
 
+type IntelRdtStats struct {
+	L3Cbm uint64 `json:"l3_cbm,omitempty"`
+}
+
 type Stats struct {
 	CpuStats    CpuStats    `json:"cpu_stats,omitempty"`
 	MemoryStats MemoryStats `json:"memory_stats,omitempty"`
 	BlkioStats  BlkioStats  `json:"blkio_stats,omitempty"`
 	// the map is in the format "size of hugepage: stats of the hugepage"
-	HugetlbStats map[string]HugetlbStats `json:"hugetlb_stats,omitempty"`
+	HugetlbStats  map[string]HugetlbStats `json:"hugetlb_stats,omitempty"`
+	IntelRdtStats IntelRdtStats           `json:"intel_rdt_stats,omitempty"`
 }
 
 func NewStats() *Stats {

libcontainer/cgroups/systemd/apply_systemd.go

@@ -62,6 +62,7 @@ var subsystems = subsystemSet{
 	&fs.NetPrioGroup{},
 	&fs.NetClsGroup{},
 	&fs.NameGroup{GroupName: "name=systemd"},
+	&fs.IntelRdtGroup{},
 }
 
 const (
@@ -233,7 +234,7 @@ func (m *Manager) Apply(pid int) error {
 		return err
 	}
 
-	// we need to manually join the freezer, net_cls, net_prio and cpuset cgroup in systemd
+	// we need to manually join the freezer, net_cls, net_prio cpuset and intel_rdt cgroup in systemd
 	// because it does not currently support it via the dbus api.
 	if err := joinFreezer(c, pid); err != nil {
 		return err
@@ -257,6 +258,11 @@ func (m *Manager) Apply(pid int) error {
 	if err := joinPerfEvent(c, pid); err != nil {
 		return err
 	}
+
+	if err := joinIntelRdt(c, pid); err != nil {
+		return err
+	}
+
 	// FIXME: Systemd does have `BlockIODeviceWeight` property, but we got problem
 	// using that (at least on systemd 208, see https://github.com/opencontainers/runc/libcontainer/pull/354),
 	// so use fs work around for now.
@@ -642,3 +648,24 @@ func joinPerfEvent(c *configs.Cgroup, pid int) error {
 	}
 	return perfEvent.Set(path, c)
 }
+
+func joinIntelRdt(c *configs.Cgroup, pid int) error {
+	path, err := join(c, "intel_rdt", pid)
+	if err != nil {
+		if !cgroups.IsNotFound(err) {
+			return err
+		}
+		// We will not return err here when:
+		// 1. The h/w platform doesn't support Intel RDT/CAT feature,
+		//    intel_rdt cgroup is not enabled in kernel.
+		// 2. intel_rdt cgroup is not mounted
+		return nil
+	}
+
+	IntelRdt, err := subsystems.Get("intel_rdt")
+	if err != nil {
+		return err
+	}
+
+	return IntelRdt.Set(path, c)
+}

libcontainer/configs/cgroup_unix.go

@@ -102,4 +102,7 @@ type Resources struct {
 
 	// Set class identifier for container's network packets
 	NetClsClassid string `json:"net_cls_classid"`
+
+	// L3 cache allocation CBM (cache bit mask) for container
+	IntelRdtL3Cbm uint64 `json:"intel_rdt_l3_cbm"`
 }

spec.go

@@ -492,6 +492,8 @@ func createCgroupConfig(name string, spec *specs.LinuxRuntimeSpec, devices []*co
 			Priority:  m.Priority,
 		})
 	}
+	c.Resources.IntelRdtL3Cbm = r.IntelRdt.L3Cbm
+
 	return c, nil
 }