From 48f52dc21f8eac15f0a9525a491cb6defaeb3cea Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Thu, 27 Jan 2022 10:33:16 -0600 Subject: [PATCH] deps: switch to runc/libcontainer/cgroups cgroup manager Add a libcontainer/cgroups based manager that implements the existing cgroups.Manager API. Remove the containerd/cgroups based managers. Signed-off-by: Edita Kizinevic --- LICENSE_DEPENDENCIES.md | 14 +- go.mod | 2 +- go.sum | 1 - internal/pkg/cgroups/config_linux.go | 2 - .../pkg/cgroups/manager_libcontainer_linux.go | 278 ++++++++++++++++++ internal/pkg/cgroups/manager_linux.go | 63 +--- internal/pkg/cgroups/managerv1_linux.go | 169 ----------- internal/pkg/cgroups/managerv1_linux_test.go | 28 +- internal/pkg/cgroups/managerv2_linux.go | 221 -------------- internal/pkg/cgroups/managerv2_linux_test.go | 34 ++- .../pkg/runtime/engine/config/oci/config.go | 4 +- .../pkg/runtime/engine/oci/create_linux.go | 3 - internal/pkg/test/tool/require/require.go | 37 +-- pkg/util/slice/slice.go | 10 + pkg/util/slice/slice_test.go | 55 ++++ 15 files changed, 421 insertions(+), 500 deletions(-) create mode 100644 internal/pkg/cgroups/manager_libcontainer_linux.go delete mode 100644 internal/pkg/cgroups/managerv1_linux.go delete mode 100644 internal/pkg/cgroups/managerv2_linux.go diff --git a/LICENSE_DEPENDENCIES.md b/LICENSE_DEPENDENCIES.md index 7f517e6e44..fe900acecf 100644 --- a/LICENSE_DEPENDENCIES.md +++ b/LICENSE_DEPENDENCIES.md @@ -17,12 +17,6 @@ The dependencies and their licenses are as follows: **License URL:** -## github.com/containerd/cgroups - -**License:** Apache-2.0 - -**License URL:** - ## github.com/containerd/containerd **License:** Apache-2.0 @@ -179,11 +173,11 @@ The dependencies and their licenses are as follows: **License URL:** -## github.com/opencontainers/runc/libcontainer/user +## github.com/opencontainers/runc/libcontainer **License:** Apache-2.0 -**License URL:** +**License URL:** ## github.com/opencontainers/runtime-spec/specs-go @@ -371,11 +365,11 @@ The dependencies and their licenses are as follows: **License URL:** -## github.com/gogo/protobuf +## github.com/gogo/protobuf/proto **License:** BSD-3-Clause -**License URL:** +**License URL:** ## github.com/golang/protobuf diff --git a/go.mod b/go.mod index b6bfe479cb..c5eb6f20d7 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,6 @@ require ( github.com/blang/semver/v4 v4.0.0 github.com/buger/jsonparser v1.1.1 github.com/cenkalti/backoff/v4 v4.1.2 - github.com/containerd/cgroups v1.0.3 github.com/containerd/containerd v1.6.1 github.com/containernetworking/cni v1.0.1 github.com/containernetworking/plugins v1.1.0 @@ -27,6 +26,7 @@ require ( github.com/moby/sys/mount v0.3.0 // indirect github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.0.3-0.20211202193544-a5463b7f9c84 + github.com/opencontainers/runc v1.1.0 github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 github.com/opencontainers/runtime-tools v0.9.1-0.20210326182921-59cdde06764b github.com/opencontainers/selinux v1.10.0 diff --git a/go.sum b/go.sum index 35c6ae240d..206c1691d9 100644 --- a/go.sum +++ b/go.sum @@ -1204,7 +1204,6 @@ go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= diff --git a/internal/pkg/cgroups/config_linux.go b/internal/pkg/cgroups/config_linux.go index 4458272e16..947263179b 100644 --- a/internal/pkg/cgroups/config_linux.go +++ b/internal/pkg/cgroups/config_linux.go @@ -21,8 +21,6 @@ func Int64ptr(i int) *int64 { return &t } -var wildcard = Int64ptr(-1) - // LinuxHugepageLimit structure corresponds to limiting kernel hugepages type LinuxHugepageLimit struct { // Pagesize is the hugepage size diff --git a/internal/pkg/cgroups/manager_libcontainer_linux.go b/internal/pkg/cgroups/manager_libcontainer_linux.go new file mode 100644 index 0000000000..7db6069f94 --- /dev/null +++ b/internal/pkg/cgroups/manager_libcontainer_linux.go @@ -0,0 +1,278 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2021-2022, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package cgroups + +import ( + "fmt" + "path/filepath" + "strings" + + lccgroups "github.com/opencontainers/runc/libcontainer/cgroups" + lcmanager "github.com/opencontainers/runc/libcontainer/cgroups/manager" + "github.com/opencontainers/runc/libcontainer/configs" + "github.com/opencontainers/runc/libcontainer/specconv" + specs "github.com/opencontainers/runtime-spec/specs-go" +) + +const unifiedMountPoint = "/sys/fs/cgroup" + +// ManagerLibcontainer manages a cgroup 'Group', using the runc/libcontainer packages +type ManagerLC struct { + group string + pid int + cgroup lccgroups.Manager +} + +func (m *ManagerLC) load() (err error) { + if m.group != "" { + return m.loadFromPath() + } + return m.loadFromPid() +} + +func (m *ManagerLC) loadFromPid() (err error) { + if m.pid == 0 { + return fmt.Errorf("cannot load from pid - no process ID specified") + } + + pidCGFile := fmt.Sprintf("/proc/%d/cgroup", m.pid) + paths, err := lccgroups.ParseCgroupFile(pidCGFile) + if err != nil { + return fmt.Errorf("cannot read %s: %w", pidCGFile, err) + } + + // cgroups v2 path is always given by the unified "" subsystem + ok := false + if lccgroups.IsCgroup2UnifiedMode() { + m.group, ok = paths[""] + if !ok { + return fmt.Errorf("could not find cgroups v2 unified path") + } + return m.loadFromPath() + } + + // For cgroups v1 we are relying on fetching the 'devices' subsystem path. + // The devices subsystem is needed for our OCI engine and its presence is + // enforced in runc/libcontainer/cgroups/fs initialization without 'skipDevices'. + // This means we never explicitly put a container into a cgroup without a + // set 'devices' path. + m.group, ok = paths["devices"] + if !ok { + return fmt.Errorf("could not find cgroups v1 path (using devices subsystem)") + } + return m.loadFromPath() +} + +func (m *ManagerLC) loadFromPath() (err error) { + if m.group == "" { + return fmt.Errorf("cannot load from path - no path specified") + } + + lcConfig := &configs.Cgroup{ + Path: m.group, + Resources: &configs.Resources{}, + } + + m.cgroup, err = lcmanager.New(lcConfig) + if err != nil { + return fmt.Errorf("while creating cgroup manager: %w", err) + } + + return nil +} + +// GetCgroupRootPath returns cgroup root path +// TODO - this returns "" on error which needs to be checked for +// carefully. Should return an actual error instead. +func (m *ManagerLC) GetCgroupRootPath() string { + if m.cgroup == nil { + return "" + } + + // v2 - has a single fixed mountpoint for the root cgroup + if lccgroups.IsCgroup2UnifiedMode() { + return unifiedMountPoint + } + + // v1 - Get absolute paths to cgroup by subsystem + subPaths := m.cgroup.GetPaths() + + // For cgroups v1 we are relying on fetching the 'devices' subsystem path. + // The devices subsystem is needed for our OCI engine and its presence is + // enforced in runc/libcontainer/cgroups/fs initialization without 'skipDevices'. + // This means we never explicitly put a container into a cgroup without a + // set 'devices' path. + devicePath, ok := subPaths["devices"] + if !ok { + return "" + } + + // Take the piece before the first occurrence of "devices" as the root. + // I.E. /sys/fs/cgroup/devices/singularity/196219 -> /sys/fs/cgroup + pathParts := strings.Split(devicePath, "devices") + if len(pathParts) != 2 { + return "" + } + + return filepath.Clean(pathParts[0]) +} + +// ApplyFromSpec applies a cgroups configuration from an OCI LinuxResources spec +// struct, creating a new group if necessary, and places the process with +// Manager.Pid into the cgroup. The `Unified` key for native v2 cgroup +// specifications is not yet supported. +func (m *ManagerLC) ApplyFromSpec(resources *specs.LinuxResources) (err error) { + if m.group == "" { + return fmt.Errorf("path must be specified when creating a cgroup") + } + if m.pid == 0 { + return fmt.Errorf("pid must be specified when creating a cgroup") + } + + spec := &specs.Spec{ + Linux: &specs.Linux{ + CgroupsPath: m.group, + Resources: resources, + }, + } + + opts := &specconv.CreateOpts{ + CgroupName: m.group, + UseSystemdCgroup: false, + RootlessCgroups: false, + Spec: spec, + } + + lcConfig, err := specconv.CreateCgroupConfig(opts, nil) + if err != nil { + return fmt.Errorf("could not create cgroup config: %w", err) + } + + m.cgroup, err = lcmanager.New(lcConfig) + if err != nil { + return fmt.Errorf("while creating cgroup manager: %w", err) + } + + err = m.cgroup.Apply(m.pid) + if err != nil { + return fmt.Errorf("while creating cgroup: %w", err) + } + + err = m.cgroup.Set(lcConfig.Resources) + if err != nil { + return fmt.Errorf("while setting cgroup limits: %w", err) + } + + return nil +} + +// ApplyFromFile applies a cgroup configuration from a toml file, creating a new +// group if necessary, and places the process with Manager.Pid into the cgroup. +// The `Unified` key for native v2 cgroup specifications is not yet supported. +func (m *ManagerLC) ApplyFromFile(path string) error { + spec, err := readSpecFromFile(path) + if err != nil { + return err + } + return m.ApplyFromSpec(&spec) +} + +// UpdateFromSpec updates the existing managed cgroup using configuration from +// an OCI LinuxResources spec struct. The `Unified` key for native v2 cgroup +// specifications is not yet supported. +func (m *ManagerLC) UpdateFromSpec(resources *specs.LinuxResources) (err error) { + if m.cgroup == nil { + err = m.load() + if err != nil { + return fmt.Errorf("while creating cgroup manager: %w", err) + } + } + if m.group == "" { + return fmt.Errorf("cgroup path not set on manager, cannot update") + } + + spec := &specs.Spec{ + Linux: &specs.Linux{ + CgroupsPath: m.group, + Resources: resources, + }, + } + + opts := &specconv.CreateOpts{ + CgroupName: m.group, + UseSystemdCgroup: false, + RootlessCgroups: false, + Spec: spec, + } + + lcConfig, err := specconv.CreateCgroupConfig(opts, nil) + if err != nil { + return fmt.Errorf("could not create cgroup config: %w", err) + } + + err = m.cgroup.Set(lcConfig.Resources) + if err != nil { + return fmt.Errorf("while setting cgroup limits: %w", err) + } + + return nil +} + +// UpdateFromFile updates the existing managed cgroup using configuration +// from a toml file. +func (m *ManagerLC) UpdateFromFile(path string) error { + spec, err := readSpecFromFile(path) + if err != nil { + return err + } + return m.UpdateFromSpec(&spec) +} + +// Remove deletes the managed cgroup. +func (m *ManagerLC) Remove() (err error) { + if m.cgroup == nil { + if err := m.load(); err != nil { + return err + } + } + return m.cgroup.Destroy() +} + +func (m *ManagerLC) AddProc(pid int) (err error) { + if pid == 0 { + return fmt.Errorf("cannot add a zero pid to cgroup") + } + if m.cgroup == nil { + if err := m.load(); err != nil { + return err + } + } + return m.cgroup.Apply(pid) +} + +// Pause freezes processes in the managed cgroup. +func (m *ManagerLC) Pause() (err error) { + if m.cgroup == nil { + if err := m.load(); err != nil { + return err + } + } + return m.cgroup.Freeze(configs.Frozen) +} + +// Resume unfreezes process in the managed cgroup. +func (m *ManagerLC) Resume() (err error) { + if m.cgroup == nil { + if err := m.load(); err != nil { + return err + } + } + return m.cgroup.Freeze(configs.Thawed) +} diff --git a/internal/pkg/cgroups/manager_linux.go b/internal/pkg/cgroups/manager_linux.go index ecddec46a8..c28319b0ce 100644 --- a/internal/pkg/cgroups/manager_linux.go +++ b/internal/pkg/cgroups/manager_linux.go @@ -2,7 +2,7 @@ // Apptainer a Series of LF Projects LLC. // For website terms of use, trademark policy, privacy policy and other // project policies see https://lfprojects.org/policies -// Copyright (c) 2021, Sylabs Inc. All rights reserved. +// Copyright (c) 2021-2022, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -14,17 +14,12 @@ import ( "path/filepath" "strconv" - "github.com/apptainer/apptainer/pkg/sylog" - "github.com/containerd/cgroups" "github.com/opencontainers/runtime-spec/specs-go" ) // Manager is used to work with cgroups resource restrictions. It is an // interface satisfied by different implementations for v1 and v2 cgroups. type Manager interface { - // GetVersion returns the version of the cgroups interface in use by - // the manager. - GetVersion() int // GetCgroupRootPath returns the path to the root of the cgroup on the // filesystem. GetCgroupRootPath() string @@ -59,15 +54,11 @@ func NewManagerFromFile(specPath string, pid int, group string) (manager Manager if group == "" { group = filepath.Join("/apptainer", strconv.Itoa(pid)) } - if cgroups.Mode() == cgroups.Unified { - sylog.Debugf("Applying cgroups v2 configuration") - mgrv2 := ManagerV2{pid: pid, group: group} - return &mgrv2, mgrv2.ApplyFromFile(specPath) + mgr := ManagerLC{pid: pid, group: group} + if err := mgr.ApplyFromFile(specPath); err != nil { + return nil, err } - - sylog.Debugf("Applying cgroups v1 configuration") - mgrv1 := ManagerV1{pid: pid, path: group} - return &mgrv1, mgrv1.ApplyFromFile(specPath) + return &mgr, err } // NewManagerFromSpec creates a Manager, applies the configuration in spec, and adds pid to the cgroup. @@ -78,53 +69,29 @@ func NewManagerFromSpec(spec *specs.LinuxResources, pid int, group string) (mana group = filepath.Join("/apptainer", strconv.Itoa(pid)) } - if cgroups.Mode() == cgroups.Unified { - sylog.Debugf("Applying cgroups v2 configuration") - mgrv2 := ManagerV2{pid: pid, group: group} - return &mgrv2, mgrv2.ApplyFromSpec(spec) + mgr := ManagerLC{pid: pid, group: group} + if err := mgr.ApplyFromSpec(spec); err != nil { + return nil, err } - - sylog.Debugf("Applying cgroups v1 configuration") - mgrv1 := ManagerV1{pid: pid, path: group} - return &mgrv1, mgrv1.ApplyFromSpec(spec) + return &mgr, err } // GetManager returns a Manager for the provided cgroup name/path. func GetManager(group string) (manager Manager, err error) { - if cgroups.Mode() == cgroups.Unified { - sylog.Debugf("Fetching cgroups v2 configuration") - mgrv2 := ManagerV2{group: group} - if err := mgrv2.loadFromGroup(); err != nil { - return nil, err - } - return &mgrv2, nil - } - - sylog.Debugf("Fetching cgroups v1 configuration") - mgrv1 := ManagerV1{path: group} - if err := mgrv1.loadFromPath(); err != nil { + mgr := ManagerLC{group: group} + if err := mgr.load(); err != nil { return nil, err } - return &mgrv1, nil + return &mgr, nil } // GetManagerFromPid returns a Manager for the cgroup that pid is a member of. func GetManagerFromPid(pid int) (manager Manager, err error) { - if cgroups.Mode() == cgroups.Unified { - sylog.Debugf("Fetching cgroups v2 configuration") - mgrv2 := ManagerV2{pid: pid} - if err := mgrv2.loadFromPid(); err != nil { - return nil, err - } - return &mgrv2, nil - } - - sylog.Debugf("Fetching cgroups v1 configuration") - mgrv1 := ManagerV1{pid: pid} - if err := mgrv1.loadFromPid(); err != nil { + mgr := ManagerLC{pid: pid} + if err := mgr.load(); err != nil { return nil, err } - return &mgrv1, nil + return &mgr, nil } // readSpecFromFile loads a TOML file containing a specs.LinuxResources cgroups configuration. diff --git a/internal/pkg/cgroups/managerv1_linux.go b/internal/pkg/cgroups/managerv1_linux.go deleted file mode 100644 index 20d84a7fc2..0000000000 --- a/internal/pkg/cgroups/managerv1_linux.go +++ /dev/null @@ -1,169 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2021, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -package cgroups - -import ( - "fmt" - "path/filepath" - "strings" - - "github.com/containerd/cgroups" - specs "github.com/opencontainers/runtime-spec/specs-go" -) - -// ManagerV1 manages a cgroup 'Path', containing process 'Pid' for a v1 cgroups hierarchy. -type ManagerV1 struct { - path string - pid int - cgroup cgroups.Cgroup -} - -func (m *ManagerV1) GetVersion() int { - return 1 -} - -func (m *ManagerV1) load() (err error) { - if m.path != "" { - return m.loadFromPath() - } - return m.loadFromPid() -} - -func (m *ManagerV1) loadFromPid() (err error) { - if m.pid == 0 { - return fmt.Errorf("cannot load from pid - no process ID specified") - } - path := cgroups.PidPath(m.pid) - m.cgroup, err = cgroups.Load(cgroups.V1, path) - return err -} - -func (m *ManagerV1) loadFromPath() (err error) { - if m.path == "" { - return fmt.Errorf("cannot load from path - no path specified") - } - path := cgroups.StaticPath(m.path) - m.cgroup, err = cgroups.Load(cgroups.V1, path) - return err -} - -// GetCgroupRootPath returns the path to the root of the cgroup on the -// filesystem. -func (m *ManagerV1) GetCgroupRootPath() string { - if m.cgroup == nil { - return "" - } - - for _, sub := range m.cgroup.Subsystems() { - processes, err := m.cgroup.Processes(sub.Name(), false) - if len(processes) == 0 || err != nil { - continue - } - process := processes[0] - cgroupPath := strings.Split(process.Path, string(sub.Name()))[0] - return filepath.Clean(cgroupPath) - } - - return "" -} - -// ApplyFromSpec applies a cgroups configuration from an OCI LinuxResources -// spec struct, creating a new group if necessary, and places the process -// with Manager.Pid into the cgroup. -func (m *ManagerV1) ApplyFromSpec(spec *specs.LinuxResources) (err error) { - var path cgroups.Path - - if !filepath.IsAbs(m.path) { - return fmt.Errorf("cgroup path must be an absolute path") - } - - path = cgroups.StaticPath(m.path) - - s := spec - if s == nil { - s = &specs.LinuxResources{} - } - - // creates cgroup - m.cgroup, err = cgroups.New(cgroups.V1, path, s) - if err != nil { - return err - } - - return m.cgroup.Add(cgroups.Process{Pid: m.pid}) -} - -// ApplyFromFile applies a cgroup configuration from a toml file, creating a -// new group if necessary, and places the process with Manager.Pid into the -// cgroup. -func (m *ManagerV1) ApplyFromFile(path string) error { - spec, err := readSpecFromFile(path) - if err != nil { - return err - } - return m.ApplyFromSpec(&spec) -} - -// UpdateFromSpec updates the existing managed cgroup using configuration -// from an OCI LinuxResources spec struct. -func (m *ManagerV1) UpdateFromSpec(spec *specs.LinuxResources) (err error) { - if m.cgroup == nil { - if err = m.load(); err != nil { - return - } - } - err = m.cgroup.Update(spec) - return -} - -// UpdateFromFile updates the existing managed cgroup using configuration -// from a toml file. -func (m *ManagerV1) UpdateFromFile(path string) error { - spec, err := readSpecFromFile(path) - if err != nil { - return err - } - return m.UpdateFromSpec(&spec) -} - -func (m *ManagerV1) AddProc(pid int) (err error) { - if m.cgroup == nil { - if err := m.load(); err != nil { - return err - } - } - return m.cgroup.Add(cgroups.Process{Pid: pid}) -} - -// Remove deletes the managed cgroup. -func (m *ManagerV1) Remove() error { - // deletes subgroup - return m.cgroup.Delete() -} - -// Pause freezes processes in the managed cgroup. -func (m *ManagerV1) Pause() error { - if m.cgroup == nil { - if err := m.load(); err != nil { - return err - } - } - return m.cgroup.Freeze() -} - -// Resume unfreezes process in the managed cgroup. -func (m *ManagerV1) Resume() error { - if m.cgroup == nil { - if err := m.load(); err != nil { - return err - } - } - return m.cgroup.Thaw() -} diff --git a/internal/pkg/cgroups/managerv1_linux_test.go b/internal/pkg/cgroups/managerv1_linux_test.go index 9b048fd8e5..9f32873178 100644 --- a/internal/pkg/cgroups/managerv1_linux_test.go +++ b/internal/pkg/cgroups/managerv1_linux_test.go @@ -2,7 +2,7 @@ // Apptainer a Series of LF Projects LLC. // For website terms of use, trademark policy, privacy policy and other // project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2021, Sylabs Inc. All rights reserved. +// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -21,6 +21,7 @@ import ( "github.com/apptainer/apptainer/internal/pkg/test/tool/require" ) +//nolint:dupl func TestCgroupsV1(t *testing.T) { test.EnsurePrivilege(t) require.CgroupsV1(t) @@ -29,13 +30,18 @@ func TestCgroupsV1(t *testing.T) { if err := cmd.Start(); err != nil { t.Fatal(err) } - defer cmd.Process.Kill() pid := cmd.Process.Pid strPid := strconv.Itoa(pid) path := filepath.Join("/apptainer", strPid) - manager := &ManagerV1{pid: pid, path: path} + manager := &ManagerLC{pid: pid, group: path} + + defer func() { + cmd.Process.Kill() + cmd.Process.Wait() + manager.Remove() + }() cgroupsToml := "example/cgroups.toml" // Some systems, e.g. ppc64le may not have a 2MB page size, so don't @@ -49,7 +55,6 @@ func TestCgroupsV1(t *testing.T) { if err := manager.ApplyFromFile(cgroupsToml); err != nil { t.Fatal(err) } - defer manager.Remove() rootPath := manager.GetCgroupRootPath() if rootPath == "" { @@ -73,7 +78,7 @@ func TestCgroupsV1(t *testing.T) { } // test update/load from PID - manager = &ManagerV1{pid: pid} + manager = &ManagerLC{pid: pid} if err := manager.UpdateFromFile(tmpfile.Name()); err != nil { t.Fatal(err) @@ -81,11 +86,12 @@ func TestCgroupsV1(t *testing.T) { ensureIntInFile(t, cpuShares, 512) } +//nolint:dupl func TestPauseResumeV1(t *testing.T) { test.EnsurePrivilege(t) require.CgroupsV1(t) - manager := &ManagerV1{} + manager := &ManagerLC{} if err := manager.Pause(); err == nil { t.Errorf("unexpected success with PID 0") } @@ -97,15 +103,19 @@ func TestPauseResumeV1(t *testing.T) { if err := cmd.Start(); err != nil { t.Fatal(err) } - defer cmd.Process.Kill() manager.pid = cmd.Process.Pid - manager.path = filepath.Join("/apptainer", strconv.Itoa(manager.pid)) + manager.group = filepath.Join("/apptainer", strconv.Itoa(manager.pid)) + + defer func() { + cmd.Process.Kill() + cmd.Process.Wait() + manager.Remove() + }() if err := manager.ApplyFromFile("example/cgroups.toml"); err != nil { t.Fatal(err) } - defer manager.Remove() manager.Pause() // cgroups v1 freeze is to uninterruptible sleep diff --git a/internal/pkg/cgroups/managerv2_linux.go b/internal/pkg/cgroups/managerv2_linux.go deleted file mode 100644 index 0a34ea66a9..0000000000 --- a/internal/pkg/cgroups/managerv2_linux.go +++ /dev/null @@ -1,221 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2021, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -package cgroups - -import ( - "fmt" - "path" - - "github.com/apptainer/apptainer/pkg/sylog" - cgroupsv2 "github.com/containerd/cgroups/v2" - specs "github.com/opencontainers/runtime-spec/specs-go" -) - -const mountPoint = "/sys/fs/cgroup" - -// ManagerV2 manages a cgroup 'Group', containing process 'Pid' for a v2 unified cgroups hierarchy. -type ManagerV2 struct { - group string - pid int - cgroup *cgroupsv2.Manager -} - -func (m *ManagerV2) load() (err error) { - if m.group != "" { - return m.loadFromGroup() - } - return m.loadFromPid() -} - -func (m *ManagerV2) loadFromPid() (err error) { - if m.pid == 0 { - return fmt.Errorf("cannot load from pid - no process ID specified") - } - group, err := cgroupsv2.PidGroupPath(m.pid) - if err != nil { - return fmt.Errorf("could not find group for pid %d: %v", m.pid, err) - } - m.cgroup, err = cgroupsv2.LoadManager(mountPoint, group) - return err -} - -func (m *ManagerV2) loadFromGroup() (err error) { - if m.group == "" { - return fmt.Errorf("cannot load from group - no group specified") - } - m.cgroup, err = cgroupsv2.LoadManager(mountPoint, m.group) - return err -} - -func (m *ManagerV2) GetVersion() int { - return 2 -} - -// GetCgroupRootPath returns cgroup root path -func (m *ManagerV2) GetCgroupRootPath() string { - if m.group == "" { - return "" - } - return path.Join(mountPoint, m.group) -} - -// ApplyFromSpec applies a cgroups configuration from an OCI LinuxResources spec -// struct, creating a new group if necessary, and places the process with -// Manager.Pid into the cgroup. The `Unified` key for native v2 cgroup -// specifications is not yet supported. -func (m *ManagerV2) ApplyFromSpec(spec *specs.LinuxResources) (err error) { - if len(spec.Unified) > 0 { - sylog.Warningf("Unified cgroup resource specifications are not supported, and will not be applied.") - } - if m.group == "" { - return fmt.Errorf("group must be specified when creating a cgroup") - } - if m.pid == 0 { - return fmt.Errorf("pid must be specified when creating a cgroup") - } - - s := spec - if s == nil { - s = &specs.LinuxResources{} - } - - // translate the LinuxResources cgroups v1 / OCI spec to v2 Resources - res := cgroupsv2.ToResources(s) - // v1 device restrictions have to manually be brought across into the v2 - // Resources struct, as ToResources(s) doesn't do this. They will then be - // converted to ebpf programs and attached when the cgroup is created. - res.Devices = v2FixDevices(s.Devices) - - // creates cgroup - m.cgroup, err = cgroupsv2.NewManager(mountPoint, m.group, res) - if err != nil { - return err - } - - return m.cgroup.AddProc(uint64(m.pid)) -} - -// ApplyFromFile applies a cgroup configuration from a toml file, creating a new -// group if necessary, and places the process with Manager.Pid into the cgroup. -// The `Unified` key for native v2 cgroup specifications is not yet supported. -func (m *ManagerV2) ApplyFromFile(path string) error { - spec, err := readSpecFromFile(path) - if err != nil { - return err - } - return m.ApplyFromSpec(&spec) -} - -// UpdateFromSpec updates the existing managed cgroup using configuration from -// an OCI LinuxResources spec struct. The `Unified` key for native v2 cgroup -// specifications is not yet supported. -func (m *ManagerV2) UpdateFromSpec(spec *specs.LinuxResources) (err error) { - if len(spec.Unified) > 0 { - sylog.Warningf("Unified cgroup resource specifications are not supported, and will not be applied.") - } - if m.group == "" { - if m.pid == 0 { - return fmt.Errorf("pid must be provided if group is not known") - } - m.group, err = cgroupsv2.PidGroupPath(m.pid) - if err != nil { - return fmt.Errorf("could not find group for pid %d: %v", m.pid, err) - } - } - - s := spec - if s == nil { - s = &specs.LinuxResources{} - } - - // translate the LinuxResources cgroupsv1 / OCI spec to v2 Resources - res := cgroupsv2.ToResources(s) - // v1 device restrictions have to manually be brought across into the v2 Resources struct, - // as ToResources doesn't do this. They will then be converted to ebpf programs and attached. - res.Devices = v2FixDevices(s.Devices) - - // updates existing cgroup - m.cgroup, err = cgroupsv2.NewManager(mountPoint, m.group, res) - if err != nil { - return err - } - - return err -} - -// UpdateFromFile updates the existing managed cgroup using configuration -// from a toml file. -func (m *ManagerV2) UpdateFromFile(path string) error { - spec, err := readSpecFromFile(path) - if err != nil { - return err - } - return m.UpdateFromSpec(&spec) -} - -// Remove deletes the managed cgroup. -func (m *ManagerV2) Remove() (err error) { - // deletes subgroup - return m.cgroup.Delete() -} - -func (m *ManagerV2) AddProc(pid int) (err error) { - if m.cgroup == nil { - if err := m.load(); err != nil { - return err - } - } - return m.cgroup.AddProc(uint64(pid)) -} - -// Pause freezes processes in the managed cgroup. -func (m *ManagerV2) Pause() (err error) { - if m.cgroup == nil { - if err := m.load(); err != nil { - return err - } - } - return m.cgroup.Freeze() -} - -// Resume unfreezes process in the managed cgroup. -func (m *ManagerV2) Resume() (err error) { - if m.cgroup == nil { - if err := m.load(); err != nil { - return err - } - } - return m.cgroup.Thaw() -} - -// v2FixDevices modifies device entries to use an explicit, rather than implied -// wildcard. -// -// containerd/cgroups v1 device handling accepts: -// "" for type, which is replaced as "a" -// nil for major/minor, which is replaced as -1 -// -// containerd/cgroups v2 will not handle the "" and nil, and the explicit -// wildcard is needed. -func v2FixDevices(devs []specs.LinuxDeviceCgroup) []specs.LinuxDeviceCgroup { - for i, d := range devs { - if d.Type == "" { - d.Type = "a" - } - if d.Major == nil { - d.Major = wildcard - } - if d.Minor == nil { - d.Minor = wildcard - } - devs[i] = d - } - return devs -} diff --git a/internal/pkg/cgroups/managerv2_linux_test.go b/internal/pkg/cgroups/managerv2_linux_test.go index f1a44fac6e..03047968f2 100644 --- a/internal/pkg/cgroups/managerv2_linux_test.go +++ b/internal/pkg/cgroups/managerv2_linux_test.go @@ -2,7 +2,7 @@ // Apptainer a Series of LF Projects LLC. // For website terms of use, trademark policy, privacy policy and other // project policies see https://lfprojects.org/policies -// Copyright (c) 2021, Sylabs Inc. All rights reserved. +// Copyright (c) 2021-2022, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -22,22 +22,28 @@ import ( "github.com/apptainer/apptainer/internal/pkg/test/tool/require" ) +//nolint:dupl func TestCgroupsV2(t *testing.T) { test.EnsurePrivilege(t) - require.CgroupsV2(t) + require.CgroupsV2Unified(t) // Create process to put into a cgroup cmd := exec.Command("/bin/cat", "/dev/zero") if err := cmd.Start(); err != nil { t.Fatal(err) } - defer cmd.Process.Kill() pid := cmd.Process.Pid strPid := strconv.Itoa(pid) group := filepath.Join("/apptainer", strPid) - manager := &ManagerV2{pid: pid, group: group} + manager := &ManagerLC{pid: pid, group: group} + + defer func() { + cmd.Process.Kill() + cmd.Process.Wait() + manager.Remove() + }() // Example sets various things - we will check [pids] limit = 1024 cgroupsToml := "example/cgroups.toml" @@ -53,11 +59,10 @@ func TestCgroupsV2(t *testing.T) { if err := manager.ApplyFromFile(cgroupsToml); err != nil { t.Fatal(err) } - defer manager.Remove() // For cgroups v2 [pids] limit -> pids.max // Check for correct 1024 value - pidsMax := filepath.Join(mountPoint, group, "pids.max") + pidsMax := filepath.Join("/sys/fs/cgroup", group, "pids.max") ensureIntInFile(t, pidsMax, 1024) // Write a new config with [pids] limit = 512 @@ -75,7 +80,7 @@ func TestCgroupsV2(t *testing.T) { } // test update/load from PID - manager = &ManagerV2{pid: pid} + manager = &ManagerLC{pid: pid} // Update existing cgroup from new config if err := manager.UpdateFromFile(tmpfile.Name()); err != nil { @@ -86,11 +91,12 @@ func TestCgroupsV2(t *testing.T) { ensureIntInFile(t, pidsMax, 512) } +//nolint:dupl func TestPauseResumeV2(t *testing.T) { test.EnsurePrivilege(t) - require.CgroupsV2(t) + require.CgroupsV2Unified(t) - manager := &ManagerV2{} + manager := &ManagerLC{} if err := manager.Pause(); err == nil { t.Errorf("unexpected success with PID 0") } @@ -102,22 +108,26 @@ func TestPauseResumeV2(t *testing.T) { if err := cmd.Start(); err != nil { t.Fatal(err) } - defer cmd.Process.Kill() manager.pid = cmd.Process.Pid manager.group = filepath.Join("/apptainer", strconv.Itoa(manager.pid)) + defer func() { + cmd.Process.Kill() + cmd.Process.Wait() + manager.Remove() + }() + if err := manager.ApplyFromFile("example/cgroups.toml"); err != nil { t.Fatal(err) } - defer manager.Remove() manager.Pause() // cgroups v2 freeze is to interruptible sleep, which could actually occur // for our cat /dev/zero while it's running, so check freeze marker as well // as the process state here. ensureState(t, manager.pid, "S") - freezePath := path.Join(mountPoint, manager.group, "cgroup.freeze") + freezePath := path.Join("/sys/fs/cgroup", manager.group, "cgroup.freeze") ensureIntInFile(t, freezePath, 1) manager.Resume() diff --git a/internal/pkg/runtime/engine/config/oci/config.go b/internal/pkg/runtime/engine/config/oci/config.go index 4aeeb7c8c4..4acd44aa07 100644 --- a/internal/pkg/runtime/engine/config/oci/config.go +++ b/internal/pkg/runtime/engine/config/oci/config.go @@ -15,7 +15,7 @@ import ( "github.com/apptainer/apptainer/internal/pkg/runtime/engine/config/oci/generate" "github.com/apptainer/apptainer/internal/pkg/security/seccomp" - "github.com/containerd/cgroups" + "github.com/opencontainers/runc/libcontainer/cgroups" "github.com/opencontainers/runtime-spec/specs-go" cseccomp "github.com/seccomp/containers-golang" ) @@ -43,7 +43,7 @@ func (c *Config) UnmarshalJSON(b []byte) error { // DefaultConfig returns an OCI config generator with a // default OCI configuration for cgroups v1 or v2 dependent on the current host. func DefaultConfig() (*generate.Generator, error) { - if cgroups.Mode() == cgroups.Unified { + if cgroups.IsCgroup2HybridMode() { return DefaultConfigV2() } return DefaultConfigV1() diff --git a/internal/pkg/runtime/engine/oci/create_linux.go b/internal/pkg/runtime/engine/oci/create_linux.go index 875a16f8d3..6f75a70c34 100644 --- a/internal/pkg/runtime/engine/oci/create_linux.go +++ b/internal/pkg/runtime/engine/oci/create_linux.go @@ -810,9 +810,6 @@ func (c *container) addDevices(system *mount.System) error { } // cgroupDevices are essential for operation, so must be allowed *prior* to a configured wildcard deny. - // containerd/cgroups/v2 device filtering via eBPF is written such that it stops at the wildcard. - // See: https://github.com/containerd/cgroups/blob/ddda8a174e9ae86b31366812ae2d0f9f9570a7f1/v2/devicefilter.go#L93 - // https://github.com/containerd/cgroups/blob/ddda8a174e9ae86b31366812ae2d0f9f9570a7f1/v2/devicefilter.go#L164 c.engine.EngineConfig.OciConfig.Linux.Resources.Devices = append(cgroupDevices, c.engine.EngineConfig.OciConfig.Linux.Resources.Devices...) } diff --git a/internal/pkg/test/tool/require/require.go b/internal/pkg/test/tool/require/require.go index 967561a53c..c71ff4fafe 100644 --- a/internal/pkg/test/tool/require/require.go +++ b/internal/pkg/test/tool/require/require.go @@ -25,7 +25,8 @@ import ( "github.com/apptainer/apptainer/internal/pkg/security/seccomp" "github.com/apptainer/apptainer/pkg/network" "github.com/apptainer/apptainer/pkg/util/fs/proc" - "github.com/containerd/cgroups" + "github.com/apptainer/apptainer/pkg/util/slice" + "github.com/opencontainers/runc/libcontainer/cgroups" ) var ( @@ -129,26 +130,25 @@ func Network(t *testing.T) { // Cgroups checks that any cgroups version is enabled, if not the // current test is skipped with a message. func Cgroups(t *testing.T) { - mode := cgroups.Mode() - if mode == cgroups.Unavailable { + subsystems, err := cgroups.GetAllSubsystems() + if err != nil || len(subsystems) == 0 { t.Skipf("cgroups not available") } } -// CgroupsV1 checks that cgroups v1 is enabled, if not the +// CgroupsV1 checks that legacy cgroups is enabled, if not the // current test is skipped with a message. func CgroupsV1(t *testing.T) { - mode := cgroups.Mode() - if mode != cgroups.Legacy && mode != cgroups.Hybrid { - t.Skipf("cgroups v1 not available") + Cgroups(t) + if cgroups.IsCgroup2UnifiedMode() || cgroups.IsCgroup2HybridMode() { + t.Skipf("cgroups v1 legacy mode not available") } } -// CgroupsV2 checks that cgroups v2 is enabled, if not the +// CgroupsV2 checks that cgroups v2 unified mode is enabled, if not the // current test is skipped with a message. -func CgroupsV2(t *testing.T) { - mode := cgroups.Mode() - if mode != cgroups.Unified { +func CgroupsV2Unified(t *testing.T) { + if !cgroups.IsCgroup2UnifiedMode() { t.Skipf("cgroups v2 unified mode not available") } } @@ -157,20 +157,13 @@ func CgroupsV2(t *testing.T) { // available, if not the current test is skipped with a // message func CgroupsFreezer(t *testing.T) { - if cgroups.Mode() == cgroups.Unified { - return - } - - subSys, err := cgroups.V1() + subsystems, err := cgroups.GetAllSubsystems() if err != nil { - t.Skipf("cgroups disabled") + t.Skipf("couldn't get cgroups subsystems: %v", err) } - for _, s := range subSys { - if s.Name() == "freezer" { - return - } + if !slice.ContainsString(subsystems, "freezer") { + t.Skipf("no cgroups freezer subsystem available") } - t.Skipf("no cgroups freezer subsystem available") } // Nvidia checks that an NVIDIA stack is available diff --git a/pkg/util/slice/slice.go b/pkg/util/slice/slice.go index f6eb89dd3a..c56b4dcfd5 100644 --- a/pkg/util/slice/slice.go +++ b/pkg/util/slice/slice.go @@ -30,3 +30,13 @@ func ContainsAnyString(s []string, matches []string) bool { } return false } + +// ContainsInt returns true if int slice s contains match +func ContainsInt(s []int, match int) bool { + for _, a := range s { + if a == match { + return true + } + } + return false +} diff --git a/pkg/util/slice/slice_test.go b/pkg/util/slice/slice_test.go index 3e666a5618..cee3b42e55 100644 --- a/pkg/util/slice/slice_test.go +++ b/pkg/util/slice/slice_test.go @@ -135,3 +135,58 @@ func TestContainsAnyString(t *testing.T) { }) } } + +func TestContainsInt(t *testing.T) { + type args struct { + s []int + match int + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "NoMatchSingle", + args: args{[]int{1}, 0}, + want: false, + }, + { + name: "NoMatchMulti", + args: args{[]int{1, 2, 3}, 0}, + want: false, + }, + { + name: "NoMatchEmpty", + args: args{[]int{}, 0}, + want: false, + }, + { + name: "MatchSingle", + args: args{[]int{1}, 1}, + want: true, + }, + { + name: "MatchMultiStart", + args: args{[]int{1, 2, 3}, 1}, + want: true, + }, + { + name: "MatchMultiMid", + args: args{[]int{1, 2, 3}, 2}, + want: true, + }, + { + name: "MatchMultiEnd", + args: args{[]int{1, 2, 3}, 2}, + want: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ContainsInt(tt.args.s, tt.args.match); got != tt.want { + t.Errorf("ContainsInt() = %v, want %v", got, tt.want) + } + }) + } +}