From eda54d52ef8c33cad83046c8af8340ad14a93b98 Mon Sep 17 00:00:00 2001 From: Bin Tang Date: Fri, 2 Jun 2023 11:07:01 +0800 Subject: [PATCH 01/15] pkg: introduce pkg/cgroup package This PR introduce a cgroup interface for nydusd daemon Signed-off-by: Bin Tang --- pkg/cgroup/cgroup.go | 61 +++++++++++++++++++++++++++++++++++++++++++ pkg/cgroup/manager.go | 51 ++++++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+) create mode 100644 pkg/cgroup/cgroup.go create mode 100644 pkg/cgroup/manager.go diff --git a/pkg/cgroup/cgroup.go b/pkg/cgroup/cgroup.go new file mode 100644 index 0000000000..c74b572b97 --- /dev/null +++ b/pkg/cgroup/cgroup.go @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2023. Nydus Developers. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package cgroup + +import ( + "errors" + + "github.com/containerd/cgroups" + v1 "github.com/containerd/nydus-snapshotter/pkg/cgroup/v1" + v2 "github.com/containerd/nydus-snapshotter/pkg/cgroup/v2" +) + +const ( + defaultSlice = "system.slice" +) + +var ( + ErrCgroupNotSupported = errors.New("cgroups: cgroup not supported") +) + +type Config struct { + MemoryLimitInBytes int64 +} + +type DaemonCgroup interface { + // Delete the current cgroup. + Delete() error + // Add a process to current cgroup. + AddProc(pid int) error +} + +func createCgroup(name string, config Config) (DaemonCgroup, error) { + if cgroups.Mode() == cgroups.Unified { + return v2.NewCgroup(defaultSlice, name, config.MemoryLimitInBytes) + } + + return v1.NewCgroup(defaultSlice, name, config.MemoryLimitInBytes) +} + +func supported() bool { + return cgroups.Mode() != cgroups.Unavailable +} + +func displayMode() string { + switch cgroups.Mode() { + case cgroups.Legacy: + return "legacy" + case cgroups.Hybrid: + return "hybrid" + case cgroups.Unified: + return "unified" + case cgroups.Unavailable: + return "unavailable" + default: + return "unknown" + } +} diff --git a/pkg/cgroup/manager.go b/pkg/cgroup/manager.go new file mode 100644 index 0000000000..0ca9dd0b63 --- /dev/null +++ b/pkg/cgroup/manager.go @@ -0,0 +1,51 @@ +/* + * Copyright (c) 2023. Nydus Developers. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package cgroup + +import ( + "github.com/containerd/containerd/log" +) + +type Manager struct { + name string + config Config + cgroup DaemonCgroup +} + +type Opt struct { + Name string + Config Config +} + +func NewManager(opt Opt) (*Manager, error) { + if !supported() { + log.L.Warn("cgroup is not supported") + return nil, ErrCgroupNotSupported + } + + log.L.Infof("cgroup mode: %s", displayMode()) + cg, err := createCgroup(opt.Name, opt.Config) + if err != nil { + return nil, err + } + + return &Manager{ + name: opt.Name, + config: opt.Config, + cgroup: cg, + }, nil +} + +// Please make sure the *Manager is not null. +func (m *Manager) AddProc(pid int) error { + return m.cgroup.AddProc(pid) +} + +// Please make sure the *Manager is not null. +func (m *Manager) Delete() error { + return m.cgroup.Delete() +} From 9b373c0f0888ba11a7019cef9868e4a39d6616c9 Mon Sep 17 00:00:00 2001 From: Bin Tang Date: Fri, 2 Jun 2023 11:10:01 +0800 Subject: [PATCH 02/15] cgroup: add cgroup v1 implementation Signed-off-by: Bin Tang --- pkg/cgroup/v1/v1.go | 86 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 pkg/cgroup/v1/v1.go diff --git a/pkg/cgroup/v1/v1.go b/pkg/cgroup/v1/v1.go new file mode 100644 index 0000000000..4dab8498b5 --- /dev/null +++ b/pkg/cgroup/v1/v1.go @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2023. Nydus Developers. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package v1 + +import ( + "github.com/containerd/cgroups/v3/cgroup1" + "github.com/containerd/containerd/log" + "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" +) + +type Cgroup struct { + controller cgroup1.Cgroup +} + +func generateHierarchy() cgroup1.Hierarchy { + return cgroup1.SingleSubsystem(cgroup1.Default, cgroup1.Memory) +} + +func NewCgroup(slice, name string, memoryLimitInBytes int64) (Cgroup, error) { + hierarchy := generateHierarchy() + specResources := &specs.LinuxResources{ + Memory: &specs.LinuxMemory{ + Limit: &memoryLimitInBytes, + }, + } + + controller, err := cgroup1.Load(cgroup1.Slice(slice, name), cgroup1.WithHiearchy(hierarchy)) + if err != nil && err != cgroup1.ErrCgroupDeleted { + return Cgroup{}, err + } + + if controller != nil { + processes, err := controller.Processes(cgroup1.Memory, true) + if err != nil { + return Cgroup{}, err + } + if len(processes) > 0 { + log.L.Infof("target cgroup is existed with processes %v", processes) + if err := controller.Update(specResources); err != nil { + return Cgroup{}, err + } + return Cgroup{ + controller: controller, + }, nil + } + if err := controller.Delete(); err != nil { + return Cgroup{}, err + } + } + + controller, err = cgroup1.New(cgroup1.Slice(slice, name), specResources, cgroup1.WithHiearchy(hierarchy)) + if err != nil { + return Cgroup{}, errors.Wrapf(err, "create cgroup") + } + log.L.Infof("create cgroup (v1) successful, state: %v", controller.State()) + + return Cgroup{ + controller: controller, + }, nil +} + +func (cg Cgroup) Delete() error { + processes, err := cg.controller.Processes(cgroup1.Memory, true) + if err != nil { + return err + } + if len(processes) > 0 { + log.L.Infof("skip destroy cgroup because of running daemon %v", processes) + return nil + } + return cg.controller.Delete() +} +func (cg Cgroup) AddProc(pid int) error { + err := cg.controller.AddProc(uint64(pid), cgroup1.Memory) + if err != nil { + return err + } + + log.L.Infof("add process %d to daemon cgroup successful", pid) + return nil +} From 57509f4d4ab26d2c09cfa1b664e623614afa15db Mon Sep 17 00:00:00 2001 From: Bin Tang Date: Fri, 2 Jun 2023 11:10:09 +0800 Subject: [PATCH 03/15] cgroup: add cgroup v2 implementation Signed-off-by: Bin Tang --- pkg/cgroup/v2/v2.go | 94 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 pkg/cgroup/v2/v2.go diff --git a/pkg/cgroup/v2/v2.go b/pkg/cgroup/v2/v2.go new file mode 100644 index 0000000000..719295cc90 --- /dev/null +++ b/pkg/cgroup/v2/v2.go @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2023. Nydus Developers. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package v2 + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/containerd/cgroups/v3/cgroup2" + "github.com/containerd/containerd/log" + "golang.org/x/exp/slices" +) + +const ( + defaultRoot = "/sys/fs/cgroup" +) + +var ( + ErrRootMemorySubtreeControllerDisabled = errors.New("cgroups v2: root subtree controller for memory is disabled") +) + +type Cgroup struct { + manager *cgroup2.Manager +} + +func readSubtreeControllers(dir string) ([]string, error) { + b, err := os.ReadFile(filepath.Join(dir, "cgroup.subtree_control")) + if err != nil { + return nil, err + } + return strings.Fields(string(b)), nil +} + +func NewCgroup(slice, name string, memoryLimitInBytes int64) (Cgroup, error) { + resources := &cgroup2.Resources{ + Memory: &cgroup2.Memory{}, + } + if memoryLimitInBytes > -1 { + resources = &cgroup2.Resources{ + Memory: &cgroup2.Memory{ + Max: &memoryLimitInBytes, + }, + } + } + + rootSubtreeControllers, err := readSubtreeControllers(defaultRoot) + if err != nil { + return Cgroup{}, err + } + log.L.Infof("root subtree controllers: %s", rootSubtreeControllers) + + if !slices.Contains(rootSubtreeControllers, "memory") { + return Cgroup{}, ErrRootMemorySubtreeControllerDisabled + } + + m, err := cgroup2.NewManager(defaultRoot, fmt.Sprintf("/%s/%s", slice, name), resources) + if err != nil { + return Cgroup{}, err + } + + controllers, err := m.Controllers() + if err != nil { + return Cgroup{}, err + } + log.L.Infof("create cgroup (v2) successful, controllers: %v", controllers) + + return Cgroup{ + manager: m, + }, nil +} + +func (cg Cgroup) Delete() error { + if cg.manager != nil { + return cg.manager.Delete() + } + return nil +} +func (cg Cgroup) AddProc(pid int) error { + if cg.manager != nil { + err := cg.manager.AddProc(uint64(pid)) + if err != nil { + return err + } + log.L.Infof("add process %d to daemon cgroup successful", pid) + } + return nil +} From 54fc5bc5cd2ffbf6bc5a118c994252a0433b560e Mon Sep 17 00:00:00 2001 From: Bin Tang Date: Fri, 2 Jun 2023 11:16:56 +0800 Subject: [PATCH 04/15] config: add cgroup config entry for daemon Signed-off-by: Bin Tang --- config/config.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/config/config.go b/config/config.go index 83c2266416..c7328578d5 100644 --- a/config/config.go +++ b/config/config.go @@ -106,6 +106,10 @@ type Experimental struct { EnableReferrerDetect bool `toml:"enable_referrer_detect"` } +type CgroupConfig struct { + MemoryLimit string `toml:"memory_limit"` +} + // Configure how to start and recover nydusd daemons type DaemonConfig struct { NydusdPath string `toml:"nydusd_path"` @@ -203,6 +207,7 @@ type SnapshotterConfig struct { ImageConfig ImageConfig `toml:"image"` CacheManagerConfig CacheManagerConfig `toml:"cache_manager"` LoggingConfig LoggingConfig `toml:"log"` + CgroupConfig CgroupConfig `toml:"cgroup"` Experimental Experimental `toml:"experimental"` } From 6030cab456610dc337475bd8260c69b3549cc9ca Mon Sep 17 00:00:00 2001 From: Bin Tang Date: Fri, 2 Jun 2023 11:33:57 +0800 Subject: [PATCH 05/15] pkg: introduce pkg/parser package The parser package aims to parse configuration data to expected units Signed-off-by: Bin Tang --- pkg/utils/parser/parser.go | 77 ++++++++++++++++++++++++++++++++++++ pkg/utils/sysinfo/sysinfo.go | 38 ++++++++++++++++++ 2 files changed, 115 insertions(+) create mode 100644 pkg/utils/parser/parser.go create mode 100644 pkg/utils/sysinfo/sysinfo.go diff --git a/pkg/utils/parser/parser.go b/pkg/utils/parser/parser.go new file mode 100644 index 0000000000..14c7b04803 --- /dev/null +++ b/pkg/utils/parser/parser.go @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2023. Nydus Developers. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package parser + +import ( + "regexp" + "strconv" + "sync" + + "github.com/pkg/errors" +) + +var ( + unitMultipliers map[string]int64 + unitMultipliersOnce sync.Once +) + +func InitUnitMultipliers() { + unitMultipliers = make(map[string]int64, 10) + + unitMultipliers["KiB"] = 1024 + unitMultipliers["MiB"] = unitMultipliers["KiB"] * 1024 + unitMultipliers["GiB"] = unitMultipliers["MiB"] * 1024 + unitMultipliers["TiB"] = unitMultipliers["GiB"] * 1024 + unitMultipliers["PiB"] = unitMultipliers["TiB"] * 1024 + + unitMultipliers["Ki"] = 1024 + unitMultipliers["Mi"] = unitMultipliers["Ki"] * 1024 + unitMultipliers["Gi"] = unitMultipliers["Mi"] * 1024 + unitMultipliers["Ti"] = unitMultipliers["Gi"] * 1024 + unitMultipliers["Pi"] = unitMultipliers["Ti"] * 1024 +} + +func MemoryConfigToBytes(data string, totalMemoryBytes int) (int64, error) { + if data == "" { + return -1, nil + } + + // Memory value without unit. + value, err := strconv.ParseFloat(data, 64) + if err == nil { + return int64(value), nil + } + + re := regexp.MustCompile(`(\d*\.?\d+)([a-zA-Z\%]+)`) + matches := re.FindStringSubmatch(data) + if len(matches) != 3 { + return 0, errors.Errorf("Falied to convert data to bytes: Unknown unit in %s", data) + } + + // Parse memory value and unit. + valueString, unit := matches[1], matches[2] + value, err = strconv.ParseFloat(valueString, 64) + if err != nil { + return 0, errors.Wrap(err, "Failed to parse memory limit") + } + + // Return if the unit is byte. + if unit == "B" { + return int64(value), nil + } + + // Calculate value if the unit is "%". + if unit == "%" { + limitMemory := float64(totalMemoryBytes) * value / 100 + return int64(limitMemory + 0.5), nil + } + + unitMultipliersOnce.Do(InitUnitMultipliers) + + multiplier := unitMultipliers[unit] + return int64(value * float64(multiplier)), nil +} diff --git a/pkg/utils/sysinfo/sysinfo.go b/pkg/utils/sysinfo/sysinfo.go new file mode 100644 index 0000000000..cf8562c1dd --- /dev/null +++ b/pkg/utils/sysinfo/sysinfo.go @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2023. Nydus Developers. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package sysinfo + +import ( + "sync" + "syscall" +) + +var ( + sysinfo *syscall.Sysinfo_t + sysinfoOnce sync.Once + sysinfoErr error +) + +func GetSysinfo() { + var info syscall.Sysinfo_t + err := syscall.Sysinfo(&info) + if err != nil { + sysinfoErr = err + return + } + sysinfo = &info + sysinfoErr = nil +} + +func GetTotalMemoryBytes() (int, error) { + sysinfoOnce.Do(GetSysinfo) + if sysinfo == nil { + return 0, sysinfoErr + } + + return int(sysinfo.Totalram), nil +} From 7b055ff55063772bc7c3d484deb82b3a4da9f947 Mon Sep 17 00:00:00 2001 From: Bin Tang Date: Fri, 2 Jun 2023 11:36:30 +0800 Subject: [PATCH 06/15] config: parse cgroup configuration to expected units Signed-off-by: Bin Tang --- config/config.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/config/config.go b/config/config.go index c7328578d5..20e0e23585 100644 --- a/config/config.go +++ b/config/config.go @@ -16,8 +16,11 @@ import ( "github.com/containerd/nydus-snapshotter/internal/constant" "github.com/containerd/nydus-snapshotter/internal/flags" + "github.com/containerd/nydus-snapshotter/pkg/cgroup" "github.com/containerd/nydus-snapshotter/pkg/errdefs" "github.com/containerd/nydus-snapshotter/pkg/utils/file" + "github.com/containerd/nydus-snapshotter/pkg/utils/parser" + "github.com/containerd/nydus-snapshotter/pkg/utils/sysinfo" ) func init() { @@ -341,3 +344,19 @@ func ParseParameters(args *flags.Args, cfg *SnapshotterConfig) error { return nil } + +func ParseCgroupConfig(config CgroupConfig) (cgroup.Config, error) { + totalMemory, err := sysinfo.GetTotalMemoryBytes() + if err != nil { + return cgroup.Config{}, errors.Wrap(err, "Failed to get total memory bytes") + } + + memoryLimitInBytes, err := parser.MemoryConfigToBytes(config.MemoryLimit, totalMemory) + if err != nil { + return cgroup.Config{}, err + } + + return cgroup.Config{ + MemoryLimitInBytes: memoryLimitInBytes, + }, nil +} From 8d70c00db91e07960bb59a9145f5fbda8a8d008b Mon Sep 17 00:00:00 2001 From: Bin Tang Date: Fri, 2 Jun 2023 11:46:47 +0800 Subject: [PATCH 07/15] config: add test for ParseCgroupConfig Signed-off-by: Bin Tang --- pkg/utils/parser/parser_test.go | 89 +++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 pkg/utils/parser/parser_test.go diff --git a/pkg/utils/parser/parser_test.go b/pkg/utils/parser/parser_test.go new file mode 100644 index 0000000000..15846d8783 --- /dev/null +++ b/pkg/utils/parser/parser_test.go @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2023. Nydus Developers. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package parser + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMemoryLimitToBytes(t *testing.T) { + totalMemoryBytes := 10000 + + for desc, test := range map[string]struct { + MemoryLimit string + expected int64 + }{ + "memory limit is zero": { + MemoryLimit: "", + expected: -1, + }, + "memory limit is a percentage": { + MemoryLimit: "20%", + expected: 2000, + }, + "memory limit is a float percentage": { + MemoryLimit: "0.2%", + expected: 20, + }, + "memory limit is a value without unit": { + MemoryLimit: "10240", + expected: 10240, + }, + "memory limit is a value with Byte unit": { + MemoryLimit: "10240B", + expected: 10240, + }, + "memory limit is a value with KiB unit": { + MemoryLimit: "30KiB", + expected: 30 * 1024, + }, + "memory limit is a value with MiB unit": { + MemoryLimit: "30MiB", + expected: 30 * 1024 * 1024, + }, + "memory limit is a value with GiB unit": { + MemoryLimit: "30GiB", + expected: 30 * 1024 * 1024 * 1024, + }, + "memory limit is a value with TiB unit": { + MemoryLimit: "30TiB", + expected: 30 * 1024 * 1024 * 1024 * 1024, + }, + "memory limit is a value with PiB unit": { + MemoryLimit: "30PiB", + expected: 30 * 1024 * 1024 * 1024 * 1024 * 1024, + }, + "memory limit is a value with Ki unit": { + MemoryLimit: "30Ki", + expected: 30 * 1024, + }, + "memory limit is a value with Mi unit": { + MemoryLimit: "30Mi", + expected: 30 * 1024 * 1024, + }, + "memory limit is a value with Gi unit": { + MemoryLimit: "30Gi", + expected: 30 * 1024 * 1024 * 1024, + }, + "memory limit is a value with Ti unit": { + MemoryLimit: "30Ti", + expected: 30 * 1024 * 1024 * 1024 * 1024, + }, + "memory limit is a value with Pi unit": { + MemoryLimit: "30Pi", + expected: 30 * 1024 * 1024 * 1024 * 1024 * 1024, + }, + } { + t.Logf("TestCase %q", desc) + + memoryLimitInBytes, err := MemoryConfigToBytes(test.MemoryLimit, totalMemoryBytes) + assert.NoError(t, err) + assert.Equal(t, memoryLimitInBytes, test.expected) + } +} From d8d6dbe0e77e6e163dd50fb0338f194625671949 Mon Sep 17 00:00:00 2001 From: Bin Tang Date: Fri, 2 Jun 2023 11:41:43 +0800 Subject: [PATCH 08/15] daemon: add cgroup manager to daemon manager The cgroup manager aims to manager the lifetime of daemon's cgroup and add nydusd to this cgroup for memory resource limitation and reclaim. Signed-off-by: Bin Tang --- pkg/manager/daemon_adaptor.go | 7 +++++++ pkg/manager/manager.go | 12 +++++++++++- snapshot/snapshot.go | 22 ++++++++++++++++++++++ 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/pkg/manager/daemon_adaptor.go b/pkg/manager/daemon_adaptor.go index d4fd7f1369..1b711aee0a 100644 --- a/pkg/manager/daemon_adaptor.go +++ b/pkg/manager/daemon_adaptor.go @@ -95,6 +95,13 @@ func (m *Manager) StartDaemon(d *daemon.Daemon) error { collector.NewDaemonEventCollector(types.DaemonStateRunning).Collect() + if m.CgroupMgr != nil { + if err := m.CgroupMgr.AddProc(d.States.ProcessID); err != nil { + log.L.WithError(err).Errorf("add daemon %s to cgroup failed", d.ID()) + return + } + } + d.Lock() collector.NewDaemonInfoCollector(&d.Version, 1).Collect() d.Unlock() diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go index 246aedcb55..d896734c99 100644 --- a/pkg/manager/manager.go +++ b/pkg/manager/manager.go @@ -20,6 +20,7 @@ import ( "github.com/containerd/nydus-snapshotter/config" "github.com/containerd/nydus-snapshotter/config/daemonconfig" + "github.com/containerd/nydus-snapshotter/pkg/cgroup" "github.com/containerd/nydus-snapshotter/pkg/daemon" "github.com/containerd/nydus-snapshotter/pkg/daemon/types" "github.com/containerd/nydus-snapshotter/pkg/errdefs" @@ -135,6 +136,9 @@ type Manager struct { // A basic configuration template loaded from the file DaemonConfig daemonconfig.DaemonConfig + // Cgroup manager for nydusd + CgroupMgr *cgroup.Manager + // In order to validate daemon fs driver is consistent with the latest snapshotter boot FsDriver string @@ -150,6 +154,7 @@ type Opt struct { // Nydus-snapshotter work directory RootDir string DaemonConfig daemonconfig.DaemonConfig + CgroupMgr *cgroup.Manager // In order to validate daemon fs driver is consistent with the latest snapshotter boot FsDriver string } @@ -278,6 +283,7 @@ func NewManager(opt Opt) (*Manager, error) { RecoverPolicy: opt.RecoverPolicy, SupervisorSet: supervisorSet, DaemonConfig: opt.DaemonConfig, + CgroupMgr: opt.CgroupMgr, FsDriver: opt.FsDriver, } @@ -444,7 +450,6 @@ func (m *Manager) DestroyDaemon(d *daemon.Daemon) error { if err := d.Wait(); err != nil { log.L.Warnf("Failed to wait for daemon, %v", err) } - collector.NewDaemonEventCollector(types.DaemonStateDestroyed).Collect() d.Lock() collector.NewDaemonInfoCollector(&d.Version, -1).Collect() @@ -514,6 +519,11 @@ func (m *Manager) Recover(ctx context.Context) (map[string]*daemon.Daemon, map[s log.L.Infof("found RUNNING daemon %s during reconnecting", d.ID()) liveDaemons[d.ID()] = d + if m.CgroupMgr != nil { + if err := m.CgroupMgr.AddProc(d.States.ProcessID); err != nil { + return errors.Wrapf(err, "add daemon %s to cgroup failed", d.ID()) + } + } d.Lock() collector.NewDaemonInfoCollector(&d.Version, 1).Collect() d.Unlock() diff --git a/snapshot/snapshot.go b/snapshot/snapshot.go index c5e66635e5..74b58f2abd 100644 --- a/snapshot/snapshot.go +++ b/snapshot/snapshot.go @@ -29,6 +29,8 @@ import ( "github.com/containerd/nydus-snapshotter/config/daemonconfig" "github.com/containerd/nydus-snapshotter/pkg/cache" + "github.com/containerd/nydus-snapshotter/pkg/cgroup" + v2 "github.com/containerd/nydus-snapshotter/pkg/cgroup/v2" "github.com/containerd/nydus-snapshotter/pkg/daemon" "github.com/containerd/nydus-snapshotter/pkg/errdefs" "github.com/containerd/nydus-snapshotter/pkg/layout" @@ -82,6 +84,19 @@ func NewSnapshotter(ctx context.Context, cfg *config.SnapshotterConfig) (snapsho return nil, errors.Wrap(err, "parse recover policy") } + cgroupConfig, err := config.ParseCgroupConfig(cfg.CgroupConfig) + if err != nil { + return nil, errors.Wrap(err, "parse cgroup configuration") + } + log.L.Infof("parsed cgroup config: %#v", cgroupConfig) + cgroupMgr, err := cgroup.NewManager(cgroup.Opt{ + Name: "nydusd", + Config: cgroupConfig, + }) + if err != nil && (err != cgroup.ErrCgroupNotSupported || err != v2.ErrRootMemorySubtreeControllerDisabled) { + return nil, errors.Wrap(err, "create cgroup manager") + } + manager, err := mgr.NewManager(mgr.Opt{ NydusdBinaryPath: cfg.DaemonConfig.NydusdPath, Database: db, @@ -90,6 +105,7 @@ func NewSnapshotter(ctx context.Context, cfg *config.SnapshotterConfig) (snapsho RecoverPolicy: rp, FsDriver: config.GetFsDriver(), DaemonConfig: daemonConfig, + CgroupMgr: cgroupMgr, }) if err != nil { return nil, errors.Wrap(err, "create daemons manager") @@ -562,6 +578,12 @@ func (o *snapshotter) Close() error { o.fs.TryStopSharedDaemon() + if o.manager.CgroupMgr != nil { + if err := o.manager.CgroupMgr.Delete(); err != nil { + log.L.Errorf("failed to destroy cgroup, err %v", err) + } + } + return o.ms.Close() } From 18542aae2c324d10964e81047a5350b923d5dcc1 Mon Sep 17 00:00:00 2001 From: Bin Tang Date: Fri, 2 Jun 2023 11:43:12 +0800 Subject: [PATCH 09/15] update go.mod to adapt pkg/cgroup package Signed-off-by: Bin Tang --- go.mod | 13 +++++++++++-- go.sum | 14 ++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index e3852a4a82..4937a5eb80 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/aws/aws-sdk-go-v2/config v1.18.16 github.com/aws/aws-sdk-go-v2/credentials v1.13.16 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.56 + github.com/containerd/cgroups/v3 v3.0.1 github.com/containerd/containerd v1.7.0 github.com/containerd/continuity v0.3.0 github.com/containerd/fifo v1.1.0 @@ -34,6 +35,7 @@ require ( github.com/stretchr/testify v1.8.2 github.com/urfave/cli/v2 v2.25.0 go.etcd.io/bbolt v1.3.7 + golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 golang.org/x/net v0.8.0 golang.org/x/sync v0.1.0 golang.org/x/sys v0.6.0 @@ -47,6 +49,13 @@ require ( github.com/containers/ocicrypt v1.1.7 ) +require ( + github.com/cilium/ebpf v0.9.1 // indirect + github.com/coreos/go-systemd/v22 v22.5.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/godbus/dbus/v5 v5.1.0 // indirect +) + require ( github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20221215162035-5330a85ea652 // indirect github.com/Microsoft/go-winio v0.6.0 // indirect @@ -68,7 +77,7 @@ require ( github.com/aws/smithy-go v1.13.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/containerd/cgroups v1.1.0 // indirect + github.com/containerd/cgroups v1.1.0 github.com/containerd/ttrpc v1.2.1 // indirect github.com/containerd/typeurl/v2 v2.1.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect @@ -112,7 +121,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/runc v1.1.5 // indirect - github.com/opencontainers/runtime-spec v1.1.0-rc.1 // indirect + github.com/opencontainers/runtime-spec v1.1.0-rc.1 github.com/opencontainers/selinux v1.11.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/common v0.42.0 // indirect diff --git a/go.sum b/go.sum index d395b8ab6b..33bd917f39 100644 --- a/go.sum +++ b/go.sum @@ -62,12 +62,16 @@ github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= +github.com/cilium/ebpf v0.9.1 h1:64sn2K3UKw8NbP/blsixRpF3nXuyhz/VjRlRzvlBRu4= +github.com/cilium/ebpf v0.9.1/go.mod h1:+OhNOIXx/Fnu1IE8bJz2dzOA+VSfyTfdNUVdlQnxUFY= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= +github.com/containerd/cgroups/v3 v3.0.1 h1:4hfGvu8rfGIwVIDd+nLzn/B9ZXx4BcCjzt5ToenJRaE= +github.com/containerd/cgroups/v3 v3.0.1/go.mod h1:/vtwk1VXrtoa5AaZLkypuOJgA/6DyPMZHJPGQNtlHnw= github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= github.com/containerd/containerd v1.7.0 h1:G/ZQr3gMZs6ZT0qPUZ15znx5QSdQdASW11nXTLTM2Pg= github.com/containerd/containerd v1.7.0/go.mod h1:QfR7Efgb/6X2BDpTPJRvPTYDE9rsF0FsXX9J8sIs/sc= @@ -88,6 +92,8 @@ github.com/containerd/typeurl/v2 v2.1.0/go.mod h1:IDp2JFvbwZ31H8dQbEIY7sDl2L3o3H github.com/containers/ocicrypt v1.1.7 h1:thhNr4fu2ltyGz8aMx8u48Ae0Pnbip3ePP9/mzkZ/3U= github.com/containers/ocicrypt v1.1.7/go.mod h1:7CAhjcj2H8AYp5YvEie7oVSK2AhBY8NscCYRawuDNtw= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= @@ -108,6 +114,8 @@ github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNk github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8= github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/emicklei/go-restful/v3 v3.10.2 h1:hIovbnmBTLjHXkqEBUz3HGpXZdM7ZrE9fJIZIqlJLqE= github.com/emicklei/go-restful/v3 v3.10.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= @@ -122,6 +130,7 @@ github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= +github.com/frankban/quicktest v1.14.0 h1:+cqqvzZV87b4adx/5ayVOaYZ2CrvM4ejQvUdBzPPUss= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -137,6 +146,8 @@ github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/ github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= +github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -343,6 +354,7 @@ go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188Wl go.opentelemetry.io/otel/trace v1.14.0 h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyKcFq/M= go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -350,6 +362,8 @@ golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc= +golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= From c6e22ec9166b7cea3e8ec4e31676f66787908822 Mon Sep 17 00:00:00 2001 From: Bin Tang Date: Fri, 2 Jun 2023 11:44:39 +0800 Subject: [PATCH 10/15] misc: add cgroup configuration for daemon Signed-off-by: Bin Tang --- config/config_test.go | 3 +++ misc/snapshotter/config.toml | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/config/config_test.go b/config/config_test.go index 2cb13b00b4..b8fd3d10ec 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -83,6 +83,9 @@ func TestLoadSnapshotterTOMLConfig(t *testing.T) { MetricsConfig: MetricsConfig{ Address: ":9110", }, + CgroupConfig: CgroupConfig{ + MemoryLimit: "", + }, } A.EqualValues(cfg, &exampleConfig) diff --git a/misc/snapshotter/config.toml b/misc/snapshotter/config.toml index ecc8ea6b42..2e43fa8286 100644 --- a/misc/snapshotter/config.toml +++ b/misc/snapshotter/config.toml @@ -33,6 +33,12 @@ recover_policy = "restart" # Setting to 0 will use the default configuration of nydusd. threads_number = 4 +[cgroup] +# The memory limit for nydusd cgroup, which contains all nydusd processes. +# Percentage is supported as well, please ensure it is end with "%". +# The default unit is bytes. Acceptable values include "209715200", "200MiB", "200Mi" and "10%". +memory_limit = "" + [log] # Print logs to stdout rather than logging files log_to_stdout = false From 59641087febc0f589a37c6595f54c538b65089dd Mon Sep 17 00:00:00 2001 From: Bin Tang Date: Mon, 5 Jun 2023 14:38:14 +0800 Subject: [PATCH 11/15] workflow: add iontegration test for cgroup v2 environment Signed-off-by: Bin Tang --- .github/workflows/e2e.yml | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 89d3560dbd..7031d4a23f 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -19,8 +19,37 @@ env: REGISTRY: ghcr.io jobs: - run-e2e: - runs-on: ubuntu-latest + run-e2e-for-cgroups-v1: + runs-on: ubuntu-20.04 + steps: + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: "1.19.6" + - name: Checkout repository + uses: actions/checkout@v3 + - name: cache go mod + uses: actions/cache@v3 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('go.sum') }} + restore-keys: | + ${{ runner.os }}-go + - name: Log in to container registry + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Run e2e test + run: | + TAG=$GITHUB_REF_NAME + [ "$TAG" == "main" ] && TAG="latest" + [ "$GITHUB_EVENT_NAME" == "pull_request" ] && TAG="local" + make integration + + run-e2e-for-cgroups-v2: + runs-on: ubuntu-22.04 steps: - name: Set up Go uses: actions/setup-go@v3 From 51615c4c2531253265dd25872435ff02fd83c942 Mon Sep 17 00:00:00 2001 From: Bin Tang Date: Mon, 5 Jun 2023 19:40:26 +0800 Subject: [PATCH 12/15] makefile(integration): enable cgroup capability for container Signed-off-by: Bin Tang --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a2c5b8952b..32313a4355 100644 --- a/Makefile +++ b/Makefile @@ -130,7 +130,7 @@ smoke: integration: CGO_ENABLED=1 ${PROXY} GOOS=${GOOS} GOARCH=${GOARCH} go build -ldflags '-X "${PKG}/version.Version=${VERSION}" -extldflags "-static"' -race -v -o bin/containerd-nydus-grpc ./cmd/containerd-nydus-grpc $(SUDO) DOCKER_BUILDKIT=1 docker build ${BUILD_ARG_E2E_DOWNLOADS_MIRROR} -t nydus-snapshotter-e2e:0.1 -f integration/Dockerfile . - $(SUDO) docker run --name nydus-snapshotter_e2e --rm --privileged -v /root/.docker:/root/.docker -v `go env GOMODCACHE`:/go/pkg/mod \ + $(SUDO) docker run --cap-add SYS_ADMIN --security-opt seccomp=unconfined --cgroup-parent=system.slice --cgroupns private --name nydus-snapshotter_e2e --rm --privileged -v /root/.docker:/root/.docker -v `go env GOMODCACHE`:/go/pkg/mod \ -v `go env GOCACHE`:/root/.cache/go-build -v `pwd`:/nydus-snapshotter \ -v /usr/src/linux-headers-${KERNEL_VER}:/usr/src/linux-headers-${KERNEL_VER} \ ${ENV_TARGET_IMAGES_FILE} \ From a85f4fd52f5475d190ffc38ea4e225855e3e7111 Mon Sep 17 00:00:00 2001 From: Bin Tang Date: Mon, 5 Jun 2023 20:13:38 +0800 Subject: [PATCH 13/15] integration: enable nesting for cgroup v2 refer to https://github.com/moby/moby/issues/43093 Signed-off-by: Bin Tang --- integration/entrypoint.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/integration/entrypoint.sh b/integration/entrypoint.sh index 94e4bf180e..5e49adc468 100755 --- a/integration/entrypoint.sh +++ b/integration/entrypoint.sh @@ -563,6 +563,18 @@ function kill_multiple_nydusd_recover_failover { detect_go_race } +# Refer to https://github.com/moby/moby/blob/088afc99e4bf8adb78e29733396182417d67ada2/hack/dind#L28-L38 +function enable_nesting_for_cgroup_v2() { + if [ -f /sys/fs/cgroup/cgroup.controllers ]; then + mkdir -p /sys/fs/cgroup/init + xargs -rn1 < /sys/fs/cgroup/cgroup.procs > /sys/fs/cgroup/init/cgroup.procs || : + sed -e 's/ / +/g' -e 's/^/-/' < /sys/fs/cgroup/cgroup.controllers \ + > /sys/fs/cgroup/cgroup.subtree_control + fi +} + +enable_nesting_for_cgroup_v2 + reboot_containerd multiple start_single_container_multiple_daemons From a5c6821b175901b69c9b8e27f9de958472c0f150 Mon Sep 17 00:00:00 2001 From: Bin Tang Date: Thu, 15 Jun 2023 20:10:03 +0800 Subject: [PATCH 14/15] config: add enable entry for nydusd cgroup Signed-off-by: Bin Tang --- config/config.go | 1 + snapshot/snapshot.go | 26 +++++++++++++++----------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/config/config.go b/config/config.go index 20e0e23585..c407518bdb 100644 --- a/config/config.go +++ b/config/config.go @@ -110,6 +110,7 @@ type Experimental struct { } type CgroupConfig struct { + Enable bool `toml:"enable"` MemoryLimit string `toml:"memory_limit"` } diff --git a/snapshot/snapshot.go b/snapshot/snapshot.go index 74b58f2abd..d6e0661329 100644 --- a/snapshot/snapshot.go +++ b/snapshot/snapshot.go @@ -84,17 +84,21 @@ func NewSnapshotter(ctx context.Context, cfg *config.SnapshotterConfig) (snapsho return nil, errors.Wrap(err, "parse recover policy") } - cgroupConfig, err := config.ParseCgroupConfig(cfg.CgroupConfig) - if err != nil { - return nil, errors.Wrap(err, "parse cgroup configuration") - } - log.L.Infof("parsed cgroup config: %#v", cgroupConfig) - cgroupMgr, err := cgroup.NewManager(cgroup.Opt{ - Name: "nydusd", - Config: cgroupConfig, - }) - if err != nil && (err != cgroup.ErrCgroupNotSupported || err != v2.ErrRootMemorySubtreeControllerDisabled) { - return nil, errors.Wrap(err, "create cgroup manager") + var cgroupMgr *cgroup.Manager + if cfg.CgroupConfig.Enable { + cgroupConfig, err := config.ParseCgroupConfig(cfg.CgroupConfig) + if err != nil { + return nil, errors.Wrap(err, "parse cgroup configuration") + } + log.L.Infof("parsed cgroup config: %#v", cgroupConfig) + + cgroupMgr, err = cgroup.NewManager(cgroup.Opt{ + Name: "nydusd", + Config: cgroupConfig, + }) + if err != nil && (err != cgroup.ErrCgroupNotSupported || err != v2.ErrRootMemorySubtreeControllerDisabled) { + return nil, errors.Wrap(err, "create cgroup manager") + } } manager, err := mgr.NewManager(mgr.Opt{ From 5d9bdba059bdd2d2262ebeededbd0a6a288f4f38 Mon Sep 17 00:00:00 2001 From: Bin Tang Date: Thu, 15 Jun 2023 20:15:53 +0800 Subject: [PATCH 15/15] misc: add enable entry for nydusd cgroup Signed-off-by: Bin Tang --- config/config_test.go | 1 + misc/snapshotter/config.toml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/config/config_test.go b/config/config_test.go index b8fd3d10ec..3dc05c9d89 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -84,6 +84,7 @@ func TestLoadSnapshotterTOMLConfig(t *testing.T) { Address: ":9110", }, CgroupConfig: CgroupConfig{ + Enable: true, MemoryLimit: "", }, } diff --git a/misc/snapshotter/config.toml b/misc/snapshotter/config.toml index 2e43fa8286..3ad3c6f43d 100644 --- a/misc/snapshotter/config.toml +++ b/misc/snapshotter/config.toml @@ -34,6 +34,8 @@ recover_policy = "restart" threads_number = 4 [cgroup] +# Whether to use separate cgroup for nydusd. +enable = true # The memory limit for nydusd cgroup, which contains all nydusd processes. # Percentage is supported as well, please ensure it is end with "%". # The default unit is bytes. Acceptable values include "209715200", "200MiB", "200Mi" and "10%".