Skip to content

Commit

Permalink
daemon: add nydusd process to specific cgroup
Browse files Browse the repository at this point in the history
Signed-off-by: Bin Tang <tangbin.bin@bytedance.com>
  • Loading branch information
sctb512 committed Jun 1, 2023
1 parent 2c1de89 commit caab0fb
Show file tree
Hide file tree
Showing 16 changed files with 556 additions and 11 deletions.
88 changes: 82 additions & 6 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,15 +9,19 @@ package config

import (
"os"
"regexp"
"strconv"

"github.com/imdario/mergo"
"github.com/pelletier/go-toml"
"github.com/pkg/errors"

"github.com/containerd/nydus-snapshotter/internal/constant"
"github.com/containerd/nydus-snapshotter/internal/flags"
"github.com/containerd/nydus-snapshotter/pkg/cgroup"
"github.com/containerd/nydus-snapshotter/pkg/errdefs"
"github.com/containerd/nydus-snapshotter/pkg/utils/file"
"github.com/containerd/nydus-snapshotter/pkg/utils/sysinfo"
)

func init() {
Expand Down Expand Up @@ -46,6 +50,20 @@ const (
DaemonModeInvalid DaemonMode = DaemonMode(constant.DaemonModeInvalid)
)

var unitMultipliers = InitUnitMultipliers()

func InitUnitMultipliers() map[string]int64 {
unitMultipliers := make(map[string]int64, 5)

unitMultipliers["KiB"] = 1024
unitMultipliers["MiB"] = unitMultipliers["KiB"] * 1024
unitMultipliers["GiB"] = unitMultipliers["MiB"] * 1024
unitMultipliers["TiB"] = unitMultipliers["GiB"] * 1024
unitMultipliers["PiB"] = unitMultipliers["TiB"] * 1024

return unitMultipliers
}

func parseDaemonMode(m string) (DaemonMode, error) {
switch m {
case string(DaemonModeMultiple):
Expand Down Expand Up @@ -106,14 +124,20 @@ type Experimental struct {
EnableReferrerDetect bool `toml:"enable_referrer_detect"`
}

type CgroupConfig struct {
MemoryLimit string `toml:"memory_limit"`
MemoryWatermarkScaleFactor int64 `toml:"memory_watermark_scale_factor"`
}

// Configure how to start and recover nydusd daemons
type DaemonConfig struct {
NydusdPath string `toml:"nydusd_path"`
NydusdConfigPath string `toml:"nydusd_config"`
NydusImagePath string `toml:"nydusimage_path"`
RecoverPolicy string `toml:"recover_policy"`
FsDriver string `toml:"fs_driver"`
ThreadsNumber int `toml:"threads_number"`
NydusdPath string `toml:"nydusd_path"`
NydusdConfigPath string `toml:"nydusd_config"`
NydusImagePath string `toml:"nydusimage_path"`
RecoverPolicy string `toml:"recover_policy"`
FsDriver string `toml:"fs_driver"`
ThreadsNumber int `toml:"threads_number"`
CgroupConfig CgroupConfig `toml:"cgroup"`
}

type LoggingConfig struct {
Expand Down Expand Up @@ -336,3 +360,55 @@ func ParseParameters(args *flags.Args, cfg *SnapshotterConfig) error {

return nil
}

func memoryConfigToBytes(s string, totalMemoryBytes int) (int64, error) {
if s == "" {
return 0, nil
}

value, err := strconv.ParseFloat(s, 64)
if err == nil {
return int64(value), nil
}

re := regexp.MustCompile(`(\d+)([a-zA-Z\%]+)`)
matches := re.FindStringSubmatch(s)

if len(matches) == 3 {
valueString, unit := matches[1], matches[2]
value, err := strconv.ParseFloat(valueString, 64)
if err != nil {
return 0, errors.Wrap(err, "Failed to parse memory limit")
}
if unit == "B" {
return int64(value), nil
}

if unit == "%%" {
limitMemory := float64(totalMemoryBytes) * value / 100
return int64(limitMemory + 0.5), nil
}

multiplier := unitMultipliers[unit]
return int64(value * float64(multiplier)), nil
}

return 0, errors.Errorf("Falied to convert memory limit:Unknown unit in %s", s)
}

func ParseCgroupConfig(config CgroupConfig) (cgroup.Config, error) {
totalMemory, err := sysinfo.GetTotalMemoryBytes()
if err != nil {
return cgroup.Config{}, errors.Wrap(err, "Failed to get total memory bytes")
}

memoryLimitInBytes, err := memoryConfigToBytes(config.MemoryLimit, totalMemory)
if err != nil {
return cgroup.Config{}, err
}

return cgroup.Config{
MemoryLimitInBytes: memoryLimitInBytes,
MemoryWatermarkScaleFactor: config.MemoryWatermarkScaleFactor,
}, nil
}
56 changes: 56 additions & 0 deletions config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,10 @@ func TestLoadSnapshotterTOMLConfig(t *testing.T) {
RecoverPolicy: "restart",
NydusdConfigPath: "/etc/nydus/nydusd-config.fusedev.json",
ThreadsNumber: 4,
CgroupConfig: CgroupConfig{
MemoryLimit: "",
MemoryWatermarkScaleFactor: 1000,
},
},
SnapshotsConfig: SnapshotConfig{
EnableNydusOverlayFS: false,
Expand Down Expand Up @@ -248,3 +252,55 @@ func TestProcessConfigurations(t *testing.T) {
err = ProcessConfigurations(&snapshotterConfig3)
A.NoError(err)
}

func TestMemoryLimitToBytes(t *testing.T) {
totalMemoryBytes := 10000

for desc, test := range map[string]struct {
MemoryLimit string
expected int64
}{
"memory limit is zero": {
MemoryLimit: "",
expected: 0,
},
"memory limit is a percentage": {
MemoryLimit: "20%%",
expected: 2000,
},
"memory limit is a value without unit": {
MemoryLimit: "10240",
expected: 10240,
},
"memory limit is a value with Byte unit": {
MemoryLimit: "10240B",
expected: 10240,
},
"memory limit is a value with KiB unit": {
MemoryLimit: "30KiB",
expected: 30 * 1024,
},
"memory limit is a value with MiB unit": {
MemoryLimit: "30MiB",
expected: 30 * 1024 * 1024,
},
"memory limit is a value with GiB unit": {
MemoryLimit: "30GiB",
expected: 30 * 1024 * 1024 * 1024,
},
"memory limit is a value with TiB unit": {
MemoryLimit: "30TiB",
expected: 30 * 1024 * 1024 * 1024 * 1024,
},
"memory limit is a value with PiB unit": {
MemoryLimit: "30PiB",
expected: 30 * 1024 * 1024 * 1024 * 1024 * 1024,
},
} {
t.Logf("TestCase %q", desc)

memoryLimitInBytes, err := memoryConfigToBytes(test.MemoryLimit, totalMemoryBytes)
assert.NoError(t, err)
assert.Equal(t, memoryLimitInBytes, test.expected)
}
}
12 changes: 10 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ require (
github.com/aws/aws-sdk-go-v2/config v1.18.16
github.com/aws/aws-sdk-go-v2/credentials v1.13.16
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.56
github.com/containerd/cgroups/v3 v3.0.1
github.com/containerd/containerd v1.7.0
github.com/containerd/continuity v0.3.0
github.com/containerd/fifo v1.1.0
Expand Down Expand Up @@ -46,6 +47,13 @@ require (
k8s.io/cri-api v0.27.0-alpha.3
)

require (
github.com/cilium/ebpf v0.9.1 // indirect
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/godbus/dbus/v5 v5.1.0 // indirect
)

require (
github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20221215162035-5330a85ea652 // indirect
github.com/Microsoft/go-winio v0.6.0 // indirect
Expand All @@ -67,7 +75,7 @@ require (
github.com/aws/smithy-go v1.13.5 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/containerd/cgroups v1.1.0 // indirect
github.com/containerd/cgroups v1.1.0
github.com/containerd/ttrpc v1.2.1 // indirect
github.com/containerd/typeurl/v2 v2.1.0 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
Expand Down Expand Up @@ -111,7 +119,7 @@ require (
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/opencontainers/runc v1.1.5 // indirect
github.com/opencontainers/runtime-spec v1.1.0-rc.1 // indirect
github.com/opencontainers/runtime-spec v1.1.0-rc.1
github.com/opencontainers/selinux v1.11.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/common v0.42.0 // indirect
Expand Down
12 changes: 12 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -62,12 +62,16 @@ github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E=
github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA=
github.com/cilium/ebpf v0.9.1 h1:64sn2K3UKw8NbP/blsixRpF3nXuyhz/VjRlRzvlBRu4=
github.com/cilium/ebpf v0.9.1/go.mod h1:+OhNOIXx/Fnu1IE8bJz2dzOA+VSfyTfdNUVdlQnxUFY=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM=
github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw=
github.com/containerd/cgroups/v3 v3.0.1 h1:4hfGvu8rfGIwVIDd+nLzn/B9ZXx4BcCjzt5ToenJRaE=
github.com/containerd/cgroups/v3 v3.0.1/go.mod h1:/vtwk1VXrtoa5AaZLkypuOJgA/6DyPMZHJPGQNtlHnw=
github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U=
github.com/containerd/containerd v1.7.0 h1:G/ZQr3gMZs6ZT0qPUZ15znx5QSdQdASW11nXTLTM2Pg=
github.com/containerd/containerd v1.7.0/go.mod h1:QfR7Efgb/6X2BDpTPJRvPTYDE9rsF0FsXX9J8sIs/sc=
Expand All @@ -86,6 +90,8 @@ github.com/containerd/ttrpc v1.2.1/go.mod h1:sIT6l32Ph/H9cvnJsfXM5drIVzTr5A2flTf
github.com/containerd/typeurl/v2 v2.1.0 h1:yNAhJvbNEANt7ck48IlEGOxP7YAp6LLpGn5jZACDNIE=
github.com/containerd/typeurl/v2 v2.1.0/go.mod h1:IDp2JFvbwZ31H8dQbEIY7sDl2L3o3HZj1hsSQlywkQ0=
github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
Expand All @@ -106,6 +112,8 @@ github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNk
github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8=
github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA=
github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
github.com/emicklei/go-restful/v3 v3.10.2 h1:hIovbnmBTLjHXkqEBUz3HGpXZdM7ZrE9fJIZIqlJLqE=
github.com/emicklei/go-restful/v3 v3.10.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
Expand All @@ -120,6 +128,7 @@ github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w=
github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg=
github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0=
github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
github.com/frankban/quicktest v1.14.0 h1:+cqqvzZV87b4adx/5ayVOaYZ2CrvM4ejQvUdBzPPUss=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
Expand All @@ -135,6 +144,8 @@ github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/
github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk=
github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
Expand Down Expand Up @@ -333,6 +344,7 @@ go.opentelemetry.io/otel v1.14.0/go.mod h1:o4buv+dJzx8rohcUeRmWUZhqupFvzWis188Wl
go.opentelemetry.io/otel/trace v1.14.0 h1:wp2Mmvj41tDsyAJXiWDWpfNsOiIyd38fy85pyKcFq/M=
go.opentelemetry.io/otel/trace v1.14.0/go.mod h1:8avnQLK+CG77yNLUae4ea2JDQ6iT+gozhnZjy/rw9G8=
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
Expand Down
8 changes: 8 additions & 0 deletions misc/snapshotter/config.toml
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,14 @@ recover_policy = "restart"
# Setting to 0 will use the default configuration of nydusd.
threads_number = 4

[daemon.cgroup]
# The memory limit for nydusd.slice, which contains all nydusd processes.
# Percentage is supported as well, please ensure it is end with '%'.
memory_limit = ""
# The memory watermark scale factor for nydusd.slice. This entry is invalid
# when the per-memcg reclaim is not supported.
memory_watermark_scale_factor = 1000

[log]
# Print logs to stdout rather than logging files
log_to_stdout = false
Expand Down
1 change: 1 addition & 0 deletions misc/snapshotter/nydus-snapshotter.fusedev.service
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ KillMode=process
OOMScoreAdjust=-999
StandardOutput=journal
StandardError=journal
MemoryMax=100M

[Install]
WantedBy=multi-user.target
41 changes: 41 additions & 0 deletions pkg/cgroup/cgroup.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
/*
* Copyright (c) 2023. Nydus Developers. All rights reserved.
*
* SPDX-License-Identifier: Apache-2.0
*/

package cgroup

import (
"github.com/containerd/cgroups"
v1 "github.com/containerd/nydus-snapshotter/pkg/cgroup/v1"
v2 "github.com/containerd/nydus-snapshotter/pkg/cgroup/v2"
)

const (
defaultSlice = "system.slice"
)

type Config struct {
MemoryLimitInBytes int64
MemoryWatermarkScaleFactor int64
}

type DaemonCgroup interface {
// Delete the current cgroup.
Delete() error
// Add a process to current cgroup.
AddProc(pid int) error
}

func createCgroup(name string, config Config) (DaemonCgroup, error) {
if cgroups.Mode() == cgroups.Unified {
return v2.NewCgroup(defaultSlice, name, config.MemoryLimitInBytes, config.MemoryWatermarkScaleFactor)
}

return v1.NewCgroup(defaultSlice, name, config.MemoryLimitInBytes, config.MemoryWatermarkScaleFactor)
}

func Supported() bool {
return cgroups.Mode() != cgroups.Unavailable
}
41 changes: 41 additions & 0 deletions pkg/cgroup/manager.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
/*
* Copyright (c) 2023. Nydus Developers. All rights reserved.
*
* SPDX-License-Identifier: Apache-2.0
*/

package cgroup

type Manager struct {
name string
config Config
cgroup DaemonCgroup
}

type Opt struct {
Name string
Config Config
}

func NewManager(opt Opt) (*Manager, error) {
cg, err := createCgroup(opt.Name, opt.Config)
if err != nil {
return nil, err
}

return &Manager{
name: opt.Name,
config: opt.Config,
cgroup: cg,
}, nil
}

// Please make sure the *Manager is not null.
func (m *Manager) AddProc(pid int) error {
return m.cgroup.AddProc(pid)
}

// Please make sure the *Manager is not null.
func (m *Manager) Delete() error {
return m.cgroup.Delete()
}
Loading

0 comments on commit caab0fb

Please sign in to comment.