diff --git a/misc/snapshotter/nydus-snapshotter.fusedev.service b/misc/snapshotter/nydus-snapshotter.fusedev.service index 3abdb55598..62cc98737c 100644 --- a/misc/snapshotter/nydus-snapshotter.fusedev.service +++ b/misc/snapshotter/nydus-snapshotter.fusedev.service @@ -10,9 +10,10 @@ ExecStart=/usr/local/bin/containerd-nydus-grpc --config /etc/nydus/config.toml Restart=always RestartSec=1 KillMode=process -OOMScoreAdjust=-999 +OOMScoreAdjust=-1000 StandardOutput=journal StandardError=journal +MemoryMax=2G [Install] WantedBy=multi-user.target diff --git a/pkg/manager/daemon_adaptor.go b/pkg/manager/daemon_adaptor.go index 63882e1d42..0064d5b88f 100644 --- a/pkg/manager/daemon_adaptor.go +++ b/pkg/manager/daemon_adaptor.go @@ -23,6 +23,7 @@ import ( "github.com/containerd/nydus-snapshotter/pkg/errdefs" "github.com/containerd/nydus-snapshotter/pkg/metrics/collector" metrics "github.com/containerd/nydus-snapshotter/pkg/metrics/tool" + "github.com/containerd/nydus-snapshotter/pkg/utils/oom" ) // Fork the nydusd daemon with the process PID decided @@ -40,6 +41,10 @@ func (m *Manager) StartDaemon(d *daemon.Daemon) error { defer d.Unlock() d.States.ProcessID = cmd.Process.Pid + err = oom.ChangeDaemonOOMScoreAdj(d.States.ProcessID, -999) + if err != nil { + return errors.Wrapf(err, "change oom score adj for %d", cmd.Process.Pid) + } // Profile nydusd daemon CPU usage during its startup. if config.GetDaemonProfileCPUDuration() > 0 { diff --git a/pkg/system/system.go b/pkg/system/system.go index 03cdd219cd..5fbfbe705f 100644 --- a/pkg/system/system.go +++ b/pkg/system/system.go @@ -29,6 +29,7 @@ import ( "github.com/containerd/nydus-snapshotter/pkg/filesystem" "github.com/containerd/nydus-snapshotter/pkg/manager" metrics "github.com/containerd/nydus-snapshotter/pkg/metrics/tool" + "github.com/containerd/nydus-snapshotter/pkg/utils/oom" ) const ( @@ -326,6 +327,10 @@ func (sc *Controller) upgradeNydusDaemon(d *daemon.Daemon, c upgradeRequest) err if err := cmd.Start(); err != nil { return errors.Wrap(err, "start process") } + err = oom.ChangeDaemonOOMScoreAdj(cmd.Process.Pid, -999) + if err != nil { + return errors.Wrapf(err, "change oom score adj for %d", cmd.Process.Pid) + } if err := new.WaitUntilState(types.DaemonStateInit); err != nil { return errors.Wrap(err, "wait until init state") diff --git a/pkg/utils/oom/oom.go b/pkg/utils/oom/oom.go new file mode 100644 index 0000000000..abf9263caf --- /dev/null +++ b/pkg/utils/oom/oom.go @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2023. Nydus Developers. All rights reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package oom + +import ( + "fmt" + "os" + "strconv" + "strings" + + "github.com/pkg/errors" +) + +var ( + OOMScoreAdjMin = -1000 + OOMScoreAdjMax = 1000 +) + +func ReadOOMScoreAdj(path string) (int, error) { + oomBuf, err := os.ReadFile(path) + if err != nil { + return 0, errors.Wrapf(err, "read file %s", path) + } + + oom, err := strconv.Atoi(strings.ReplaceAll(string(oomBuf), "\n", "")) + if err != nil { + return 0, errors.Wrapf(err, "convert %s to integer", string(oomBuf)) + } + return oom, nil +} + +func WriteOOMScoreAdj(path string, oomScoreAdj int) error { + return os.WriteFile(path, []byte(fmt.Sprint(oomScoreAdj)), 0644) +} + +// Change the oom_score_adj of target process if the oom_score_adj of +// current process is equal to OOM_SCORE_ADJ_MIN. +// If the target's oom_score_adj is already greater than OOM_SCORE_ADJ_MIN, skip it. +func ChangeDaemonOOMScoreAdj(pid, scodeAdj int) error { + currentOOMPath := "/proc/self/oom_score_adj" + currentOOM, err := ReadOOMScoreAdj(currentOOMPath) + if err != nil { + return errors.Wrapf(err, "read oom_score_adj file %s", currentOOMPath) + } + if currentOOM > OOMScoreAdjMin { + return nil + } + + daemonOOMPath := fmt.Sprintf("/proc/%d/oom_score_adj", pid) + daemonOOM, err := ReadOOMScoreAdj(currentOOMPath) + if err != nil { + return errors.Wrapf(err, "read oom_score_adj file %s", daemonOOMPath) + } + if daemonOOM > OOMScoreAdjMin { + return nil + } + + return WriteOOMScoreAdj(daemonOOMPath, scodeAdj) +}