From 7c4a4c4fe7beadb01cd892529b17c77915c6afd8 Mon Sep 17 00:00:00 2001 From: Joseph Gooch Date: Wed, 24 Jun 2020 18:42:03 +0000 Subject: [PATCH] Implement SD-NOTIFY proxy in conmon This leverages conmon's ability to proxy the SD-NOTIFY socket. This prevents locking caused by OCI runtime blocking, waiting for SD-NOTIFY messages, and instead passes the messages directly up to the host. Signed-off-by: Joseph Gooch --- libpod/container_internal_linux.go | 24 ++++++++++++++++++++++++ libpod/diff.go | 1 + libpod/oci_conmon_exec_linux.go | 2 +- libpod/oci_conmon_linux.go | 20 ++++++++------------ 4 files changed, 34 insertions(+), 13 deletions(-) diff --git a/libpod/container_internal_linux.go b/libpod/container_internal_linux.go index a1b4334fb8..8ef8f469a0 100644 --- a/libpod/container_internal_linux.go +++ b/libpod/container_internal_linux.go @@ -349,6 +349,7 @@ func (c *Container) generateSpec(ctx context.Context) (*spec.Spec, error) { newMount.Options = append(newMount.Options, "ro", "nosuid", "noexec", "nodev") } if !MountExists(g.Mounts(), dstPath) { + logrus.Warnf("Mounting %q to %q", srcPath, dstPath) g.AddMount(newMount) } else { logrus.Warnf("User mount overriding libpod mount at %q", dstPath) @@ -544,6 +545,11 @@ func (c *Container) generateSpec(ctx context.Context) (*spec.Spec, error) { g.AddProcessEnv("container", "libpod") } + // Set notify + if c.config.SdNotifyMode == define.SdNotifyModeContainer { + g.AddProcessEnv("NOTIFY_SOCKET", "/run/notify/notify.sock") + } + cgroupPath, err := c.getOCICgroupPath() if err != nil { return nil, err @@ -1384,6 +1390,24 @@ func (c *Container) makeBindMounts() error { } } + // Bind notify proxy + if c.config.SdNotifyMode == define.SdNotifyModeContainer { + notifyDir := filepath.Join(c.bundlePath(), "notify") + logrus.Debugf("checking notify %q dir", notifyDir) + if err := os.MkdirAll(notifyDir, 0755); err != nil { + if !os.IsExist(err) { + return errors.Wrapf(err, "unable to create notify %q dir", notifyDir) + } + } + if err := label.Relabel(notifyDir, c.MountLabel(), true); err != nil { + return errors.Wrapf(err, "relabel failed %q", notifyDir) + } + logrus.Debugf("add bindmount notify %q dir", notifyDir) + if _, ok := c.state.BindMounts["/run/notify"]; !ok { + c.state.BindMounts["/run/notify"] = notifyDir + } + } + return nil } diff --git a/libpod/diff.go b/libpod/diff.go index 43f4d2e967..47640a912d 100644 --- a/libpod/diff.go +++ b/libpod/diff.go @@ -15,6 +15,7 @@ var containerMounts = map[string]bool{ "/etc/resolv.conf": true, "/proc": true, "/run": true, + "/run/notify": true, "/run/.containerenv": true, "/run/secrets": true, "/sys": true, diff --git a/libpod/oci_conmon_exec_linux.go b/libpod/oci_conmon_exec_linux.go index 8651c1dc55..e05dfd9f37 100644 --- a/libpod/oci_conmon_exec_linux.go +++ b/libpod/oci_conmon_exec_linux.go @@ -444,7 +444,7 @@ func (r *ConmonOCIRuntime) startExec(c *Container, sessionID string, options *Ex // } // } - conmonEnv, extraFiles, err := r.configureConmonEnv(c, runtimeDir) + conmonEnv, extraFiles, err := r.configureConmonEnv(runtimeDir) if err != nil { return nil, nil, err } diff --git a/libpod/oci_conmon_linux.go b/libpod/oci_conmon_linux.go index 94630e57bf..1fab6e62cf 100644 --- a/libpod/oci_conmon_linux.go +++ b/libpod/oci_conmon_linux.go @@ -364,11 +364,6 @@ func (r *ConmonOCIRuntime) StartContainer(ctr *Container) error { return err } env := []string{fmt.Sprintf("XDG_RUNTIME_DIR=%s", runtimeDir)} - if ctr.config.SdNotifyMode == define.SdNotifyModeContainer { - if notify, ok := os.LookupEnv("NOTIFY_SOCKET"); ok { - env = append(env, fmt.Sprintf("NOTIFY_SOCKET=%s", notify)) - } - } if path, ok := os.LookupEnv("PATH"); ok { env = append(env, fmt.Sprintf("PATH=%s", path)) } @@ -1065,7 +1060,7 @@ func (r *ConmonOCIRuntime) createOCIContainer(ctr *Container, restoreOptions *Co } // 0, 1 and 2 are stdin, stdout and stderr - conmonEnv, envFiles, err := r.configureConmonEnv(ctr, runtimeDir) + conmonEnv, envFiles, err := r.configureConmonEnv(runtimeDir) if err != nil { return err } @@ -1268,7 +1263,7 @@ func prepareProcessExec(c *Container, cmd, env []string, tty bool, cwd, user, se // configureConmonEnv gets the environment values to add to conmon's exec struct // TODO this may want to be less hardcoded/more configurable in the future -func (r *ConmonOCIRuntime) configureConmonEnv(ctr *Container, runtimeDir string) ([]string, []*os.File, error) { +func (r *ConmonOCIRuntime) configureConmonEnv(runtimeDir string) ([]string, []*os.File, error) { env := make([]string, 0, 6) env = append(env, fmt.Sprintf("XDG_RUNTIME_DIR=%s", runtimeDir)) env = append(env, fmt.Sprintf("_CONTAINERS_USERNS_CONFIGURED=%s", os.Getenv("_CONTAINERS_USERNS_CONFIGURED"))) @@ -1280,11 +1275,6 @@ func (r *ConmonOCIRuntime) configureConmonEnv(ctr *Container, runtimeDir string) env = append(env, fmt.Sprintf("HOME=%s", home)) extraFiles := make([]*os.File, 0) - if ctr.config.SdNotifyMode == define.SdNotifyModeContainer { - if notify, ok := os.LookupEnv("NOTIFY_SOCKET"); ok { - env = append(env, fmt.Sprintf("NOTIFY_SOCKET=%s", notify)) - } - } if !r.sdNotify { if listenfds, ok := os.LookupEnv("LISTEN_FDS"); ok { env = append(env, fmt.Sprintf("LISTEN_FDS=%s", listenfds), "LISTEN_PID=1") @@ -1319,6 +1309,12 @@ func (r *ConmonOCIRuntime) sharedConmonArgs(ctr *Container, cuuid, bundlePath, p args = append(args, rFlags...) } + if ctr.config.SdNotifyMode == define.SdNotifyModeContainer { + if notify, ok := os.LookupEnv("NOTIFY_SOCKET"); ok { + args = append(args, fmt.Sprintf("--sdnotify-socket=%s", notify)) + } + } + if ctr.CgroupManager() == config.SystemdCgroupsManager && !ctr.config.NoCgroups && ctr.config.CgroupsMode != cgroupSplit { args = append(args, "-s") }